proxy: add panic recovery to Process.ProxyRequest (#363)

Switching to use httputil.ReverseProxy in #342 introduced a possible
panic if a client disconnects while streaming the body. Since llama-swap
does not use http.Server the recover() is not automatically there.

- introduce a recover() in Process.ProxyRequest to recover and log the
  event
- add TestProcess_ReverseProxyPanicIsHandled to reproduce and test the
  fix

fixes: #362

Makefile

@@ -33,7 +33,7 @@ test: proxy/ui_dist/placeholder.txt
 
 # for CI - full test (takes longer)
 test-all: proxy/ui_dist/placeholder.txt
-	go test -count=1 ./proxy/...
+	go test -race -count=1 ./proxy/...
 
 ui/node_modules:
 	cd ui && npm install
@@ -61,12 +61,12 @@ windows: ui
 # for testing proxy.Process
 simple-responder:
 	@echo "Building simple responder"
-	GOOS=darwin GOARCH=arm64 go build -o $(BUILD_DIR)/simple-responder_darwin_arm64 misc/simple-responder/simple-responder.go
+	GOOS=darwin GOARCH=arm64 go build -o $(BUILD_DIR)/simple-responder_darwin_arm64 cmd/simple-responder/simple-responder.go
-	GOOS=linux GOARCH=amd64 go build -o $(BUILD_DIR)/simple-responder_linux_amd64 misc/simple-responder/simple-responder.go
+	GOOS=linux GOARCH=amd64 go build -o $(BUILD_DIR)/simple-responder_linux_amd64 cmd/simple-responder/simple-responder.go
 
 simple-responder-windows:
 	@echo "Building simple responder for windows"
-	GOOS=windows GOARCH=amd64 go build -o $(BUILD_DIR)/simple-responder.exe misc/simple-responder/simple-responder.go
+	GOOS=windows GOARCH=amd64 go build -o $(BUILD_DIR)/simple-responder.exe cmd/simple-responder/simple-responder.go
 
 # Ensure build directory exists
 $(BUILD_DIR):
@@ -86,5 +86,11 @@ release:
 	echo "tagging new version: $$new_tag"; \
 	git tag "$$new_tag";
 
+GOOS ?= $(shell go env GOOS 2>/dev/null || echo linux)
+GOARCH ?= $(shell go env GOARCH 2>/dev/null || echo amd64)
+wol-proxy: $(BUILD_DIR)
+	@echo "Building wol-proxy"
+	go build -o $(BUILD_DIR)/wol-proxy-$(GOOS)-$(GOARCH)-$(shell date +%Y-%m-%d) cmd/wol-proxy/wol-proxy.go
+
 # Phony targets
-.PHONY: all clean ui mac linux windows simple-responder test test-all test-dev
+.PHONY: all clean ui mac linux windows simple-responder simple-responder-windows test test-all test-dev wol-proxy

README.md

@@ -5,74 +5,165 @@
 
 # llama-swap
 
-llama-swap is a light weight, transparent proxy server that provides automatic model swapping to llama.cpp's server.
+Run multiple LLM models on your machine and hot-swap between them as needed. llama-swap works with any OpenAI API-compatible server, giving you the flexibility to switch models without restarting your applications.
 
-Written in golang, it is very easy to install (single binary with no dependencies) and configure (single yaml file). To get started, download a pre-built binary, a provided docker images or Homebrew.
+Built in Go for performance and simplicity, llama-swap has zero dependencies and is incredibly easy to set up. Get started in minutes - just one binary and one configuration file.
 
 ## Features:
 
-- ✅ Easy to deploy: single binary with no dependencies
+- ✅ Easy to deploy and configure: one binary, one configuration file. no external dependencies
-- ✅ Easy to config: single yaml file
 - ✅ On-demand model switching
+- ✅ Use any local OpenAI compatible server (llama.cpp, vllm, tabbyAPI, etc)
+  - future proof, upgrade your inference servers at any time.
 - ✅ OpenAI API supported endpoints:
   - `v1/completions`
   - `v1/chat/completions`
   - `v1/embeddings`
   - `v1/audio/speech` ([#36](https://github.com/mostlygeek/llama-swap/issues/36))
   - `v1/audio/transcriptions` ([docs](https://github.com/mostlygeek/llama-swap/issues/41#issuecomment-2722637867))
-- ✅ llama-server (llama.cpp) supported endpoints:
+- ✅ llama-server (llama.cpp) supported endpoints
   - `v1/rerank`, `v1/reranking`, `/rerank`
   - `/infill` - for code infilling
   - `/completion` - for completion endpoint
-- ✅ llama-swap custom API endpoints
+- ✅ llama-swap API
   - `/ui` - web UI
-  - `/log` - remote log monitoring
+  - `/upstream/:model_id` - direct access to upstream server ([demo](https://github.com/mostlygeek/llama-swap/pull/31))
-  - `/upstream/:model_id` - direct access to upstream HTTP server ([demo](https://github.com/mostlygeek/llama-swap/pull/31))
+  - `/models/unload` - manually unload running models ([#58](https://github.com/mostlygeek/llama-swap/issues/58))
-  - `/unload` - manually unload running models ([#58](https://github.com/mostlygeek/llama-swap/issues/58))
   - `/running` - list currently running models ([#61](https://github.com/mostlygeek/llama-swap/issues/61))
+  - `/log` - remote log monitoring
   - `/health` - just returns "OK"
-- ✅ Run multiple models at once with `Groups` ([#107](https://github.com/mostlygeek/llama-swap/issues/107))
+- ✅ Customizable
-- ✅ Automatic unloading of models after timeout by setting a `ttl`
+  - Run multiple models at once with `Groups` ([#107](https://github.com/mostlygeek/llama-swap/issues/107))
-- ✅ Use any local OpenAI compatible server (llama.cpp, vllm, tabbyAPI, etc)
+  - Automatic unloading of models after timeout by setting a `ttl`
-- ✅ Reliable Docker and Podman support using `cmd` and `cmdStop` together
+  - Reliable Docker and Podman support using `cmd` and `cmdStop` together
-- ✅ Full control over server settings per model
+  - Preload models on startup with `hooks` ([#235](https://github.com/mostlygeek/llama-swap/pull/235))
-- ✅ Preload models on startup with `hooks` ([#235](https://github.com/mostlygeek/llama-swap/pull/235))
+
+### Web UI
+
+llama-swap includes a real time web interface for monitoring logs and controlling models:
+
+<img width="1360" height="963" alt="image" src="https://github.com/user-attachments/assets/adef4a8e-de0b-49db-885a-8f6dedae6799" />
+
+The Activity Page shows recent requests:
+
+<img width="1360" height="963" alt="image" src="https://github.com/user-attachments/assets/5f3edee6-d03a-4ae5-ae06-b20ac1f135bd" />
+
+## Installation
+
+llama-swap can be installed in multiple ways
+
+1. Docker
+2. Homebrew (OSX and Linux)
+3. WinGet
+4. From release binaries
+5. From source
+
+### Docker Install ([download images](https://github.com/mostlygeek/llama-swap/pkgs/container/llama-swap))
+
+Nightly container images with llama-swap and llama-server are built for multiple platforms (cuda, vulkan, intel, etc).
+
+```shell
+$ docker pull ghcr.io/mostlygeek/llama-swap:cuda
+
+# run with a custom configuration and models directory
+$ docker run -it --rm --runtime nvidia -p 9292:8080 \
+ -v /path/to/models:/models \
+ -v /path/to/custom/config.yaml:/app/config.yaml \
+ ghcr.io/mostlygeek/llama-swap:cuda
+```
+
+<details>
+<summary>
+more examples
+</summary>
+
+```shell
+# pull latest images per platform
+docker pull ghcr.io/mostlygeek/llama-swap:cpu
+docker pull ghcr.io/mostlygeek/llama-swap:cuda
+docker pull ghcr.io/mostlygeek/llama-swap:vulkan
+docker pull ghcr.io/mostlygeek/llama-swap:intel
+docker pull ghcr.io/mostlygeek/llama-swap:musa
+
+# tagged llama-swap, platform and llama-server version images
+docker pull ghcr.io/mostlygeek/llama-swap:v166-cuda-b6795
+
+```
+
+</details>
+
+### Homebrew Install (macOS/Linux)
+
+```shell
+brew tap mostlygeek/llama-swap
+brew install llama-swap
+llama-swap --config path/to/config.yaml --listen localhost:8080
+```
+
+### WinGet Install (Windows)
+
+> [!NOTE]
+> WinGet is maintained by community contributor [Dvd-Znf](https://github.com/Dvd-Znf) ([#327](https://github.com/mostlygeek/llama-swap/issues/327)). It is not an official part of llama-swap.
+
+```shell
+# install
+C:\> winget install llama-swap
+
+# upgrade
+C:\> winget upgrade llama-swap
+```
+
+### Pre-built Binaries
+
+Binaries are available on the [release](https://github.com/mostlygeek/llama-swap/releases) page for Linux, Mac, Windows and FreeBSD.
+
+### Building from source
+
+1. Building requires Go and Node.js (for UI).
+1. `git clone https://github.com/mostlygeek/llama-swap.git`
+1. `make clean all`
+1. look in the `build/` subdirectory for the llama-swap binary
+
+## Configuration
+
+```yaml
+# minimum viable config.yaml
+
+models:
+  model1:
+    cmd: llama-server --port ${PORT} --model /path/to/model.gguf
+```
+
+That's all you need to get started:
+
+1. `models` - holds all model configurations
+2. `model1` - the ID used in API calls
+3. `cmd` - the command to run to start the server.
+4. `${PORT}` - an automatically assigned port number
+
+Almost all configuration settings are optional and can be added one step at a time:
+
+- Advanced features
+  - `groups` to run multiple models at once
+  - `hooks` to run things on startup
+  - `macros` reusable snippets
+- Model customization
+  - `ttl` to automatically unload models
+  - `aliases` to use familiar model names (e.g., "gpt-4o-mini")
+  - `env` to pass custom environment variables to inference servers
+  - `cmdStop` gracefully stop Docker/Podman containers
+  - `useModelName` to override model names sent to upstream servers
+  - `${PORT}` automatic port variables for dynamic port assignment
+  - `filters` rewrite parts of requests before sending to the upstream server
+
+See the [configuration documentation](docs/configuration.md) for all options.
 
 ## How does llama-swap work?
 
-When a request is made to an OpenAI compatible endpoint, llama-swap will extract the `model` value and load the appropriate server configuration to serve it. If the wrong upstream server is running, it will be replaced with the correct one. This is where the "swap" part comes in. The upstream server is automatically swapped to the correct one to serve the request.
+When a request is made to an OpenAI compatible endpoint, llama-swap will extract the `model` value and load the appropriate server configuration to serve it. If the wrong upstream server is running, it will be replaced with the correct one. This is where the "swap" part comes in. The upstream server is automatically swapped to handle the request correctly.
 
 In the most basic configuration llama-swap handles one model at a time. For more advanced use cases, the `groups` feature allows multiple models to be loaded at the same time. You have complete control over how your system resources are used.
 
-## config.yaml
-
-llama-swap is managed entirely through a yaml configuration file.
-
-It can be very minimal to start:
-
-```yaml
-models:
-  "qwen2.5":
-    cmd: |
-      /path/to/llama-server
-      -hf bartowski/Qwen2.5-0.5B-Instruct-GGUF:Q4_K_M
-      --port ${PORT}
-```
-
-However, there are many more capabilities that llama-swap supports:
-
-- `groups` to run multiple models at once
-- `ttl` to automatically unload models
-- `macros` for reusable snippets
-- `aliases` to use familiar model names (e.g., "gpt-4o-mini")
-- `env` to pass custom environment variables to inference servers
-- `cmdStop` for to gracefully stop Docker/Podman containers
-- `useModelName` to override model names sent to upstream servers
-- `healthCheckTimeout` to control model startup wait times
-- `${PORT}` automatic port variables for dynamic port assignment
-
-See the [configuration documentation](https://github.com/mostlygeek/llama-swap/wiki/Configuration) in the wiki all options and examples.
-
 ## Reverse Proxy Configuration (nginx)
 
 If you deploy llama-swap behind nginx, disable response buffering for streaming endpoints. By default, nginx buffers responses which breaks Server‑Sent Events (SSE) and streaming chat completion. ([#236](https://github.com/mostlygeek/llama-swap/issues/236))
@@ -97,111 +188,7 @@ location /v1/chat/completions {
 
 As a safeguard, llama-swap also sets `X-Accel-Buffering: no` on SSE responses. However, explicitly disabling `proxy_buffering` at your reverse proxy is still recommended for reliable streaming behavior.
 
-## Web UI
+## Monitoring Logs on the CLI
-
-llama-swap includes a real time web interface for monitoring logs and models:
-
-<img width="1360" height="963" alt="image" src="https://github.com/user-attachments/assets/adef4a8e-de0b-49db-885a-8f6dedae6799" />
-
-The Activity Page shows recent requests:
-
-<img width="1360" height="963" alt="image" src="https://github.com/user-attachments/assets/5f3edee6-d03a-4ae5-ae06-b20ac1f135bd" />
-
-## Installation
-
-llama-swap can be installed in multiple ways
-
-1. Docker
-2. Homebrew (OSX and Linux)
-3. From release binaries
-4. From source
-
-### Docker Install ([download images](https://github.com/mostlygeek/llama-swap/pkgs/container/llama-swap))
-
-Docker images with llama-swap and llama-server are built nightly.
-
-```shell
-# use CPU inference comes with the example config above
-$ docker run -it --rm -p 9292:8080 ghcr.io/mostlygeek/llama-swap:cpu
-
-# qwen2.5 0.5B
-$ curl -s http://localhost:9292/v1/chat/completions \
-    -H "Content-Type: application/json" \
-    -H "Authorization: Bearer no-key" \
-    -d '{"model":"qwen2.5","messages": [{"role": "user","content": "tell me a joke"}]}' | \
-    jq -r '.choices[0].message.content'
-
-# SmolLM2 135M
-$ curl -s http://localhost:9292/v1/chat/completions \
-    -H "Content-Type: application/json" \
-    -H "Authorization: Bearer no-key" \
-    -d '{"model":"smollm2","messages": [{"role": "user","content": "tell me a joke"}]}' | \
-    jq -r '.choices[0].message.content'
-```
-
-<details>
-<summary>Docker images are built nightly with llama-server for cuda, intel, vulcan and musa.</summary>
-
-They include:
-
-- `ghcr.io/mostlygeek/llama-swap:cpu`
-- `ghcr.io/mostlygeek/llama-swap:cuda`
-- `ghcr.io/mostlygeek/llama-swap:intel`
-- `ghcr.io/mostlygeek/llama-swap:vulkan`
-- ROCm disabled until fixed in llama.cpp container
-
-Specific versions are also available and are tagged with the llama-swap, architecture and llama.cpp versions. For example: `ghcr.io/mostlygeek/llama-swap:v89-cuda-b4716`
-
-Beyond the demo you will likely want to run the containers with your downloaded models and custom configuration.
-
-```shell
-$ docker run -it --rm --runtime nvidia -p 9292:8080 \
-  -v /path/to/models:/models \
-  -v /path/to/custom/config.yaml:/app/config.yaml \
-  ghcr.io/mostlygeek/llama-swap:cuda
-```
-
-</details>
-
-### Homebrew Install (macOS/Linux)
-
-The latest release of `llama-swap` can be installed via [Homebrew](https://brew.sh).
-
-```shell
-# Set up tap and install formula
-brew tap mostlygeek/llama-swap
-brew install llama-swap
-# Run llama-swap
-llama-swap --config path/to/config.yaml --listen localhost:8080
-```
-
-This will install the `llama-swap` binary and make it available in your path. See the [configuration documentation](https://github.com/mostlygeek/llama-swap/wiki/Configuration)
-
-### Pre-built Binaries ([download](https://github.com/mostlygeek/llama-swap/releases))
-
-Binaries are available for Linux, Mac, Windows and FreeBSD. These are automatically published and are likely a few hours ahead of the docker releases. The binary install works with any OpenAI compatible server, not just llama-server.
-
-1. Download a [release](https://github.com/mostlygeek/llama-swap/releases) appropriate for your OS and architecture.
-1. Create a configuration file, see the [configuration documentation](https://github.com/mostlygeek/llama-swap/wiki/Configuration).
-1. Run the binary with `llama-swap --config path/to/config.yaml --listen localhost:8080`.
-   Available flags:
-   - `--config`: Path to the configuration file (default: `config.yaml`).
-   - `--listen`: Address and port to listen on (default: `:8080`).
-   - `--version`: Show version information and exit.
-   - `--watch-config`: Automatically reload the configuration file when it changes. This will wait for in-flight requests to complete then stop all running models (default: `false`).
-
-### Building from source
-
-1. Build requires golang and nodejs for the user interface.
-1. `git clone https://github.com/mostlygeek/llama-swap.git`
-1. `make clean all`
-1. Binaries will be in `build/` subdirectory
-
-## Monitoring Logs
-
-Open the `http://<host>:<port>/` with your browser to get a web interface with streaming logs.
-
-CLI access is also supported:
 
 ```shell
 # sends up to the last 10KB of logs
@@ -227,11 +214,11 @@ curl -Ns 'http://host/logs/stream?no-history'
 
 Any OpenAI compatible server would work. llama-swap was originally designed for llama-server and it is the best supported.
 
-For Python based inference servers like vllm or tabbyAPI it is recommended to run them via podman or docker. This provides clean environment isolation as well as responding correctly to `SIGTERM` signals to shutdown.
+For Python based inference servers like vllm or tabbyAPI it is recommended to run them via podman or docker. This provides clean environment isolation as well as responding correctly to `SIGTERM` signals for proper shutdown.
 
 ## Star History
 
 > [!NOTE]
-> ⭐️ Star this project to help others discover it! 
+> ⭐️ Star this project to help others discover it!
 
 [![Star History Chart](https://api.star-history.com/svg?repos=mostlygeek/llama-swap&type=Date)](https://www.star-history.com/#mostlygeek/llama-swap&Date)

cmd/wol-proxy/README.md

@@ -0,0 +1,27 @@
+# wol-proxy
+
+wol-proxy automatically wakes up a suspended llama-swap server using Wake-on-LAN when requests are received.
+
+When a request arrives and llama-swap is unavailable, wol-proxy sends a WOL packet and holds the request until the server becomes available. If the server doesn't respond within the timeout period (default: 60 seconds), the request is dropped.
+
+This utility helps conserve energy by allowing GPU-heavy servers to remain suspended when idle, as they can consume hundreds of watts even when not actively processing requests.
+
+## Usage
+
+```shell
+# minimal
+$ ./wol-proxy -mac BA:DC:0F:FE:E0:00 -upstream http://192.168.1.13:8080
+
+# everything
+$ ./wol-proxy -mac BA:DC:0F:FE:E0:00 -upstream http://192.168.1.13:8080 \
+    # use debug log level
+    -log debug \
+    # altenerative listening port
+    -listen localhost:9999 \
+    # seconds to hold requests waiting for upstream to be ready
+    -timeout 30
+```
+
+## API
+
+`GET /status` - that's it. Everything else is proxied to the upstream server.

cmd/wol-proxy/wol-proxy.go

@@ -0,0 +1,319 @@
+package main
+
+import (
+	"bufio"
+	"context"
+	"errors"
+	"flag"
+	"fmt"
+	"io"
+	"log/slog"
+	"net"
+	"net/http"
+	"net/http/httputil"
+	"net/url"
+	"os"
+	"os/signal"
+	"strings"
+	"sync"
+	"time"
+)
+
+var (
+	flagMac      = flag.String("mac", "", "mac address to send WoL packet to")
+	flagUpstream = flag.String("upstream", "", "upstream proxy address to send requests to")
+	flagListen   = flag.String("listen", ":8080", "listen address to listen on")
+	flagLog      = flag.String("log", "info", "log level (debug, info, warn, error)")
+	flagTimeout  = flag.Int("timeout", 60, "seconds requests wait for upstream response before failing")
+)
+
+func main() {
+	flag.Parse()
+
+	switch *flagLog {
+	case "debug":
+		slog.SetLogLoggerLevel(slog.LevelDebug)
+	case "info":
+		slog.SetLogLoggerLevel(slog.LevelInfo)
+	case "warn":
+		slog.SetLogLoggerLevel(slog.LevelWarn)
+	case "error":
+		slog.SetLogLoggerLevel(slog.LevelError)
+	default:
+		slog.Error("invalid log level", "logLevel", *flagLog)
+		return
+	}
+
+	// Validate flags
+	if *flagListen == "" {
+		slog.Error("listen address is required")
+		return
+	}
+
+	if *flagMac == "" {
+		slog.Error("mac address is required")
+		return
+	}
+
+	if *flagTimeout < 1 {
+		slog.Error("timeout must be greater than 0")
+		return
+	}
+
+	var upstreamURL *url.URL
+	var err error
+	// validate mac address
+	if _, err = net.ParseMAC(*flagMac); err != nil {
+		slog.Error("invalid mac address", "error", err)
+		return
+	}
+
+	if *flagUpstream == "" {
+		slog.Error("upstream proxy address is required")
+		return
+	} else {
+		upstreamURL, err = url.ParseRequestURI(*flagUpstream)
+		if err != nil {
+			slog.Error("error parsing upstream url", "error", err)
+			return
+		}
+	}
+
+	proxy := newProxy(upstreamURL)
+	server := &http.Server{
+		Addr:    *flagListen,
+		Handler: proxy,
+	}
+
+	// start the server
+	go func() {
+		slog.Info("server starting on", "address", *flagListen)
+		if err := server.ListenAndServe(); err != nil {
+			slog.Error("error starting server", "error", err)
+		}
+	}()
+
+	// graceful shutdown
+	ctx, _ := signal.NotifyContext(context.Background(), os.Interrupt)
+	<-ctx.Done()
+	server.Close()
+}
+
+type upstreamStatus string
+
+const (
+	notready upstreamStatus = "not ready"
+	ready    upstreamStatus = "ready"
+)
+
+type proxyServer struct {
+	upstreamProxy *httputil.ReverseProxy
+	failCount     int
+	statusMutex   sync.RWMutex
+	status        upstreamStatus
+}
+
+func newProxy(url *url.URL) *proxyServer {
+	p := httputil.NewSingleHostReverseProxy(url)
+	proxy := &proxyServer{
+		upstreamProxy: p,
+		status:        notready,
+		failCount:     0,
+	}
+
+	// start a goroutine to monitor upstream status via SSE
+	go func() {
+		eventsUrl := url.Scheme + "://" + url.Host + "/api/events"
+		client := &http.Client{
+			Timeout: 0, // No timeout for SSE connection
+		}
+
+		waitDuration := 10 * time.Second
+
+		for {
+			slog.Debug("connecting to SSE endpoint", "url", eventsUrl)
+
+			req, err := http.NewRequest("GET", eventsUrl, nil)
+			if err != nil {
+				slog.Warn("failed to create SSE request", "error", err)
+				proxy.setStatus(notready)
+				proxy.incFail(1)
+				time.Sleep(waitDuration)
+				continue
+			}
+
+			req.Header.Set("Accept", "text/event-stream")
+			req.Header.Set("Cache-Control", "no-cache")
+			req.Header.Set("Connection", "keep-alive")
+
+			resp, err := client.Do(req)
+			if err != nil {
+				slog.Error("failed to connect to SSE endpoint", "error", err)
+				proxy.setStatus(notready)
+				proxy.incFail(1)
+				time.Sleep(10 * time.Second)
+				continue
+			}
+
+			if resp.StatusCode != http.StatusOK {
+				slog.Warn("SSE endpoint returned non-OK status", "status", resp.StatusCode)
+				_, _ = io.Copy(io.Discard, resp.Body)
+				_ = resp.Body.Close()
+				proxy.setStatus(notready)
+				proxy.incFail(1)
+				time.Sleep(10 * time.Second)
+				continue
+			}
+
+			// Successfully connected to SSE endpoint
+			slog.Info("connected to SSE endpoint, upstream ready")
+			proxy.setStatus(ready)
+			proxy.resetFailures()
+
+			// Read from the SSE stream to detect disconnection
+			scanner := bufio.NewScanner(resp.Body)
+
+			// use a fairly large buffer to avoid scanner errors when reading large SSE events
+			buf := make([]byte, 0, 1024*1024*2)
+			scanner.Buffer(buf, 1024*1024*2)
+			events := 0
+			if slog.Default().Enabled(context.Background(), slog.LevelDebug) {
+				fmt.Print("Events: ")
+			}
+			for scanner.Scan() {
+				if slog.Default().Enabled(context.Background(), slog.LevelDebug) {
+					// Just read the events to keep connection alive
+					// We don't need to process the event data
+					events++
+					fmt.Printf("%d, ", events)
+				}
+			}
+			fmt.Println()
+			if err := scanner.Err(); err != nil {
+				slog.Error("error reading from SSE stream", "error", err)
+			}
+
+			// Connection closed or error occurred
+			_ = resp.Body.Close()
+			slog.Info("SSE connection closed, upstream not ready")
+			proxy.setStatus(notready)
+			proxy.incFail(1)
+
+			// Wait before reconnecting
+			time.Sleep(waitDuration)
+		}
+	}()
+
+	return proxy
+}
+
+func (p *proxyServer) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	if r.Method == "GET" && r.URL.Path == "/status" {
+		status := string(p.getStatus())
+		failCount := p.getFailures()
+		w.Header().Set("Content-Type", "text/plain")
+		w.WriteHeader(200)
+		fmt.Fprintf(w, "status: %s\n", status)
+		fmt.Fprintf(w, "failures: %d\n", failCount)
+		return
+	}
+
+	if p.getStatus() == notready {
+		path := r.URL.Path
+		if strings.HasPrefix(path, "/api/events") {
+			slog.Debug("Skipping wake up", "req", path)
+			w.WriteHeader(http.StatusNoContent)
+			return
+		}
+
+		slog.Info("upstream not ready, sending magic packet", "req", path, "from", r.RemoteAddr)
+		if err := sendMagicPacket(*flagMac); err != nil {
+			slog.Warn("failed to send magic WoL packet", "error", err)
+		}
+		ticker := time.NewTicker(250 * time.Millisecond)
+		timeout, cancel := context.WithTimeout(context.Background(), time.Duration(*flagTimeout)*time.Second)
+		defer cancel()
+	loop:
+		for {
+			select {
+			case <-timeout.Done():
+				slog.Info("timeout waiting for upstream to be ready")
+				http.Error(w, "timeout", http.StatusRequestTimeout)
+				return
+			case <-ticker.C:
+				if p.getStatus() == ready {
+					ticker.Stop()
+					break loop
+				}
+			}
+		}
+	}
+
+	p.upstreamProxy.ServeHTTP(w, r)
+}
+
+func (p *proxyServer) getStatus() upstreamStatus {
+	p.statusMutex.RLock()
+	defer p.statusMutex.RUnlock()
+	return p.status
+}
+
+func (p *proxyServer) setStatus(status upstreamStatus) {
+	p.statusMutex.Lock()
+	defer p.statusMutex.Unlock()
+	p.status = status
+}
+
+func (p *proxyServer) incFail(num int) {
+	p.statusMutex.Lock()
+	defer p.statusMutex.Unlock()
+	p.failCount += num
+}
+
+func (p *proxyServer) getFailures() int {
+	p.statusMutex.RLock()
+	defer p.statusMutex.RUnlock()
+	return p.failCount
+}
+
+func (p *proxyServer) resetFailures() {
+	p.statusMutex.Lock()
+	defer p.statusMutex.Unlock()
+	p.failCount = 0
+}
+
+func sendMagicPacket(macAddr string) error {
+	hwAddr, err := net.ParseMAC(macAddr)
+	if err != nil {
+		return err
+	}
+
+	if len(hwAddr) != 6 {
+		return errors.New("invalid MAC address")
+	}
+
+	// Create the magic packet.
+	packet := make([]byte, 102)
+	// Add 6 bytes of 0xFF.
+	for i := 0; i < 6; i++ {
+		packet[i] = 0xFF
+	}
+	// Repeat the MAC address 16 times.
+	for i := 1; i <= 16; i++ {
+		copy(packet[i*6:], hwAddr)
+	}
+
+	// Send the packet using UDP.
+	addr := net.UDPAddr{
+		IP:   net.IPv4bcast,
+		Port: 9,
+	}
+	conn, err := net.DialUDP("udp", nil, &addr)
+	if err != nil {
+		return err
+	}
+	defer conn.Close()
+
+	_, err = conn.Write(packet)
+	return err
+}

docs/configuration.md

@@ -0,0 +1,386 @@
+# config.yaml
+
+llama-swap is designed to be very simple: one binary, one configuration file.
+
+## minimal viable config
+
+```yaml
+models:
+  model1:
+    cmd: llama-server --port ${PORT} --model /path/to/model.gguf
+```
+
+This is enough to launch `llama-server` to serve `model1`. Of course, llama-swap is about making it possible to serve many models:
+
+```yaml
+models:
+  model1:
+    cmd: llama-server --port ${PORT} -m /path/to/model.gguf
+  model2:
+    cmd: llama-server --port ${PORT} -m /path/to/another_model.gguf
+  model3:
+    cmd: llama-server --port ${PORT} -m /path/to/third_model.gguf
+```
+
+With this configuration models will be hot swapped and loaded on demand. The special `${PORT}` macro provides a unique port per model. Useful if you want to run multiple models at the same time with the `groups` feature.
+
+## Advanced control with `cmd`
+
+llama-swap is also about customizability. You can use any CLI flag available:
+
+```yaml
+models:
+  model1:
+    cmd: | # support for multi-line
+      llama-server --PORT ${PORT} -m /path/to/model.gguf
+      --ctx-size 8192
+      --jinja
+      --cache-type-k q8_0
+      --cache-type-v q8_0
+```
+
+## Support for any OpenAI API compatible server
+
+llama-swap supports any OpenAI API compatible server. If you can run it on the CLI llama-swap will be able to manage it. Even if it's run in Docker or Podman containers.
+
+```yaml
+models:
+  "Q3-30B-CODER-VLLM":
+    name: "Qwen3 30B Coder vllm AWQ (Q3-30B-CODER-VLLM)"
+    # cmdStop provides a reliable way to stop containers
+    cmdStop: docker stop vllm-coder
+    cmd: |
+      docker run --init --rm --name vllm-coder
+        --runtime=nvidia --gpus '"device=2,3"'
+        --shm-size=16g
+        -v /mnt/nvme/vllm-cache:/root/.cache
+        -v /mnt/ssd-extra/models:/models -p ${PORT}:8000
+        vllm/vllm-openai:v0.10.0
+        --model "/models/cpatonn/Qwen3-Coder-30B-A3B-Instruct-AWQ"
+        --served-model-name "Q3-30B-CODER-VLLM"
+        --enable-expert-parallel
+        --swap-space 16
+        --max-num-seqs 512
+        --max-model-len 65536
+        --max-seq-len-to-capture 65536
+        --gpu-memory-utilization 0.9
+        --tensor-parallel-size 2
+        --trust-remote-code
+```
+
+## Many more features..
+
+llama-swap supports many more features to customize how you want to manage your environment.
+
+| Feature   | Description                                    |
+| --------- | ---------------------------------------------- |
+| `ttl`     | automatic unloading of models after a timeout  |
+| `macros`  | reusable snippets to use in configurations     |
+| `groups`  | run multiple models at a time                  |
+| `hooks`   | event driven functionality                     |
+| `env`     | define environment variables per model         |
+| `aliases` | serve a model with different names             |
+| `filters` | modify requests before sending to the upstream |
+| `...`     | And many more tweaks                           |
+
+## Full Configuration Example
+
+> [!NOTE]
+> This is a copy of `config.example.yaml`. Always check that for the most up to date examples.
+
+```yaml
+# llama-swap YAML configuration example
+# -------------------------------------
+#
+# 💡 Tip - Use an LLM with this file!
+# ====================================
+#  This example configuration is written to be LLM friendly. Try
+#  copying this file into an LLM and asking it to explain or generate
+#  sections for you.
+# ====================================
+
+# Usage notes:
+# - Below are all the available configuration options for llama-swap.
+# - Settings noted as "required" must be in your configuration file
+# - Settings noted as "optional" can be omitted
+
+# healthCheckTimeout: number of seconds to wait for a model to be ready to serve requests
+# - optional, default: 120
+# - minimum value is 15 seconds, anything less will be set to this value
+healthCheckTimeout: 500
+
+# logLevel: sets the logging value
+# - optional, default: info
+# - Valid log levels: debug, info, warn, error
+logLevel: info
+
+# metricsMaxInMemory: maximum number of metrics to keep in memory
+# - optional, default: 1000
+# - controls how many metrics are stored in memory before older ones are discarded
+# - useful for limiting memory usage when processing large volumes of metrics
+metricsMaxInMemory: 1000
+
+# startPort: sets the starting port number for the automatic ${PORT} macro.
+# - optional, default: 5800
+# - the ${PORT} macro can be used in model.cmd and model.proxy settings
+# - it is automatically incremented for every model that uses it
+startPort: 10001
+
+# macros: a dictionary of string substitutions
+# - optional, default: empty dictionary
+# - macros are reusable snippets
+# - used in a model's cmd, cmdStop, proxy, checkEndpoint, filters.stripParams
+# - useful for reducing common configuration settings
+# - macro names are strings and must be less than 64 characters
+# - macro names must match the regex ^[a-zA-Z0-9_-]+$
+# - macro names must not be a reserved name: PORT or MODEL_ID
+# - macro values can be numbers, bools, or strings
+# - macros can contain other macros, but they must be defined before they are used
+macros:
+  # Example of a multi-line macro
+  "latest-llama": >
+    /path/to/llama-server/llama-server-ec9e0301
+    --port ${PORT}
+
+  "default_ctx": 4096
+
+  # Example of macro-in-macro usage. macros can contain other macros
+  # but they must be previously declared.
+  "default_args": "--ctx-size ${default_ctx}"
+
+# models: a dictionary of model configurations
+# - required
+# - each key is the model's ID, used in API requests
+# - model settings have default values that are used if they are not defined here
+# - the model's ID is available in the ${MODEL_ID} macro, also available in macros defined above
+# - below are examples of the all the settings a model can have
+models:
+  # keys are the model names used in API requests
+  "llama":
+    # macros: a dictionary of string substitutions specific to this model
+    # - optional, default: empty dictionary
+    # - macros defined here override macros defined in the global macros section
+    # - model level macros follow the same rules as global macros
+    macros:
+      "default_ctx": 16384
+      "temp": 0.7
+
+    # cmd: the command to run to start the inference server.
+    # - required
+    # - it is just a string, similar to what you would run on the CLI
+    # - using `|` allows for comments in the command, these will be parsed out
+    # - macros can be used within cmd
+    cmd: |
+      # ${latest-llama} is a macro that is defined above
+      ${latest-llama}
+      --model path/to/llama-8B-Q4_K_M.gguf
+      --ctx-size ${default_ctx}
+      --temperature ${temp}
+
+    # name: a display name for the model
+    # - optional, default: empty string
+    # - if set, it will be used in the v1/models API response
+    # - if not set, it will be omitted in the JSON model record
+    name: "llama 3.1 8B"
+
+    # description: a description for the model
+    # - optional, default: empty string
+    # - if set, it will be used in the v1/models API response
+    # - if not set, it will be omitted in the JSON model record
+    description: "A small but capable model used for quick testing"
+
+    # env: define an array of environment variables to inject into cmd's environment
+    # - optional, default: empty array
+    # - each value is a single string
+    # - in the format: ENV_NAME=value
+    env:
+      - "CUDA_VISIBLE_DEVICES=0,1,2"
+
+    # proxy: the URL where llama-swap routes API requests
+    # - optional, default: http://localhost:${PORT}
+    # - if you used ${PORT} in cmd this can be omitted
+    # - if you use a custom port in cmd this *must* be set
+    proxy: http://127.0.0.1:8999
+
+    # aliases: alternative model names that this model configuration is used for
+    # - optional, default: empty array
+    # - aliases must be unique globally
+    # - useful for impersonating a specific model
+    aliases:
+      - "gpt-4o-mini"
+      - "gpt-3.5-turbo"
+
+    # checkEndpoint: URL path to check if the server is ready
+    # - optional, default: /health
+    # - endpoint is expected to return an HTTP 200 response
+    # - all requests wait until the endpoint is ready or fails
+    # - use "none" to skip endpoint health checking
+    checkEndpoint: /custom-endpoint
+
+    # ttl: automatically unload the model after ttl seconds
+    # - optional, default: 0
+    # - ttl values must be a value greater than 0
+    # - a value of 0 disables automatic unloading of the model
+    ttl: 60
+
+    # useModelName: override the model name that is sent to upstream server
+    # - optional, default: ""
+    # - useful for when the upstream server expects a specific model name that
+    #   is different from the model's ID
+    useModelName: "qwen:qwq"
+
+    # filters: a dictionary of filter settings
+    # - optional, default: empty dictionary
+    # - only stripParams is currently supported
+    filters:
+      # stripParams: a comma separated list of parameters to remove from the request
+      # - optional, default: ""
+      # - useful for server side enforcement of sampling parameters
+      # - the `model` parameter can never be removed
+      # - can be any JSON key in the request body
+      # - recommended to stick to sampling parameters
+      stripParams: "temperature, top_p, top_k"
+
+    # metadata: a dictionary of arbitrary values that are included in /v1/models
+    # - optional, default: empty dictionary
+    # - while metadata can contains complex types it is recommended to keep it simple
+    # - metadata is only passed through in /v1/models responses
+    metadata:
+      # port will remain an integer
+      port: ${PORT}
+
+      # the ${temp} macro will remain a float
+      temperature: ${temp}
+      note: "The ${MODEL_ID} is running on port ${PORT} temp=${temp}, context=${default_ctx}"
+
+      a_list:
+        - 1
+        - 1.23
+        - "macros are OK in list and dictionary types: ${MODEL_ID}"
+
+      an_obj:
+        a: "1"
+        b: 2
+        # objects can contain complex types with macro substitution
+        # becomes: c: [0.7, false, "model: llama"]
+        c: ["${temp}", false, "model: ${MODEL_ID}"]
+
+    # concurrencyLimit: overrides the allowed number of active parallel requests to a model
+    # - optional, default: 0
+    # - useful for limiting the number of active parallel requests a model can process
+    # - must be set per model
+    # - any number greater than 0 will override the internal default value of 10
+    # - any requests that exceeds the limit will receive an HTTP 429 Too Many Requests response
+    # - recommended to be omitted and the default used
+    concurrencyLimit: 0
+
+  # Unlisted model example:
+  "qwen-unlisted":
+    # unlisted: boolean, true or false
+    # - optional, default: false
+    # - unlisted models do not show up in /v1/models api requests
+    # - can be requested as normal through all apis
+    unlisted: true
+    cmd: llama-server --port ${PORT} -m Llama-3.2-1B-Instruct-Q4_K_M.gguf -ngl 0
+
+  # Docker example:
+  # container runtimes like Docker and Podman can be used reliably with
+  # a combination of cmd, cmdStop, and ${MODEL_ID}
+  "docker-llama":
+    proxy: "http://127.0.0.1:${PORT}"
+    cmd: |
+      docker run --name ${MODEL_ID}
+      --init --rm -p ${PORT}:8080 -v /mnt/nvme/models:/models
+      ghcr.io/ggml-org/llama.cpp:server
+      --model '/models/Qwen2.5-Coder-0.5B-Instruct-Q4_K_M.gguf'
+
+    # cmdStop: command to run to stop the model gracefully
+    # - optional, default: ""
+    # - useful for stopping commands managed by another system
+    # - the upstream's process id is available in the ${PID} macro
+    #
+    # When empty, llama-swap has this default behaviour:
+    # - on POSIX systems: a SIGTERM signal is sent
+    # - on Windows, calls taskkill to stop the process
+    # - processes have 5 seconds to shutdown until forceful termination is attempted
+    cmdStop: docker stop ${MODEL_ID}
+
+# groups: a dictionary of group settings
+# - optional, default: empty dictionary
+# - provides advanced controls over model swapping behaviour
+# - using groups some models can be kept loaded indefinitely, while others are swapped out
+# - model IDs must be defined in the Models section
+# - a model can only be a member of one group
+# - group behaviour is controlled via the `swap`, `exclusive` and `persistent` fields
+# - see issue #109 for details
+#
+# NOTE: the example below uses model names that are not defined above for demonstration purposes
+groups:
+  # group1 works the same as the default behaviour of llama-swap where only one model is allowed
+  # to run a time across the whole llama-swap instance
+  "group1":
+    # swap: controls the model swapping behaviour in within the group
+    # - optional, default: true
+    # - true : only one model is allowed to run at a time
+    # - false: all models can run together, no swapping
+    swap: true
+
+    # exclusive: controls how the group affects other groups
+    # - optional, default: true
+    # - true: causes all other groups to unload when this group runs a model
+    # - false: does not affect other groups
+    exclusive: true
+
+    # members references the models defined above
+    # required
+    members:
+      - "llama"
+      - "qwen-unlisted"
+
+  # Example:
+  # - in group2 all models can run at the same time
+  # - when a different group is loaded it causes all running models in this group to unload
+  "group2":
+    swap: false
+
+    # exclusive: false does not unload other groups when a model in group2 is requested
+    # - the models in group2 will be loaded but will not unload any other groups
+    exclusive: false
+    members:
+      - "docker-llama"
+      - "modelA"
+      - "modelB"
+
+  # Example:
+  # - a persistent group, prevents other groups from unloading it
+  "forever":
+    # persistent: prevents over groups from unloading the models in this group
+    # - optional, default: false
+    # - does not affect individual model behaviour
+    persistent: true
+
+    # set swap/exclusive to false to prevent swapping inside the group
+    # and the unloading of other groups
+    swap: false
+    exclusive: false
+    members:
+      - "forever-modelA"
+      - "forever-modelB"
+      - "forever-modelc"
+
+# hooks: a dictionary of event triggers and actions
+# - optional, default: empty dictionary
+# - the only supported hook is on_startup
+hooks:
+  # on_startup: a dictionary of actions to perform on startup
+  # - optional, default: empty dictionary
+  # - the only supported action is preload
+  on_startup:
+    # preload: a list of model ids to load on startup
+    # - optional, default: empty list
+    # - model names must match keys in the models sections
+    # - when preloading multiple models at once, define a group
+    #   otherwise models will be loaded and swapped out
+    preload:
+      - "llama"
+```

llama-swap.go

@@ -28,7 +28,9 @@ var (
 func main() {
 	// Define a command-line flag for the port
 	configPath := flag.String("config", "config.yaml", "config file name")
-	listenStr := flag.String("listen", ":8080", "listen ip/port")
+	listenStr := flag.String("listen", "", "listen ip/port")
+	certFile := flag.String("tls-cert-file", "", "TLS certificate file")
+	keyFile := flag.String("tls-key-file", "", "TLS key file")
 	showVersion := flag.Bool("version", false, "show version of build")
 	watchConfig := flag.Bool("watch-config", false, "Automatically reload config file on change")
 
@@ -55,6 +57,23 @@ func main() {
 		gin.SetMode(gin.ReleaseMode)
 	}
 
+	// Validate TLS flags.
+	var useTLS = (*certFile != "" && *keyFile != "")
+	if (*certFile != "" && *keyFile == "") ||
+		(*certFile == "" && *keyFile != "") {
+		fmt.Println("Error: Both --tls-cert-file and --tls-key-file must be provided for TLS.")
+		os.Exit(1)
+	}
+
+	// Set default ports.
+	if *listenStr == "" {
+		defaultPort := ":8080"
+		if useTLS {
+			defaultPort = ":8443"
+		}
+		listenStr = &defaultPort
+	}
+
 	// Setup channels for server management
 	exitChan := make(chan struct{})
 	sigChan := make(chan os.Signal, 1)
@@ -167,9 +186,16 @@ func main() {
 	}()
 
 	// Start server
-	fmt.Printf("llama-swap listening on %s\n", *listenStr)
 	go func() {
-		if err := srv.ListenAndServe(); err != nil && err != http.ErrServerClosed {
+		var err error
+		if useTLS {
+			fmt.Printf("llama-swap listening with TLS on https://%s\n", *listenStr)
+			err = srv.ListenAndServeTLS(*certFile, *keyFile)
+		} else {
+			fmt.Printf("llama-swap listening on http://%s\n", *listenStr)
+			err = srv.ListenAndServe()
+		}
+		if err != nil && err != http.ErrServerClosed {
 			log.Fatalf("Fatal server error: %v\n", err)
 		}
 	}()

proxy/config/config.go

@@ -3,6 +3,7 @@ package config
 import (
 	"fmt"
 	"io"
+	"net/url"
 	"os"
 	"regexp"
 	"runtime"
@@ -342,6 +343,13 @@ func LoadConfigFromReader(r io.Reader) (Config, error) {
 			}
 		}
 
+		// Validate the proxy URL.
+		if _, err := url.Parse(modelConfig.Proxy); err != nil {
+			return Config{}, fmt.Errorf(
+				"model %s: invalid proxy URL: %w", modelId, err,
+			)
+		}
+
 		config.Models[modelId] = modelConfig
 	}
 

proxy/metrics_middleware.go

@@ -1,184 +0,0 @@
-package proxy
-
-import (
-	"bytes"
-	"fmt"
-	"io"
-	"net/http"
-	"strings"
-	"time"
-
-	"github.com/gin-gonic/gin"
-	"github.com/tidwall/gjson"
-)
-
-type MetricsRecorder struct {
-	metricsMonitor *MetricsMonitor
-	realModelName  string
-	//	isStreaming    bool
-	startTime time.Time
-}
-
-// MetricsMiddleware sets up the MetricsResponseWriter for capturing upstream requests
-func MetricsMiddleware(pm *ProxyManager) gin.HandlerFunc {
-	return func(c *gin.Context) {
-		bodyBytes, err := io.ReadAll(c.Request.Body)
-		if err != nil {
-			pm.sendErrorResponse(c, http.StatusBadRequest, "could not ready request body")
-			c.Abort()
-			return
-		}
-		c.Request.Body = io.NopCloser(bytes.NewBuffer(bodyBytes))
-
-		requestedModel := gjson.GetBytes(bodyBytes, "model").String()
-		if requestedModel == "" {
-			pm.sendErrorResponse(c, http.StatusBadRequest, "missing or invalid 'model' key")
-			c.Abort()
-			return
-		}
-
-		realModelName, found := pm.config.RealModelName(requestedModel)
-		if !found {
-			pm.sendErrorResponse(c, http.StatusBadRequest, fmt.Sprintf("could not find real modelID for %s", requestedModel))
-			c.Abort()
-			return
-		}
-
-		writer := &MetricsResponseWriter{
-			ResponseWriter: c.Writer,
-			metricsRecorder: &MetricsRecorder{
-				metricsMonitor: pm.metricsMonitor,
-				realModelName:  realModelName,
-				startTime:      time.Now(),
-			},
-		}
-		c.Writer = writer
-		c.Next()
-
-		// check for streaming response
-		if strings.Contains(c.Writer.Header().Get("Content-Type"), "text/event-stream") {
-			writer.metricsRecorder.processStreamingResponse(writer.body)
-		} else {
-			writer.metricsRecorder.processNonStreamingResponse(writer.body)
-		}
-	}
-}
-
-func (rec *MetricsRecorder) parseAndRecordMetrics(jsonData gjson.Result) bool {
-	usage := jsonData.Get("usage")
-	timings := jsonData.Get("timings")
-	if !usage.Exists() && !timings.Exists() {
-		return false
-	}
-
-	// default values
-	cachedTokens := -1 // unknown or missing data
-	outputTokens := 0
-	inputTokens := 0
-
-	// timings data
-	tokensPerSecond := -1.0
-	promptPerSecond := -1.0
-	durationMs := int(time.Since(rec.startTime).Milliseconds())
-
-	if usage.Exists() {
-		outputTokens = int(jsonData.Get("usage.completion_tokens").Int())
-		inputTokens = int(jsonData.Get("usage.prompt_tokens").Int())
-	}
-
-	// use llama-server's timing data for tok/sec and duration as it is more accurate
-	if timings.Exists() {
-		inputTokens = int(jsonData.Get("timings.prompt_n").Int())
-		outputTokens = int(jsonData.Get("timings.predicted_n").Int())
-		promptPerSecond = jsonData.Get("timings.prompt_per_second").Float()
-		tokensPerSecond = jsonData.Get("timings.predicted_per_second").Float()
-		durationMs = int(jsonData.Get("timings.prompt_ms").Float() + jsonData.Get("timings.predicted_ms").Float())
-
-		if cachedValue := jsonData.Get("timings.cache_n"); cachedValue.Exists() {
-			cachedTokens = int(cachedValue.Int())
-		}
-	}
-
-	rec.metricsMonitor.addMetrics(TokenMetrics{
-		Timestamp:       time.Now(),
-		Model:           rec.realModelName,
-		CachedTokens:    cachedTokens,
-		InputTokens:     inputTokens,
-		OutputTokens:    outputTokens,
-		PromptPerSecond: promptPerSecond,
-		TokensPerSecond: tokensPerSecond,
-		DurationMs:      durationMs,
-	})
-
-	return true
-}
-
-func (rec *MetricsRecorder) processStreamingResponse(body []byte) {
-	// Iterate **backwards** through the lines looking for the data payload with
-	// usage data
-	lines := bytes.Split(body, []byte("\n"))
-
-	for i := len(lines) - 1; i >= 0; i-- {
-		line := bytes.TrimSpace(lines[i])
-		if len(line) == 0 {
-			continue
-		}
-
-		// SSE payload always follows "data:"
-		prefix := []byte("data:")
-		if !bytes.HasPrefix(line, prefix) {
-			continue
-		}
-		data := bytes.TrimSpace(line[len(prefix):])
-
-		if len(data) == 0 {
-			continue
-		}
-
-		if bytes.Equal(data, []byte("[DONE]")) {
-			// [DONE] line itself contains nothing of interest.
-			continue
-		}
-
-		if gjson.ValidBytes(data) {
-			if rec.parseAndRecordMetrics(gjson.ParseBytes(data)) {
-				return // short circuit if a metric was recorded
-			}
-		}
-	}
-}
-
-func (rec *MetricsRecorder) processNonStreamingResponse(body []byte) {
-	if len(body) == 0 {
-		return
-	}
-
-	// Parse JSON to extract usage information
-	if gjson.ValidBytes(body) {
-		rec.parseAndRecordMetrics(gjson.ParseBytes(body))
-	}
-}
-
-// MetricsResponseWriter captures the entire response for non-streaming
-type MetricsResponseWriter struct {
-	gin.ResponseWriter
-	body            []byte
-	metricsRecorder *MetricsRecorder
-}
-
-func (w *MetricsResponseWriter) Write(b []byte) (int, error) {
-	n, err := w.ResponseWriter.Write(b)
-	if err != nil {
-		return n, err
-	}
-	w.body = append(w.body, b...)
-	return n, nil
-}
-
-func (w *MetricsResponseWriter) WriteHeader(statusCode int) {
-	w.ResponseWriter.WriteHeader(statusCode)
-}
-
-func (w *MetricsResponseWriter) Header() http.Header {
-	return w.ResponseWriter.Header()
-}

proxy/metrics_monitor.go

@@ -1,12 +1,18 @@
 package proxy
 
 import (
+	"bytes"
 	"encoding/json"
+	"fmt"
+	"io"
+	"net/http"
+	"strings"
 	"sync"
 	"time"
 
+	"github.com/gin-gonic/gin"
 	"github.com/mostlygeek/llama-swap/event"
-	"github.com/mostlygeek/llama-swap/proxy/config"
+	"github.com/tidwall/gjson"
 )
 
 // TokenMetrics represents parsed token statistics from llama-server logs
@@ -31,21 +37,18 @@ func (e TokenMetricsEvent) Type() uint32 {
 	return TokenMetricsEventID // defined in events.go
 }
 
-// MetricsMonitor parses llama-server output for token statistics
+// metricsMonitor parses llama-server output for token statistics
-type MetricsMonitor struct {
+type metricsMonitor struct {
 	mu         sync.RWMutex
 	metrics    []TokenMetrics
 	maxMetrics int
 	nextID     int
+	logger     *LogMonitor
 }
 
-func NewMetricsMonitor(config *config.Config) *MetricsMonitor {
+func newMetricsMonitor(logger *LogMonitor, maxMetrics int) *metricsMonitor {
-	maxMetrics := config.MetricsMaxInMemory
+	mp := &metricsMonitor{
-	if maxMetrics <= 0 {
+		logger:     logger,
-		maxMetrics = 1000 // Default fallback
-	}
-
-	mp := &MetricsMonitor{
 		maxMetrics: maxMetrics,
 	}
 
@@ -53,7 +56,7 @@ func NewMetricsMonitor(config *config.Config) *MetricsMonitor {
 }
 
 // addMetrics adds a new metric to the collection and publishes an event
-func (mp *MetricsMonitor) addMetrics(metric TokenMetrics) {
+func (mp *metricsMonitor) addMetrics(metric TokenMetrics) {
 	mp.mu.Lock()
 	defer mp.mu.Unlock()
 
@@ -66,8 +69,8 @@ func (mp *MetricsMonitor) addMetrics(metric TokenMetrics) {
 	event.Emit(TokenMetricsEvent{Metrics: metric})
 }
 
-// GetMetrics returns a copy of the current metrics
+// getMetrics returns a copy of the current metrics
-func (mp *MetricsMonitor) GetMetrics() []TokenMetrics {
+func (mp *metricsMonitor) getMetrics() []TokenMetrics {
 	mp.mu.RLock()
 	defer mp.mu.RUnlock()
 
@@ -76,9 +79,189 @@ func (mp *MetricsMonitor) GetMetrics() []TokenMetrics {
 	return result
 }
 
-// GetMetricsJSON returns metrics as JSON
+// getMetricsJSON returns metrics as JSON
-func (mp *MetricsMonitor) GetMetricsJSON() ([]byte, error) {
+func (mp *metricsMonitor) getMetricsJSON() ([]byte, error) {
 	mp.mu.RLock()
 	defer mp.mu.RUnlock()
 	return json.Marshal(mp.metrics)
 }
+
+// wrapHandler wraps the proxy handler to extract token metrics
+// if wrapHandler returns an error it is safe to assume that no
+// data was sent to the client
+func (mp *metricsMonitor) wrapHandler(
+	modelID string,
+	writer gin.ResponseWriter,
+	request *http.Request,
+	next func(modelID string, w http.ResponseWriter, r *http.Request) error,
+) error {
+	recorder := newBodyCopier(writer)
+	if err := next(modelID, recorder, request); err != nil {
+		return err
+	}
+
+	// after this point we have to assume that data was sent to the client
+	// and we can only log errors but not send them to clients
+
+	if recorder.Status() != http.StatusOK {
+		mp.logger.Warnf("metrics skipped, HTTP status=%d, path=%s", recorder.Status(), request.URL.Path)
+		return nil
+	}
+
+	body := recorder.body.Bytes()
+	if len(body) == 0 {
+		mp.logger.Warn("metrics skipped, empty body")
+		return nil
+	}
+
+	if strings.Contains(recorder.Header().Get("Content-Type"), "text/event-stream") {
+		if tm, err := processStreamingResponse(modelID, recorder.StartTime(), body); err != nil {
+			mp.logger.Warnf("error processing streaming response: %v, path=%s", err, request.URL.Path)
+		} else {
+			mp.addMetrics(tm)
+		}
+	} else {
+		if gjson.ValidBytes(body) {
+			if tm, err := parseMetrics(modelID, recorder.StartTime(), gjson.ParseBytes(body)); err != nil {
+				mp.logger.Warnf("error parsing metrics: %v, path=%s", err, request.URL.Path)
+			} else {
+				mp.addMetrics(tm)
+			}
+		} else {
+			mp.logger.Warnf("metrics skipped, invalid JSON in response body path=%s", request.URL.Path)
+		}
+	}
+
+	return nil
+}
+
+func processStreamingResponse(modelID string, start time.Time, body []byte) (TokenMetrics, error) {
+	// Iterate **backwards** through the body looking for the data payload with
+	// usage data. This avoids allocating a slice of all lines via bytes.Split.
+
+	// Start from the end of the body and scan backwards for newlines
+	pos := len(body)
+	for pos > 0 {
+		// Find the previous newline (or start of body)
+		lineStart := bytes.LastIndexByte(body[:pos], '\n')
+		if lineStart == -1 {
+			lineStart = 0
+		} else {
+			lineStart++ // Move past the newline
+		}
+
+		line := bytes.TrimSpace(body[lineStart:pos])
+		pos = lineStart - 1 // Move position before the newline for next iteration
+
+		if len(line) == 0 {
+			continue
+		}
+
+		// SSE payload always follows "data:"
+		prefix := []byte("data:")
+		if !bytes.HasPrefix(line, prefix) {
+			continue
+		}
+		data := bytes.TrimSpace(line[len(prefix):])
+
+		if len(data) == 0 {
+			continue
+		}
+
+		if bytes.Equal(data, []byte("[DONE]")) {
+			// [DONE] line itself contains nothing of interest.
+			continue
+		}
+
+		if gjson.ValidBytes(data) {
+			return parseMetrics(modelID, start, gjson.ParseBytes(data))
+		}
+	}
+
+	return TokenMetrics{}, fmt.Errorf("no valid JSON data found in stream")
+}
+
+func parseMetrics(modelID string, start time.Time, jsonData gjson.Result) (TokenMetrics, error) {
+	usage := jsonData.Get("usage")
+	timings := jsonData.Get("timings")
+	if !usage.Exists() && !timings.Exists() {
+		return TokenMetrics{}, fmt.Errorf("no usage or timings data found")
+	}
+	// default values
+	cachedTokens := -1 // unknown or missing data
+	outputTokens := 0
+	inputTokens := 0
+
+	// timings data
+	tokensPerSecond := -1.0
+	promptPerSecond := -1.0
+	durationMs := int(time.Since(start).Milliseconds())
+
+	if usage.Exists() {
+		outputTokens = int(jsonData.Get("usage.completion_tokens").Int())
+		inputTokens = int(jsonData.Get("usage.prompt_tokens").Int())
+	}
+
+	// use llama-server's timing data for tok/sec and duration as it is more accurate
+	if timings.Exists() {
+		inputTokens = int(jsonData.Get("timings.prompt_n").Int())
+		outputTokens = int(jsonData.Get("timings.predicted_n").Int())
+		promptPerSecond = jsonData.Get("timings.prompt_per_second").Float()
+		tokensPerSecond = jsonData.Get("timings.predicted_per_second").Float()
+		durationMs = int(jsonData.Get("timings.prompt_ms").Float() + jsonData.Get("timings.predicted_ms").Float())
+
+		if cachedValue := jsonData.Get("timings.cache_n"); cachedValue.Exists() {
+			cachedTokens = int(cachedValue.Int())
+		}
+	}
+
+	return TokenMetrics{
+		Timestamp:       time.Now(),
+		Model:           modelID,
+		CachedTokens:    cachedTokens,
+		InputTokens:     inputTokens,
+		OutputTokens:    outputTokens,
+		PromptPerSecond: promptPerSecond,
+		TokensPerSecond: tokensPerSecond,
+		DurationMs:      durationMs,
+	}, nil
+}
+
+// responseBodyCopier records the response body and writes to the original response writer
+// while also capturing it in a buffer for later processing
+type responseBodyCopier struct {
+	gin.ResponseWriter
+	body  *bytes.Buffer
+	tee   io.Writer
+	start time.Time
+}
+
+func newBodyCopier(w gin.ResponseWriter) *responseBodyCopier {
+	bodyBuffer := &bytes.Buffer{}
+	return &responseBodyCopier{
+		ResponseWriter: w,
+		body:           bodyBuffer,
+		tee:            io.MultiWriter(w, bodyBuffer),
+	}
+}
+
+func (w *responseBodyCopier) Write(b []byte) (int, error) {
+	if w.start.IsZero() {
+		w.start = time.Now()
+	}
+
+	// Single write operation that writes to both the response and buffer
+	return w.tee.Write(b)
+}
+
+func (w *responseBodyCopier) WriteHeader(statusCode int) {
+	w.ResponseWriter.WriteHeader(statusCode)
+}
+
+func (w *responseBodyCopier) Header() http.Header {
+	return w.ResponseWriter.Header()
+}
+
+func (w *responseBodyCopier) StartTime() time.Time {
+	return w.start
+}

proxy/metrics_monitor_test.go

@@ -0,0 +1,693 @@
+package proxy
+
+import (
+	"encoding/json"
+	"net/http"
+	"net/http/httptest"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/gin-gonic/gin"
+	"github.com/mostlygeek/llama-swap/event"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestMetricsMonitor_AddMetrics(t *testing.T) {
+	t.Run("adds metrics and assigns ID", func(t *testing.T) {
+		mm := newMetricsMonitor(testLogger, 10)
+
+		metric := TokenMetrics{
+			Model:        "test-model",
+			InputTokens:  100,
+			OutputTokens: 50,
+		}
+
+		mm.addMetrics(metric)
+
+		metrics := mm.getMetrics()
+		assert.Equal(t, 1, len(metrics))
+		assert.Equal(t, 0, metrics[0].ID)
+		assert.Equal(t, "test-model", metrics[0].Model)
+		assert.Equal(t, 100, metrics[0].InputTokens)
+		assert.Equal(t, 50, metrics[0].OutputTokens)
+	})
+
+	t.Run("increments ID for each metric", func(t *testing.T) {
+		mm := newMetricsMonitor(testLogger, 10)
+
+		for i := 0; i < 5; i++ {
+			mm.addMetrics(TokenMetrics{Model: "model"})
+		}
+
+		metrics := mm.getMetrics()
+		assert.Equal(t, 5, len(metrics))
+		for i := 0; i < 5; i++ {
+			assert.Equal(t, i, metrics[i].ID)
+		}
+	})
+
+	t.Run("respects max metrics limit", func(t *testing.T) {
+		mm := newMetricsMonitor(testLogger, 3)
+
+		// Add 5 metrics
+		for i := 0; i < 5; i++ {
+			mm.addMetrics(TokenMetrics{
+				Model:       "model",
+				InputTokens: i,
+			})
+		}
+
+		metrics := mm.getMetrics()
+		assert.Equal(t, 3, len(metrics))
+
+		// Should keep the last 3 metrics (IDs 2, 3, 4)
+		assert.Equal(t, 2, metrics[0].ID)
+		assert.Equal(t, 3, metrics[1].ID)
+		assert.Equal(t, 4, metrics[2].ID)
+	})
+
+	t.Run("emits TokenMetricsEvent", func(t *testing.T) {
+		mm := newMetricsMonitor(testLogger, 10)
+
+		receivedEvent := make(chan TokenMetricsEvent, 1)
+		cancel := event.On(func(e TokenMetricsEvent) {
+			receivedEvent <- e
+		})
+		defer cancel()
+
+		metric := TokenMetrics{
+			Model:        "test-model",
+			InputTokens:  100,
+			OutputTokens: 50,
+		}
+
+		mm.addMetrics(metric)
+
+		select {
+		case evt := <-receivedEvent:
+			assert.Equal(t, 0, evt.Metrics.ID)
+			assert.Equal(t, "test-model", evt.Metrics.Model)
+			assert.Equal(t, 100, evt.Metrics.InputTokens)
+			assert.Equal(t, 50, evt.Metrics.OutputTokens)
+		case <-time.After(1 * time.Second):
+			t.Fatal("timeout waiting for event")
+		}
+	})
+}
+
+func TestMetricsMonitor_GetMetrics(t *testing.T) {
+	t.Run("returns empty slice when no metrics", func(t *testing.T) {
+		mm := newMetricsMonitor(testLogger, 10)
+		metrics := mm.getMetrics()
+		assert.NotNil(t, metrics)
+		assert.Equal(t, 0, len(metrics))
+	})
+
+	t.Run("returns copy of metrics", func(t *testing.T) {
+		mm := newMetricsMonitor(testLogger, 10)
+		mm.addMetrics(TokenMetrics{Model: "model1"})
+		mm.addMetrics(TokenMetrics{Model: "model2"})
+
+		metrics1 := mm.getMetrics()
+		metrics2 := mm.getMetrics()
+
+		// Verify we got copies
+		assert.Equal(t, 2, len(metrics1))
+		assert.Equal(t, 2, len(metrics2))
+
+		// Modify the returned slice shouldn't affect the original
+		metrics1[0].Model = "modified"
+		metrics3 := mm.getMetrics()
+		assert.Equal(t, "model1", metrics3[0].Model)
+	})
+}
+
+func TestMetricsMonitor_GetMetricsJSON(t *testing.T) {
+	t.Run("returns valid JSON for empty metrics", func(t *testing.T) {
+		mm := newMetricsMonitor(testLogger, 10)
+		jsonData, err := mm.getMetricsJSON()
+		assert.NoError(t, err)
+		assert.NotNil(t, jsonData)
+
+		var metrics []TokenMetrics
+		err = json.Unmarshal(jsonData, &metrics)
+		assert.NoError(t, err)
+		assert.Equal(t, 0, len(metrics))
+	})
+
+	t.Run("returns valid JSON with metrics", func(t *testing.T) {
+		mm := newMetricsMonitor(testLogger, 10)
+		mm.addMetrics(TokenMetrics{
+			Model:           "model1",
+			InputTokens:     100,
+			OutputTokens:    50,
+			TokensPerSecond: 25.5,
+		})
+		mm.addMetrics(TokenMetrics{
+			Model:           "model2",
+			InputTokens:     200,
+			OutputTokens:    100,
+			TokensPerSecond: 30.0,
+		})
+
+		jsonData, err := mm.getMetricsJSON()
+		assert.NoError(t, err)
+
+		var metrics []TokenMetrics
+		err = json.Unmarshal(jsonData, &metrics)
+		assert.NoError(t, err)
+		assert.Equal(t, 2, len(metrics))
+		assert.Equal(t, "model1", metrics[0].Model)
+		assert.Equal(t, "model2", metrics[1].Model)
+	})
+}
+
+func TestMetricsMonitor_WrapHandler(t *testing.T) {
+	t.Run("successful non-streaming request with usage data", func(t *testing.T) {
+		mm := newMetricsMonitor(testLogger, 10)
+
+		responseBody := `{
+			"usage": {
+				"prompt_tokens": 100,
+				"completion_tokens": 50
+			}
+		}`
+
+		nextHandler := func(modelID string, w http.ResponseWriter, r *http.Request) error {
+			w.Header().Set("Content-Type", "application/json")
+			w.WriteHeader(http.StatusOK)
+			w.Write([]byte(responseBody))
+			return nil
+		}
+
+		req := httptest.NewRequest("POST", "/test", nil)
+		rec := httptest.NewRecorder()
+		ginCtx, _ := gin.CreateTestContext(rec)
+
+		err := mm.wrapHandler("test-model", ginCtx.Writer, req, nextHandler)
+		assert.NoError(t, err)
+
+		metrics := mm.getMetrics()
+		assert.Equal(t, 1, len(metrics))
+		assert.Equal(t, "test-model", metrics[0].Model)
+		assert.Equal(t, 100, metrics[0].InputTokens)
+		assert.Equal(t, 50, metrics[0].OutputTokens)
+	})
+
+	t.Run("successful request with timings data", func(t *testing.T) {
+		mm := newMetricsMonitor(testLogger, 10)
+
+		responseBody := `{
+			"timings": {
+				"prompt_n": 100,
+				"predicted_n": 50,
+				"prompt_per_second": 150.5,
+				"predicted_per_second": 25.5,
+				"prompt_ms": 500.0,
+				"predicted_ms": 1500.0,
+				"cache_n": 20
+			}
+		}`
+
+		nextHandler := func(modelID string, w http.ResponseWriter, r *http.Request) error {
+			w.Header().Set("Content-Type", "application/json")
+			w.WriteHeader(http.StatusOK)
+			w.Write([]byte(responseBody))
+			return nil
+		}
+
+		req := httptest.NewRequest("POST", "/test", nil)
+		rec := httptest.NewRecorder()
+		ginCtx, _ := gin.CreateTestContext(rec)
+
+		err := mm.wrapHandler("test-model", ginCtx.Writer, req, nextHandler)
+		assert.NoError(t, err)
+
+		metrics := mm.getMetrics()
+		assert.Equal(t, 1, len(metrics))
+		assert.Equal(t, "test-model", metrics[0].Model)
+		assert.Equal(t, 100, metrics[0].InputTokens)
+		assert.Equal(t, 50, metrics[0].OutputTokens)
+		assert.Equal(t, 20, metrics[0].CachedTokens)
+		assert.Equal(t, 150.5, metrics[0].PromptPerSecond)
+		assert.Equal(t, 25.5, metrics[0].TokensPerSecond)
+		assert.Equal(t, 2000, metrics[0].DurationMs) // 500 + 1500
+	})
+
+	t.Run("streaming request with SSE format", func(t *testing.T) {
+		mm := newMetricsMonitor(testLogger, 10)
+
+		// Note: SSE format requires proper line breaks - each data line followed by blank line
+		responseBody := `data: {"choices":[{"text":"Hello"}]}
+
+data: {"choices":[{"text":" World"}]}
+
+data: {"usage":{"prompt_tokens":10,"completion_tokens":20},"timings":{"prompt_n":10,"predicted_n":20,"prompt_per_second":100.0,"predicted_per_second":50.0,"prompt_ms":100.0,"predicted_ms":400.0}}
+
+data: [DONE]
+
+`
+
+		nextHandler := func(modelID string, w http.ResponseWriter, r *http.Request) error {
+			w.Header().Set("Content-Type", "text/event-stream")
+			w.WriteHeader(http.StatusOK)
+			w.Write([]byte(responseBody))
+			return nil
+		}
+
+		req := httptest.NewRequest("POST", "/test", nil)
+		rec := httptest.NewRecorder()
+		ginCtx, _ := gin.CreateTestContext(rec)
+
+		err := mm.wrapHandler("test-model", ginCtx.Writer, req, nextHandler)
+		assert.NoError(t, err)
+
+		metrics := mm.getMetrics()
+		assert.Equal(t, 1, len(metrics))
+		assert.Equal(t, "test-model", metrics[0].Model)
+		// When timings data is present, it takes precedence
+		assert.Equal(t, 10, metrics[0].InputTokens)
+		assert.Equal(t, 20, metrics[0].OutputTokens)
+	})
+
+	t.Run("non-OK status code does not record metrics", func(t *testing.T) {
+		mm := newMetricsMonitor(testLogger, 10)
+
+		nextHandler := func(modelID string, w http.ResponseWriter, r *http.Request) error {
+			w.WriteHeader(http.StatusBadRequest)
+			w.Write([]byte("error"))
+			return nil
+		}
+
+		req := httptest.NewRequest("POST", "/test", nil)
+		rec := httptest.NewRecorder()
+		ginCtx, _ := gin.CreateTestContext(rec)
+
+		err := mm.wrapHandler("test-model", ginCtx.Writer, req, nextHandler)
+		assert.NoError(t, err)
+
+		metrics := mm.getMetrics()
+		assert.Equal(t, 0, len(metrics))
+	})
+
+	t.Run("empty response body does not record metrics", func(t *testing.T) {
+		mm := newMetricsMonitor(testLogger, 10)
+
+		nextHandler := func(modelID string, w http.ResponseWriter, r *http.Request) error {
+			w.WriteHeader(http.StatusOK)
+			return nil
+		}
+
+		req := httptest.NewRequest("POST", "/test", nil)
+		rec := httptest.NewRecorder()
+		ginCtx, _ := gin.CreateTestContext(rec)
+
+		err := mm.wrapHandler("test-model", ginCtx.Writer, req, nextHandler)
+		assert.NoError(t, err)
+
+		metrics := mm.getMetrics()
+		assert.Equal(t, 0, len(metrics))
+	})
+
+	t.Run("invalid JSON does not record metrics", func(t *testing.T) {
+		mm := newMetricsMonitor(testLogger, 10)
+
+		nextHandler := func(modelID string, w http.ResponseWriter, r *http.Request) error {
+			w.Header().Set("Content-Type", "application/json")
+			w.WriteHeader(http.StatusOK)
+			w.Write([]byte("not valid json"))
+			return nil
+		}
+
+		req := httptest.NewRequest("POST", "/test", nil)
+		rec := httptest.NewRecorder()
+		ginCtx, _ := gin.CreateTestContext(rec)
+
+		err := mm.wrapHandler("test-model", ginCtx.Writer, req, nextHandler)
+		assert.NoError(t, err) // Errors after response is sent are logged, not returned
+
+		metrics := mm.getMetrics()
+		assert.Equal(t, 0, len(metrics))
+	})
+
+	t.Run("next handler error is propagated", func(t *testing.T) {
+		mm := newMetricsMonitor(testLogger, 10)
+
+		expectedErr := assert.AnError
+		nextHandler := func(modelID string, w http.ResponseWriter, r *http.Request) error {
+			return expectedErr
+		}
+
+		req := httptest.NewRequest("POST", "/test", nil)
+		rec := httptest.NewRecorder()
+		ginCtx, _ := gin.CreateTestContext(rec)
+
+		err := mm.wrapHandler("test-model", ginCtx.Writer, req, nextHandler)
+		assert.Equal(t, expectedErr, err)
+
+		metrics := mm.getMetrics()
+		assert.Equal(t, 0, len(metrics))
+	})
+
+	t.Run("response without usage or timings does not record metrics", func(t *testing.T) {
+		mm := newMetricsMonitor(testLogger, 10)
+
+		responseBody := `{"result": "ok"}`
+
+		nextHandler := func(modelID string, w http.ResponseWriter, r *http.Request) error {
+			w.Header().Set("Content-Type", "application/json")
+			w.WriteHeader(http.StatusOK)
+			w.Write([]byte(responseBody))
+			return nil
+		}
+
+		req := httptest.NewRequest("POST", "/test", nil)
+		rec := httptest.NewRecorder()
+		ginCtx, _ := gin.CreateTestContext(rec)
+
+		err := mm.wrapHandler("test-model", ginCtx.Writer, req, nextHandler)
+		assert.NoError(t, err) // Errors after response is sent are logged, not returned
+
+		metrics := mm.getMetrics()
+		assert.Equal(t, 0, len(metrics))
+	})
+}
+
+func TestMetricsMonitor_ResponseBodyCopier(t *testing.T) {
+	t.Run("captures response body", func(t *testing.T) {
+		rec := httptest.NewRecorder()
+		ginCtx, _ := gin.CreateTestContext(rec)
+		copier := newBodyCopier(ginCtx.Writer)
+
+		testData := []byte("test response body")
+		n, err := copier.Write(testData)
+
+		assert.NoError(t, err)
+		assert.Equal(t, len(testData), n)
+		assert.Equal(t, testData, copier.body.Bytes())
+		assert.Equal(t, string(testData), rec.Body.String())
+	})
+
+	t.Run("sets start time on first write", func(t *testing.T) {
+		rec := httptest.NewRecorder()
+		ginCtx, _ := gin.CreateTestContext(rec)
+		copier := newBodyCopier(ginCtx.Writer)
+
+		assert.True(t, copier.StartTime().IsZero())
+
+		copier.Write([]byte("test"))
+
+		assert.False(t, copier.StartTime().IsZero())
+	})
+
+	t.Run("preserves headers", func(t *testing.T) {
+		rec := httptest.NewRecorder()
+		ginCtx, _ := gin.CreateTestContext(rec)
+		copier := newBodyCopier(ginCtx.Writer)
+
+		copier.Header().Set("X-Test", "value")
+
+		assert.Equal(t, "value", rec.Header().Get("X-Test"))
+	})
+
+	t.Run("preserves status code", func(t *testing.T) {
+		rec := httptest.NewRecorder()
+		ginCtx, _ := gin.CreateTestContext(rec)
+		copier := newBodyCopier(ginCtx.Writer)
+
+		copier.WriteHeader(http.StatusCreated)
+
+		// Gin's ResponseWriter tracks status internally
+		assert.Equal(t, http.StatusCreated, copier.Status())
+	})
+}
+
+func TestMetricsMonitor_Concurrent(t *testing.T) {
+	t.Run("concurrent addMetrics is safe", func(t *testing.T) {
+		mm := newMetricsMonitor(testLogger, 1000)
+
+		var wg sync.WaitGroup
+		numGoroutines := 10
+		metricsPerGoroutine := 100
+
+		for i := 0; i < numGoroutines; i++ {
+			wg.Add(1)
+			go func(id int) {
+				defer wg.Done()
+				for j := 0; j < metricsPerGoroutine; j++ {
+					mm.addMetrics(TokenMetrics{
+						Model:        "test-model",
+						InputTokens:  id*1000 + j,
+						OutputTokens: j,
+					})
+				}
+			}(i)
+		}
+
+		wg.Wait()
+
+		metrics := mm.getMetrics()
+		assert.Equal(t, numGoroutines*metricsPerGoroutine, len(metrics))
+	})
+
+	t.Run("concurrent reads and writes are safe", func(t *testing.T) {
+		mm := newMetricsMonitor(testLogger, 100)
+
+		done := make(chan bool)
+
+		// Writer goroutine
+		go func() {
+			for i := 0; i < 50; i++ {
+				mm.addMetrics(TokenMetrics{Model: "test-model"})
+				time.Sleep(1 * time.Millisecond)
+			}
+			done <- true
+		}()
+
+		// Multiple reader goroutines
+		var wg sync.WaitGroup
+		for i := 0; i < 5; i++ {
+			wg.Add(1)
+			go func() {
+				defer wg.Done()
+				for j := 0; j < 20; j++ {
+					_ = mm.getMetrics()
+					_, _ = mm.getMetricsJSON()
+					time.Sleep(2 * time.Millisecond)
+				}
+			}()
+		}
+
+		<-done
+		wg.Wait()
+
+		// Final check
+		metrics := mm.getMetrics()
+		assert.Equal(t, 50, len(metrics))
+	})
+}
+
+func TestMetricsMonitor_ParseMetrics(t *testing.T) {
+	t.Run("prefers timings over usage data", func(t *testing.T) {
+		mm := newMetricsMonitor(testLogger, 10)
+
+		// Timings should take precedence over usage
+		responseBody := `{
+			"usage": {
+				"prompt_tokens": 50,
+				"completion_tokens": 25
+			},
+			"timings": {
+				"prompt_n": 100,
+				"predicted_n": 50,
+				"prompt_per_second": 150.5,
+				"predicted_per_second": 25.5,
+				"prompt_ms": 500.0,
+				"predicted_ms": 1500.0
+			}
+		}`
+
+		nextHandler := func(modelID string, w http.ResponseWriter, r *http.Request) error {
+			w.Header().Set("Content-Type", "application/json")
+			w.WriteHeader(http.StatusOK)
+			w.Write([]byte(responseBody))
+			return nil
+		}
+
+		req := httptest.NewRequest("POST", "/test", nil)
+		rec := httptest.NewRecorder()
+		ginCtx, _ := gin.CreateTestContext(rec)
+
+		err := mm.wrapHandler("test-model", ginCtx.Writer, req, nextHandler)
+		assert.NoError(t, err)
+
+		metrics := mm.getMetrics()
+		assert.Equal(t, 1, len(metrics))
+		// Should use timings values, not usage values
+		assert.Equal(t, 100, metrics[0].InputTokens)
+		assert.Equal(t, 50, metrics[0].OutputTokens)
+	})
+
+	t.Run("handles missing cache_n in timings", func(t *testing.T) {
+		mm := newMetricsMonitor(testLogger, 10)
+
+		responseBody := `{
+			"timings": {
+				"prompt_n": 100,
+				"predicted_n": 50,
+				"prompt_per_second": 150.5,
+				"predicted_per_second": 25.5,
+				"prompt_ms": 500.0,
+				"predicted_ms": 1500.0
+			}
+		}`
+
+		nextHandler := func(modelID string, w http.ResponseWriter, r *http.Request) error {
+			w.Header().Set("Content-Type", "application/json")
+			w.WriteHeader(http.StatusOK)
+			w.Write([]byte(responseBody))
+			return nil
+		}
+
+		req := httptest.NewRequest("POST", "/test", nil)
+		rec := httptest.NewRecorder()
+		ginCtx, _ := gin.CreateTestContext(rec)
+
+		err := mm.wrapHandler("test-model", ginCtx.Writer, req, nextHandler)
+		assert.NoError(t, err)
+
+		metrics := mm.getMetrics()
+		assert.Equal(t, 1, len(metrics))
+		assert.Equal(t, -1, metrics[0].CachedTokens) // Default value when not present
+	})
+}
+
+func TestMetricsMonitor_StreamingResponse(t *testing.T) {
+	t.Run("finds metrics in last valid SSE data", func(t *testing.T) {
+		mm := newMetricsMonitor(testLogger, 10)
+
+		// Metrics should be found in the last data line before [DONE]
+		responseBody := `data: {"choices":[{"text":"First"}]}
+
+data: {"choices":[{"text":"Second"}]}
+
+data: {"usage":{"prompt_tokens":100,"completion_tokens":50}}
+
+data: [DONE]
+
+`
+
+		nextHandler := func(modelID string, w http.ResponseWriter, r *http.Request) error {
+			w.Header().Set("Content-Type", "text/event-stream")
+			w.WriteHeader(http.StatusOK)
+			w.Write([]byte(responseBody))
+			return nil
+		}
+
+		req := httptest.NewRequest("POST", "/test", nil)
+		rec := httptest.NewRecorder()
+		ginCtx, _ := gin.CreateTestContext(rec)
+
+		err := mm.wrapHandler("test-model", ginCtx.Writer, req, nextHandler)
+		assert.NoError(t, err)
+
+		metrics := mm.getMetrics()
+		assert.Equal(t, 1, len(metrics))
+		assert.Equal(t, 100, metrics[0].InputTokens)
+		assert.Equal(t, 50, metrics[0].OutputTokens)
+	})
+
+	t.Run("handles streaming with no valid JSON", func(t *testing.T) {
+		mm := newMetricsMonitor(testLogger, 10)
+
+		responseBody := `data: not json
+
+data: [DONE]
+
+`
+
+		nextHandler := func(modelID string, w http.ResponseWriter, r *http.Request) error {
+			w.Header().Set("Content-Type", "text/event-stream")
+			w.WriteHeader(http.StatusOK)
+			w.Write([]byte(responseBody))
+			return nil
+		}
+
+		req := httptest.NewRequest("POST", "/test", nil)
+		rec := httptest.NewRecorder()
+		ginCtx, _ := gin.CreateTestContext(rec)
+
+		err := mm.wrapHandler("test-model", ginCtx.Writer, req, nextHandler)
+		assert.NoError(t, err) // Errors after response is sent are logged, not returned
+
+		metrics := mm.getMetrics()
+		assert.Equal(t, 0, len(metrics))
+	})
+
+	t.Run("handles empty streaming response", func(t *testing.T) {
+		mm := newMetricsMonitor(testLogger, 10)
+
+		responseBody := ``
+
+		nextHandler := func(modelID string, w http.ResponseWriter, r *http.Request) error {
+			w.Header().Set("Content-Type", "text/event-stream")
+			w.WriteHeader(http.StatusOK)
+			w.Write([]byte(responseBody))
+			return nil
+		}
+
+		req := httptest.NewRequest("POST", "/test", nil)
+		rec := httptest.NewRecorder()
+		ginCtx, _ := gin.CreateTestContext(rec)
+
+		err := mm.wrapHandler("test-model", ginCtx.Writer, req, nextHandler)
+		// Empty body should not trigger WrapHandler processing
+		assert.NoError(t, err)
+
+		metrics := mm.getMetrics()
+		assert.Equal(t, 0, len(metrics))
+	})
+}
+
+// Benchmark tests
+func BenchmarkMetricsMonitor_AddMetrics(b *testing.B) {
+	mm := newMetricsMonitor(testLogger, 1000)
+
+	metric := TokenMetrics{
+		Model:           "test-model",
+		CachedTokens:    100,
+		InputTokens:     500,
+		OutputTokens:    250,
+		PromptPerSecond: 1200.5,
+		TokensPerSecond: 45.8,
+		DurationMs:      5000,
+		Timestamp:       time.Now(),
+	}
+
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		mm.addMetrics(metric)
+	}
+}
+
+func BenchmarkMetricsMonitor_AddMetrics_SmallBuffer(b *testing.B) {
+	// Test performance with a smaller buffer where wrapping occurs more frequently
+	mm := newMetricsMonitor(testLogger, 100)
+
+	metric := TokenMetrics{
+		Model:           "test-model",
+		CachedTokens:    100,
+		InputTokens:     500,
+		OutputTokens:    250,
+		PromptPerSecond: 1200.5,
+		TokensPerSecond: 45.8,
+		DurationMs:      5000,
+		Timestamp:       time.Now(),
+	}
+
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		mm.addMetrics(metric)
+	}
+}

proxy/process.go

@@ -4,14 +4,14 @@ import (
 	"context"
 	"errors"
 	"fmt"
-	"io"
 	"net"
 	"net/http"
+	"net/http/httputil"
 	"net/url"
 	"os/exec"
-	"strconv"
 	"strings"
 	"sync"
+	"sync/atomic"
 	"syscall"
 	"time"
 
@@ -39,11 +39,13 @@ const (
 )
 
 type Process struct {
-	ID     string
+	ID           string
-	config config.ModelConfig
+	config       config.ModelConfig
-	cmd    *exec.Cmd
+	cmd          *exec.Cmd
+	reverseProxy *httputil.ReverseProxy
 
 	// PR #155 called to cancel the upstream process
+	cmdMutex       sync.RWMutex
 	cancelUpstream context.CancelFunc
 
 	// closed when command exits
@@ -55,12 +57,14 @@ type Process struct {
 	healthCheckTimeout      int
 	healthCheckLoopInterval time.Duration
 
-	lastRequestHandled time.Time
+	lastRequestHandledMutex sync.RWMutex
+	lastRequestHandled      time.Time
 
 	stateMutex sync.RWMutex
 	state      ProcessState
 
-	inFlightRequests sync.WaitGroup
+	inFlightRequests      sync.WaitGroup
+	inFlightRequestsCount atomic.Int32
 
 	// used to block on multiple start() calls
 	waitStarting sync.WaitGroup
@@ -81,10 +85,29 @@ func NewProcess(ID string, healthCheckTimeout int, config config.ModelConfig, pr
 		concurrentLimit = config.ConcurrencyLimit
 	}
 
+	// Setup the reverse proxy.
+	proxyURL, err := url.Parse(config.Proxy)
+	if err != nil {
+		proxyLogger.Errorf("<%s> invalid proxy URL %q: %v", ID, config.Proxy, err)
+	}
+
+	var reverseProxy *httputil.ReverseProxy
+	if proxyURL != nil {
+		reverseProxy = httputil.NewSingleHostReverseProxy(proxyURL)
+		reverseProxy.ModifyResponse = func(resp *http.Response) error {
+			// prevent nginx from buffering streaming responses (e.g., SSE)
+			if strings.Contains(strings.ToLower(resp.Header.Get("Content-Type")), "text/event-stream") {
+				resp.Header.Set("X-Accel-Buffering", "no")
+			}
+			return nil
+		}
+	}
+
 	return &Process{
 		ID:                      ID,
 		config:                  config,
 		cmd:                     nil,
+		reverseProxy:            reverseProxy,
 		cancelUpstream:          nil,
 		processLogger:           processLogger,
 		proxyLogger:             proxyLogger,
@@ -107,6 +130,20 @@ func (p *Process) LogMonitor() *LogMonitor {
 	return p.processLogger
 }
 
+// setLastRequestHandled sets the last request handled time in a thread-safe manner.
+func (p *Process) setLastRequestHandled(t time.Time) {
+	p.lastRequestHandledMutex.Lock()
+	defer p.lastRequestHandledMutex.Unlock()
+	p.lastRequestHandled = t
+}
+
+// getLastRequestHandled gets the last request handled time in a thread-safe manner.
+func (p *Process) getLastRequestHandled() time.Time {
+	p.lastRequestHandledMutex.RLock()
+	defer p.lastRequestHandledMutex.RUnlock()
+	return p.lastRequestHandled
+}
+
 // custom error types for swapping state
 var (
 	ErrExpectedStateMismatch  = errors.New("expected state mismatch")
@@ -130,6 +167,13 @@ func (p *Process) swapState(expectedState, newState ProcessState) (ProcessState,
 	}
 
 	p.state = newState
+
+	// Atomically increment waitStarting when entering StateStarting
+	// This ensures any thread that sees StateStarting will also see the WaitGroup counter incremented
+	if newState == StateStarting {
+		p.waitStarting.Add(1)
+	}
+
 	p.proxyLogger.Debugf("<%s> swapState() State transitioned from %s to %s", p.ID, expectedState, newState)
 	event.Emit(ProcessStateChangeEvent{ProcessName: p.ID, NewState: newState, OldState: expectedState})
 	return p.state, nil
@@ -158,6 +202,15 @@ func (p *Process) CurrentState() ProcessState {
 	return p.state
 }
 
+// forceState forces the process state to the new state with mutex protection.
+// This should only be used in exceptional cases where the normal state transition
+// validation via swapState() cannot be used.
+func (p *Process) forceState(newState ProcessState) {
+	p.stateMutex.Lock()
+	defer p.stateMutex.Unlock()
+	p.state = newState
+}
+
 // start starts the upstream command, checks the health endpoint, and sets the state to Ready
 // it is a private method because starting is automatic but stopping can be called
 // at any time.
@@ -191,7 +244,7 @@ func (p *Process) start() error {
 		}
 	}
 
-	p.waitStarting.Add(1)
+	// waitStarting.Add(1) is now called atomically in swapState() when transitioning to StateStarting
 	defer p.waitStarting.Done()
 	cmdContext, ctxCancelUpstream := context.WithCancel(context.Background())
 
@@ -201,8 +254,11 @@ func (p *Process) start() error {
 	p.cmd.Env = append(p.cmd.Environ(), p.config.Env...)
 	p.cmd.Cancel = p.cmdStopUpstreamProcess
 	p.cmd.WaitDelay = p.gracefulStopTimeout
+
+	p.cmdMutex.Lock()
 	p.cancelUpstream = ctxCancelUpstream
 	p.cmdWaitChan = make(chan struct{})
+	p.cmdMutex.Unlock()
 
 	p.failedStartCount++ // this will be reset to zero when the process has successfully started
 
@@ -212,7 +268,7 @@ func (p *Process) start() error {
 	// Set process state to failed
 	if err != nil {
 		if curState, swapErr := p.swapState(StateStarting, StateStopped); swapErr != nil {
-			p.state = StateStopped // force it into a stopped state
+			p.forceState(StateStopped) // force it into a stopped state
 			return fmt.Errorf(
 				"failed to start command '%s' and state swap failed. command error: %v, current state: %v, state swap error: %v",
 				strings.Join(args, " "), err, curState, swapErr,
@@ -285,10 +341,12 @@ func (p *Process) start() error {
 					return
 				}
 
-				// wait for all inflight requests to complete and ticker
+				// skip the TTL check if there are inflight requests
-				p.inFlightRequests.Wait()
+				if p.inFlightRequestsCount.Load() != 0 {
+					continue
+				}
 
-				if time.Since(p.lastRequestHandled) > maxDuration {
+				if time.Since(p.getLastRequestHandled()) > maxDuration {
 					p.proxyLogger.Infof("<%s> Unloading model, TTL of %ds reached", p.ID, p.config.UnloadAfter)
 					p.Stop()
 					return
@@ -344,7 +402,7 @@ func (p *Process) Shutdown() {
 
 	p.stopCommand()
 	// just force it to this state since there is no recovery from shutdown
-	p.state = StateShutdown
+	p.forceState(StateShutdown)
 }
 
 // stopCommand will send a SIGTERM to the process and wait for it to exit.
@@ -355,13 +413,18 @@ func (p *Process) stopCommand() {
 		p.proxyLogger.Debugf("<%s> stopCommand took %v", p.ID, time.Since(stopStartTime))
 	}()
 
-	if p.cancelUpstream == nil {
+	p.cmdMutex.RLock()
+	cancelUpstream := p.cancelUpstream
+	cmdWaitChan := p.cmdWaitChan
+	p.cmdMutex.RUnlock()
+
+	if cancelUpstream == nil {
 		p.proxyLogger.Errorf("<%s> stopCommand has a nil p.cancelUpstream()", p.ID)
 		return
 	}
 
-	p.cancelUpstream()
+	cancelUpstream()
-	<-p.cmdWaitChan
+	<-cmdWaitChan
 }
 
 func (p *Process) checkHealthEndpoint(healthURL string) error {
@@ -418,8 +481,10 @@ func (p *Process) ProxyRequest(w http.ResponseWriter, r *http.Request) {
 	}
 
 	p.inFlightRequests.Add(1)
+	p.inFlightRequestsCount.Add(1)
 	defer func() {
-		p.lastRequestHandled = time.Now()
+		p.setLastRequestHandled(time.Now())
+		p.inFlightRequestsCount.Add(-1)
 		p.inFlightRequests.Done()
 	}()
 
@@ -434,56 +499,22 @@ func (p *Process) ProxyRequest(w http.ResponseWriter, r *http.Request) {
 		startDuration = time.Since(beginStartTime)
 	}
 
-	proxyTo := p.config.Proxy
+	// recover from http.ErrAbortHandler panics that can occur when the client
-	client := &http.Client{}
+	// disconnects before the response is sent
-	req, err := http.NewRequestWithContext(r.Context(), r.Method, proxyTo+r.URL.String(), r.Body)
+	defer func() {
-	if err != nil {
+		if r := recover(); r != nil {
-		http.Error(w, err.Error(), http.StatusInternalServerError)
+			if r == http.ErrAbortHandler {
-		return
+				p.proxyLogger.Infof("<%s> recovered from client disconnection during streaming", p.ID)
-	}
+			} else {
-	req.Header = r.Header.Clone()
+				p.proxyLogger.Infof("<%s> recovered from panic: %v", p.ID, r)
-
-	contentLength, err := strconv.ParseInt(req.Header.Get("content-length"), 10, 64)
-	if err == nil {
-		req.ContentLength = contentLength
-	}
-
-	resp, err := client.Do(req)
-	if err != nil {
-		http.Error(w, err.Error(), http.StatusBadGateway)
-		return
-	}
-	defer resp.Body.Close()
-	for k, vv := range resp.Header {
-		for _, v := range vv {
-			w.Header().Add(k, v)
-		}
-	}
-	// prevent nginx from buffering streaming responses (e.g., SSE)
-	if strings.Contains(strings.ToLower(resp.Header.Get("Content-Type")), "text/event-stream") {
-		w.Header().Set("X-Accel-Buffering", "no")
-	}
-	w.WriteHeader(resp.StatusCode)
-
-	// faster than io.Copy when streaming
-	buf := make([]byte, 32*1024)
-	for {
-		n, err := resp.Body.Read(buf)
-		if n > 0 {
-			if _, writeErr := w.Write(buf[:n]); writeErr != nil {
-				return
-			}
-			if flusher, ok := w.(http.Flusher); ok {
-				flusher.Flush()
 			}
 		}
-		if err == io.EOF {
+	}()
-			break
+
-		}
+	if p.reverseProxy != nil {
-		if err != nil {
+		p.reverseProxy.ServeHTTP(w, r)
-			http.Error(w, err.Error(), http.StatusBadGateway)
+	} else {
-			return
+		http.Error(w, fmt.Sprintf("No reverse proxy available for %s", p.ID), http.StatusInternalServerError)
-		}
 	}
 
 	totalTime := time.Since(requestBeginTime)
@@ -519,13 +550,16 @@ func (p *Process) waitForCmd() {
 	case StateStopping:
 		if curState, err := p.swapState(StateStopping, StateStopped); err != nil {
 			p.proxyLogger.Errorf("<%s> Process exited but could not swap to StateStopped. curState=%s, err: %v", p.ID, curState, err)
-			p.state = StateStopped
+			p.forceState(StateStopped)
 		}
 	default:
 		p.proxyLogger.Infof("<%s> process exited but not StateStopping, current state: %s", p.ID, currentState)
-		p.state = StateStopped // force it to be in this state
+		p.forceState(StateStopped) // force it to be in this state
 	}
+
+	p.cmdMutex.Lock()
 	close(p.cmdWaitChan)
+	p.cmdMutex.Unlock()
 }
 
 // cmdStopUpstreamProcess attemps to stop the upstream process gracefully

proxy/process_test.go

@@ -436,7 +436,9 @@ func TestProcess_ForceStopWithKill(t *testing.T) {
 		if runtime.GOOS == "windows" {
 			assert.Contains(t, w.Body.String(), "wsarecv: An existing connection was forcibly closed by the remote host")
 		} else {
-			assert.Contains(t, w.Body.String(), "unexpected EOF")
+			// Upstream may be killed mid-response.
+			// Assert an incomplete or partial response.
+			assert.NotEqual(t, "12345", w.Body.String())
 		}
 
 		close(waitChan)
@@ -492,3 +494,74 @@ func TestProcess_EnvironmentSetCorrectly(t *testing.T) {
 	assert.Equal(t, len(process1.cmd.Environ())+2, len(process2.cmd.Environ()), "process2 should have 2 more environment variables than process1")
 
 }
+
+// TestProcess_ReverseProxyPanicIsHandled tests that panics from
+// httputil.ReverseProxy in Process.ProxyRequest(w, r) do not bubble up and are
+// handled appropriately.
+//
+// httputil.ReverseProxy will panic with http.ErrAbortHandler when it has sent headers
+// can't copy the body. This can be caused by a client disconnecting before the full
+// response is sent from some reason.
+//
+// bug: https://github.com/mostlygeek/llama-swap/issues/362
+// see: https://github.com/golang/go/issues/23643 (where panic was added to httputil.ReverseProxy)
+func TestProcess_ReverseProxyPanicIsHandled(t *testing.T) {
+	// Add defer/recover to catch any panics that aren't handled by ProxyRequest
+	// If this recover() is hit, it means ProxyRequest didn't handle the panic properly
+	defer func() {
+		if r := recover(); r != nil {
+			t.Fatalf("ProxyRequest should handle panics from reverseProxy.ServeHTTP, but panic was not caught: %v", r)
+		}
+	}()
+
+	expectedMessage := "panic_test"
+	config := getTestSimpleResponderConfig(expectedMessage)
+
+	process := NewProcess("panic-test", 5, config, debugLogger, debugLogger)
+	defer process.Stop()
+
+	// Start the process
+	err := process.start()
+	assert.Nil(t, err)
+	assert.Equal(t, StateReady, process.CurrentState())
+
+	// Create a custom ResponseWriter that simulates a client disconnect
+	// by panicking when Write is called after headers are sent
+	panicWriter := &panicOnWriteResponseWriter{
+		ResponseRecorder: httptest.NewRecorder(),
+		shouldPanic:      true,
+	}
+
+	// Make a request that will trigger the panic
+	req := httptest.NewRequest("GET", "/slow-respond?echo=test&delay=100ms", nil)
+
+	// This should panic inside reverseProxy.ServeHTTP when the panicWriter.Write() is called.
+	// ProxyRequest should catch and handle this panic gracefully.
+	process.ProxyRequest(panicWriter, req)
+
+	// If we get here, the panic was properly recovered in ProxyRequest
+	// The process should still be in a ready state
+	assert.Equal(t, StateReady, process.CurrentState())
+}
+
+// panicOnWriteResponseWriter is a ResponseWriter that panics on Write
+// to simulate a client disconnect after headers are sent
+// used by: TestProcess_ReverseProxyPanicIsHandled
+type panicOnWriteResponseWriter struct {
+	*httptest.ResponseRecorder
+	shouldPanic   bool
+	headerWritten bool
+}
+
+func (w *panicOnWriteResponseWriter) WriteHeader(statusCode int) {
+	w.headerWritten = true
+	w.ResponseRecorder.WriteHeader(statusCode)
+}
+
+func (w *panicOnWriteResponseWriter) Write(b []byte) (int, error) {
+	if w.shouldPanic && w.headerWritten {
+		// Simulate the panic that httputil.ReverseProxy throws
+		panic(http.ErrAbortHandler)
+	}
+	return w.ResponseRecorder.Write(b)
+}

proxy/proxymanager.go

@@ -36,7 +36,7 @@ type ProxyManager struct {
 	upstreamLogger *LogMonitor
 	muxLogger      *LogMonitor
 
-	metricsMonitor *MetricsMonitor
+	metricsMonitor *metricsMonitor
 
 	processGroups map[string]*ProcessGroup
 
@@ -75,6 +75,13 @@ func New(config config.Config) *ProxyManager {
 
 	shutdownCtx, shutdownCancel := context.WithCancel(context.Background())
 
+	var maxMetrics int
+	if config.MetricsMaxInMemory <= 0 {
+		maxMetrics = 1000 // Default fallback
+	} else {
+		maxMetrics = config.MetricsMaxInMemory
+	}
+
 	pm := &ProxyManager{
 		config:    config,
 		ginEngine: gin.New(),
@@ -83,7 +90,7 @@ func New(config config.Config) *ProxyManager {
 		muxLogger:      stdoutLogger,
 		upstreamLogger: upstreamLogger,
 
-		metricsMonitor: NewMetricsMonitor(&config),
+		metricsMonitor: newMetricsMonitor(proxyLogger, maxMetrics),
 
 		processGroups: make(map[string]*ProcessGroup),
 
@@ -131,7 +138,15 @@ func New(config config.Config) *ProxyManager {
 }
 
 func (pm *ProxyManager) setupGinEngine() {
+
 	pm.ginEngine.Use(func(c *gin.Context) {
+
+		// don't log the Wake on Lan proxy health check
+		if c.Request.URL.Path == "/wol-health" {
+			c.Next()
+			return
+		}
+
 		// Start timer
 		start := time.Now()
 
@@ -185,27 +200,25 @@ func (pm *ProxyManager) setupGinEngine() {
 		c.Next()
 	})
 
-	mm := MetricsMiddleware(pm)
-
 	// Set up routes using the Gin engine
-	pm.ginEngine.POST("/v1/chat/completions", mm, pm.proxyOAIHandler)
+	pm.ginEngine.POST("/v1/chat/completions", pm.proxyOAIHandler)
 	// Support legacy /v1/completions api, see issue #12
-	pm.ginEngine.POST("/v1/completions", mm, pm.proxyOAIHandler)
+	pm.ginEngine.POST("/v1/completions", pm.proxyOAIHandler)
 
 	// Support embeddings and reranking
-	pm.ginEngine.POST("/v1/embeddings", mm, pm.proxyOAIHandler)
+	pm.ginEngine.POST("/v1/embeddings", pm.proxyOAIHandler)
 
 	// llama-server's /reranking endpoint + aliases
-	pm.ginEngine.POST("/reranking", mm, pm.proxyOAIHandler)
+	pm.ginEngine.POST("/reranking", pm.proxyOAIHandler)
-	pm.ginEngine.POST("/rerank", mm, pm.proxyOAIHandler)
+	pm.ginEngine.POST("/rerank", pm.proxyOAIHandler)
-	pm.ginEngine.POST("/v1/rerank", mm, pm.proxyOAIHandler)
+	pm.ginEngine.POST("/v1/rerank", pm.proxyOAIHandler)
-	pm.ginEngine.POST("/v1/reranking", mm, pm.proxyOAIHandler)
+	pm.ginEngine.POST("/v1/reranking", pm.proxyOAIHandler)
 
 	// llama-server's /infill endpoint for code infilling
-	pm.ginEngine.POST("/infill", mm, pm.proxyOAIHandler)
+	pm.ginEngine.POST("/infill", pm.proxyOAIHandler)
 
 	// llama-server's /completion endpoint
-	pm.ginEngine.POST("/completion", mm, pm.proxyOAIHandler)
+	pm.ginEngine.POST("/completion", pm.proxyOAIHandler)
 
 	// Support audio/speech endpoint
 	pm.ginEngine.POST("/v1/audio/speech", pm.proxyOAIHandler)
@@ -235,6 +248,11 @@ func (pm *ProxyManager) setupGinEngine() {
 		c.String(http.StatusOK, "OK")
 	})
 
+	// see cmd/wol-proxy/wol-proxy.go, not logged
+	pm.ginEngine.GET("/wol-health", func(c *gin.Context) {
+		c.String(http.StatusOK, "OK")
+	})
+
 	pm.ginEngine.GET("/favicon.ico", func(c *gin.Context) {
 		if data, err := reactStaticFS.ReadFile("ui_dist/favicon.ico"); err == nil {
 			c.Data(http.StatusOK, "image/x-icon", data)
@@ -461,8 +479,23 @@ func (pm *ProxyManager) proxyToUpstream(c *gin.Context) {
 	}
 
 	// rewrite the path
+	originalPath := c.Request.URL.Path
 	c.Request.URL.Path = remainingPath
-	processGroup.ProxyRequest(realModelName, c.Writer, c.Request)
+
+	// attempt to record metrics if it is a POST request
+	if pm.metricsMonitor != nil && c.Request.Method == "POST" {
+		if err := pm.metricsMonitor.wrapHandler(realModelName, c.Writer, c.Request, processGroup.ProxyRequest); err != nil {
+			pm.sendErrorResponse(c, http.StatusInternalServerError, fmt.Sprintf("error proxying metrics wrapped request: %s", err.Error()))
+			pm.proxyLogger.Errorf("Error proxying wrapped upstream request for model %s, path=%s", realModelName, originalPath)
+			return
+		}
+	} else {
+		if err := processGroup.ProxyRequest(realModelName, c.Writer, c.Request); err != nil {
+			pm.sendErrorResponse(c, http.StatusInternalServerError, fmt.Sprintf("error proxying request: %s", err.Error()))
+			pm.proxyLogger.Errorf("Error proxying upstream request for model %s, path=%s", realModelName, originalPath)
+			return
+		}
+	}
 }
 
 func (pm *ProxyManager) proxyOAIHandler(c *gin.Context) {
@@ -522,10 +555,18 @@ func (pm *ProxyManager) proxyOAIHandler(c *gin.Context) {
 	c.Request.Header.Set("content-length", strconv.Itoa(len(bodyBytes)))
 	c.Request.ContentLength = int64(len(bodyBytes))
 
-	if err := processGroup.ProxyRequest(realModelName, c.Writer, c.Request); err != nil {
+	if pm.metricsMonitor != nil && c.Request.Method == "POST" {
-		pm.sendErrorResponse(c, http.StatusInternalServerError, fmt.Sprintf("error proxying request: %s", err.Error()))
+		if err := pm.metricsMonitor.wrapHandler(realModelName, c.Writer, c.Request, processGroup.ProxyRequest); err != nil {
-		pm.proxyLogger.Errorf("Error Proxying Request for processGroup %s and model %s", processGroup.id, realModelName)
+			pm.sendErrorResponse(c, http.StatusInternalServerError, fmt.Sprintf("error proxying metrics wrapped request: %s", err.Error()))
-		return
+			pm.proxyLogger.Errorf("Error Proxying Metrics Wrapped Request for processGroup %s and model %s", processGroup.id, realModelName)
+			return
+		}
+	} else {
+		if err := processGroup.ProxyRequest(realModelName, c.Writer, c.Request); err != nil {
+			pm.sendErrorResponse(c, http.StatusInternalServerError, fmt.Sprintf("error proxying request: %s", err.Error()))
+			pm.proxyLogger.Errorf("Error Proxying Request for processGroup %s and model %s", processGroup.id, realModelName)
+			return
+		}
 	}
 }
 

proxy/proxymanager_api.go

@@ -180,7 +180,7 @@ func (pm *ProxyManager) apiSendEvents(c *gin.Context) {
 	sendLogData("proxy", pm.proxyLogger.GetHistory())
 	sendLogData("upstream", pm.upstreamLogger.GetHistory())
 	sendModels()
-	sendMetrics(pm.metricsMonitor.GetMetrics())
+	sendMetrics(pm.metricsMonitor.getMetrics())
 
 	for {
 		select {
@@ -198,7 +198,7 @@ func (pm *ProxyManager) apiSendEvents(c *gin.Context) {
 }
 
 func (pm *ProxyManager) apiGetMetrics(c *gin.Context) {
-	jsonData, err := pm.metricsMonitor.GetMetricsJSON()
+	jsonData, err := pm.metricsMonitor.getMetricsJSON()
 	if err != nil {
 		c.JSON(http.StatusInternalServerError, gin.H{"error": "failed to get metrics"})
 		return

proxy/proxymanager_test.go

@@ -21,6 +21,32 @@ import (
 	"github.com/tidwall/gjson"
 )
 
+// TestResponseRecorder adds CloseNotify to httptest.ResponseRecorder.
+// "If you want to write your own tests around streams you will need a Recorder that can handle CloseNotifier."
+// The tests can panic otherwise:
+// panic: interface conversion: *httptest.ResponseRecorder is not http.CloseNotifier: missing method CloseNotify
+// See: https://github.com/gin-gonic/gin/issues/1815
+// TestResponseRecorder is taken from gin's own tests: https://github.com/gin-gonic/gin/blob/ce20f107f5dc498ec7489d7739541a25dcd48463/context_test.go#L1747-L1765
+type TestResponseRecorder struct {
+	*httptest.ResponseRecorder
+	closeChannel chan bool
+}
+
+func (r *TestResponseRecorder) CloseNotify() <-chan bool {
+	return r.closeChannel
+}
+
+func (r *TestResponseRecorder) closeClient() {
+	r.closeChannel <- true
+}
+
+func CreateTestResponseRecorder() *TestResponseRecorder {
+	return &TestResponseRecorder{
+		httptest.NewRecorder(),
+		make(chan bool, 1),
+	}
+}
+
 func TestProxyManager_SwapProcessCorrectly(t *testing.T) {
 	config := config.AddDefaultGroupToConfig(config.Config{
 		HealthCheckTimeout: 15,
@@ -37,7 +63,7 @@ func TestProxyManager_SwapProcessCorrectly(t *testing.T) {
 	for _, modelName := range []string{"model1", "model2"} {
 		reqBody := fmt.Sprintf(`{"model":"%s"}`, modelName)
 		req := httptest.NewRequest("POST", "/v1/chat/completions", bytes.NewBufferString(reqBody))
-		w := httptest.NewRecorder()
+		w := CreateTestResponseRecorder()
 
 		proxy.ServeHTTP(w, req)
 		assert.Equal(t, http.StatusOK, w.Code)
@@ -74,7 +100,7 @@ func TestProxyManager_SwapMultiProcess(t *testing.T) {
 		t.Run(requestedModel, func(t *testing.T) {
 			reqBody := fmt.Sprintf(`{"model":"%s"}`, requestedModel)
 			req := httptest.NewRequest("POST", "/v1/chat/completions", bytes.NewBufferString(reqBody))
-			w := httptest.NewRecorder()
+			w := CreateTestResponseRecorder()
 
 			proxy.ServeHTTP(w, req)
 			assert.Equal(t, http.StatusOK, w.Code)
@@ -116,7 +142,7 @@ func TestProxyManager_PersistentGroupsAreNotSwapped(t *testing.T) {
 	for _, requestedModel := range tests {
 		reqBody := fmt.Sprintf(`{"model":"%s"}`, requestedModel)
 		req := httptest.NewRequest("POST", "/v1/chat/completions", bytes.NewBufferString(reqBody))
-		w := httptest.NewRecorder()
+		w := CreateTestResponseRecorder()
 
 		proxy.ServeHTTP(w, req)
 		assert.Equal(t, http.StatusOK, w.Code)
@@ -159,7 +185,7 @@ func TestProxyManager_SwapMultiProcessParallelRequests(t *testing.T) {
 
 			reqBody := fmt.Sprintf(`{"model":"%s"}`, key)
 			req := httptest.NewRequest("POST", "/v1/chat/completions?wait=1000ms", bytes.NewBufferString(reqBody))
-			w := httptest.NewRecorder()
+			w := CreateTestResponseRecorder()
 
 			proxy.ServeHTTP(w, req)
 
@@ -212,7 +238,7 @@ func TestProxyManager_ListModelsHandler(t *testing.T) {
 	// Create a test request
 	req := httptest.NewRequest("GET", "/v1/models", nil)
 	req.Header.Add("Origin", "i-am-the-origin")
-	w := httptest.NewRecorder()
+	w := CreateTestResponseRecorder()
 
 	// Call the listModelsHandler
 	proxy.ServeHTTP(w, req)
@@ -311,7 +337,7 @@ models:
 	proxy := New(processedConfig)
 
 	req := httptest.NewRequest("GET", "/v1/models", nil)
-	w := httptest.NewRecorder()
+	w := CreateTestResponseRecorder()
 	proxy.ServeHTTP(w, req)
 
 	assert.Equal(t, http.StatusOK, w.Code)
@@ -387,7 +413,7 @@ func TestProxyManager_ListModelsHandler_SortedByID(t *testing.T) {
 
 	// Request models list
 	req := httptest.NewRequest("GET", "/v1/models", nil)
-	w := httptest.NewRecorder()
+	w := CreateTestResponseRecorder()
 	proxy.ServeHTTP(w, req)
 
 	assert.Equal(t, http.StatusOK, w.Code)
@@ -448,7 +474,7 @@ func TestProxyManager_Shutdown(t *testing.T) {
 			defer wg.Done()
 			reqBody := fmt.Sprintf(`{"model":"%s"}`, modelName)
 			req := httptest.NewRequest("POST", "/v1/chat/completions", bytes.NewBufferString(reqBody))
-			w := httptest.NewRecorder()
+			w := CreateTestResponseRecorder()
 
 			// send a request to trigger the proxy to load ... this should hang waiting for start up
 			proxy.ServeHTTP(w, req)
@@ -476,12 +502,12 @@ func TestProxyManager_Unload(t *testing.T) {
 	proxy := New(conf)
 	reqBody := fmt.Sprintf(`{"model":"%s"}`, "model1")
 	req := httptest.NewRequest("POST", "/v1/chat/completions", bytes.NewBufferString(reqBody))
-	w := httptest.NewRecorder()
+	w := CreateTestResponseRecorder()
 	proxy.ServeHTTP(w, req)
 
 	assert.Equal(t, proxy.processGroups[config.DEFAULT_GROUP_ID].processes["model1"].CurrentState(), StateReady)
 	req = httptest.NewRequest("GET", "/unload", nil)
-	w = httptest.NewRecorder()
+	w = CreateTestResponseRecorder()
 	proxy.ServeHTTP(w, req)
 	assert.Equal(t, http.StatusOK, w.Code)
 	assert.Equal(t, w.Body.String(), "OK")
@@ -519,7 +545,7 @@ func TestProxyManager_UnloadSingleModel(t *testing.T) {
 	for _, modelName := range []string{"model1", "model2"} {
 		reqBody := fmt.Sprintf(`{"model":"%s"}`, modelName)
 		req := httptest.NewRequest("POST", "/v1/chat/completions", bytes.NewBufferString(reqBody))
-		w := httptest.NewRecorder()
+		w := CreateTestResponseRecorder()
 		proxy.ServeHTTP(w, req)
 	}
 
@@ -527,7 +553,7 @@ func TestProxyManager_UnloadSingleModel(t *testing.T) {
 	assert.Equal(t, StateReady, proxy.processGroups[testGroupId].processes["model2"].CurrentState())
 
 	req := httptest.NewRequest("POST", "/api/models/unload/model1", nil)
-	w := httptest.NewRecorder()
+	w := CreateTestResponseRecorder()
 	proxy.ServeHTTP(w, req)
 	assert.Equal(t, http.StatusOK, w.Code)
 	if !assert.Equal(t, w.Body.String(), "OK") {
@@ -571,7 +597,7 @@ func TestProxyManager_RunningEndpoint(t *testing.T) {
 
 	t.Run("no models loaded", func(t *testing.T) {
 		req := httptest.NewRequest("GET", "/running", nil)
-		w := httptest.NewRecorder()
+		w := CreateTestResponseRecorder()
 		proxy.ServeHTTP(w, req)
 
 		assert.Equal(t, http.StatusOK, w.Code)
@@ -589,13 +615,13 @@ func TestProxyManager_RunningEndpoint(t *testing.T) {
 		// Load just a model.
 		reqBody := `{"model":"model1"}`
 		req := httptest.NewRequest("POST", "/v1/chat/completions", bytes.NewBufferString(reqBody))
-		w := httptest.NewRecorder()
+		w := CreateTestResponseRecorder()
 		proxy.ServeHTTP(w, req)
 		assert.Equal(t, http.StatusOK, w.Code)
 
 		// Simulate browser call for the `/running` endpoint.
 		req = httptest.NewRequest("GET", "/running", nil)
-		w = httptest.NewRecorder()
+		w = CreateTestResponseRecorder()
 		proxy.ServeHTTP(w, req)
 
 		var response RunningResponse
@@ -647,7 +673,7 @@ func TestProxyManager_AudioTranscriptionHandler(t *testing.T) {
 	// Create the request with the multipart form data
 	req := httptest.NewRequest("POST", "/v1/audio/transcriptions", &b)
 	req.Header.Set("Content-Type", w.FormDataContentType())
-	rec := httptest.NewRecorder()
+	rec := CreateTestResponseRecorder()
 	proxy.ServeHTTP(rec, req)
 
 	// Verify the response
@@ -682,7 +708,7 @@ func TestProxyManager_UseModelName(t *testing.T) {
 	t.Run("useModelName over rides requested model: /v1/chat/completions", func(t *testing.T) {
 		reqBody := fmt.Sprintf(`{"model":"%s"}`, requestedModel)
 		req := httptest.NewRequest("POST", "/v1/chat/completions", bytes.NewBufferString(reqBody))
-		w := httptest.NewRecorder()
+		w := CreateTestResponseRecorder()
 
 		proxy.ServeHTTP(w, req)
 		assert.Equal(t, http.StatusOK, w.Code)
@@ -716,7 +742,7 @@ func TestProxyManager_UseModelName(t *testing.T) {
 		// Create the request with the multipart form data
 		req := httptest.NewRequest("POST", "/v1/audio/transcriptions", &b)
 		req.Header.Set("Content-Type", w.FormDataContentType())
-		rec := httptest.NewRecorder()
+		rec := CreateTestResponseRecorder()
 		proxy.ServeHTTP(rec, req)
 
 		// Verify the response
@@ -784,7 +810,7 @@ func TestProxyManager_CORSOptionsHandler(t *testing.T) {
 				req.Header.Set(k, v)
 			}
 
-			w := httptest.NewRecorder()
+			w := CreateTestResponseRecorder()
 			proxy.ServeHTTP(w, req)
 
 			assert.Equal(t, tt.expectedStatus, w.Code)
@@ -812,7 +838,7 @@ models:
 	defer proxy.StopProcesses(StopWaitForInflightRequest)
 	t.Run("main model name", func(t *testing.T) {
 		req := httptest.NewRequest("GET", "/upstream/model1/test", nil)
-		rec := httptest.NewRecorder()
+		rec := CreateTestResponseRecorder()
 		proxy.ServeHTTP(rec, req)
 		assert.Equal(t, http.StatusOK, rec.Code)
 		assert.Equal(t, "model1", rec.Body.String())
@@ -820,7 +846,7 @@ models:
 
 	t.Run("model alias", func(t *testing.T) {
 		req := httptest.NewRequest("GET", "/upstream/model-alias/test", nil)
-		rec := httptest.NewRecorder()
+		rec := CreateTestResponseRecorder()
 		proxy.ServeHTTP(rec, req)
 		assert.Equal(t, http.StatusOK, rec.Code)
 		assert.Equal(t, "model1", rec.Body.String())
@@ -841,7 +867,7 @@ func TestProxyManager_ChatContentLength(t *testing.T) {
 
 	reqBody := fmt.Sprintf(`{"model":"%s", "x": "this is just some content to push the length out a bit"}`, "model1")
 	req := httptest.NewRequest("POST", "/v1/chat/completions", bytes.NewBufferString(reqBody))
-	w := httptest.NewRecorder()
+	w := CreateTestResponseRecorder()
 
 	proxy.ServeHTTP(w, req)
 	assert.Equal(t, http.StatusOK, w.Code)
@@ -869,7 +895,7 @@ func TestProxyManager_FiltersStripParams(t *testing.T) {
 	defer proxy.StopProcesses(StopWaitForInflightRequest)
 	reqBody := `{"model":"model1", "temperature":0.1, "x_param":"123", "y_param":"abc", "stream":true}`
 	req := httptest.NewRequest("POST", "/v1/chat/completions", bytes.NewBufferString(reqBody))
-	w := httptest.NewRecorder()
+	w := CreateTestResponseRecorder()
 
 	proxy.ServeHTTP(w, req)
 	assert.Equal(t, http.StatusOK, w.Code)
@@ -885,76 +911,6 @@ func TestProxyManager_FiltersStripParams(t *testing.T) {
 	// t.Logf("%v", response)
 }
 
-func TestProxyManager_MiddlewareWritesMetrics_NonStreaming(t *testing.T) {
-	config := config.AddDefaultGroupToConfig(config.Config{
-		HealthCheckTimeout: 15,
-		Models: map[string]config.ModelConfig{
-			"model1": getTestSimpleResponderConfig("model1"),
-		},
-		LogLevel: "error",
-	})
-
-	proxy := New(config)
-	defer proxy.StopProcesses(StopWaitForInflightRequest)
-
-	// Make a non-streaming request
-	reqBody := `{"model":"model1", "stream": false}`
-	req := httptest.NewRequest("POST", "/v1/chat/completions", bytes.NewBufferString(reqBody))
-	w := httptest.NewRecorder()
-
-	proxy.ServeHTTP(w, req)
-	assert.Equal(t, http.StatusOK, w.Code)
-
-	// Check that metrics were recorded
-	metrics := proxy.metricsMonitor.GetMetrics()
-	if !assert.NotEmpty(t, metrics, "metrics should be recorded for non-streaming request") {
-		return
-	}
-
-	// Verify the last metric has the correct model
-	lastMetric := metrics[len(metrics)-1]
-	assert.Equal(t, "model1", lastMetric.Model)
-	assert.Equal(t, 25, lastMetric.InputTokens, "input tokens should be 25")
-	assert.Equal(t, 10, lastMetric.OutputTokens, "output tokens should be 10")
-	assert.Greater(t, lastMetric.TokensPerSecond, 0.0, "tokens per second should be greater than 0")
-	assert.Greater(t, lastMetric.DurationMs, 0, "duration should be greater than 0")
-}
-
-func TestProxyManager_MiddlewareWritesMetrics_Streaming(t *testing.T) {
-	config := config.AddDefaultGroupToConfig(config.Config{
-		HealthCheckTimeout: 15,
-		Models: map[string]config.ModelConfig{
-			"model1": getTestSimpleResponderConfig("model1"),
-		},
-		LogLevel: "error",
-	})
-
-	proxy := New(config)
-	defer proxy.StopProcesses(StopWaitForInflightRequest)
-
-	// Make a streaming request
-	reqBody := `{"model":"model1", "stream": true}`
-	req := httptest.NewRequest("POST", "/v1/chat/completions?stream=true", bytes.NewBufferString(reqBody))
-	w := httptest.NewRecorder()
-
-	proxy.ServeHTTP(w, req)
-	assert.Equal(t, http.StatusOK, w.Code)
-
-	// Check that metrics were recorded
-	metrics := proxy.metricsMonitor.GetMetrics()
-	if !assert.NotEmpty(t, metrics, "metrics should be recorded for streaming request") {
-		return
-	}
-
-	// Verify the last metric has the correct model
-	lastMetric := metrics[len(metrics)-1]
-	assert.Equal(t, "model1", lastMetric.Model)
-	assert.Equal(t, 25, lastMetric.InputTokens, "input tokens should be 25")
-	assert.Equal(t, 10, lastMetric.OutputTokens, "output tokens should be 10")
-	assert.Greater(t, lastMetric.TokensPerSecond, 0.0, "tokens per second should be greater than 0")
-	assert.Greater(t, lastMetric.DurationMs, 0, "duration should be greater than 0")
-}
-
 func TestProxyManager_HealthEndpoint(t *testing.T) {
 	config := config.AddDefaultGroupToConfig(config.Config{
 		HealthCheckTimeout: 15,
@@ -967,7 +923,7 @@ func TestProxyManager_HealthEndpoint(t *testing.T) {
 	proxy := New(config)
 	defer proxy.StopProcesses(StopWaitForInflightRequest)
 	req := httptest.NewRequest("GET", "/health", nil)
-	rec := httptest.NewRecorder()
+	rec := CreateTestResponseRecorder()
 	proxy.ServeHTTP(rec, req)
 	assert.Equal(t, http.StatusOK, rec.Code)
 	assert.Equal(t, "OK", rec.Body.String())
@@ -988,7 +944,7 @@ func TestProxyManager_CompletionEndpoint(t *testing.T) {
 
 	reqBody := `{"model":"model1"}`
 	req := httptest.NewRequest("POST", "/completion", bytes.NewBufferString(reqBody))
-	w := httptest.NewRecorder()
+	w := CreateTestResponseRecorder()
 
 	proxy.ServeHTTP(w, req)
 	assert.Equal(t, http.StatusOK, w.Code)
@@ -1075,18 +1031,28 @@ func TestProxyManager_StreamingEndpointsReturnNoBufferingHeader(t *testing.T) {
 
 	for _, endpoint := range endpoints {
 		t.Run(endpoint, func(t *testing.T) {
-			ctx, cancel := context.WithCancel(context.Background())
+			ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond)
 			defer cancel()
 
 			req := httptest.NewRequest("GET", endpoint, nil)
 			req = req.WithContext(ctx)
-			rec := httptest.NewRecorder()
+			rec := CreateTestResponseRecorder()
 
-			// We don't need the handler to fully complete, just to set the headers
+			// Run handler in goroutine and wait for context timeout
-			// so run it in a goroutine and check the headers after a short delay
+			done := make(chan struct{})
-			go proxy.ServeHTTP(rec, req)
+			go func() {
-			time.Sleep(10 * time.Millisecond) // give it time to start and write headers
+				defer close(done)
+				proxy.ServeHTTP(rec, req)
+			}()
 
+			// Wait for either the handler to complete or context to timeout
+			<-ctx.Done()
+
+			// At this point, the handler has either finished or been cancelled
+			// Wait for the goroutine to fully exit before reading
+			<-done
+
+			// Now it's safe to read from rec - no more concurrent writes
 			assert.Equal(t, http.StatusOK, rec.Code)
 			assert.Equal(t, "no", rec.Header().Get("X-Accel-Buffering"))
 		})
@@ -1109,7 +1075,7 @@ func TestProxyManager_ProxiedStreamingEndpointReturnsNoBufferingHeader(t *testin
 	reqBody := `{"model":"streaming-model"}`
 	// simple-responder will return text/event-stream when stream=true is in the query
 	req := httptest.NewRequest("POST", "/v1/chat/completions?stream=true", bytes.NewBufferString(reqBody))
-	rec := httptest.NewRecorder()
+	rec := CreateTestResponseRecorder()
 
 	proxy.ServeHTTP(rec, req)
 

ui/package-lock.json

@@ -3975,9 +3975,9 @@
       }
     },
     "node_modules/vite": {
-      "version": "6.3.5",
+      "version": "6.4.1",
-      "resolved": "https://registry.npmjs.org/vite/-/vite-6.3.5.tgz",
+      "resolved": "https://registry.npmjs.org/vite/-/vite-6.4.1.tgz",
-      "integrity": "sha512-cZn6NDFE7wdTpINgs++ZJ4N49W2vRp8LCKrn3Ob1kYNtOo21vfDoaV5GzBfLU4MovSAB8uNRm4jgzVQZ+mBzPQ==",
+      "integrity": "sha512-+Oxm7q9hDoLMyJOYfUYBuHQo+dkAloi33apOPP56pzj+vsdJDzr+j1NISE5pyaAuKL4A3UD34qd0lx5+kfKp2g==",
       "dev": true,
       "license": "MIT",
       "dependencies": {

ui/src/pages/Models.tsx

@@ -191,42 +191,300 @@ function ModelsPanel() {
   );
 }
 
+interface HistogramData {
+  bins: number[];
+  min: number;
+  max: number;
+  binSize: number;
+  p99: number;
+  p95: number;
+  p50: number;
+}
+
+function TokenHistogram({ data }: { data: HistogramData }) {
+  const { bins, min, max, p50, p95, p99 } = data;
+  const maxCount = Math.max(...bins);
+
+  const height = 120;
+  const padding = { top: 10, right: 15, bottom: 25, left: 45 };
+
+  // Use viewBox for responsive sizing
+  const viewBoxWidth = 600;
+  const chartWidth = viewBoxWidth - padding.left - padding.right;
+  const chartHeight = height - padding.top - padding.bottom;
+
+  const barWidth = chartWidth / bins.length;
+  const range = max - min;
+
+  // Calculate x position for a given value
+  const getXPosition = (value: number) => {
+    return padding.left + ((value - min) / range) * chartWidth;
+  };
+
+  return (
+    <div className="mt-2 w-full">
+      <svg
+        viewBox={`0 0 ${viewBoxWidth} ${height}`}
+        className="w-full h-auto"
+        preserveAspectRatio="xMidYMid meet"
+      >
+        {/* Y-axis */}
+        <line
+          x1={padding.left}
+          y1={padding.top}
+          x2={padding.left}
+          y2={height - padding.bottom}
+          stroke="currentColor"
+          strokeWidth="1"
+          opacity="0.3"
+        />
+
+        {/* X-axis */}
+        <line
+          x1={padding.left}
+          y1={height - padding.bottom}
+          x2={viewBoxWidth - padding.right}
+          y2={height - padding.bottom}
+          stroke="currentColor"
+          strokeWidth="1"
+          opacity="0.3"
+        />
+
+        {/* Histogram bars */}
+        {bins.map((count, i) => {
+          const barHeight = maxCount > 0 ? (count / maxCount) * chartHeight : 0;
+          const x = padding.left + i * barWidth;
+          const y = height - padding.bottom - barHeight;
+          const binStart = min + i * data.binSize;
+          const binEnd = binStart + data.binSize;
+
+          return (
+            <g key={i}>
+              <rect
+                x={x}
+                y={y}
+                width={Math.max(barWidth - 1, 1)}
+                height={barHeight}
+                fill="currentColor"
+                opacity="0.6"
+                className="text-blue-500 dark:text-blue-400 hover:opacity-90 transition-opacity cursor-pointer"
+              />
+              <title>{`${binStart.toFixed(1)} - ${binEnd.toFixed(1)} tokens/sec\nCount: ${count}`}</title>
+            </g>
+          );
+        })}
+
+        {/* Percentile lines */}
+        <line
+          x1={getXPosition(p50)}
+          y1={padding.top}
+          x2={getXPosition(p50)}
+          y2={height - padding.bottom}
+          stroke="currentColor"
+          strokeWidth="2"
+          strokeDasharray="4 2"
+          opacity="0.7"
+          className="text-gray-600 dark:text-gray-400"
+        />
+
+        <line
+          x1={getXPosition(p95)}
+          y1={padding.top}
+          x2={getXPosition(p95)}
+          y2={height - padding.bottom}
+          stroke="currentColor"
+          strokeWidth="2"
+          strokeDasharray="4 2"
+          opacity="0.7"
+          className="text-orange-500 dark:text-orange-400"
+        />
+
+        <line
+          x1={getXPosition(p99)}
+          y1={padding.top}
+          x2={getXPosition(p99)}
+          y2={height - padding.bottom}
+          stroke="currentColor"
+          strokeWidth="2"
+          strokeDasharray="4 2"
+          opacity="0.7"
+          className="text-green-500 dark:text-green-400"
+        />
+
+        {/* X-axis labels */}
+        <text
+          x={padding.left}
+          y={height - 5}
+          fontSize="10"
+          fill="currentColor"
+          opacity="0.6"
+          textAnchor="start"
+        >
+          {min.toFixed(1)}
+        </text>
+
+        <text
+          x={viewBoxWidth - padding.right}
+          y={height - 5}
+          fontSize="10"
+          fill="currentColor"
+          opacity="0.6"
+          textAnchor="end"
+        >
+          {max.toFixed(1)}
+        </text>
+
+        {/* X-axis label */}
+        <text
+          x={padding.left + chartWidth / 2}
+          y={height - 2}
+          fontSize="10"
+          fill="currentColor"
+          opacity="0.6"
+          textAnchor="middle"
+        >
+          Tokens/Second Distribution
+        </text>
+      </svg>
+    </div>
+  );
+}
+
 function StatsPanel() {
   const { metrics } = useAPI();
 
-  const [totalRequests, totalInputTokens, totalOutputTokens, avgTokensPerSecond] = useMemo(() => {
+  const [totalRequests, totalInputTokens, totalOutputTokens, tokenStats, histogramData] = useMemo(() => {
     const totalRequests = metrics.length;
     if (totalRequests === 0) {
-      return [0, 0, 0];
+      return [0, 0, 0, { p99: 0, p95: 0, p50: 0 }, null];
     }
     const totalInputTokens = metrics.reduce((sum, m) => sum + m.input_tokens, 0);
     const totalOutputTokens = metrics.reduce((sum, m) => sum + m.output_tokens, 0);
-    const avgTokensPerSecond = (metrics.reduce((sum, m) => sum + m.tokens_per_second, 0) / totalRequests).toFixed(2);
+
-    return [totalRequests, totalInputTokens, totalOutputTokens, avgTokensPerSecond];
+    // Calculate token statistics using output_tokens and duration_ms
+    // Filter out metrics with invalid duration or output tokens
+    const validMetrics = metrics.filter((m) => m.duration_ms > 0 && m.output_tokens > 0);
+    if (validMetrics.length === 0) {
+      return [totalRequests, totalInputTokens, totalOutputTokens, { p99: 0, p95: 0, p50: 0 }, null];
+    }
+
+    // Calculate tokens/second for each valid metric
+    const tokensPerSecond = validMetrics.map((m) => m.output_tokens / (m.duration_ms / 1000));
+
+    // Sort for percentile calculation
+    const sortedTokensPerSecond = [...tokensPerSecond].sort((a, b) => a - b);
+
+    // Calculate percentiles - showing speed thresholds where X% of requests are SLOWER (below)
+    // P99: 99% of requests are slower than this speed (99th percentile - fast requests)
+    // P95: 95% of requests are slower than this speed (95th percentile)
+    // P50: 50% of requests are slower than this speed (median)
+    const p99 = sortedTokensPerSecond[Math.floor(sortedTokensPerSecond.length * 0.99)];
+    const p95 = sortedTokensPerSecond[Math.floor(sortedTokensPerSecond.length * 0.95)];
+    const p50 = sortedTokensPerSecond[Math.floor(sortedTokensPerSecond.length * 0.5)];
+
+    // Create histogram data
+    const min = Math.min(...tokensPerSecond);
+    const max = Math.max(...tokensPerSecond);
+    const binCount = Math.min(30, Math.max(10, Math.floor(tokensPerSecond.length / 5))); // Adaptive bin count
+    const binSize = (max - min) / binCount;
+
+    const bins = Array(binCount).fill(0);
+    tokensPerSecond.forEach((value) => {
+      const binIndex = Math.min(Math.floor((value - min) / binSize), binCount - 1);
+      bins[binIndex]++;
+    });
+
+    const histogramData = {
+      bins,
+      min,
+      max,
+      binSize,
+      p99,
+      p95,
+      p50,
+    };
+
+    return [
+      totalRequests,
+      totalInputTokens,
+      totalOutputTokens,
+      {
+        p99: p99.toFixed(2),
+        p95: p95.toFixed(2),
+        p50: p50.toFixed(2),
+      },
+      histogramData,
+    ];
   }, [metrics]);
 
+  const nf = new Intl.NumberFormat();
+
   return (
     <div className="card">
-      <div className="rounded-lg overflow-hidden border border-gray-200 dark:border-white/10">
+      <div className="rounded-lg overflow-hidden border border-card-border-inner">
-        <table className="w-full">
+        <table className="min-w-full divide-y divide-card-border-inner">
-          <thead>
+          <thead className="bg-secondary">
-            <tr className="border-b border-gray-200 dark:border-white/10 text-right">
+            <tr>
-              <th>Requests</th>
+              <th className="px-4 py-3 text-left text-xs font-semibold uppercase tracking-wider text-txtmain">
-              <th className="border-l border-gray-200 dark:border-white/10">Processed</th>
+                Requests
-              <th className="border-l border-gray-200 dark:border-white/10">Generated</th>
+              </th>
-              <th className="border-l border-gray-200 dark:border-white/10">Tokens/Sec</th>
+              <th className="px-4 py-3 text-left text-xs font-semibold uppercase tracking-wider text-txtmain border-l border-card-border-inner">
+                Processed
+              </th>
+              <th className="px-4 py-3 text-left text-xs font-semibold uppercase tracking-wider text-txtmain border-l border-card-border-inner">
+                Generated
+              </th>
+              <th className="px-4 py-3 text-left text-xs font-semibold uppercase tracking-wider text-txtmain border-l border-card-border-inner">
+                Token Stats (tokens/sec)
+              </th>
             </tr>
           </thead>
-          <tbody>
+
-            <tr className="text-right">
+          <tbody className="bg-surface divide-y divide-card-border-inner">
-              <td className="border-r border-gray-200 dark:border-white/10">{totalRequests}</td>
+            <tr className="hover:bg-secondary">
-              <td className="border-r border-gray-200 dark:border-white/10">
+              <td className="px-4 py-4 text-sm font-semibold text-gray-900 dark:text-white">{totalRequests}</td>
-                {new Intl.NumberFormat().format(totalInputTokens)}
+
+              <td className="px-4 py-4 text-sm text-gray-700 dark:text-gray-300 border-l border-gray-200 dark:border-white/10">
+                <div className="flex items-center gap-2">
+                  <span className="text-sm font-medium">{nf.format(totalInputTokens)}</span>
+                  <span className="text-xs text-gray-500 dark:text-gray-400">tokens</span>
+                </div>
               </td>
-              <td className="border-r border-gray-200 dark:border-white/10">
+
-                {new Intl.NumberFormat().format(totalOutputTokens)}
+              <td className="px-4 py-4 text-sm text-gray-700 dark:text-gray-300 border-l border-gray-200 dark:border-white/10">
+                <div className="flex items-center gap-2">
+                  <span className="text-sm font-medium">{nf.format(totalOutputTokens)}</span>
+                  <span className="text-xs text-gray-500 dark:text-gray-400">tokens</span>
+                </div>
+              </td>
+
+              <td className="px-4 py-4 border-l border-gray-200 dark:border-white/10">
+                <div className="space-y-3">
+                  <div className="grid grid-cols-3 gap-2 items-center">
+                    <div className="text-center">
+                      <div className="text-xs text-gray-500 dark:text-gray-400">P50</div>
+                      <div className="mt-1 inline-block rounded-full bg-gray-100 dark:bg-white/5 px-3 py-1 text-sm font-semibold text-gray-800 dark:text-white">
+                        {tokenStats.p50}
+                      </div>
+                    </div>
+
+                    <div className="text-center">
+                      <div className="text-xs text-gray-500 dark:text-gray-400">P95</div>
+                      <div className="mt-1 inline-block rounded-full bg-gray-100 dark:bg-white/5 px-3 py-1 text-sm font-semibold text-gray-800 dark:text-white">
+                        {tokenStats.p95}
+                      </div>
+                    </div>
+
+                    <div className="text-center">
+                      <div className="text-xs text-gray-500 dark:text-gray-400">P99</div>
+                      <div className="mt-1 inline-block rounded-full bg-gray-100 dark:bg-white/5 px-3 py-1 text-sm font-semibold text-gray-800 dark:text-white">
+                        {tokenStats.p99}
+                      </div>
+                    </div>
+                  </div>
+                  {histogramData && <TokenHistogram data={histogramData} />}
+                </div>
               </td>
-              <td>{avgTokensPerSecond}</td>
             </tr>
           </tbody>
         </table>