support TTS /v1/audio/speech (#36)

Improve README documentation.

README.md

@@ -3,33 +3,37 @@
 ![llama-swap header image](header.jpeg)
 
 # Introduction
-llama-swap is an OpenAI API compatible server that gives you complete control over how you use your hardware. It automatically swaps to the configuration of your choice for serving a model. Since [llama.cpp's server](https://github.com/ggerganov/llama.cpp/tree/master/examples/server) can't swap models, let's swap the server instead!
+llama-swap is a light weight, transparent proxy server that provides automatic model swapping to llama.cpp's server.
 
-Features:
+Written in golang, it is very easy to install (single binary with no dependancies) and configure (single yaml file). Download a pre-built [release](https://github.com/mostlygeek/llama-swap/releases) or built it yourself from source with `make clean all`.
+
+## How does it work?
+When a request is made to an OpenAI compatible endpoints, lama-swap will extract the `model` value load the appropriate server configuration to serve it. If a server is already running it will stop it and start a new one. This is where the "swap" part comes in. The upstream server is automatically swapped to the correct one to serve the request.
+
+In the most basic configuration llama-swap handles one model at a time. For more advanced use cases, the `profiles` feature can load multiple models at the same time. You have complete control over how your system resources are used.
+
+## Do I need to use llama.cpp's server (llama-server)?
+Any OpenAI compatible server would work. llama-swap was originally designed for llama-server and it is the best supported. For Python based inference servers like vllm or tabbyAPI it is recommended to run them via podman. This provides clean environment isolation as well as responding correctly to `SIGTERM` signals to shutdown.
+
+## Features:
 
 - ✅ Easy to deploy: single binary with no dependencies
 - ✅ Easy to config: single yaml file
 - ✅ On-demand model switching
 - ✅ Full control over server settings per model
-- ✅ OpenAI API support (`v1/completions` and `v1/chat/completions`)
+- ✅ OpenAI API supported endpoints:
+  - `v1/completions`
+  - `v1/chat/completions`
+  - `v1/embeddings`
+  - `v1/rerank`
+  - `v1/audio/speech`
 - ✅ Multiple GPU support
 - ✅ Run multiple models at once with `profiles`
 - ✅ Remote log monitoring at `/log`
 - ✅ Automatic unloading of models from GPUs after timeout
-- ✅ Use any local OpenAI compatible server (llama.cpp, vllm, tabblyAPI, etc)
+- ✅ Use any local OpenAI compatible server (llama.cpp, vllm, tabbyAPI, etc)
 - ✅ Direct access to upstream HTTP server via `/upstream/:model_id` ([demo](https://github.com/mostlygeek/llama-swap/pull/31))
 
-## Releases
-
-Builds for Linux and OSX are available on the [Releases](https://github.com/mostlygeek/llama-swap/releases) page.
-
-### Building from source
-
-1. Install golang for your system
-1. `git clone git@github.com:mostlygeek/llama-swap.git`
-1. `make clean all`
-1. Binaries will be in `build/` subdirectory
-
 ## config.yaml
 
 llama-swap's configuration is purposefully simple.
@@ -39,6 +43,9 @@ llama-swap's configuration is purposefully simple.
 # Default (and minimum) is 15 seconds
 healthCheckTimeout: 60
 
+# Write HTTP logs (useful for troubleshooting), defaults to false
+logRequests: true
+
 # define valid model values and the upstream server start
 models:
   "llama":
@@ -92,19 +99,26 @@ profiles:
     - "llama"
 ```
 
-**Guides and examples**
+### Advanced Examples
 
 - [config.example.yaml](config.example.yaml) includes example for supporting `v1/embeddings` and `v1/rerank` endpoints
 - [Speculative Decoding](examples/speculative-decoding/README.md) - using a small draft model can increase inference speeds from 20% to 40%. This example includes a configurations Qwen2.5-Coder-32B (2.5x increase) and Llama-3.1-70B (1.4x increase) in the best cases.
 - [Optimizing Code Generation](examples/benchmark-snakegame/README.md) - find the optimal settings for your machine. This example demonstrates defining multiple configurations and testing which one is fastest.
 
-## Installation
+### Installation
 
 1. Create a configuration file, see [config.example.yaml](config.example.yaml)
 1. Download a [release](https://github.com/mostlygeek/llama-swap/releases) appropriate for your OS and architecture.
     * _Note: Windows currently untested._
 1. Run the binary with `llama-swap --config path/to/config.yaml`
 
+### Building from source
+
+1. Install golang for your system
+1. `git clone git@github.com:mostlygeek/llama-swap.git`
+1. `make clean all`
+1. Binaries will be in `build/` subdirectory
+
 ## Monitoring Logs
 
 Open the `http://<host>/logs` with your browser to get a web interface with streaming logs.

config.example.yaml

@@ -2,6 +2,9 @@
 # Default (and minimum): 15 seconds
 healthCheckTimeout: 15
 
+# Log HTTP requests helpful for troubleshoot, defaults to False
+logRequests: true
+
 models:
   "llama":
     cmd: >

go.mod

@@ -33,7 +33,7 @@ require (
 	github.com/ugorji/go/codec v1.2.12 // indirect
 	golang.org/x/arch v0.8.0 // indirect
 	golang.org/x/crypto v0.31.0 // indirect
-	golang.org/x/net v0.25.0 // indirect
+	golang.org/x/net v0.33.0 // indirect
 	golang.org/x/sys v0.28.0 // indirect
 	golang.org/x/text v0.21.0 // indirect
 	google.golang.org/protobuf v1.34.1 // indirect

go.sum

@@ -70,6 +70,8 @@ golang.org/x/crypto v0.31.0 h1:ihbySMvVjLAeSH1IbfcRTkD/iNscyz8rGzjF/E5hV6U=
 golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk=
 golang.org/x/net v0.25.0 h1:d/OCCoBEUq33pjydKrGQhw7IlUPI2Oylr+8qLx49kac=
 golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM=
+golang.org/x/net v0.33.0 h1:74SYHlV8BIgHIFC/LrYkOGIwL19eTYXQ5wc6TBuO36I=
+golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4=
 golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.20.0 h1:Od9JTbYCk261bKm4M/mw7AklTlFYIa0bIp9BgSm1S8Y=

proxy/config.go

@@ -25,6 +25,7 @@ func (m *ModelConfig) SanitizedCommand() ([]string, error) {
 
 type Config struct {
 	HealthCheckTimeout int                    `yaml:"healthCheckTimeout"`
+	LogRequests        bool                   `yaml:"logRequests"`
 	Models             map[string]ModelConfig `yaml:"models"`
 	Profiles           map[string][]string    `yaml:"profiles"`
 

proxy/html/index.html

@@ -0,0 +1,14 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+    <meta charset="UTF-8">
+    <meta name="viewport" content="width=device-width, initial-scale=1.0">
+    <title>llama-swap</title>
+</head>
+<body>
+    <h1>llama-swap</h1>
+    <p>
+        <a href="/logs">view logs</a> | <a href="/upstream">configured models</a> | <a href="https://github.com/mostlygeek/llama-swap">github</a>
+    </p>
+</body>
+</html>

proxy/html/logs.html

@@ -12,42 +12,134 @@
             flex-direction: column;
             font-family: "Courier New", Courier, monospace;
         }
+        #log-controls {
+            margin: 0.5em;
+            display: flex;
+            align-items: center;
+            justify-content: space-between; /* Spaces out elements evenly */
+        }
+        #log-controls input {
+            flex: 1;
+        }
+        #log-controls input:focus {
+           outline: none; /* Ensures no outline is shown when the input is focused */
+        }
         #log-stream {
             flex: 1;
-            margin: 1em;
-            padding: 10px;
+            margin: 0.5em;
+            padding: 1em;
             background: #f4f4f4;
             overflow-y: auto;
             white-space: pre-wrap; /* Ensures line wrapping */
             word-wrap: break-word; /* Ensures long words wrap */
         }
+
+        .regex-error {
+            background-color: #ff0000 !important;
+        }
+
+        /* Dark mode styles */
+        @media (prefers-color-scheme: dark) {
+            body {
+                background-color: #333;
+                color: #fff;
+            }
+
+            #log-stream {
+                background: #444;
+                color: #fff;
+            }
+
+            #log-controls input {
+                background: #555;
+                color: #fff;
+                border: 1px solid #777;
+            }
+
+            #log-controls button {
+                background: #555;
+                color: #fff;
+                border: 1px solid #777;
+            }
+        }
     </style>
 </head>
 <body>
-    <pre id="log-stream">Waiting for logs...
-</pre>
-
+    <pre id="log-stream">Waiting for logs...</pre>
+    <div id="log-controls">
+        <input type="text" id="filter-input" placeholder="regex filter">
+        <button id="clear-button">clear</button>
+    </div>
     <script>
-        // Establish an EventSource connection to the SSE endpoint
-        if (typeof(EventSource) !== "undefined") {
-            const eventSource = new EventSource("/logs/streamSSE");
+        const logStream = document.getElementById('log-stream');
+        const filterInput = document.getElementById('filter-input');
+        var logData = "";
+        let regexFilter = null;
 
-            eventSource.onmessage = function(event) {
-                // Append the new log message to the <pre> element
-                const logStream = document.getElementById('log-stream');
+        function setupEventSource() {
+            if (typeof(EventSource) !== "undefined") {
+                const eventSource = new EventSource("/logs/streamSSE");
 
-                logStream.textContent += event.data;
+                eventSource.onmessage = function(event) {
+                    logData += event.data;
+                    render()
+                };
 
-                // Auto-scroll to the bottom
-                logStream.scrollTop = logStream.scrollHeight;
-            };
-
-            eventSource.onerror = function(err) {
-                console.error("EventSource failed:", err);
-            };
-        } else {
-            console.error("SSE not supported by this browser.");
+                eventSource.onerror = function(err) {
+                    logData = "EventSource failed: " + err.message;
+                };
+            } else {
+                logData = "SSE Not supported by this browser."
+            }
         }
+
+        // poor-ai's react  ¯\_(ツ)_/¯
+        function render() {
+            if (regexFilter) {
+                const lines = logData.split('\n');
+                const filteredLines = lines.filter(line => {
+                    return regexFilter === null || regexFilter.test(line);
+                });
+
+                if (filteredLines.length > 0) {
+                    logStream.textContent = filteredLines.join('\n') + '\n';
+                } else {
+                    logStream.textContent = "";
+                }
+            } else {
+                logStream.textContent = logData;
+            }
+
+            logStream.scrollTop = logStream.scrollHeight;
+        }
+
+        function updateFilter() {
+            const pattern = filterInput.value.trim();
+            filterInput.classList.remove('regex-error');
+            if (pattern) {
+                try {
+                    regexFilter = new RegExp(pattern);
+                } catch (e) {
+                    console.error("Invalid regex pattern:", e);
+                    regexFilter = null;
+                    filterInput.classList.add('regex-error');
+                    return
+                }
+            } else {
+                regexFilter = null;
+            }
+
+            render();
+        }
+
+        filterInput.addEventListener('input', updateFilter);
+        document.getElementById('clear-button').addEventListener('click', () => {
+            filterInput.value = "";
+            regexFilter = null;
+            render();
+        });
+        setupEventSource();
+        updateFilter();
     </script>
 </body>
 </html>

proxy/html_files.go

@@ -0,0 +1,10 @@
+package proxy
+
+import "embed"
+
+//go:embed html
+var htmlFiles embed.FS
+
+func getHTMLFile(path string) ([]byte, error) {
+	return htmlFiles.ReadFile("html/" + path)
+}

proxy/proxymanager.go

@@ -2,7 +2,6 @@ package proxy
 
 import (
 	"bytes"
-	"embed"
 	"encoding/json"
 	"fmt"
 	"io"
@@ -20,15 +19,6 @@ const (
 	PROFILE_SPLIT_CHAR = ":"
 )
 
-//go:embed html/favicon.ico
-var faviconData []byte
-
-//go:embed html/logs.html
-var logsHTML []byte
-
-// make sure embed is kept there by the IDE auto-package importer
-var _ = embed.FS{}
-
 type ProxyManager struct {
 	sync.Mutex
 
@@ -46,6 +36,39 @@ func New(config *Config) *ProxyManager {
 		ginEngine:        gin.New(),
 	}
 
+	if config.LogRequests {
+		pm.ginEngine.Use(func(c *gin.Context) {
+			// Start timer
+			start := time.Now()
+
+			// capture these because /upstream/:model rewrites them in c.Next()
+			clientIP := c.ClientIP()
+			method := c.Request.Method
+			path := c.Request.URL.Path
+
+			// Process request
+			c.Next()
+
+			// Stop timer
+			duration := time.Since(start)
+
+			statusCode := c.Writer.Status()
+			bodySize := c.Writer.Size()
+
+			fmt.Fprintf(pm.logMonitor, "[llama-swap] %s [%s] \"%s %s %s\" %d %d \"%s\" %v\n",
+				clientIP,
+				time.Now().Format("2006-01-02 15:04:05"),
+				method,
+				path,
+				c.Request.Proto,
+				statusCode,
+				bodySize,
+				c.Request.UserAgent(),
+				duration,
+			)
+		})
+	}
+
 	// Set up routes using the Gin engine
 	pm.ginEngine.POST("/v1/chat/completions", pm.proxyOAIHandler)
 	// Support legacy /v1/completions api, see issue #12
@@ -55,6 +78,9 @@ func New(config *Config) *ProxyManager {
 	pm.ginEngine.POST("/v1/embeddings", pm.proxyOAIHandler)
 	pm.ginEngine.POST("/v1/rerank", pm.proxyOAIHandler)
 
+	// Support audio/speech endpoint
+	pm.ginEngine.POST("/v1/audio/speech", pm.proxyOAIHandler)
+
 	pm.ginEngine.GET("/v1/models", pm.listModelsHandler)
 
 	// in proxymanager_loghandlers.go
@@ -65,8 +91,29 @@ func New(config *Config) *ProxyManager {
 	pm.ginEngine.GET("/upstream", pm.upstreamIndex)
 	pm.ginEngine.Any("/upstream/:model_id/*upstreamPath", pm.proxyToUpstream)
 
+	pm.ginEngine.GET("/", func(c *gin.Context) {
+		// Set the Content-Type header to text/html
+		c.Header("Content-Type", "text/html")
+
+		// Write the embedded HTML content to the response
+		htmlData, err := getHTMLFile("index.html")
+		if err != nil {
+			c.String(http.StatusInternalServerError, err.Error())
+			return
+		}
+		_, err = c.Writer.Write(htmlData)
+		if err != nil {
+			c.String(http.StatusInternalServerError, fmt.Sprintf("failed to write response: %v", err))
+			return
+		}
+	})
+
 	pm.ginEngine.GET("/favicon.ico", func(c *gin.Context) {
-		c.Data(http.StatusOK, "image/x-icon", faviconData)
+		if data, err := getHTMLFile("favicon.ico"); err == nil {
+			c.Data(http.StatusOK, "image/x-icon", data)
+		} else {
+			c.String(http.StatusInternalServerError, err.Error())
+		}
 	})
 
 	// Disable console color for testing
@@ -127,7 +174,7 @@ func (pm *ProxyManager) listModelsHandler(c *gin.Context) {
 
 	// Encode the data as JSON and write it to the response writer
 	if err := json.NewEncoder(c.Writer).Encode(map[string]interface{}{"data": data}); err != nil {
-		c.AbortWithError(http.StatusInternalServerError, fmt.Errorf("error encoding JSON"))
+		pm.sendErrorResponse(c, http.StatusInternalServerError, fmt.Sprintf("error encoding JSON %s", err.Error()))
 		return
 	}
 }
@@ -197,12 +244,12 @@ func (pm *ProxyManager) proxyToUpstream(c *gin.Context) {
 	requestedModel := c.Param("model_id")
 
 	if requestedModel == "" {
-		c.AbortWithError(http.StatusBadRequest, fmt.Errorf("model id required in path"))
+		pm.sendErrorResponse(c, http.StatusBadRequest, "model id required in path")
 		return
 	}
 
 	if process, err := pm.swapModel(requestedModel); err != nil {
-		c.AbortWithError(http.StatusNotFound, fmt.Errorf("unable to swap to model, %s", err.Error()))
+		pm.sendErrorResponse(c, http.StatusNotFound, fmt.Sprintf("unable to swap to model, %s", err.Error()))
 	} else {
 		// rewrite the path
 		c.Request.URL.Path = c.Param("upstreamPath")
@@ -238,22 +285,23 @@ func (pm *ProxyManager) upstreamIndex(c *gin.Context) {
 func (pm *ProxyManager) proxyOAIHandler(c *gin.Context) {
 	bodyBytes, err := io.ReadAll(c.Request.Body)
 	if err != nil {
-		c.AbortWithError(http.StatusBadRequest, fmt.Errorf("invalid JSON"))
+		pm.sendErrorResponse(c, http.StatusBadRequest, "could not ready request body")
 		return
 	}
+
 	var requestBody map[string]interface{}
 	if err := json.Unmarshal(bodyBytes, &requestBody); err != nil {
-		c.AbortWithError(http.StatusBadRequest, fmt.Errorf("invalid JSON"))
+		pm.sendErrorResponse(c, http.StatusBadRequest, fmt.Sprintf("invalid JSON: %s", err.Error()))
 		return
 	}
 	model, ok := requestBody["model"].(string)
 	if !ok {
-		c.AbortWithError(http.StatusBadRequest, fmt.Errorf("missing or invalid 'model' key"))
+		pm.sendErrorResponse(c, http.StatusBadRequest, "missing or invalid 'model' key")
 		return
 	}
 
 	if process, err := pm.swapModel(model); err != nil {
-		c.AbortWithError(http.StatusNotFound, fmt.Errorf("unable to swap to model, %s", err.Error()))
+		pm.sendErrorResponse(c, http.StatusNotFound, fmt.Sprintf("unable to swap to model, %s", err.Error()))
 		return
 	} else {
 		c.Request.Body = io.NopCloser(bytes.NewBuffer(bodyBytes))
@@ -266,6 +314,16 @@ func (pm *ProxyManager) proxyOAIHandler(c *gin.Context) {
 	}
 }
 
+func (pm *ProxyManager) sendErrorResponse(c *gin.Context, statusCode int, message string) {
+	acceptHeader := c.GetHeader("Accept")
+
+	if strings.Contains(acceptHeader, "application/json") {
+		c.JSON(statusCode, gin.H{"error": message})
+	} else {
+		c.String(statusCode, message)
+	}
+}
+
 func ProcessKeyName(groupName, modelName string) string {
 	return groupName + PROFILE_SPLIT_CHAR + modelName
 }

proxy/proxymanager_loghandlers.go

@@ -16,9 +16,14 @@ func (pm *ProxyManager) sendLogsHandlers(c *gin.Context) {
 		c.Header("Content-Type", "text/html")
 
 		// Write the embedded HTML content to the response
-		_, err := c.Writer.Write(logsHTML)
+		logsHTML, err := getHTMLFile("logs.html")
 		if err != nil {
-			c.AbortWithError(http.StatusInternalServerError, fmt.Errorf("failed to write response: %v", err))
+			c.String(http.StatusInternalServerError, err.Error())
+			return
+		}
+		_, err = c.Writer.Write(logsHTML)
+		if err != nil {
+			c.String(http.StatusInternalServerError, fmt.Sprintf("failed to write response: %v", err))
 			return
 		}
 	} else {
@@ -43,7 +48,7 @@ func (pm *ProxyManager) streamLogsHandler(c *gin.Context) {
 	notify := c.Request.Context().Done()
 	flusher, ok := c.Writer.(http.Flusher)
 	if !ok {
-		c.AbortWithError(http.StatusInternalServerError, fmt.Errorf("Streaming unsupported"))
+		c.AbortWithError(http.StatusInternalServerError, fmt.Errorf("streaming unsupported"))
 		return
 	}
 
@@ -53,11 +58,7 @@ func (pm *ProxyManager) streamLogsHandler(c *gin.Context) {
 	if !skipHistory {
 		history := pm.logMonitor.GetHistory()
 		if len(history) != 0 {
-			_, err := c.Writer.Write(history)
-			if err != nil {
-				c.AbortWithError(http.StatusInternalServerError, err)
-				return
-			}
+			c.Writer.Write(history)
 			flusher.Flush()
 		}
 	}
@@ -68,7 +69,7 @@ func (pm *ProxyManager) streamLogsHandler(c *gin.Context) {
 		case msg := <-ch:
 			_, err := c.Writer.Write(msg)
 			if err != nil {
-				c.AbortWithError(http.StatusInternalServerError, err)
+				// just break the loop if we can't write for some reason
 				return
 			}
 			flusher.Flush()