Add support for sending a custom model name to upstream (#69) (#71)

* add test for splitRequestedModel()
* Add `useModelName` parameter to model configuration
* add docs to README

.github/workflows/closeinactive.yml

@@ -0,0 +1,23 @@
+# https://docs.github.com/en/actions/use-cases-and-examples/project-management/closing-inactive-issues
+name: Close inactive issues
+on:
+  schedule:
+    - cron: "32 1 * * *"
+
+jobs:
+  close-issues:
+    runs-on: ubuntu-latest
+    permissions:
+      issues: write
+      pull-requests: write
+    steps:
+      - uses: actions/stale@v9
+        with:
+          days-before-issue-stale: 30
+          days-before-issue-close: 14
+          stale-issue-label: "stale"
+          stale-issue-message: "This issue is stale because it has been open for 30 days with no activity."
+          close-issue-message: "This issue was closed because it has been inactive for 14 days since being marked as stale."
+          days-before-pr-stale: -1
+          days-before-pr-close: -1
+          repo-token: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/containers.yml

@@ -16,6 +16,7 @@ jobs:
     strategy:
       matrix:
         platform: [intel, cuda, vulkan, cpu, musa]
+      fail-fast: false
     steps:
       - name: Checkout code
         uses: actions/checkout@v4

.github/workflows/go-ci.yml

@@ -0,0 +1,32 @@
+# This workflow will build a golang project
+
+name: CI
+
+on:
+  push:
+    branches: [ "main" ]
+
+  pull_request:
+    branches: [ "main" ]
+
+  # Allows manual triggering of the workflow
+  workflow_dispatch:
+
+jobs:
+
+  run-tests:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v4
+
+    - name: Set up Go
+      uses: actions/setup-go@v4
+      with:
+        go-version: '1.23'
+
+    # necessary for testing proxy/Process swapping
+    - name: Create simple-responder
+      run: make simple-responder
+
+    - name: Test all
+      run: make test-all

README.md

@@ -11,20 +11,23 @@ Written in golang, it is very easy to install (single binary with no dependancie
 - ✅ Easy to deploy: single binary with no dependencies
 - ✅ Easy to config: single yaml file
 - ✅ On-demand model switching
-- ✅ Full control over server settings per model
 - ✅ OpenAI API supported endpoints:
   - `v1/completions`
   - `v1/chat/completions`
   - `v1/embeddings`
   - `v1/rerank`
   - `v1/audio/speech` ([#36](https://github.com/mostlygeek/llama-swap/issues/36))
+  - `v1/audio/transcriptions` ([docs](https://github.com/mostlygeek/llama-swap/issues/41#issuecomment-2722637867))
+- ✅ llama-swap custom API endpoints
+  - `/log` - remote log monitoring
+  - `/upstream/:model_id` - direct access to upstream HTTP server ([demo](https://github.com/mostlygeek/llama-swap/pull/31))
+  - `/unload` - manually unload running models ([#58](https://github.com/mostlygeek/llama-swap/issues/58))
+  - `/running` - list currently running models ([#61](https://github.com/mostlygeek/llama-swap/issues/61))
 - ✅ Run multiple models at once with `profiles` ([docs](https://github.com/mostlygeek/llama-swap/issues/53#issuecomment-2660761741))
-- ✅ Remote log monitoring at `/log`
-- ✅ Direct access to upstream HTTP server via `/upstream/:model_id` ([demo](https://github.com/mostlygeek/llama-swap/pull/31))
-- ✅ Manually unload models via `/unload` endpoint ([#58](https://github.com/mostlygeek/llama-swap/issues/58))
 - ✅ Automatic unloading of models after timeout by setting a `ttl`
 - ✅ Use any local OpenAI compatible server (llama.cpp, vllm, tabbyAPI, etc)
 - ✅ Docker and Podman support
+- ✅ Full control over server settings per model
 
 ## How does llama-swap work?
 
@@ -114,6 +117,13 @@ models:
       ghcr.io/ggerganov/llama.cpp:server
       --model '/models/Qwen2.5-Coder-0.5B-Instruct-Q4_K_M.gguf'
 
+  # `useModelName` will send a specific model name to the upstream server
+  # overriding whatever was set in the request
+  "qwq":
+    proxy: http://127.0.0.1:11434
+    cmd: my-server
+    useModelName: "qwen:qwq"
+
 # profiles make it easy to managing multi model (and gpu) configurations.
 #
 # Tips:

docker/build-container.sh

@@ -38,6 +38,12 @@ else
         | jq -r --arg arch "$ARCH" '.[] | select(.metadata.container.tags[] | startswith("server-\($arch)")) | .metadata.container.tags[]' \
         | sort -r | head -n1 | awk -F '-' '{print $3}')
 
+    # Abort if LCPP_TAG is empty.
+    if [[ -z "$LCPP_TAG" ]]; then
+        echo "Abort: Could not find llama-server container for arch: $ARCH"
+        exit 1
+    fi
+
     CONTAINER_TAG="ghcr.io/mostlygeek/llama-swap:v${LS_VER}-${ARCH}-${LCPP_TAG}"
     CONTAINER_LATEST="ghcr.io/mostlygeek/llama-swap:${ARCH}"
     echo "Building ${CONTAINER_TAG} $LS_VER"

go.mod

@@ -3,7 +3,11 @@ module github.com/mostlygeek/llama-swap
 go 1.23.0
 
 require (
+	github.com/gin-gonic/gin v1.10.0
+	github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510
 	github.com/stretchr/testify v1.9.0
+	github.com/tidwall/gjson v1.18.0
+	github.com/tidwall/sjson v1.2.5
 	gopkg.in/yaml.v3 v3.0.1
 )
 
@@ -15,12 +19,10 @@ require (
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/gabriel-vasile/mimetype v1.4.3 // indirect
 	github.com/gin-contrib/sse v0.1.0 // indirect
-	github.com/gin-gonic/gin v1.10.0 // indirect
 	github.com/go-playground/locales v0.14.1 // indirect
 	github.com/go-playground/universal-translator v0.18.1 // indirect
 	github.com/go-playground/validator/v10 v10.20.0 // indirect
 	github.com/goccy/go-json v0.10.2 // indirect
-	github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect
 	github.com/klauspost/cpuid/v2 v2.2.7 // indirect
 	github.com/leodido/go-urn v1.4.0 // indirect
@@ -29,16 +31,14 @@ require (
 	github.com/modern-go/reflect2 v1.0.2 // indirect
 	github.com/pelletier/go-toml/v2 v2.2.2 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
-	github.com/tidwall/gjson v1.18.0 // indirect
 	github.com/tidwall/match v1.1.1 // indirect
 	github.com/tidwall/pretty v1.2.1 // indirect
-	github.com/tidwall/sjson v1.2.5 // indirect
 	github.com/twitchyliquid64/golang-asm v0.15.1 // indirect
 	github.com/ugorji/go/codec v1.2.12 // indirect
 	golang.org/x/arch v0.8.0 // indirect
-	golang.org/x/crypto v0.31.0 // indirect
-	golang.org/x/net v0.33.0 // indirect
-	golang.org/x/sys v0.28.0 // indirect
-	golang.org/x/text v0.21.0 // indirect
+	golang.org/x/crypto v0.36.0 // indirect
+	golang.org/x/net v0.37.0 // indirect
+	golang.org/x/sys v0.31.0 // indirect
+	golang.org/x/text v0.23.0 // indirect
 	google.golang.org/protobuf v1.34.1 // indirect
 )

go.sum

@@ -78,20 +78,28 @@ golang.org/x/crypto v0.23.0 h1:dIJU/v2J8Mdglj/8rJ6UUOM3Zc9zLZxVZwwxMooUSAI=
 golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8=
 golang.org/x/crypto v0.31.0 h1:ihbySMvVjLAeSH1IbfcRTkD/iNscyz8rGzjF/E5hV6U=
 golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk=
+golang.org/x/crypto v0.36.0 h1:AnAEvhDddvBdpY+uR+MyHmuZzzNqXSe/GvuDeob5L34=
+golang.org/x/crypto v0.36.0/go.mod h1:Y4J0ReaxCR1IMaabaSMugxJES1EpwhBHhv2bDHklZvc=
 golang.org/x/net v0.25.0 h1:d/OCCoBEUq33pjydKrGQhw7IlUPI2Oylr+8qLx49kac=
 golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM=
 golang.org/x/net v0.33.0 h1:74SYHlV8BIgHIFC/LrYkOGIwL19eTYXQ5wc6TBuO36I=
 golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4=
+golang.org/x/net v0.37.0 h1:1zLorHbz+LYj7MQlSf1+2tPIIgibq2eL5xkrGk6f+2c=
+golang.org/x/net v0.37.0/go.mod h1:ivrbrMbzFq5J41QOQh0siUuly180yBYtLp+CKbEaFx8=
 golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.20.0 h1:Od9JTbYCk261bKm4M/mw7AklTlFYIa0bIp9BgSm1S8Y=
 golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA=
 golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/sys v0.31.0 h1:ioabZlmFYtWhL+TRYpcnNlLwhyxaM9kWTDEmfnprqik=
+golang.org/x/sys v0.31.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k=
 golang.org/x/text v0.15.0 h1:h1V/4gjBv8v9cjcR6+AR5+/cIYK5N/WAgiv4xlsEtAk=
 golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
 golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo=
 golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ=
+golang.org/x/text v0.23.0 h1:D71I7dUrlY+VX0gQShAThNGHFxZ13dGLBHQLVl1mJlY=
+golang.org/x/text v0.23.0/go.mod h1:/BLNzu4aZCJ1+kcD0DNRotWKage4q2rGVAg4o22unh4=
 google.golang.org/protobuf v1.34.1 h1:9ddQBjfCyZPOHPUiPxpYESBLc+T8P3E+Vo4IbKZgFWg=
 google.golang.org/protobuf v1.34.1/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=

misc/simple-responder/simple-responder.go

@@ -67,6 +67,46 @@ func main() {
 		c.String(200, *responseMessage)
 	})
 
+	// issue #41
+	r.POST("/v1/audio/transcriptions", func(c *gin.Context) {
+		// Parse the multipart form
+		if err := c.Request.ParseMultipartForm(10 << 20); err != nil { // 10 MB max memory
+			c.JSON(http.StatusBadRequest, gin.H{"error": fmt.Sprintf("Error parsing multipart form: %s", err)})
+			return
+		}
+
+		// Get the model from the form values
+		model := c.Request.FormValue("model")
+
+		if model == "" {
+			c.JSON(http.StatusBadRequest, gin.H{"error": "Missing model parameter"})
+			return
+		}
+
+		// Get the file from the form
+		file, _, err := c.Request.FormFile("file")
+		if err != nil {
+			c.JSON(http.StatusBadRequest, gin.H{"error": fmt.Sprintf("Error getting file: %s", err)})
+			return
+		}
+		defer file.Close()
+
+		// Read the file content to get its size
+		fileBytes, err := io.ReadAll(file)
+		if err != nil {
+			c.JSON(http.StatusInternalServerError, gin.H{"error": fmt.Sprintf("Error reading file: %s", err)})
+			return
+		}
+
+		fileSize := len(fileBytes)
+
+		// Return a JSON response with the model and transcription text including file size
+		c.JSON(http.StatusOK, gin.H{
+			"text":  fmt.Sprintf("The length of the file is %d bytes", fileSize),
+			"model": model,
+		})
+	})
+
 	r.GET("/slow-respond", func(c *gin.Context) {
 		echo := c.Query("echo")
 		delay := c.Query("delay")

proxy/config.go

@@ -17,6 +17,7 @@ type ModelConfig struct {
 	CheckEndpoint string   `yaml:"checkEndpoint"`
 	UnloadAfter   int      `yaml:"ttl"`
 	Unlisted      bool     `yaml:"unlisted"`
+	UseModelName  string   `yaml:"useModelName"`
 }
 
 func (m *ModelConfig) SanitizedCommand() ([]string, error) {

proxy/process.go

@@ -116,7 +116,9 @@ func (p *Process) start() error {
 		return fmt.Errorf("can not start(), upstream proxy missing")
 	}
 
-	// wait for the other start() to complete
+	// multiple start() calls will wait for the one that is actually starting to
+	// complete before proceeding.
+	// ===========
 	curState := p.CurrentState()
 
 	if curState == StateReady {
@@ -132,10 +134,26 @@ func (p *Process) start() error {
 
 		return nil
 	}
+	// ===========
+
+	// There is the possibility of a hard to replicate race condition where
+	// curState *WAS* StateStopped but by the time we get to the p.stateMutex.Lock()
+	// below, it's value has changed!
 
 	p.stateMutex.Lock()
 	defer p.stateMutex.Unlock()
 
+	// with the exclusive lock, check if p.state is StateStopped, which is the only valid state
+	// to transition from to StateReady
+
+	if p.state != StateStopped {
+		if p.state == StateReady {
+			return nil
+		} else {
+			return fmt.Errorf("start() can not proceed expected StateReady but process is in %v", p.state)
+		}
+	}
+
 	if err := p.setState(StateStarting); err != nil {
 		return err
 	}

proxy/process_test.go

@@ -169,6 +169,8 @@ func TestProcess_LowTTLValue(t *testing.T) {
 }
 
 // issue #19
+// This test makes sure using Process.Stop() does not affect pending HTTP
+// requests. All HTTP requests in this test should complete successfully.
 func TestProcess_HTTPRequestsHaveTimeToFinish(t *testing.T) {
 	if testing.Short() {
 		t.Skip("skipping slow test")
@@ -192,8 +194,9 @@ func TestProcess_HTTPRequestsHaveTimeToFinish(t *testing.T) {
 		wg.Add(1)
 		go func(key string) {
 			defer wg.Done()
-			// send a request that should take 5 * 200ms (1 second) to complete
-			req := httptest.NewRequest("GET", fmt.Sprintf("/slow-respond?echo=%s&delay=200ms", key), nil)
+			// send a request where simple-responder is will wait 300ms before responding
+			// this will simulate an in-progress request.
+			req := httptest.NewRequest("GET", fmt.Sprintf("/slow-respond?echo=%s&delay=300ms", key), nil)
 			w := httptest.NewRecorder()
 
 			process.ProxyRequest(w, req)
@@ -209,9 +212,9 @@ func TestProcess_HTTPRequestsHaveTimeToFinish(t *testing.T) {
 		}(key)
 	}
 
-	// stop the requests in the middle
+	// Stop the process while requests are still being processed
 	go func() {
-		<-time.After(500 * time.Millisecond)
+		<-time.After(150 * time.Millisecond)
 		process.Stop()
 	}()
 

proxy/proxymanager.go

@@ -5,6 +5,7 @@ import (
 	"encoding/json"
 	"fmt"
 	"io"
+	"mime/multipart"
 	"net/http"
 	"sort"
 	"strconv"
@@ -95,6 +96,7 @@ func New(config *Config) *ProxyManager {
 
 	// Support audio/speech endpoint
 	pm.ginEngine.POST("/v1/audio/speech", pm.proxyOAIHandler)
+	pm.ginEngine.POST("/v1/audio/transcriptions", pm.proxyOAIPostFormHandler)
 
 	pm.ginEngine.GET("/v1/models", pm.listModelsHandler)
 
@@ -108,6 +110,8 @@ func New(config *Config) *ProxyManager {
 
 	pm.ginEngine.GET("/unload", pm.unloadAllModelsHandler)
 
+	pm.ginEngine.GET("/running", pm.listRunningProcessesHandler)
+
 	pm.ginEngine.GET("/", func(c *gin.Context) {
 		// Set the Content-Type header to text/html
 		c.Header("Content-Type", "text/html")
@@ -347,12 +351,21 @@ func (pm *ProxyManager) proxyOAIHandler(c *gin.Context) {
 		pm.sendErrorResponse(c, http.StatusBadRequest, "missing or invalid 'model' key")
 	}
 
-	if process, err := pm.swapModel(requestedModel); err != nil {
+	process, err := pm.swapModel(requestedModel)
+
+	if err != nil {
 		pm.sendErrorResponse(c, http.StatusNotFound, fmt.Sprintf("unable to swap to model, %s", err.Error()))
 		return
-	} else {
+	}
 
-		// strip
+	// issue #69 allow custom model names to be sent to upstream
+	if process.config.UseModelName != "" {
+		bodyBytes, err = sjson.SetBytes(bodyBytes, "model", process.config.UseModelName)
+		if err != nil {
+			pm.sendErrorResponse(c, http.StatusInternalServerError, fmt.Sprintf("error updating JSON: %s", err.Error()))
+			return
+		}
+	} else {
 		profileName, modelName := splitRequestedModel(requestedModel)
 		if profileName != "" {
 			bodyBytes, err = sjson.SetBytes(bodyBytes, "model", modelName)
@@ -362,14 +375,119 @@ func (pm *ProxyManager) proxyOAIHandler(c *gin.Context) {
 			}
 		}
 
-		c.Request.Body = io.NopCloser(bytes.NewBuffer(bodyBytes))
-
-		// dechunk it as we already have all the body bytes see issue #11
-		c.Request.Header.Del("transfer-encoding")
-		c.Request.Header.Add("content-length", strconv.Itoa(len(bodyBytes)))
-
-		process.ProxyRequest(c.Writer, c.Request)
 	}
+
+	c.Request.Body = io.NopCloser(bytes.NewBuffer(bodyBytes))
+
+	// dechunk it as we already have all the body bytes see issue #11
+	c.Request.Header.Del("transfer-encoding")
+	c.Request.Header.Add("content-length", strconv.Itoa(len(bodyBytes)))
+
+	process.ProxyRequest(c.Writer, c.Request)
+
+}
+
+func (pm *ProxyManager) proxyOAIPostFormHandler(c *gin.Context) {
+	// We need to reconstruct the multipart form in any case since the body is consumed
+	// Create a new buffer for the reconstructed request
+	var requestBuffer bytes.Buffer
+	multipartWriter := multipart.NewWriter(&requestBuffer)
+
+	// Parse multipart form
+	if err := c.Request.ParseMultipartForm(32 << 20); err != nil { // 32MB max memory, larger files go to tmp disk
+		pm.sendErrorResponse(c, http.StatusBadRequest, fmt.Sprintf("error parsing multipart form: %s", err.Error()))
+		return
+	}
+
+	// Get model parameter from the form
+	requestedModel := c.Request.FormValue("model")
+	if requestedModel == "" {
+		pm.sendErrorResponse(c, http.StatusBadRequest, "missing or invalid 'model' parameter in form data")
+		return
+	}
+
+	// Swap to the requested model
+	process, err := pm.swapModel(requestedModel)
+	if err != nil {
+		pm.sendErrorResponse(c, http.StatusNotFound, fmt.Sprintf("unable to swap to model, %s", err.Error()))
+		return
+	}
+
+	// Get profile name and model name from the requested model
+	profileName, modelName := splitRequestedModel(requestedModel)
+
+	// Copy all form values
+	for key, values := range c.Request.MultipartForm.Value {
+		for _, value := range values {
+			fieldValue := value
+			// If this is the model field and we have a profile, use just the model name
+			if key == "model" {
+				if process.config.UseModelName != "" {
+					fieldValue = process.config.UseModelName
+				} else if profileName != "" {
+					fieldValue = modelName
+				}
+			}
+			field, err := multipartWriter.CreateFormField(key)
+			if err != nil {
+				pm.sendErrorResponse(c, http.StatusInternalServerError, "error recreating form field")
+				return
+			}
+			if _, err = field.Write([]byte(fieldValue)); err != nil {
+				pm.sendErrorResponse(c, http.StatusInternalServerError, "error writing form field")
+				return
+			}
+		}
+	}
+
+	// Copy all files from the original request
+	for key, fileHeaders := range c.Request.MultipartForm.File {
+		for _, fileHeader := range fileHeaders {
+			formFile, err := multipartWriter.CreateFormFile(key, fileHeader.Filename)
+			if err != nil {
+				pm.sendErrorResponse(c, http.StatusInternalServerError, "error recreating form file")
+				return
+			}
+
+			file, err := fileHeader.Open()
+			if err != nil {
+				pm.sendErrorResponse(c, http.StatusInternalServerError, "error opening uploaded file")
+				return
+			}
+
+			if _, err = io.Copy(formFile, file); err != nil {
+				file.Close()
+				pm.sendErrorResponse(c, http.StatusInternalServerError, "error copying file data")
+				return
+			}
+			file.Close()
+		}
+	}
+
+	// Close the multipart writer to finalize the form
+	if err := multipartWriter.Close(); err != nil {
+		pm.sendErrorResponse(c, http.StatusInternalServerError, "error finalizing multipart form")
+		return
+	}
+
+	// Create a new request with the reconstructed form data
+	modifiedReq, err := http.NewRequestWithContext(
+		c.Request.Context(),
+		c.Request.Method,
+		c.Request.URL.String(),
+		&requestBuffer,
+	)
+	if err != nil {
+		pm.sendErrorResponse(c, http.StatusInternalServerError, "error creating modified request")
+		return
+	}
+
+	// Copy the headers from the original request
+	modifiedReq.Header = c.Request.Header.Clone()
+	modifiedReq.Header.Set("Content-Type", multipartWriter.FormDataContentType())
+
+	// Use the modified request for proxying
+	process.ProxyRequest(c.Writer, modifiedReq)
 }
 
 func (pm *ProxyManager) sendErrorResponse(c *gin.Context, statusCode int, message string) {
@@ -387,6 +505,28 @@ func (pm *ProxyManager) unloadAllModelsHandler(c *gin.Context) {
 	c.String(http.StatusOK, "OK")
 }
 
+func (pm *ProxyManager) listRunningProcessesHandler(context *gin.Context) {
+	context.Header("Content-Type", "application/json")
+	runningProcesses := make([]gin.H, 0) // Default to an empty response.
+
+	for _, process := range pm.currentProcesses {
+
+		// Append the process ID and State (multiple entries if profiles are being used).
+		runningProcesses = append(runningProcesses, gin.H{
+			"model": process.ID,
+			"state": process.state,
+		})
+
+	}
+
+	// Put the results under the `running` key.
+	response := gin.H{
+		"running": runningProcesses,
+	}
+
+	context.JSON(http.StatusOK, response) // Always return 200 OK
+}
+
 func ProcessKeyName(groupName, modelName string) string {
 	return groupName + PROFILE_SPLIT_CHAR + modelName
 }

proxy/proxymanager_test.go

@@ -4,6 +4,8 @@ import (
 	"bytes"
 	"encoding/json"
 	"fmt"
+	"math/rand"
+	"mime/multipart"
 	"net/http"
 	"net/http/httptest"
 	"sync"
@@ -349,3 +351,293 @@ func TestProxyManager_StripProfileSlug(t *testing.T) {
 	assert.Equal(t, http.StatusOK, w.Code)
 	assert.Contains(t, w.Body.String(), "ok")
 }
+
+// Test issue #61 `Listing the current list of models and the loaded model.`
+func TestProxyManager_RunningEndpoint(t *testing.T) {
+
+	// Shared configuration
+	config := &Config{
+		HealthCheckTimeout: 15,
+		Models: map[string]ModelConfig{
+			"model1": getTestSimpleResponderConfig("model1"),
+			"model2": getTestSimpleResponderConfig("model2"),
+		},
+		Profiles: map[string][]string{
+			"test": {"model1", "model2"},
+		},
+	}
+
+	// Define a helper struct to parse the JSON response.
+	type RunningResponse struct {
+		Running []struct {
+			Model string `json:"model"`
+			State string `json:"state"`
+		} `json:"running"`
+	}
+
+	// Create proxy once for all tests
+	proxy := New(config)
+	defer proxy.StopProcesses()
+
+	t.Run("no models loaded", func(t *testing.T) {
+		req := httptest.NewRequest("GET", "/running", nil)
+		w := httptest.NewRecorder()
+		proxy.HandlerFunc(w, req)
+
+		assert.Equal(t, http.StatusOK, w.Code)
+
+		var response RunningResponse
+
+		// Check if this is a valid JSON object.
+		assert.NoError(t, json.Unmarshal(w.Body.Bytes(), &response))
+
+		// We should have an empty running array here.
+		assert.Empty(t, response.Running, "expected no running models")
+	})
+
+	t.Run("single model loaded", func(t *testing.T) {
+		// Load just a model.
+		reqBody := `{"model":"model1"}`
+		req := httptest.NewRequest("POST", "/v1/chat/completions", bytes.NewBufferString(reqBody))
+		w := httptest.NewRecorder()
+		proxy.HandlerFunc(w, req)
+		assert.Equal(t, http.StatusOK, w.Code)
+
+		// Simulate browser call for the `/running` endpoint.
+		req = httptest.NewRequest("GET", "/running", nil)
+		w = httptest.NewRecorder()
+		proxy.HandlerFunc(w, req)
+
+		var response RunningResponse
+		assert.NoError(t, json.Unmarshal(w.Body.Bytes(), &response))
+
+		// Check if we have a single array element.
+		assert.Len(t, response.Running, 1)
+
+		// Is this the right model?
+		assert.Equal(t, "model1", response.Running[0].Model)
+
+		// Is the model loaded?
+		assert.Equal(t, "ready", response.Running[0].State)
+	})
+
+	t.Run("multiple models via profile", func(t *testing.T) {
+		// Load more than one model.
+		for _, model := range []string{"model1", "model2"} {
+			profileModel := ProcessKeyName("test", model)
+			reqBody := fmt.Sprintf(`{"model":"%s"}`, profileModel)
+			req := httptest.NewRequest("POST", "/v1/chat/completions", bytes.NewBufferString(reqBody))
+			w := httptest.NewRecorder()
+			proxy.HandlerFunc(w, req)
+			assert.Equal(t, http.StatusOK, w.Code)
+		}
+
+		// Simulate the browser call.
+		req := httptest.NewRequest("GET", "/running", nil)
+		w := httptest.NewRecorder()
+		proxy.HandlerFunc(w, req)
+
+		var response RunningResponse
+
+		// The JSON response must be valid.
+		assert.NoError(t, json.Unmarshal(w.Body.Bytes(), &response))
+
+		// The response should contain 2 models.
+		assert.Len(t, response.Running, 2)
+
+		expectedModels := map[string]struct{}{
+			"model1": {},
+			"model2": {},
+		}
+
+		// Iterate through the models and check their states as well.
+		for _, entry := range response.Running {
+			_, exists := expectedModels[entry.Model]
+			assert.True(t, exists, "unexpected model %s", entry.Model)
+			assert.Equal(t, "ready", entry.State)
+			delete(expectedModels, entry.Model)
+		}
+
+		// Since we deleted each model while testing for its validity we should have no more models in the response.
+		assert.Empty(t, expectedModels, "unexpected additional models in response")
+	})
+}
+
+func TestProxyManager_AudioTranscriptionHandler(t *testing.T) {
+	config := &Config{
+		HealthCheckTimeout: 15,
+		Profiles: map[string][]string{
+			"test": {"TheExpectedModel"},
+		},
+		Models: map[string]ModelConfig{
+			"TheExpectedModel": getTestSimpleResponderConfig("TheExpectedModel"),
+		},
+	}
+
+	proxy := New(config)
+	defer proxy.StopProcesses()
+
+	testCases := []struct {
+		name        string
+		modelInput  string
+		expectModel string
+	}{
+		{
+			name:        "With Profile Prefix",
+			modelInput:  "test:TheExpectedModel",
+			expectModel: "TheExpectedModel", // Profile prefix should be stripped
+		},
+		{
+			name:        "Without Profile Prefix",
+			modelInput:  "TheExpectedModel",
+			expectModel: "TheExpectedModel", // Should remain the same
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			// Create a buffer with multipart form data
+			var b bytes.Buffer
+			w := multipart.NewWriter(&b)
+
+			// Add the model field
+			fw, err := w.CreateFormField("model")
+			assert.NoError(t, err)
+			_, err = fw.Write([]byte(tc.modelInput))
+			assert.NoError(t, err)
+
+			// Add a file field
+			fw, err = w.CreateFormFile("file", "test.mp3")
+			assert.NoError(t, err)
+			// Generate random content length between 10 and 20
+			contentLength := rand.Intn(11) + 10 // 10 to 20
+			content := make([]byte, contentLength)
+			_, err = fw.Write(content)
+			assert.NoError(t, err)
+			w.Close()
+
+			// Create the request with the multipart form data
+			req := httptest.NewRequest("POST", "/v1/audio/transcriptions", &b)
+			req.Header.Set("Content-Type", w.FormDataContentType())
+			rec := httptest.NewRecorder()
+			proxy.HandlerFunc(rec, req)
+
+			// Verify the response
+			assert.Equal(t, http.StatusOK, rec.Code)
+			var response map[string]string
+			err = json.Unmarshal(rec.Body.Bytes(), &response)
+			assert.NoError(t, err)
+			assert.Equal(t, tc.expectModel, response["model"])
+			assert.Equal(t, response["text"], fmt.Sprintf("The length of the file is %d bytes", contentLength)) // matches simple-responder
+		})
+	}
+}
+
+func TestProxyManager_SplitRequestedModel(t *testing.T) {
+
+	tests := []struct {
+		name            string
+		requestedModel  string
+		expectedProfile string
+		expectedModel   string
+	}{
+		{"no profile", "gpt-4", "", "gpt-4"},
+		{"with profile", "profile1:gpt-4", "profile1", "gpt-4"},
+		{"only profile", "profile1:", "profile1", ""},
+		{"empty model", ":gpt-4", "", "gpt-4"},
+		{"empty profile", ":", "", ""},
+		{"no split char", "gpt-4", "", "gpt-4"},
+		{"profile and model with delimiter", "profile1:delimiter:gpt-4", "profile1", "delimiter:gpt-4"},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			profileName, modelName := splitRequestedModel(tt.requestedModel)
+			if profileName != tt.expectedProfile {
+				t.Errorf("splitRequestedModel(%q) = %q, %q; want %q, %q", tt.requestedModel, profileName, modelName, tt.expectedProfile, tt.expectedModel)
+			}
+			if modelName != tt.expectedModel {
+				t.Errorf("splitRequestedModel(%q) = %q, %q; want %q, %q", tt.requestedModel, profileName, modelName, tt.expectedProfile, tt.expectedModel)
+			}
+		})
+	}
+}
+
+// Test useModelName in configuration sends overrides what is sent to upstream
+func TestProxyManager_UseModelName(t *testing.T) {
+
+	upstreamModelName := "upstreamModel"
+
+	modelConfig := getTestSimpleResponderConfig(upstreamModelName)
+	modelConfig.UseModelName = upstreamModelName
+
+	config := &Config{
+		HealthCheckTimeout: 15,
+		Profiles: map[string][]string{
+			"test": {"model1"},
+		},
+
+		Models: map[string]ModelConfig{
+			"model1": modelConfig,
+		},
+	}
+
+	proxy := New(config)
+	defer proxy.StopProcesses()
+
+	tests := []struct {
+		description    string
+		requestedModel string
+	}{
+		{"useModelName over rides requested model", "model1"},
+		{"useModelName over rides requested profile:model", "test:model1"},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.description+": /v1/chat/completions", func(t *testing.T) {
+			reqBody := fmt.Sprintf(`{"model":"%s"}`, tt.requestedModel)
+			req := httptest.NewRequest("POST", "/v1/chat/completions", bytes.NewBufferString(reqBody))
+			w := httptest.NewRecorder()
+
+			proxy.HandlerFunc(w, req)
+			assert.Equal(t, http.StatusOK, w.Code)
+			assert.Contains(t, w.Body.String(), upstreamModelName)
+
+		})
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.description+": /v1/audio/transcriptions", func(t *testing.T) {
+			// Create a buffer with multipart form data
+			var b bytes.Buffer
+			w := multipart.NewWriter(&b)
+
+			// Add the model field
+			fw, err := w.CreateFormField("model")
+			assert.NoError(t, err)
+			_, err = fw.Write([]byte(tt.requestedModel))
+			assert.NoError(t, err)
+
+			// Add a file field
+			fw, err = w.CreateFormFile("file", "test.mp3")
+			assert.NoError(t, err)
+			_, err = fw.Write([]byte("test"))
+			assert.NoError(t, err)
+			w.Close()
+
+			// Create the request with the multipart form data
+			req := httptest.NewRequest("POST", "/v1/audio/transcriptions", &b)
+			req.Header.Set("Content-Type", w.FormDataContentType())
+			rec := httptest.NewRecorder()
+			proxy.HandlerFunc(rec, req)
+
+			// Verify the response
+			assert.Equal(t, http.StatusOK, rec.Code)
+			var response map[string]string
+			err = json.Unmarshal(rec.Body.Bytes(), &response)
+			assert.NoError(t, err)
+			assert.Equal(t, upstreamModelName, response["model"])
+		})
+	}
+
+}