add tokens processed to ui models page

Model alias values are not properly resolved and work in upstream/ path.

Related to #229.

.github/ISSUE_TEMPLATE/bug-report.md

@@ -1,11 +1,13 @@
 ---
 name: Bug Report
-about: Something is not working as expected...
+about: I found a defect
 title: ''
 labels: bug
 assignees: ''
 
 ---
+> [!IMPORTANT]
+> If you have questions about llama-swap please post in the Q&A in Discussions. Use bug reports when you've found a defect and wish to discuss a fix.
 
 **Describe the bug**
 A clear and concise description of what the bug is.

.github/workflows/go-ci.yml

@@ -22,6 +22,13 @@ jobs:
       with:
         go-version: '1.23'
 
+    # Only run in this linux based runner
+    - name: Check Formatting
+      run: |
+        if [ "$(gofmt -l . | grep -v 'event/.*_test.go' | wc -l)" -gt 0 ]; then
+          gofmt -l . | grep -v 'event/.*_test.go'
+          exit 1
+        fi
     # cache simple-responder to save the build time
     - name: Restore Simple Responder
       id: restore-simple-responder

Makefile

@@ -45,6 +45,7 @@ mac: ui
 linux: ui
 	@echo "Building Linux binary..."
 	GOOS=linux GOARCH=amd64 go build -ldflags="-X main.commit=${GIT_HASH} -X main.version=local_${GIT_HASH} -X main.date=${BUILD_DATE}" -o $(BUILD_DIR)/$(APP_NAME)-linux-amd64
+	GOOS=linux GOARCH=arm64 go build -ldflags="-X main.commit=${GIT_HASH} -X main.version=local_${GIT_HASH} -X main.date=${BUILD_DATE}" -o $(BUILD_DIR)/$(APP_NAME)-linux-arm64
 
 # Build Windows binary
 windows: ui

README.md

@@ -36,7 +36,7 @@ Written in golang, it is very easy to install (single binary with no dependencie
 
 ## How does llama-swap work?
 
-When a request is made to an OpenAI compatible endpoint, lama-swap will extract the `model` value and load the appropriate server configuration to serve it. If the wrong upstream server is running, it will be replaced with the correct one. This is where the "swap" part comes in. The upstream server is automatically swapped to the correct one to serve the request.
+When a request is made to an OpenAI compatible endpoint, llama-swap will extract the `model` value and load the appropriate server configuration to serve it. If the wrong upstream server is running, it will be replaced with the correct one. This is where the "swap" part comes in. The upstream server is automatically swapped to the correct one to serve the request.
 
 In the most basic configuration llama-swap handles one model at a time. For more advanced use cases, the `groups` feature allows multiple models to be loaded at the same time. You have complete control over how your system resources are used.
 
@@ -197,13 +197,6 @@ Any OpenAI compatible server would work. llama-swap was originally designed for
 
 For Python based inference servers like vllm or tabbyAPI it is recommended to run them via podman or docker. This provides clean environment isolation as well as responding correctly to `SIGTERM` signals to shutdown.
 
-## Contributors 
-<a href="https://github.com/mostlygeek/llama-swap/graphs/contributors">
-  <img src="https://contrib.rocks/image?repo=mostlygeek/llama-swap" />
-</a>
-
-Made with [contrib.rocks](https://contrib.rocks).
-
 ## Star History
 
 [![Star History Chart](https://api.star-history.com/svg?repos=mostlygeek/llama-swap&type=Date)](https://www.star-history.com/#mostlygeek/llama-swap&Date)

llama-swap.go

@@ -133,7 +133,7 @@ func main() {
 							ReloadingState: proxy.ReloadingStateStart,
 						})
 					} else if changeEvent.Name == filepath.Join(configDir, "..data") && changeEvent.Has(fsnotify.Create) {
-						// the change for k8s configmap 
+						// the change for k8s configmap
 						event.Emit(proxy.ConfigFileChangedEvent{
 							ReloadingState: proxy.ReloadingStateStart,
 						})

proxy/proxymanager.go

@@ -8,6 +8,7 @@ import (
 	"mime/multipart"
 	"net/http"
 	"os"
+	"sort"
 	"strconv"
 	"strings"
 	"sync"
@@ -333,6 +334,13 @@ func (pm *ProxyManager) listModelsHandler(c *gin.Context) {
 		data = append(data, record)
 	}
 
+	// Sort by the "id" key
+	sort.Slice(data, func(i, j int) bool {
+		si, _ := data[i]["id"].(string)
+		sj, _ := data[j]["id"].(string)
+		return si < sj
+	})
+
 	// Set CORS headers if origin exists
 	if origin := c.GetHeader("Origin"); origin != "" {
 		c.Header("Access-Control-Allow-Origin", origin)
@@ -353,7 +361,7 @@ func (pm *ProxyManager) proxyToUpstream(c *gin.Context) {
 		return
 	}
 
-	processGroup, _, err := pm.swapProcessGroup(requestedModel)
+	processGroup, realModelName, err := pm.swapProcessGroup(requestedModel)
 	if err != nil {
 		pm.sendErrorResponse(c, http.StatusInternalServerError, fmt.Sprintf("error swapping process group: %s", err.Error()))
 		return
@@ -361,7 +369,7 @@ func (pm *ProxyManager) proxyToUpstream(c *gin.Context) {
 
 	// rewrite the path
 	c.Request.URL.Path = c.Param("upstreamPath")
-	processGroup.ProxyRequest(requestedModel, c.Writer, c.Request)
+	processGroup.ProxyRequest(realModelName, c.Writer, c.Request)
 }
 
 func (pm *ProxyManager) proxyOAIHandler(c *gin.Context) {

proxy/proxymanager_test.go

@@ -9,6 +9,7 @@ import (
 	"net/http"
 	"net/http/httptest"
 	"strconv"
+	"strings"
 	"sync"
 	"testing"
 	"time"
@@ -279,6 +280,51 @@ func TestProxyManager_ListModelsHandler(t *testing.T) {
 	assert.Empty(t, expectedModels, "not all expected models were returned")
 }
 
+func TestProxyManager_ListModelsHandler_SortedByID(t *testing.T) {
+	// Intentionally add models in non-sorted order and with an unlisted model
+	config := Config{
+		HealthCheckTimeout: 15,
+		Models: map[string]ModelConfig{
+			"zeta":  getTestSimpleResponderConfig("zeta"),
+			"alpha": getTestSimpleResponderConfig("alpha"),
+			"beta":  getTestSimpleResponderConfig("beta"),
+			"hidden": func() ModelConfig {
+				mc := getTestSimpleResponderConfig("hidden")
+				mc.Unlisted = true
+				return mc
+			}(),
+		},
+		LogLevel: "error",
+	}
+
+	proxy := New(config)
+
+	// Request models list
+	req := httptest.NewRequest("GET", "/v1/models", nil)
+	w := httptest.NewRecorder()
+	proxy.ServeHTTP(w, req)
+
+	assert.Equal(t, http.StatusOK, w.Code)
+
+	var response struct {
+		Data []map[string]interface{} `json:"data"`
+	}
+	if err := json.Unmarshal(w.Body.Bytes(), &response); err != nil {
+		t.Fatalf("Failed to parse JSON response: %v", err)
+	}
+
+	// We expect only the listed models in sorted order by id
+	expectedOrder := []string{"alpha", "beta", "zeta"}
+	if assert.Len(t, response.Data, len(expectedOrder), "unexpected number of listed models") {
+		got := make([]string, 0, len(response.Data))
+		for _, m := range response.Data {
+			id, _ := m["id"].(string)
+			got = append(got, id)
+		}
+		assert.Equal(t, expectedOrder, got, "models should be sorted by id ascending")
+	}
+}
+
 func TestProxyManager_Shutdown(t *testing.T) {
 	// make broken model configurations
 	model1Config := getTestSimpleResponderConfigPort("model1", 9991)
@@ -611,21 +657,34 @@ func TestProxyManager_CORSOptionsHandler(t *testing.T) {
 }
 
 func TestProxyManager_Upstream(t *testing.T) {
-	config := AddDefaultGroupToConfig(Config{
-		HealthCheckTimeout: 15,
-		Models: map[string]ModelConfig{
-			"model1": getTestSimpleResponderConfig("model1"),
-		},
-		LogLevel: "error",
-	})
+	configStr := fmt.Sprintf(`
+logLevel: error
+models:
+  model1:
+    cmd: %s -port ${PORT} -silent -respond model1
+    aliases: [model-alias]
+`, getSimpleResponderPath())
+
+	config, err := LoadConfigFromReader(strings.NewReader(configStr))
+	assert.NoError(t, err)
 
 	proxy := New(config)
 	defer proxy.StopProcesses(StopWaitForInflightRequest)
-	req := httptest.NewRequest("GET", "/upstream/model1/test", nil)
-	rec := httptest.NewRecorder()
-	proxy.ServeHTTP(rec, req)
-	assert.Equal(t, http.StatusOK, rec.Code)
-	assert.Equal(t, "model1", rec.Body.String())
+	t.Run("main model name", func(t *testing.T) {
+		req := httptest.NewRequest("GET", "/upstream/model1/test", nil)
+		rec := httptest.NewRecorder()
+		proxy.ServeHTTP(rec, req)
+		assert.Equal(t, http.StatusOK, rec.Code)
+		assert.Equal(t, "model1", rec.Body.String())
+	})
+
+	t.Run("model alias", func(t *testing.T) {
+		req := httptest.NewRequest("GET", "/upstream/model-alias/test", nil)
+		rec := httptest.NewRecorder()
+		proxy.ServeHTTP(rec, req)
+		assert.Equal(t, http.StatusOK, rec.Code)
+		assert.Equal(t, "model1", rec.Body.String())
+	})
 }
 
 func TestProxyManager_ChatContentLength(t *testing.T) {

ui/src/contexts/APIProvider.tsx

@@ -83,6 +83,12 @@ export function APIProvider({ children, autoStartAPIEvents = true }: APIProvider
             case "modelStatus":
               {
                 const models = JSON.parse(message.data) as Model[];
+
+                // sort models by name and id
+                models.sort((a, b) => {
+                  return (a.name + a.id).localeCompare(b.name + b.id);
+                });
+
                 setModels(models);
               }
               break;

ui/src/pages/LogViewer.tsx

@@ -18,7 +18,7 @@ const LogViewer = () => {
   const direction = isNarrow ? "vertical" : "horizontal";
 
   return (
-    <PanelGroup direction={direction} className="gap-2" autoSaveId={`logviewer-panel-group-${direction}`}>
+    <PanelGroup direction={direction} className="gap-2" autoSaveId="logviewer-panel-group">
       <Panel id="proxy" defaultSize={50} minSize={5} maxSize={100} collapsible={true}>
         <LogPanel id="proxy" title="Proxy Logs" logData={proxyLogs} />
       </Panel>

ui/src/pages/Models.tsx

@@ -4,7 +4,7 @@ import { LogPanel } from "./LogViewer";
 import { usePersistentState } from "../hooks/usePersistentState";
 import { Panel, PanelGroup, PanelResizeHandle } from "react-resizable-panels";
 import { useTheme } from "../contexts/ThemeProvider";
-import { RiEyeFill, RiEyeOffFill, RiStopCircleLine } from "react-icons/ri";
+import { RiEyeFill, RiEyeOffFill, RiStopCircleLine, RiSwapBoxFill } from "react-icons/ri";
 
 export default function ModelsPage() {
   const { isNarrow } = useTheme();
@@ -12,7 +12,7 @@ export default function ModelsPage() {
   const { upstreamLogs } = useAPI();
 
   return (
-    <PanelGroup direction={direction} className="gap-2" autoSaveId={`models-panel-group-${direction}`}>
+    <PanelGroup direction={direction} className="gap-2" autoSaveId={"models-panel-group"}>
       <Panel id="models" defaultSize={50} minSize={isNarrow ? 0 : 25} maxSize={100} collapsible={isNarrow}>
         <ModelsPanel />
       </Panel>
@@ -40,6 +40,7 @@ function ModelsPanel() {
   const { models, loadModel, unloadAllModels } = useAPI();
   const [isUnloading, setIsUnloading] = useState(false);
   const [showUnlisted, setShowUnlisted] = usePersistentState("showUnlisted", true);
+  const [showIdorName, setShowIdorName] = usePersistentState<"id" | "name">("showIdorName", "id"); // true = show ID, false = show name
 
   const filteredModels = useMemo(() => {
     return models.filter((model) => showUnlisted || !model.unlisted);
@@ -58,18 +59,28 @@ function ModelsPanel() {
     }
   }, [unloadAllModels]);
 
+  const toggleIdorName = useCallback(() => {
+    setShowIdorName((prev) => (prev === "name" ? "id" : "name"));
+  }, [showIdorName]);
+
   return (
     <div className="card h-full flex flex-col">
       <div className="shrink-0">
         <h2>Models</h2>
         <div className="flex justify-between">
-          <button
-            className="btn flex items-center gap-2"
-            onClick={() => setShowUnlisted(!showUnlisted)}
-            style={{ lineHeight: "1.2" }}
-          >
-            {showUnlisted ? <RiEyeFill /> : <RiEyeOffFill />} unlisted
-          </button>
+          <div className="flex gap-2">
+            <button className="btn flex items-center gap-2" onClick={toggleIdorName} style={{ lineHeight: "1.2" }}>
+              <RiSwapBoxFill /> {showIdorName === "id" ? "ID" : "Name"}
+            </button>
+
+            <button
+              className="btn flex items-center gap-2"
+              onClick={() => setShowUnlisted(!showUnlisted)}
+              style={{ lineHeight: "1.2" }}
+            >
+              {showUnlisted ? <RiEyeFill /> : <RiEyeOffFill />} unlisted
+            </button>
+          </div>
           <button className="btn flex items-center gap-2" onClick={handleUnloadAllModels} disabled={isUnloading}>
             <RiStopCircleLine size="24" /> {isUnloading ? "Unloading..." : "Unload"}
           </button>
@@ -80,7 +91,7 @@ function ModelsPanel() {
         <table className="w-full">
           <thead className="sticky top-0 bg-card z-10">
             <tr className="border-b border-primary bg-surface">
-              <th className="text-left p-2">Name</th>
+              <th className="text-left p-2">{showIdorName === "id" ? "Model ID" : "Name"}</th>
               <th className="text-left p-2"></th>
               <th className="text-left p-2">State</th>
             </tr>
@@ -90,7 +101,7 @@ function ModelsPanel() {
               <tr key={model.id} className="border-b hover:bg-secondary-hover border-border">
                 <td className={`p-2 ${model.unlisted ? "text-txtsecondary" : ""}`}>
                   <a href={`/upstream/${model.id}/`} className={`underline`} target="_blank">
-                    {model.name !== "" ? model.name : model.id}
+                    {showIdorName === "id" ? model.id : model.name !== "" ? model.name : model.id}
                   </a>
                   {model.description !== "" && (
                     <p className={model.unlisted ? "text-opacity-70" : ""}>
@@ -122,35 +133,41 @@ function ModelsPanel() {
 function StatsPanel() {
   const { metrics } = useAPI();
 
-  const [totalRequests, totalTokens, avgTokensPerSecond] = useMemo(() => {
+  const [totalRequests, totalInputTokens, totalOutputTokens, avgTokensPerSecond] = useMemo(() => {
     const totalRequests = metrics.length;
     if (totalRequests === 0) {
       return [0, 0, 0];
     }
-    const totalTokens = metrics.reduce((sum, m) => sum + m.output_tokens, 0);
+    const totalInputTokens = metrics.reduce((sum, m) => sum + m.input_tokens, 0);
+    const totalOutputTokens = metrics.reduce((sum, m) => sum + m.output_tokens, 0);
     const avgTokensPerSecond = (metrics.reduce((sum, m) => sum + m.tokens_per_second, 0) / totalRequests).toFixed(2);
-    return [totalRequests, totalTokens, avgTokensPerSecond];
+    return [totalRequests, totalInputTokens, totalOutputTokens, avgTokensPerSecond];
   }, [metrics]);
 
   return (
     <div className="card">
-      <h2>Chat Activity</h2>
-      <table className="w-full border border-gray-200">
-        <tbody>
-          <tr className="border-b border-gray-200">
-            <td className="py-2 px-4 font-medium border-r border-gray-200">Requests</td>
-            <td className="py-2 px-4 text-right">{totalRequests}</td>
-          </tr>
-          <tr className="border-b border-gray-200">
-            <td className="py-2 px-4 font-medium border-r border-gray-200">Total Tokens Generated</td>
-            <td className="py-2 px-4 text-right">{totalTokens}</td>
-          </tr>
-          <tr>
-            <td className="py-2 px-4 font-medium border-r border-gray-200">Average Tokens/Second</td>
-            <td className="py-2 px-4 text-right">{avgTokensPerSecond}</td>
-          </tr>
-        </tbody>
-      </table>
+      <div className="rounded-lg overflow-hidden border border-gray-200">
+        <table className="w-full">
+          <tbody>
+            <tr>
+              <th className="p-2 font-medium border-b border-gray-200 text-right">Requests</th>
+              <th className="p-2 font-medium border-l border-b border-gray-200 text-right">Processed</th>
+              <th className="p-2 font-medium border-l border-b border-gray-200 text-right">Generated</th>
+              <th className="p-2 font-medium border-l border-b border-gray-200 text-right">Tokens/Sec</th>
+            </tr>
+            <tr>
+              <td className="p-2 text-right border-r border-gray-200">{totalRequests}</td>
+              <td className="p-2 text-right border-r border-gray-200">
+                {new Intl.NumberFormat().format(totalInputTokens)}
+              </td>
+              <td className="p-2 text-right border-r border-gray-200">
+                {new Intl.NumberFormat().format(totalOutputTokens)}
+              </td>
+              <td className="p-2 text-right">{avgTokensPerSecond}</td>
+            </tr>
+          </tbody>
+        </table>
+      </div>
     </div>
   );
 }