steve/llama-swap
1
0
Fork 0
Code Issues 6 Pull Requests Actions Packages Projects Releases Wiki Activity

internal/router: priority queues so batch jobs yield to interactive requests

Browse Source 
The GPU is a size-1 resource, so a single long job monopolises the box for its
whole duration and every interactive request queues behind it. Callers can now
declare intent with an X-LlamaSwap-Priority header and the serial scheduler
dispatches by score instead of by arrival.

- X-LlamaSwap-Priority: signed integer, 0 default, absent/unparseable means 0.
  interactive/normal/batch aliases resolve to +100/0/-100. Values are not
  clamped: the caller composes band and any per-user offset itself.
- serial dispatch score = priority + swap affinity + aging. Bands sit 100 apart
  so a small caller offset orders work inside a band without crossing one;
  aging is unbounded so low-priority work cannot starve.
- routing.scheduler.settings.serial.{agingDivisor,swapAffinityBonus}, defaulting
  to 60s/point and +10. swapAffinityBonus is capped at 99 so it can never
  promote a request into the next band.
- fifo adds the header to its per-model priority, so the header is not silently
  ignored under that scheduler.
- /metrics exports per-band queue depth, oldest wait and dispatch counts, plus
  counters for how often aging or swap affinity changed the pick. Each request
  records its priority, band, score and queue wait in the activity log.

Note swapAffinityBonus defaults to 10, so equal-priority requests for the
already-loaded model now run before older requests that need a swap. Set it to
0 for the previous strict arrival order.

fixes #9

Co-Authored-By: Claude Opus 5 (1M context) <[email protected]>
Claude-Session: https://claude.ai/code/session_01WUyhZBgv8BBCC5MduX88gE
This commit is contained in:
steve
2026-08-07 02:12:08 -04:00
co-authored by Claude Opus 5
parent 2f0bb2556e
commit 0358fe321e
22 changed files with 1515 additions and 67 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/server/api.go
+14 -4
View File
@@ -293,15 +293,25 @@ func (s *Server) startPreload() {
}()
}
// handleMetrics serves Prometheus-format performance metrics. Returns 503 when
// performance monitoring is disabled.
// handleMetrics serves Prometheus-format metrics: system/GPU performance plus
// the scheduler's request queue. Returns 503 only when neither source is
// available — performance monitoring disabled and a scheduler that does not
// report queue stats.
func (s *Server) handleMetrics(w http.ResponseWriter, r *http.Request) {
if s.perf == nil {
queueStats, hasQueueStats := s.local.SchedulerStats()
if s.perf == nil && !hasQueueStats {
w.WriteHeader(http.StatusServiceUnavailable)
w.Write([]byte("# performance monitor not available\n"))
return
}
s.perf.MetricsHandler().ServeHTTP(w, r)
w.Header().Set("Content-Type", "text/plain; version=0.0.4; charset=utf-8")
if s.perf != nil {
s.perf.MetricsHandler().ServeHTTP(w, r)
}
if hasQueueStats {
writeSchedulerMetrics(w, queueStats)
}
}
func handleHealth(w http.ResponseWriter, r *http.Request) {
internal/server/api_test.go
+42
View File
@@ -8,10 +8,12 @@ import (
"regexp"
"strings"
"testing"
"time"
"github.com/mostlygeek/llama-swap/internal/config"
"github.com/mostlygeek/llama-swap/internal/logmon"
"github.com/mostlygeek/llama-swap/internal/process"
"github.com/mostlygeek/llama-swap/internal/router/scheduler"
"github.com/mostlygeek/llama-swap/internal/shared"
)
@@ -301,6 +303,8 @@ func TestServer_HandleUpstream_MetricsSkipsGET(t *testing.T) {
}
func TestServer_HandleMetrics_Unavailable(t *testing.T) {
// No perf monitor and a scheduler that reports no queue stats: nothing to
// serve.
s := newTestServer(newStubRouter(nil, ""), newStubRouter(nil, ""))
w := httptest.NewRecorder()
@@ -310,6 +314,44 @@ func TestServer_HandleMetrics_Unavailable(t *testing.T) {
}
}
// TestServer_HandleMetrics_SchedulerQueue verifies the scheduler queue is
// exported even when performance monitoring is off, and that every band gets a
// series so the label set stays stable while the queue drains.
func TestServer_HandleMetrics_SchedulerQueue(t *testing.T) {
local := newStubRouter(nil, "")
local.queueStats = &scheduler.QueueStats{
Bands: map[string]scheduler.BandStats{
shared.BandInteractive: {Depth: 2, OldestWait: 3 * time.Second, Dispatched: 41},
shared.BandBatch: {Depth: 1, Dispatched: 7},
},
AgingReorders: 5,
AffinityReorders: 9,
}
s := newTestServer(local, newStubRouter(nil, ""))
w := httptest.NewRecorder()
s.ServeHTTP(w, httptest.NewRequest(http.MethodGet, "/metrics", nil))
if w.Code != http.StatusOK {
t.Fatalf("status = %d, want 200", w.Code)
}
body := w.Body.String()
for _, want := range []string{
`llamaswap_scheduler_queue_depth{band="interactive"} 2`,
`llamaswap_scheduler_queue_depth{band="normal"} 0`,
`llamaswap_scheduler_queue_depth{band="batch"} 1`,
`llamaswap_scheduler_queue_oldest_wait_seconds{band="interactive"} 3`,
`llamaswap_scheduler_dispatched_total{band="interactive"} 41`,
`llamaswap_scheduler_dispatched_total{band="batch"} 7`,
`llamaswap_scheduler_reorders_total{cause="aging"} 5`,
`llamaswap_scheduler_reorders_total{cause="swap_affinity"} 9`,
} {
if !strings.Contains(body, want) {
t.Errorf("missing %q in:\n%s", want, body)
}
}
}
func TestServer_Redirects(t *testing.T) {
s := newTestServer(newStubRouter(nil, ""), newStubRouter(nil, ""))
internal/server/metrics_scheduler.go
+39
View File
@@ -0,0 +1,39 @@
package server
import (
"fmt"
"io"
"github.com/mostlygeek/llama-swap/internal/router/scheduler"
"github.com/mostlygeek/llama-swap/internal/shared"
)
// writeSchedulerMetrics emits the request queue in Prometheus text format.
//
// The reorder counters are the ones that matter for tuning: they say how often
// aging or swap affinity actually changed which request went next. Without
// them, choosing agingDivisor and swapAffinityBonus is guesswork.
func writeSchedulerMetrics(w io.Writer, stats scheduler.QueueStats) {
fmt.Fprintf(w, "# HELP llamaswap_scheduler_queue_depth Requests waiting in the scheduler queue, by priority band\n")
fmt.Fprintf(w, "# TYPE llamaswap_scheduler_queue_depth gauge\n")
for _, band := range shared.PriorityBands {
fmt.Fprintf(w, "llamaswap_scheduler_queue_depth{band=%q} %d\n", band, stats.Bands[band].Depth)
}
fmt.Fprintf(w, "# HELP llamaswap_scheduler_queue_oldest_wait_seconds How long the longest-waiting queued request has waited, by priority band\n")
fmt.Fprintf(w, "# TYPE llamaswap_scheduler_queue_oldest_wait_seconds gauge\n")
for _, band := range shared.PriorityBands {
fmt.Fprintf(w, "llamaswap_scheduler_queue_oldest_wait_seconds{band=%q} %g\n", band, stats.Bands[band].OldestWait.Seconds())
}
fmt.Fprintf(w, "# HELP llamaswap_scheduler_dispatched_total Requests dispatched by the scheduler, by priority band\n")
fmt.Fprintf(w, "# TYPE llamaswap_scheduler_dispatched_total counter\n")
for _, band := range shared.PriorityBands {
fmt.Fprintf(w, "llamaswap_scheduler_dispatched_total{band=%q} %d\n", band, stats.Bands[band].Dispatched)
}
fmt.Fprintf(w, "# HELP llamaswap_scheduler_reorders_total Dispatches where a scoring term changed which request was picked\n")
fmt.Fprintf(w, "# TYPE llamaswap_scheduler_reorders_total counter\n")
fmt.Fprintf(w, "llamaswap_scheduler_reorders_total{cause=\"aging\"} %d\n", stats.AgingReorders)
fmt.Fprintf(w, "llamaswap_scheduler_reorders_total{cause=\"swap_affinity\"} %d\n", stats.AffinityReorders)
}
internal/server/server_test.go
+9
View File
@@ -18,6 +18,7 @@ import (
"github.com/mostlygeek/llama-swap/internal/logmon"
"github.com/mostlygeek/llama-swap/internal/process"
"github.com/mostlygeek/llama-swap/internal/router"
"github.com/mostlygeek/llama-swap/internal/router/scheduler"
"github.com/mostlygeek/llama-swap/internal/shared"
)
@@ -29,6 +30,7 @@ type stubRouter struct {
running map[string]process.ProcessState
unloadCalls atomic.Int32
loggers map[string]*logmon.Monitor
queueStats *scheduler.QueueStats
}
func newStubRouter(models []string, response string) *stubRouter {
@@ -57,6 +59,13 @@ func (s *stubRouter) ProcessLogger(modelID string) (*logmon.Monitor, bool) {
return nil, false
}
func (s *stubRouter) SchedulerStats() (scheduler.QueueStats, bool) {
if s.queueStats == nil {
return scheduler.QueueStats{}, false
}
return *s.queueStats, true
}
// newTestServer wires a Server with stub routers and a built mux.
func newTestServer(local router.LocalRouter, peer router.Router) *Server {
ctx, cancel := context.WithCancel(context.Background())