proxy,ui: add performance monitoring with Prometheus metrics (#743)
Add a comprehensive performance monitoring system that collects CPU, memory, swap, load average, network IO, and GPU stats. Provides both a REST API for the UI and a Prometheus /metrics endpoint. Backend changes: - New internal/perf package with configurable interval-based stats collection - GPU monitoring via LACT (Unix socket) and nvidia-smi fallback on Linux - Ring buffer (internal/ring) for time-series stat storage - Prometheus /metrics endpoint with all system and GPU metrics - Moved LogMonitor to internal/logmon package - New PerformanceConfig for hot-reloadable monitoring settings - REST /api/performance endpoint replacing SSE streaming UI changes: - New Performance page with real-time charts for CPU, memory, GPU, and network - Reusable PerformanceChart component - LLAMA_SWAP_URL environment variable support - Improved capture dialog display Other: - Example Grafana dashboard for Prometheus metrics - monitor-test standalone binary - Config schema and example updates fixes #596
This commit is contained in:
Benson Wong
GitHub
@@ -0,0 +1,210 @@
|
||||
package perf
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/mostlygeek/llama-swap/internal/logmon"
|
||||
"github.com/mostlygeek/llama-swap/internal/ring"
|
||||
"github.com/mostlygeek/llama-swap/proxy/config"
|
||||
)
|
||||
|
||||
var (
|
||||
ErrNotImplemented = errors.New("Not Implemented")
|
||||
ErrNoGpuTool = errors.New("no GPU monitoring tool available")
|
||||
)
|
||||
|
||||
type Monitor struct {
|
||||
mutex sync.RWMutex
|
||||
log *logmon.Monitor
|
||||
conf config.PerformanceConfig
|
||||
sysRing ring.Buffer[SysStat]
|
||||
gpuRing ring.Buffer[[]GpuStat]
|
||||
|
||||
stopCtx context.Context
|
||||
stopCancel context.CancelFunc
|
||||
|
||||
sysListeners map[chan SysStat]struct{}
|
||||
gpuListeners map[chan []GpuStat]struct{}
|
||||
}
|
||||
|
||||
func ringCapacity(c config.PerformanceConfig) int {
|
||||
n := int(c.MaxAge / c.Every)
|
||||
if n < 1 {
|
||||
n = 1
|
||||
}
|
||||
return n
|
||||
}
|
||||
|
||||
func New(c config.PerformanceConfig, logger *logmon.Monitor) (*Monitor, error) {
|
||||
|
||||
if c.Every < 100*time.Millisecond {
|
||||
c.Every = 100 * time.Millisecond
|
||||
}
|
||||
if c.GC < 1*time.Second {
|
||||
c.GC = 1 * time.Second
|
||||
}
|
||||
if c.MaxAge < 1*time.Minute {
|
||||
c.MaxAge = 1 * time.Minute
|
||||
}
|
||||
|
||||
if logger == nil {
|
||||
return nil, errors.New("logger is required")
|
||||
}
|
||||
|
||||
capacity := ringCapacity(c)
|
||||
return &Monitor{
|
||||
conf: c,
|
||||
log: logger,
|
||||
sysRing: ring.NewBuffer[SysStat](capacity),
|
||||
gpuRing: ring.NewBuffer[[]GpuStat](capacity),
|
||||
sysListeners: make(map[chan SysStat]struct{}),
|
||||
gpuListeners: make(map[chan []GpuStat]struct{}),
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (m *Monitor) Stop() {
|
||||
m.mutex.Lock()
|
||||
defer m.mutex.Unlock()
|
||||
if m.stopCancel == nil {
|
||||
return
|
||||
}
|
||||
m.stopCancel()
|
||||
m.stopCancel = nil
|
||||
}
|
||||
|
||||
// UpdateConfig updates the monitor configuration and restarts if changed.
|
||||
func (m *Monitor) UpdateConfig(newConf config.PerformanceConfig) {
|
||||
m.mutex.RLock()
|
||||
changed := m.conf != newConf
|
||||
m.mutex.RUnlock()
|
||||
|
||||
if !changed {
|
||||
return
|
||||
}
|
||||
|
||||
m.Stop()
|
||||
m.mutex.Lock()
|
||||
m.conf = newConf
|
||||
capacity := ringCapacity(newConf)
|
||||
m.sysRing = ring.NewBuffer[SysStat](capacity)
|
||||
m.gpuRing = ring.NewBuffer[[]GpuStat](capacity)
|
||||
m.mutex.Unlock()
|
||||
m.Start()
|
||||
}
|
||||
|
||||
// Subscribe returns channels to listen to system and GPU stats.
|
||||
func (m *Monitor) Subscribe() (chan SysStat, chan []GpuStat, func()) {
|
||||
m.mutex.Lock()
|
||||
defer m.mutex.Unlock()
|
||||
sysChan := make(chan SysStat, 1)
|
||||
gpuChan := make(chan []GpuStat, 1)
|
||||
|
||||
m.sysListeners[sysChan] = struct{}{}
|
||||
m.gpuListeners[gpuChan] = struct{}{}
|
||||
|
||||
unsub := func() {
|
||||
m.mutex.Lock()
|
||||
defer m.mutex.Unlock()
|
||||
delete(m.sysListeners, sysChan)
|
||||
delete(m.gpuListeners, gpuChan)
|
||||
}
|
||||
|
||||
return sysChan, gpuChan, unsub
|
||||
}
|
||||
|
||||
func (m *Monitor) Start() {
|
||||
m.mutex.Lock()
|
||||
defer m.mutex.Unlock()
|
||||
if m.stopCancel != nil {
|
||||
return
|
||||
}
|
||||
|
||||
m.stopCtx, m.stopCancel = context.WithCancel(context.Background())
|
||||
|
||||
go func() {
|
||||
tick := time.NewTicker(m.conf.Every)
|
||||
defer tick.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-m.stopCtx.Done():
|
||||
return
|
||||
case <-tick.C:
|
||||
s, err := ReadSysStats()
|
||||
if err != nil {
|
||||
if err != ErrNotImplemented {
|
||||
m.log.Errorf("failed to read sys stats: %s", err.Error())
|
||||
}
|
||||
continue
|
||||
}
|
||||
m.mutex.Lock()
|
||||
m.sysRing.Push(s)
|
||||
for l := range m.sysListeners {
|
||||
select {
|
||||
case l <- s:
|
||||
default:
|
||||
}
|
||||
}
|
||||
m.mutex.Unlock()
|
||||
}
|
||||
}
|
||||
}()
|
||||
|
||||
go func() {
|
||||
gpuCh, err := getGpuStats(m.stopCtx, m.conf.Every, m.log)
|
||||
if err != nil {
|
||||
if errors.Is(err, ErrNotImplemented) || errors.Is(err, ErrNoGpuTool) {
|
||||
m.log.Infof("GPU monitoring not available: %s", err.Error())
|
||||
} else {
|
||||
m.log.Errorf("failed to initialize GPU monitoring: %s", err.Error())
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-m.stopCtx.Done():
|
||||
return
|
||||
case g, ok := <-gpuCh:
|
||||
if !ok {
|
||||
m.log.Errorf("failed reading from gpuCh - stopping read goroutine")
|
||||
return
|
||||
}
|
||||
m.mutex.Lock()
|
||||
m.gpuRing.Push(g)
|
||||
for l := range m.gpuListeners {
|
||||
select {
|
||||
case l <- g:
|
||||
default:
|
||||
}
|
||||
}
|
||||
m.mutex.Unlock()
|
||||
}
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
// Current returns a copy of the current log of system and GPU stats.
|
||||
func (m *Monitor) Current() ([]SysStat, []GpuStat) {
|
||||
m.mutex.RLock()
|
||||
defer m.mutex.RUnlock()
|
||||
|
||||
sysStats := m.sysRing.Slice()
|
||||
|
||||
snapshots := m.gpuRing.Slice()
|
||||
var gpuStats []GpuStat
|
||||
for _, snapshot := range snapshots {
|
||||
gpuStats = append(gpuStats, snapshot...)
|
||||
}
|
||||
return sysStats, gpuStats
|
||||
}
|
||||
|
||||
func ReadSysStats() (SysStat, error) {
|
||||
return readSysStats()
|
||||
}
|
||||
|
||||
func GetGpuStats(ctx context.Context, every time.Duration, logger *logmon.Monitor) (chan []GpuStat, error) {
|
||||
return getGpuStats(ctx, every, logger)
|
||||
}
|
||||
@@ -0,0 +1,55 @@
|
||||
package perf
|
||||
|
||||
import (
|
||||
"context"
|
||||
"time"
|
||||
|
||||
"github.com/mostlygeek/llama-swap/internal/logmon"
|
||||
"github.com/shirou/gopsutil/v4/cpu"
|
||||
"github.com/shirou/gopsutil/v4/load"
|
||||
"github.com/shirou/gopsutil/v4/mem"
|
||||
)
|
||||
|
||||
func getGpuStats(ctx context.Context, every time.Duration, logger *logmon.Monitor) (chan []GpuStat, error) {
|
||||
return nil, ErrNotImplemented
|
||||
}
|
||||
|
||||
func readSysStats() (SysStat, error) {
|
||||
cpuPcts, err := cpu.Percent(0, true)
|
||||
if err != nil {
|
||||
return SysStat{}, err
|
||||
}
|
||||
|
||||
vmStat, err := mem.VirtualMemory()
|
||||
if err != nil {
|
||||
return SysStat{}, err
|
||||
}
|
||||
|
||||
const toMB = 1024 * 1024
|
||||
|
||||
var swapTotalMB, swapUsedMB int
|
||||
if swapStat, err := mem.SwapMemory(); err == nil {
|
||||
swapTotalMB = int(swapStat.Total / toMB)
|
||||
swapUsedMB = int(swapStat.Used / toMB)
|
||||
}
|
||||
|
||||
var loadAvg1, loadAvg5, loadAvg15 float64
|
||||
if loadStat, err := load.Avg(); err == nil {
|
||||
loadAvg1 = loadStat.Load1
|
||||
loadAvg5 = loadStat.Load5
|
||||
loadAvg15 = loadStat.Load15
|
||||
}
|
||||
|
||||
return SysStat{
|
||||
Timestamp: time.Now(),
|
||||
CpuUtilPerCore: cpuPcts,
|
||||
MemTotalMB: int(vmStat.Total / toMB),
|
||||
MemUsedMB: int(vmStat.Used / toMB),
|
||||
MemFreeMB: int(vmStat.Free / toMB),
|
||||
SwapTotalMB: swapTotalMB,
|
||||
SwapUsedMB: swapUsedMB,
|
||||
LoadAvg1: loadAvg1,
|
||||
LoadAvg5: loadAvg5,
|
||||
LoadAvg15: loadAvg15,
|
||||
}, nil
|
||||
}
|
||||
@@ -0,0 +1,238 @@
|
||||
package perf
|
||||
|
||||
import (
|
||||
"io"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/mostlygeek/llama-swap/internal/logmon"
|
||||
"github.com/mostlygeek/llama-swap/proxy/config"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func newTestLogger() *logmon.Monitor {
|
||||
return logmon.NewWriter(io.Discard)
|
||||
}
|
||||
|
||||
func TestNew_DefaultConfig(t *testing.T) {
|
||||
logger := newTestLogger()
|
||||
|
||||
m, err := New(config.PerformanceConfig{}, logger)
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, m)
|
||||
|
||||
assert.Equal(t, 100*time.Millisecond, m.conf.Every)
|
||||
assert.Equal(t, 1*time.Second, m.conf.GC)
|
||||
assert.Equal(t, 1*time.Minute, m.conf.MaxAge)
|
||||
}
|
||||
|
||||
func TestNew_CustomConfig(t *testing.T) {
|
||||
logger := newTestLogger()
|
||||
|
||||
cfg := config.PerformanceConfig{
|
||||
Enable: true,
|
||||
Every: 500 * time.Millisecond,
|
||||
GC: 5 * time.Second,
|
||||
MaxAge: 10 * time.Minute,
|
||||
}
|
||||
|
||||
m, err := New(cfg, logger)
|
||||
require.NoError(t, err)
|
||||
|
||||
assert.Equal(t, 500*time.Millisecond, m.conf.Every)
|
||||
assert.Equal(t, 5*time.Second, m.conf.GC)
|
||||
assert.Equal(t, 10*time.Minute, m.conf.MaxAge)
|
||||
}
|
||||
|
||||
func TestNew_NilLogger(t *testing.T) {
|
||||
m, err := New(config.PerformanceConfig{}, nil)
|
||||
assert.Error(t, err)
|
||||
assert.Nil(t, m)
|
||||
}
|
||||
|
||||
func TestNew_BelowMinimumConfig(t *testing.T) {
|
||||
logger := newTestLogger()
|
||||
|
||||
cfg := config.PerformanceConfig{
|
||||
Enable: true,
|
||||
Every: 1 * time.Millisecond,
|
||||
GC: 100 * time.Millisecond,
|
||||
MaxAge: 1 * time.Second,
|
||||
}
|
||||
|
||||
m, err := New(cfg, logger)
|
||||
require.NoError(t, err)
|
||||
|
||||
assert.Equal(t, 100*time.Millisecond, m.conf.Every)
|
||||
assert.Equal(t, 1*time.Second, m.conf.GC)
|
||||
assert.Equal(t, 1*time.Minute, m.conf.MaxAge)
|
||||
}
|
||||
|
||||
func TestSubscribe_ReturnsChannels(t *testing.T) {
|
||||
m, err := New(config.PerformanceConfig{}, newTestLogger())
|
||||
require.NoError(t, err)
|
||||
|
||||
sysCh, gpuCh, unsub := m.Subscribe()
|
||||
defer unsub()
|
||||
|
||||
assert.NotNil(t, sysCh)
|
||||
assert.NotNil(t, gpuCh)
|
||||
assert.NotNil(t, unsub)
|
||||
}
|
||||
|
||||
func TestSubscribe_UnsubscribeRemovesListeners(t *testing.T) {
|
||||
m, err := New(config.PerformanceConfig{}, newTestLogger())
|
||||
require.NoError(t, err)
|
||||
|
||||
_, _, unsub := m.Subscribe()
|
||||
|
||||
m.mutex.RLock()
|
||||
assert.Len(t, m.sysListeners, 1)
|
||||
assert.Len(t, m.gpuListeners, 1)
|
||||
m.mutex.RUnlock()
|
||||
|
||||
unsub()
|
||||
|
||||
m.mutex.RLock()
|
||||
assert.Len(t, m.sysListeners, 0)
|
||||
assert.Len(t, m.gpuListeners, 0)
|
||||
m.mutex.RUnlock()
|
||||
}
|
||||
|
||||
func TestSubscribe_MultipleSubscriptions(t *testing.T) {
|
||||
m, err := New(config.PerformanceConfig{}, newTestLogger())
|
||||
require.NoError(t, err)
|
||||
|
||||
sysCh1, gpuCh1, unsub1 := m.Subscribe()
|
||||
sysCh2, gpuCh2, unsub2 := m.Subscribe()
|
||||
defer unsub1()
|
||||
defer unsub2()
|
||||
|
||||
assert.NotEqual(t, sysCh1, sysCh2)
|
||||
assert.NotEqual(t, gpuCh1, gpuCh2)
|
||||
|
||||
m.mutex.RLock()
|
||||
assert.Len(t, m.sysListeners, 2)
|
||||
assert.Len(t, m.gpuListeners, 2)
|
||||
m.mutex.RUnlock()
|
||||
}
|
||||
|
||||
func TestCurrent_EmptyByDefault(t *testing.T) {
|
||||
m, err := New(config.PerformanceConfig{}, newTestLogger())
|
||||
require.NoError(t, err)
|
||||
|
||||
sysStats, gpuStats := m.Current()
|
||||
assert.Empty(t, sysStats)
|
||||
assert.Empty(t, gpuStats)
|
||||
}
|
||||
|
||||
func TestCurrent_ReturnsCopies(t *testing.T) {
|
||||
m, err := New(config.PerformanceConfig{}, newTestLogger())
|
||||
require.NoError(t, err)
|
||||
|
||||
now := time.Now()
|
||||
m.sysRing.Push(SysStat{Timestamp: now, MemTotalMB: 1024})
|
||||
m.gpuRing.Push([]GpuStat{{Timestamp: now, ID: 0, Name: "gpu0"}})
|
||||
|
||||
sysStats, gpuStats := m.Current()
|
||||
|
||||
assert.Len(t, sysStats, 1)
|
||||
assert.Len(t, gpuStats, 1)
|
||||
assert.Equal(t, 1024, sysStats[0].MemTotalMB)
|
||||
assert.Equal(t, "gpu0", gpuStats[0].Name)
|
||||
|
||||
// modifying the returned slice should not affect the original
|
||||
sysStats[0].MemTotalMB = 999
|
||||
original, _ := m.Current()
|
||||
assert.Equal(t, 1024, original[0].MemTotalMB)
|
||||
}
|
||||
|
||||
func TestStart_CollectsSysStats(t *testing.T) {
|
||||
if testing.Short() {
|
||||
t.Skip("skipping slow test")
|
||||
}
|
||||
|
||||
m, err := New(config.PerformanceConfig{Every: 100 * time.Millisecond}, newTestLogger())
|
||||
require.NoError(t, err)
|
||||
|
||||
m.Start()
|
||||
|
||||
time.Sleep(350 * time.Millisecond)
|
||||
m.Stop()
|
||||
|
||||
sysStats, _ := m.Current()
|
||||
assert.NotEmpty(t, sysStats, "expected sys stats to be collected")
|
||||
}
|
||||
|
||||
func TestStart_StopStopsGoroutines(t *testing.T) {
|
||||
if testing.Short() {
|
||||
t.Skip("skipping slow test")
|
||||
}
|
||||
|
||||
m, err := New(config.PerformanceConfig{Every: 100 * time.Millisecond}, newTestLogger())
|
||||
require.NoError(t, err)
|
||||
|
||||
m.Start()
|
||||
if m.stopCancel == nil {
|
||||
t.Error("stopCancel should not be nil after Start()")
|
||||
}
|
||||
|
||||
m.Stop()
|
||||
if m.stopCancel != nil {
|
||||
t.Error("stopCancel should be nil after Stop()")
|
||||
}
|
||||
}
|
||||
|
||||
func TestStart_SubscriberReceivesStats(t *testing.T) {
|
||||
if testing.Short() {
|
||||
t.Skip("skipping slow test")
|
||||
}
|
||||
|
||||
m, err := New(config.PerformanceConfig{Every: 100 * time.Millisecond}, newTestLogger())
|
||||
require.NoError(t, err)
|
||||
|
||||
sysCh, _, unsub := m.Subscribe()
|
||||
defer unsub()
|
||||
|
||||
m.Start()
|
||||
defer m.Stop()
|
||||
|
||||
select {
|
||||
case s := <-sysCh:
|
||||
assert.False(t, s.Timestamp.IsZero())
|
||||
assert.NotEmpty(t, s.CpuUtilPerCore)
|
||||
case <-time.After(500 * time.Millisecond):
|
||||
t.Fatal("timed out waiting for sys stats")
|
||||
}
|
||||
}
|
||||
|
||||
func TestReadSysStats(t *testing.T) {
|
||||
s, err := ReadSysStats()
|
||||
require.NoError(t, err)
|
||||
|
||||
assert.False(t, s.Timestamp.IsZero())
|
||||
assert.NotEmpty(t, s.CpuUtilPerCore)
|
||||
assert.Greater(t, s.MemTotalMB, 0)
|
||||
}
|
||||
|
||||
func TestCurrent_ConcurrentAccess(t *testing.T) {
|
||||
m, err := New(config.PerformanceConfig{}, newTestLogger())
|
||||
require.NoError(t, err)
|
||||
|
||||
m.sysRing.Push(SysStat{Timestamp: time.Now(), MemTotalMB: 1024})
|
||||
m.gpuRing.Push([]GpuStat{{Timestamp: time.Now(), ID: 0}})
|
||||
|
||||
var wg sync.WaitGroup
|
||||
for i := 0; i < 10; i++ {
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
sys, gpu := m.Current()
|
||||
assert.Len(t, sys, 1)
|
||||
assert.Len(t, gpu, 1)
|
||||
}()
|
||||
}
|
||||
wg.Wait()
|
||||
}
|
||||
@@ -0,0 +1,461 @@
|
||||
//go:build unix && !darwin
|
||||
|
||||
package perf
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net"
|
||||
"os"
|
||||
"os/exec"
|
||||
"os/user"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/mostlygeek/llama-swap/internal/logmon"
|
||||
"github.com/shirou/gopsutil/v4/cpu"
|
||||
"github.com/shirou/gopsutil/v4/load"
|
||||
"github.com/shirou/gopsutil/v4/mem"
|
||||
psnet "github.com/shirou/gopsutil/v4/net"
|
||||
)
|
||||
|
||||
func getGpuStats(ctx context.Context, every time.Duration, logger *logmon.Monitor) (chan []GpuStat, error) {
|
||||
if ch, err := tryLACT(ctx, every, logger); err == nil {
|
||||
logger.Info("using LACT for GPU monitoring")
|
||||
return ch, nil
|
||||
} else {
|
||||
logger.Debugf("LACT: %s", err.Error())
|
||||
}
|
||||
|
||||
if ch, err := tryNvidiaSmi(ctx, every, logger); err == nil {
|
||||
logger.Info("using nvidia-smi for GPU monitoring")
|
||||
return ch, nil
|
||||
} else {
|
||||
logger.Debugf("nvidia-smi: %s", err.Error())
|
||||
}
|
||||
|
||||
if ch, err := trySysfs(ctx, every, logger); err == nil {
|
||||
logger.Info("using sysfs for GPU monitoring")
|
||||
return ch, nil
|
||||
} else {
|
||||
logger.Debugf("sysfs: %s", err.Error())
|
||||
}
|
||||
|
||||
return nil, ErrNoGpuTool
|
||||
}
|
||||
|
||||
func tryLACT(ctx context.Context, every time.Duration, logger *logmon.Monitor) (chan []GpuStat, error) {
|
||||
socketPath := lactSocketPath()
|
||||
if socketPath == "" {
|
||||
return nil, ErrNoGpuTool
|
||||
}
|
||||
|
||||
conn, err := net.DialTimeout("unix", socketPath, 2*time.Second)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("cannot connect to LACT socket: %w", err)
|
||||
}
|
||||
defer conn.Close()
|
||||
|
||||
conn.SetDeadline(time.Now().Add(5 * time.Second))
|
||||
|
||||
devices, err := lactListDevices(conn)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("LACT ListDevices failed: %w", err)
|
||||
}
|
||||
|
||||
if len(devices) == 0 {
|
||||
return nil, fmt.Errorf("LACT returned no devices")
|
||||
}
|
||||
|
||||
ch := make(chan []GpuStat, 1)
|
||||
|
||||
go func() {
|
||||
defer close(ch)
|
||||
ticker := time.NewTicker(every)
|
||||
defer ticker.Stop()
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-ticker.C:
|
||||
socketPath := lactSocketPath()
|
||||
if socketPath == "" {
|
||||
continue
|
||||
}
|
||||
|
||||
conn, err := net.DialTimeout("unix", socketPath, 2*time.Second)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
conn.SetDeadline(time.Now().Add(5 * time.Second))
|
||||
|
||||
devices, err := lactListDevices(conn)
|
||||
if err != nil {
|
||||
conn.Close()
|
||||
continue
|
||||
}
|
||||
|
||||
stats := make([]GpuStat, 0, len(devices))
|
||||
for i, d := range devices {
|
||||
stat, err := lactGetDeviceStats(conn, d.ID, d.Name, i)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
stats = append(stats, stat)
|
||||
}
|
||||
conn.Close()
|
||||
|
||||
if len(stats) > 0 {
|
||||
select {
|
||||
case ch <- stats:
|
||||
default:
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}()
|
||||
|
||||
return ch, nil
|
||||
}
|
||||
|
||||
func tryNvidiaSmi(ctx context.Context, every time.Duration, logger *logmon.Monitor) (chan []GpuStat, error) {
|
||||
if _, err := exec.LookPath("nvidia-smi"); err != nil {
|
||||
return nil, ErrNoGpuTool
|
||||
}
|
||||
|
||||
sec := int(every.Seconds())
|
||||
if sec < 1 {
|
||||
sec = 1
|
||||
}
|
||||
|
||||
cmd := exec.CommandContext(ctx, "nvidia-smi",
|
||||
"--query-gpu=index,name,uuid,temperature.gpu,utilization.gpu,memory.used,memory.total,fan.speed,power.draw",
|
||||
"--format=csv,noheader,nounits",
|
||||
"-loop", fmt.Sprintf("%d", sec),
|
||||
)
|
||||
|
||||
stdout, err := cmd.StdoutPipe()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("nvidia-smi stdout pipe failed: %w", err)
|
||||
}
|
||||
|
||||
if err := cmd.Start(); err != nil {
|
||||
return nil, fmt.Errorf("nvidia-smi start failed: %w", err)
|
||||
}
|
||||
|
||||
ch := make(chan []GpuStat, 1)
|
||||
|
||||
go func() {
|
||||
defer close(ch)
|
||||
|
||||
scanner := bufio.NewScanner(stdout)
|
||||
for scanner.Scan() {
|
||||
line := strings.TrimSpace(scanner.Text())
|
||||
if line == "" {
|
||||
continue
|
||||
}
|
||||
|
||||
stat := parseNvidiaSmiLine(line)
|
||||
if stat != nil {
|
||||
select {
|
||||
case ch <- []GpuStat{*stat}:
|
||||
default:
|
||||
}
|
||||
}
|
||||
}
|
||||
cmd.Wait()
|
||||
}()
|
||||
|
||||
return ch, nil
|
||||
}
|
||||
|
||||
func parseNvidiaSmiLine(line string) *GpuStat {
|
||||
fields := strings.Split(line, ", ")
|
||||
if len(fields) < 9 {
|
||||
return nil
|
||||
}
|
||||
|
||||
id, _ := strconv.Atoi(strings.TrimSpace(fields[0]))
|
||||
name := strings.TrimSpace(fields[1])
|
||||
uuid := strings.TrimSpace(fields[2])
|
||||
tempC, _ := strconv.Atoi(strings.TrimSpace(fields[3]))
|
||||
gpuUtil, _ := strconv.ParseFloat(strings.TrimSpace(fields[4]), 64)
|
||||
memUsed, _ := strconv.Atoi(strings.TrimSpace(fields[5]))
|
||||
memTotal, _ := strconv.Atoi(strings.TrimSpace(fields[6]))
|
||||
fanSpeed, _ := strconv.ParseFloat(strings.TrimSpace(fields[7]), 64)
|
||||
powerDraw, _ := strconv.ParseFloat(strings.TrimSpace(fields[8]), 64)
|
||||
|
||||
var memUtil float64
|
||||
if memTotal > 0 {
|
||||
memUtil = float64(memUsed) / float64(memTotal) * 100
|
||||
}
|
||||
|
||||
return &GpuStat{
|
||||
Timestamp: time.Now(),
|
||||
ID: id,
|
||||
Name: name,
|
||||
UUID: uuid,
|
||||
TempC: tempC,
|
||||
GpuUtilPct: gpuUtil,
|
||||
MemUtilPct: memUtil,
|
||||
MemUsedMB: memUsed,
|
||||
MemTotalMB: memTotal,
|
||||
FanSpeedPct: fanSpeed,
|
||||
PowerDrawW: powerDraw,
|
||||
}
|
||||
}
|
||||
|
||||
func trySysfs(ctx context.Context, every time.Duration, logger *logmon.Monitor) (chan []GpuStat, error) {
|
||||
return nil, ErrNotImplemented
|
||||
}
|
||||
|
||||
func lactSocketPath() string {
|
||||
if p := os.Getenv("LACT_DAEMON_SOCKET_PATH"); p != "" {
|
||||
if _, err := os.Stat(p); err == nil {
|
||||
return p
|
||||
}
|
||||
}
|
||||
|
||||
rootPath := "/run/lactd.sock"
|
||||
if _, err := os.Stat(rootPath); err == nil {
|
||||
return rootPath
|
||||
}
|
||||
|
||||
u, err := user.Current()
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
userPath := filepath.Join("/run/user", u.Uid, "lactd.sock")
|
||||
if _, err := os.Stat(userPath); err == nil {
|
||||
return userPath
|
||||
}
|
||||
|
||||
return ""
|
||||
}
|
||||
|
||||
type lactRequest struct {
|
||||
Command string `json:"command"`
|
||||
Args interface{} `json:"args,omitempty"`
|
||||
}
|
||||
|
||||
type lactResponse struct {
|
||||
Status string `json:"status"`
|
||||
Data json.RawMessage `json:"data"`
|
||||
}
|
||||
|
||||
type lactDeviceEntry struct {
|
||||
ID string `json:"id"`
|
||||
Name string `json:"name"`
|
||||
}
|
||||
|
||||
type lactDeviceStats struct {
|
||||
Fan struct {
|
||||
PwmCurrent *uint8 `json:"pwm_current"`
|
||||
} `json:"fan"`
|
||||
Vram struct {
|
||||
Total *uint64 `json:"total"`
|
||||
Used *uint64 `json:"used"`
|
||||
} `json:"vram"`
|
||||
Power struct {
|
||||
Average *float64 `json:"average"`
|
||||
Current *float64 `json:"current"`
|
||||
} `json:"power"`
|
||||
Temps map[string]lactTempEntry `json:"temps"`
|
||||
BusyPercent *uint8 `json:"busy_percent"`
|
||||
}
|
||||
|
||||
type lactTempEntry struct {
|
||||
Current *float64 `json:"current"`
|
||||
}
|
||||
|
||||
func lactSendRequest(conn net.Conn, req lactRequest) (json.RawMessage, error) {
|
||||
data, err := json.Marshal(req)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
data = append(data, '\n')
|
||||
|
||||
if _, err := conn.Write(data); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
reader := bufio.NewReader(conn)
|
||||
line, err := reader.ReadBytes('\n')
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
var resp lactResponse
|
||||
if err := json.Unmarshal(line, &resp); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if resp.Status != "ok" {
|
||||
return nil, fmt.Errorf("LACT error: %s", string(resp.Data))
|
||||
}
|
||||
|
||||
return resp.Data, nil
|
||||
}
|
||||
|
||||
func lactListDevices(conn net.Conn) ([]lactDeviceEntry, error) {
|
||||
data, err := lactSendRequest(conn, lactRequest{Command: "list_devices"})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
var devices []lactDeviceEntry
|
||||
if err := json.Unmarshal(data, &devices); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return devices, nil
|
||||
}
|
||||
|
||||
func lactGetDeviceStats(conn net.Conn, id string, name string, index int) (GpuStat, error) {
|
||||
data, err := lactSendRequest(conn, lactRequest{
|
||||
Command: "device_stats",
|
||||
Args: struct {
|
||||
ID string `json:"id"`
|
||||
}{ID: id},
|
||||
})
|
||||
if err != nil {
|
||||
return GpuStat{}, err
|
||||
}
|
||||
|
||||
var stats lactDeviceStats
|
||||
if err := json.Unmarshal(data, &stats); err != nil {
|
||||
return GpuStat{}, err
|
||||
}
|
||||
|
||||
var memUsedMB, memTotalMB int
|
||||
if stats.Vram.Used != nil {
|
||||
memUsedMB = int(*stats.Vram.Used / 1024 / 1024)
|
||||
}
|
||||
if stats.Vram.Total != nil {
|
||||
memTotalMB = int(*stats.Vram.Total / 1024 / 1024)
|
||||
}
|
||||
|
||||
var memUtil float64
|
||||
if memTotalMB > 0 {
|
||||
memUtil = float64(memUsedMB) / float64(memTotalMB) * 100
|
||||
}
|
||||
|
||||
var gpuUtil float64
|
||||
if stats.BusyPercent != nil {
|
||||
gpuUtil = float64(*stats.BusyPercent)
|
||||
}
|
||||
|
||||
var fanSpeed float64
|
||||
if stats.Fan.PwmCurrent != nil {
|
||||
fanSpeed = float64(*stats.Fan.PwmCurrent) / 255.0 * 100.0
|
||||
}
|
||||
|
||||
var powerDraw float64
|
||||
if stats.Power.Average != nil && *stats.Power.Average > 0 {
|
||||
powerDraw = *stats.Power.Average
|
||||
} else if stats.Power.Current != nil {
|
||||
powerDraw = *stats.Power.Current
|
||||
}
|
||||
|
||||
var tempC int
|
||||
if t, ok := stats.Temps["edge"]; ok && t.Current != nil {
|
||||
tempC = int(*t.Current)
|
||||
} else if t, ok := stats.Temps["junction"]; ok && t.Current != nil {
|
||||
tempC = int(*t.Current)
|
||||
} else {
|
||||
for _, t := range stats.Temps {
|
||||
if t.Current != nil {
|
||||
tempC = int(*t.Current)
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
var vramTempC int
|
||||
// nvidia uses "VRAM", amd "mem"
|
||||
for _, key := range []string{"mem", "VRAM"} {
|
||||
if t, ok := stats.Temps[key]; ok && t.Current != nil && *t.Current > 0 {
|
||||
vramTempC = int(*t.Current)
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
return GpuStat{
|
||||
Timestamp: time.Now(),
|
||||
ID: index,
|
||||
Name: name,
|
||||
UUID: id,
|
||||
TempC: tempC,
|
||||
VramTempC: vramTempC,
|
||||
GpuUtilPct: gpuUtil,
|
||||
MemUtilPct: memUtil,
|
||||
MemUsedMB: memUsedMB,
|
||||
MemTotalMB: memTotalMB,
|
||||
FanSpeedPct: fanSpeed,
|
||||
PowerDrawW: powerDraw,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func readSysfs() ([]GpuStat, error) {
|
||||
return nil, ErrNotImplemented
|
||||
}
|
||||
|
||||
func readSysStats() (SysStat, error) {
|
||||
cpuPcts, err := cpu.Percent(0, true)
|
||||
if err != nil {
|
||||
return SysStat{}, err
|
||||
}
|
||||
|
||||
vmStat, err := mem.VirtualMemory()
|
||||
if err != nil {
|
||||
return SysStat{}, err
|
||||
}
|
||||
|
||||
const toMB = 1024 * 1024
|
||||
|
||||
var swapTotalMB, swapUsedMB int
|
||||
if swapStat, err := mem.SwapMemory(); err == nil {
|
||||
swapTotalMB = int(swapStat.Total / toMB)
|
||||
swapUsedMB = int(swapStat.Used / toMB)
|
||||
}
|
||||
|
||||
var loadAvg1, loadAvg5, loadAvg15 float64
|
||||
if loadStat, err := load.Avg(); err == nil {
|
||||
loadAvg1 = loadStat.Load1
|
||||
loadAvg5 = loadStat.Load5
|
||||
loadAvg15 = loadStat.Load15
|
||||
}
|
||||
|
||||
netIO := make([]NetIOStat, 0)
|
||||
if ioCounters, err := psnet.IOCounters(true); err == nil {
|
||||
for _, ioc := range ioCounters {
|
||||
if ioc.Name == "lo" {
|
||||
continue
|
||||
}
|
||||
netIO = append(netIO, NetIOStat{
|
||||
Name: ioc.Name,
|
||||
BytesRecv: ioc.BytesRecv,
|
||||
BytesSent: ioc.BytesSent,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
return SysStat{
|
||||
Timestamp: time.Now(),
|
||||
CpuUtilPerCore: cpuPcts,
|
||||
MemTotalMB: int(vmStat.Total / toMB),
|
||||
MemUsedMB: int(vmStat.Used / toMB),
|
||||
MemFreeMB: int(vmStat.Free / toMB),
|
||||
SwapTotalMB: swapTotalMB,
|
||||
SwapUsedMB: swapUsedMB,
|
||||
LoadAvg1: loadAvg1,
|
||||
LoadAvg5: loadAvg5,
|
||||
LoadAvg15: loadAvg15,
|
||||
NetIO: netIO,
|
||||
}, nil
|
||||
}
|
||||
@@ -0,0 +1,49 @@
|
||||
package perf
|
||||
|
||||
import (
|
||||
"context"
|
||||
"time"
|
||||
|
||||
"github.com/mostlygeek/llama-swap/internal/logmon"
|
||||
"github.com/shirou/gopsutil/v4/cpu"
|
||||
"github.com/shirou/gopsutil/v4/mem"
|
||||
"github.com/shirou/gopsutil/v4/net"
|
||||
)
|
||||
|
||||
func getGpuStats(ctx context.Context, every time.Duration, logger *logmon.Monitor) (chan []GpuStat, error) {
|
||||
return nil, ErrNotImplemented
|
||||
}
|
||||
|
||||
func readSysStats() (SysStat, error) {
|
||||
cpuPcts, err := cpu.Percent(0, true)
|
||||
if err != nil {
|
||||
return SysStat{}, err
|
||||
}
|
||||
|
||||
vmStat, err := mem.VirtualMemory()
|
||||
if err != nil {
|
||||
return SysStat{}, err
|
||||
}
|
||||
|
||||
const toMB = 1024 * 1024
|
||||
|
||||
netIO := make([]NetIOStat, 0)
|
||||
if ioCounters, err := net.IOCounters(true); err == nil {
|
||||
for _, ioc := range ioCounters {
|
||||
netIO = append(netIO, NetIOStat{
|
||||
Name: ioc.Name,
|
||||
BytesRecv: ioc.BytesRecv,
|
||||
BytesSent: ioc.BytesSent,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
return SysStat{
|
||||
Timestamp: time.Now(),
|
||||
CpuUtilPerCore: cpuPcts,
|
||||
MemTotalMB: int(vmStat.Total / toMB),
|
||||
MemUsedMB: int(vmStat.Used / toMB),
|
||||
MemFreeMB: int(vmStat.Free / toMB),
|
||||
NetIO: netIO,
|
||||
}, nil
|
||||
}
|
||||
@@ -0,0 +1,129 @@
|
||||
package perf
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"net/http"
|
||||
"sort"
|
||||
"strings"
|
||||
)
|
||||
|
||||
const mbToBytes = int64(1024 * 1024)
|
||||
|
||||
// MetricsHandler returns an http.HandlerFunc serving Prometheus text format metrics
|
||||
// with the most recent system and GPU stats.
|
||||
func (m *Monitor) MetricsHandler() http.HandlerFunc {
|
||||
return func(w http.ResponseWriter, r *http.Request) {
|
||||
sysStats, gpuStats := m.Current()
|
||||
w.Header().Set("Content-Type", "text/plain; version=0.0.4; charset=utf-8")
|
||||
|
||||
if len(sysStats) > 0 {
|
||||
writeSysMetrics(w, sysStats[len(sysStats)-1])
|
||||
}
|
||||
|
||||
if len(gpuStats) > 0 {
|
||||
writeGpuMetrics(w, latestPerGPU(gpuStats))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func writeSysMetrics(w http.ResponseWriter, s SysStat) {
|
||||
fmt.Fprintf(w, "# HELP llamaswap_cpu_util_percent CPU utilization per core (0-100)\n")
|
||||
fmt.Fprintf(w, "# TYPE llamaswap_cpu_util_percent gauge\n")
|
||||
for i, pct := range s.CpuUtilPerCore {
|
||||
fmt.Fprintf(w, "llamaswap_cpu_util_percent{core=\"%d\"} %g\n", i, pct)
|
||||
}
|
||||
|
||||
fmt.Fprintf(w, "# HELP llamaswap_memory_total_bytes Total memory in bytes\n")
|
||||
fmt.Fprintf(w, "# TYPE llamaswap_memory_total_bytes gauge\n")
|
||||
fmt.Fprintf(w, "llamaswap_memory_total_bytes %d\n", int64(s.MemTotalMB)*mbToBytes)
|
||||
|
||||
fmt.Fprintf(w, "# HELP llamaswap_memory_used_bytes Used memory in bytes\n")
|
||||
fmt.Fprintf(w, "# TYPE llamaswap_memory_used_bytes gauge\n")
|
||||
fmt.Fprintf(w, "llamaswap_memory_used_bytes %d\n", int64(s.MemUsedMB)*mbToBytes)
|
||||
|
||||
fmt.Fprintf(w, "# HELP llamaswap_memory_free_bytes Free memory in bytes\n")
|
||||
fmt.Fprintf(w, "# TYPE llamaswap_memory_free_bytes gauge\n")
|
||||
fmt.Fprintf(w, "llamaswap_memory_free_bytes %d\n", int64(s.MemFreeMB)*mbToBytes)
|
||||
|
||||
fmt.Fprintf(w, "# HELP llamaswap_swap_total_bytes Total swap in bytes\n")
|
||||
fmt.Fprintf(w, "# TYPE llamaswap_swap_total_bytes gauge\n")
|
||||
fmt.Fprintf(w, "llamaswap_swap_total_bytes %d\n", int64(s.SwapTotalMB)*mbToBytes)
|
||||
|
||||
fmt.Fprintf(w, "# HELP llamaswap_swap_used_bytes Used swap in bytes\n")
|
||||
fmt.Fprintf(w, "# TYPE llamaswap_swap_used_bytes gauge\n")
|
||||
fmt.Fprintf(w, "llamaswap_swap_used_bytes %d\n", int64(s.SwapUsedMB)*mbToBytes)
|
||||
|
||||
fmt.Fprintf(w, "# HELP llamaswap_load_average Load average\n")
|
||||
fmt.Fprintf(w, "# TYPE llamaswap_load_average gauge\n")
|
||||
fmt.Fprintf(w, "llamaswap_load_average{interval=\"1m\"} %g\n", s.LoadAvg1)
|
||||
fmt.Fprintf(w, "llamaswap_load_average{interval=\"5m\"} %g\n", s.LoadAvg5)
|
||||
fmt.Fprintf(w, "llamaswap_load_average{interval=\"15m\"} %g\n", s.LoadAvg15)
|
||||
|
||||
if len(s.NetIO) > 0 {
|
||||
fmt.Fprintf(w, "# HELP llamaswap_network_bytes_total Total network bytes transferred\n")
|
||||
fmt.Fprintf(w, "# TYPE llamaswap_network_bytes_total counter\n")
|
||||
for _, io := range s.NetIO {
|
||||
iface := sanitizeLabel(io.Name)
|
||||
fmt.Fprintf(w, "llamaswap_network_bytes_total{interface=\"%s\",direction=\"recv\"} %d\n", iface, io.BytesRecv)
|
||||
fmt.Fprintf(w, "llamaswap_network_bytes_total{interface=\"%s\",direction=\"sent\"} %d\n", iface, io.BytesSent)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func writeGpuMetrics(w http.ResponseWriter, gpus []GpuStat) {
|
||||
if len(gpus) == 0 {
|
||||
return
|
||||
}
|
||||
|
||||
type gpuMetric struct {
|
||||
help string
|
||||
name string
|
||||
value func(GpuStat) float64
|
||||
}
|
||||
|
||||
metrics := []gpuMetric{
|
||||
{"GPU temperature in Celsius", "llamaswap_gpu_temperature_celsius", func(g GpuStat) float64 { return float64(g.TempC) }},
|
||||
{"GPU VRAM temperature in Celsius", "llamaswap_gpu_vram_temperature_celsius", func(g GpuStat) float64 { return float64(g.VramTempC) }},
|
||||
{"GPU utilization percent (0-100)", "llamaswap_gpu_util_percent", func(g GpuStat) float64 { return g.GpuUtilPct }},
|
||||
{"GPU memory utilization percent (0-100)", "llamaswap_gpu_memory_util_percent", func(g GpuStat) float64 { return g.MemUtilPct }},
|
||||
{"GPU memory used in bytes", "llamaswap_gpu_memory_used_bytes", func(g GpuStat) float64 { return float64(g.MemUsedMB) * float64(mbToBytes) }},
|
||||
{"GPU memory total in bytes", "llamaswap_gpu_memory_total_bytes", func(g GpuStat) float64 { return float64(g.MemTotalMB) * float64(mbToBytes) }},
|
||||
{"GPU fan speed percent (0-100)", "llamaswap_gpu_fan_speed_percent", func(g GpuStat) float64 { return g.FanSpeedPct }},
|
||||
{"GPU power draw in watts", "llamaswap_gpu_power_draw_watts", func(g GpuStat) float64 { return g.PowerDrawW }},
|
||||
}
|
||||
|
||||
for _, m := range metrics {
|
||||
fmt.Fprintf(w, "# HELP %s %s\n", m.name, m.help)
|
||||
fmt.Fprintf(w, "# TYPE %s gauge\n", m.name)
|
||||
for _, g := range gpus {
|
||||
if g.UUID != "" {
|
||||
fmt.Fprintf(w, "%s{id=\"%d\",name=\"%s\",uuid=\"%s\"} %g\n",
|
||||
m.name, g.ID, sanitizeLabel(g.Name), sanitizeLabel(g.UUID), m.value(g))
|
||||
} else {
|
||||
fmt.Fprintf(w, "%s{id=\"%d\",name=\"%s\"} %g\n",
|
||||
m.name, g.ID, sanitizeLabel(g.Name), m.value(g))
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// latestPerGPU returns the most recent GpuStat for each GPU ID, sorted by ID.
|
||||
func latestPerGPU(stats []GpuStat) []GpuStat {
|
||||
latest := make(map[int]GpuStat)
|
||||
for _, g := range stats {
|
||||
if prev, ok := latest[g.ID]; !ok || g.Timestamp.After(prev.Timestamp) {
|
||||
latest[g.ID] = g
|
||||
}
|
||||
}
|
||||
result := make([]GpuStat, 0, len(latest))
|
||||
for _, g := range latest {
|
||||
result = append(result, g)
|
||||
}
|
||||
sort.Slice(result, func(i, j int) bool { return result[i].ID < result[j].ID })
|
||||
return result
|
||||
}
|
||||
|
||||
// sanitizeLabel escapes characters that are invalid in Prometheus label values.
|
||||
func sanitizeLabel(s string) string {
|
||||
return strings.NewReplacer(`"`, `\"`, `\`, `\\`, "\n", `\n`).Replace(s)
|
||||
}
|
||||
@@ -0,0 +1,248 @@
|
||||
package perf
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/mostlygeek/llama-swap/proxy/config"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestSanitizeLabel(t *testing.T) {
|
||||
tests := []struct {
|
||||
input string
|
||||
want string
|
||||
}{
|
||||
{"normal", "normal"},
|
||||
{"", ""},
|
||||
{`with"quote`, `with\"quote`},
|
||||
{`with\backslash`, `with\\backslash`},
|
||||
{"with\nnewline", `with\nnewline`},
|
||||
{`"both\n"`, `\"both\\n\"`},
|
||||
}
|
||||
for _, tc := range tests {
|
||||
assert.Equal(t, tc.want, sanitizeLabel(tc.input), "input: %q", tc.input)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLatestPerGPU_Empty(t *testing.T) {
|
||||
result := latestPerGPU(nil)
|
||||
assert.Empty(t, result)
|
||||
}
|
||||
|
||||
func TestLatestPerGPU_Single(t *testing.T) {
|
||||
now := time.Now()
|
||||
stats := []GpuStat{{ID: 0, Name: "gpu0", Timestamp: now}}
|
||||
result := latestPerGPU(stats)
|
||||
require.Len(t, result, 1)
|
||||
assert.Equal(t, "gpu0", result[0].Name)
|
||||
}
|
||||
|
||||
func TestLatestPerGPU_PicksLatest(t *testing.T) {
|
||||
earlier := time.Now().Add(-time.Second)
|
||||
later := time.Now()
|
||||
stats := []GpuStat{
|
||||
{ID: 0, Name: "old", TempC: 50, Timestamp: earlier},
|
||||
{ID: 0, Name: "new", TempC: 70, Timestamp: later},
|
||||
}
|
||||
result := latestPerGPU(stats)
|
||||
require.Len(t, result, 1)
|
||||
assert.Equal(t, "new", result[0].Name)
|
||||
assert.Equal(t, 70, result[0].TempC)
|
||||
}
|
||||
|
||||
func TestLatestPerGPU_MultipleGPUsSortedByID(t *testing.T) {
|
||||
now := time.Now()
|
||||
stats := []GpuStat{
|
||||
{ID: 2, Name: "gpu2", Timestamp: now},
|
||||
{ID: 0, Name: "gpu0", Timestamp: now},
|
||||
{ID: 1, Name: "gpu1", Timestamp: now},
|
||||
}
|
||||
result := latestPerGPU(stats)
|
||||
require.Len(t, result, 3)
|
||||
assert.Equal(t, 0, result[0].ID)
|
||||
assert.Equal(t, 1, result[1].ID)
|
||||
assert.Equal(t, 2, result[2].ID)
|
||||
}
|
||||
|
||||
func TestWriteSysMetrics(t *testing.T) {
|
||||
rec := httptest.NewRecorder()
|
||||
s := SysStat{
|
||||
CpuUtilPerCore: []float64{10.5, 20.0},
|
||||
MemTotalMB: 8192,
|
||||
MemUsedMB: 4096,
|
||||
MemFreeMB: 4096,
|
||||
SwapTotalMB: 2048,
|
||||
SwapUsedMB: 512,
|
||||
LoadAvg1: 1.5,
|
||||
LoadAvg5: 1.2,
|
||||
LoadAvg15: 0.9,
|
||||
NetIO: []NetIOStat{
|
||||
{Name: "eth0", BytesRecv: 1000, BytesSent: 2000},
|
||||
},
|
||||
}
|
||||
|
||||
writeSysMetrics(rec, s)
|
||||
body := rec.Body.String()
|
||||
|
||||
assert.Contains(t, body, `llamaswap_cpu_util_percent{core="0"} 10.5`)
|
||||
assert.Contains(t, body, `llamaswap_cpu_util_percent{core="1"} 20`)
|
||||
assert.Contains(t, body, "llamaswap_memory_total_bytes 8589934592")
|
||||
assert.Contains(t, body, "llamaswap_memory_used_bytes 4294967296")
|
||||
assert.Contains(t, body, "llamaswap_memory_free_bytes 4294967296")
|
||||
assert.Contains(t, body, "llamaswap_swap_total_bytes 2147483648")
|
||||
assert.Contains(t, body, "llamaswap_swap_used_bytes 536870912")
|
||||
assert.Contains(t, body, `llamaswap_load_average{interval="1m"} 1.5`)
|
||||
assert.Contains(t, body, `llamaswap_load_average{interval="5m"} 1.2`)
|
||||
assert.Contains(t, body, `llamaswap_load_average{interval="15m"} 0.9`)
|
||||
assert.Contains(t, body, `llamaswap_network_bytes_total{interface="eth0",direction="recv"} 1000`)
|
||||
assert.Contains(t, body, `llamaswap_network_bytes_total{interface="eth0",direction="sent"} 2000`)
|
||||
}
|
||||
|
||||
func TestWriteSysMetrics_NoNetIO(t *testing.T) {
|
||||
rec := httptest.NewRecorder()
|
||||
writeSysMetrics(rec, SysStat{CpuUtilPerCore: []float64{5.0}})
|
||||
body := rec.Body.String()
|
||||
assert.NotContains(t, body, "llamaswap_network_bytes_total")
|
||||
}
|
||||
|
||||
func TestWriteGpuMetrics_Empty(t *testing.T) {
|
||||
rec := httptest.NewRecorder()
|
||||
writeGpuMetrics(rec, nil)
|
||||
assert.Empty(t, rec.Body.String())
|
||||
}
|
||||
|
||||
func TestWriteGpuMetrics(t *testing.T) {
|
||||
rec := httptest.NewRecorder()
|
||||
gpus := []GpuStat{
|
||||
{
|
||||
ID: 0,
|
||||
Name: "NVIDIA RTX 4090",
|
||||
UUID: "GPU-1234",
|
||||
TempC: 75,
|
||||
GpuUtilPct: 85.5,
|
||||
MemUtilPct: 60.0,
|
||||
MemUsedMB: 8192,
|
||||
MemTotalMB: 24576,
|
||||
FanSpeedPct: 55.0,
|
||||
PowerDrawW: 300.5,
|
||||
},
|
||||
}
|
||||
|
||||
writeGpuMetrics(rec, gpus)
|
||||
body := rec.Body.String()
|
||||
|
||||
assert.Contains(t, body, `llamaswap_gpu_temperature_celsius{id="0",name="NVIDIA RTX 4090",uuid="GPU-1234"} 75`)
|
||||
assert.Contains(t, body, `llamaswap_gpu_vram_temperature_celsius{id="0",name="NVIDIA RTX 4090",uuid="GPU-1234"} 0`)
|
||||
assert.Contains(t, body, `llamaswap_gpu_util_percent{id="0",name="NVIDIA RTX 4090",uuid="GPU-1234"} 85.5`)
|
||||
assert.Contains(t, body, `llamaswap_gpu_memory_util_percent{id="0",name="NVIDIA RTX 4090",uuid="GPU-1234"} 60`)
|
||||
assert.Contains(t, body, `llamaswap_gpu_memory_used_bytes{id="0",name="NVIDIA RTX 4090",uuid="GPU-1234"}`)
|
||||
assert.Contains(t, body, `llamaswap_gpu_memory_total_bytes{id="0",name="NVIDIA RTX 4090",uuid="GPU-1234"}`)
|
||||
assert.Contains(t, body, `llamaswap_gpu_fan_speed_percent{id="0",name="NVIDIA RTX 4090",uuid="GPU-1234"} 55`)
|
||||
assert.Contains(t, body, `llamaswap_gpu_power_draw_watts{id="0",name="NVIDIA RTX 4090",uuid="GPU-1234"} 300.5`)
|
||||
}
|
||||
|
||||
func TestWriteGpuMetrics_VramTemp(t *testing.T) {
|
||||
rec := httptest.NewRecorder()
|
||||
gpus := []GpuStat{
|
||||
{ID: 0, Name: "AMD RX 7900", UUID: "GPU-5678", TempC: 70, VramTempC: 85},
|
||||
}
|
||||
writeGpuMetrics(rec, gpus)
|
||||
body := rec.Body.String()
|
||||
assert.Contains(t, body, `llamaswap_gpu_temperature_celsius{id="0",name="AMD RX 7900",uuid="GPU-5678"} 70`)
|
||||
assert.Contains(t, body, `llamaswap_gpu_vram_temperature_celsius{id="0",name="AMD RX 7900",uuid="GPU-5678"} 85`)
|
||||
}
|
||||
|
||||
func TestWriteGpuMetrics_EmptyUUID(t *testing.T) {
|
||||
rec := httptest.NewRecorder()
|
||||
gpus := []GpuStat{{ID: 3, Name: "AMD RX 7900", UUID: ""}}
|
||||
writeGpuMetrics(rec, gpus)
|
||||
body := rec.Body.String()
|
||||
assert.NotContains(t, body, "uuid=")
|
||||
assert.Contains(t, body, `name="AMD RX 7900"`)
|
||||
}
|
||||
|
||||
func TestWriteGpuMetrics_LabelSanitization(t *testing.T) {
|
||||
rec := httptest.NewRecorder()
|
||||
gpus := []GpuStat{
|
||||
{ID: 0, Name: `GPU "special"`, UUID: "uuid\nline"},
|
||||
}
|
||||
writeGpuMetrics(rec, gpus)
|
||||
body := rec.Body.String()
|
||||
assert.Contains(t, body, `name="GPU \"special\""`)
|
||||
assert.Contains(t, body, `uuid="uuid\nline"`)
|
||||
}
|
||||
|
||||
func TestMetricsHandler_ContentType(t *testing.T) {
|
||||
m, err := New(config.PerformanceConfig{}, newTestLogger())
|
||||
require.NoError(t, err)
|
||||
|
||||
req := httptest.NewRequest(http.MethodGet, "/metrics", nil)
|
||||
rec := httptest.NewRecorder()
|
||||
m.MetricsHandler()(rec, req)
|
||||
|
||||
assert.Equal(t, "text/plain; version=0.0.4; charset=utf-8", rec.Header().Get("Content-Type"))
|
||||
}
|
||||
|
||||
func TestMetricsHandler_EmptyStats(t *testing.T) {
|
||||
m, err := New(config.PerformanceConfig{}, newTestLogger())
|
||||
require.NoError(t, err)
|
||||
|
||||
req := httptest.NewRequest(http.MethodGet, "/metrics", nil)
|
||||
rec := httptest.NewRecorder()
|
||||
m.MetricsHandler()(rec, req)
|
||||
|
||||
assert.Equal(t, http.StatusOK, rec.Code)
|
||||
assert.Empty(t, strings.TrimSpace(rec.Body.String()))
|
||||
}
|
||||
|
||||
func TestMetricsHandler_WithSysStats(t *testing.T) {
|
||||
m, err := New(config.PerformanceConfig{}, newTestLogger())
|
||||
require.NoError(t, err)
|
||||
|
||||
m.sysRing.Push(SysStat{Timestamp: time.Now(), CpuUtilPerCore: []float64{25.0}, MemTotalMB: 4096, MemUsedMB: 2048, MemFreeMB: 2048})
|
||||
|
||||
req := httptest.NewRequest(http.MethodGet, "/metrics", nil)
|
||||
rec := httptest.NewRecorder()
|
||||
m.MetricsHandler()(rec, req)
|
||||
|
||||
body := rec.Body.String()
|
||||
assert.Contains(t, body, "llamaswap_cpu_util_percent")
|
||||
assert.Contains(t, body, "llamaswap_memory_total_bytes")
|
||||
}
|
||||
|
||||
func TestMetricsHandler_UsesLatestSysStat(t *testing.T) {
|
||||
m, err := New(config.PerformanceConfig{}, newTestLogger())
|
||||
require.NoError(t, err)
|
||||
|
||||
now := time.Now()
|
||||
m.sysRing.Push(SysStat{Timestamp: now.Add(-time.Second), MemTotalMB: 1000})
|
||||
m.sysRing.Push(SysStat{Timestamp: now, MemTotalMB: 8192})
|
||||
|
||||
req := httptest.NewRequest(http.MethodGet, "/metrics", nil)
|
||||
rec := httptest.NewRecorder()
|
||||
m.MetricsHandler()(rec, req)
|
||||
|
||||
body := rec.Body.String()
|
||||
// 8192 MB = 8589934592 bytes
|
||||
assert.Contains(t, body, "llamaswap_memory_total_bytes 8589934592")
|
||||
}
|
||||
|
||||
func TestMetricsHandler_WithGpuStats(t *testing.T) {
|
||||
m, err := New(config.PerformanceConfig{}, newTestLogger())
|
||||
require.NoError(t, err)
|
||||
|
||||
m.gpuRing.Push([]GpuStat{{ID: 0, Name: "TestGPU", UUID: "uuid-0", TempC: 65, Timestamp: time.Now()}})
|
||||
|
||||
req := httptest.NewRequest(http.MethodGet, "/metrics", nil)
|
||||
rec := httptest.NewRecorder()
|
||||
m.MetricsHandler()(rec, req)
|
||||
|
||||
body := rec.Body.String()
|
||||
assert.Contains(t, body, "llamaswap_gpu_temperature_celsius")
|
||||
assert.Contains(t, body, `name="TestGPU"`)
|
||||
}
|
||||
@@ -0,0 +1,40 @@
|
||||
package perf
|
||||
|
||||
import "time"
|
||||
|
||||
type GpuStat struct {
|
||||
Timestamp time.Time `json:"timestamp"`
|
||||
|
||||
ID int `json:"id"`
|
||||
Name string `json:"name"`
|
||||
UUID string `json:"uuid"`
|
||||
TempC int `json:"temp_c"`
|
||||
VramTempC int `json:"vram_temp_c"`
|
||||
GpuUtilPct float64 `json:"gpu_util_pct"`
|
||||
MemUtilPct float64 `json:"mem_util_pct"`
|
||||
MemUsedMB int `json:"mem_used_mb"`
|
||||
MemTotalMB int `json:"mem_total_mb"`
|
||||
FanSpeedPct float64 `json:"fan_speed_pct"`
|
||||
PowerDrawW float64 `json:"power_draw_w"`
|
||||
}
|
||||
|
||||
type NetIOStat struct {
|
||||
Name string `json:"name"`
|
||||
BytesRecv uint64 `json:"bytes_recv"`
|
||||
BytesSent uint64 `json:"bytes_sent"`
|
||||
}
|
||||
|
||||
type SysStat struct {
|
||||
Timestamp time.Time `json:"timestamp"`
|
||||
|
||||
CpuUtilPerCore []float64 `json:"cpu_util_per_core"`
|
||||
MemTotalMB int `json:"mem_total_mb"`
|
||||
MemUsedMB int `json:"mem_used_mb"`
|
||||
MemFreeMB int `json:"mem_free_mb"`
|
||||
SwapTotalMB int `json:"swap_total_mb"`
|
||||
SwapUsedMB int `json:"swap_used_mb"`
|
||||
LoadAvg1 float64 `json:"load_avg_1"`
|
||||
LoadAvg5 float64 `json:"load_avg_5"`
|
||||
LoadAvg15 float64 `json:"load_avg_15"`
|
||||
NetIO []NetIOStat `json:"net_io"`
|
||||
}
|
||||
Reference in New Issue
Block a user