llama-swap/internal/perf/prometheus.go

package perf

import (
	"fmt"
	"net/http"
	"sort"
	"strings"
)

const mbToBytes = int64(1024 * 1024)

// MetricsHandler returns an http.HandlerFunc serving Prometheus text format metrics
// with the most recent system and GPU stats.
func (m *Monitor) MetricsHandler() http.HandlerFunc {
	return func(w http.ResponseWriter, r *http.Request) {
		sysStats, gpuStats := m.Current()
		w.Header().Set("Content-Type", "text/plain; version=0.0.4; charset=utf-8")

		if len(sysStats) > 0 {
			writeSysMetrics(w, sysStats[len(sysStats)-1])
		}

		if len(gpuStats) > 0 {
			writeGpuMetrics(w, latestPerGPU(gpuStats))
		}
	}
}

func writeSysMetrics(w http.ResponseWriter, s SysStat) {
	fmt.Fprintf(w, "# HELP llamaswap_cpu_util_percent CPU utilization per core (0-100)\n")
	fmt.Fprintf(w, "# TYPE llamaswap_cpu_util_percent gauge\n")
	for i, pct := range s.CpuUtilPerCore {
		fmt.Fprintf(w, "llamaswap_cpu_util_percent{core=\"%d\"} %g\n", i, pct)
	}

	fmt.Fprintf(w, "# HELP llamaswap_memory_total_bytes Total memory in bytes\n")
	fmt.Fprintf(w, "# TYPE llamaswap_memory_total_bytes gauge\n")
	fmt.Fprintf(w, "llamaswap_memory_total_bytes %d\n", int64(s.MemTotalMB)*mbToBytes)

	fmt.Fprintf(w, "# HELP llamaswap_memory_used_bytes Used memory in bytes\n")
	fmt.Fprintf(w, "# TYPE llamaswap_memory_used_bytes gauge\n")
	fmt.Fprintf(w, "llamaswap_memory_used_bytes %d\n", int64(s.MemUsedMB)*mbToBytes)

	fmt.Fprintf(w, "# HELP llamaswap_memory_free_bytes Free memory in bytes\n")
	fmt.Fprintf(w, "# TYPE llamaswap_memory_free_bytes gauge\n")
	fmt.Fprintf(w, "llamaswap_memory_free_bytes %d\n", int64(s.MemFreeMB)*mbToBytes)

	fmt.Fprintf(w, "# HELP llamaswap_swap_total_bytes Total swap in bytes\n")
	fmt.Fprintf(w, "# TYPE llamaswap_swap_total_bytes gauge\n")
	fmt.Fprintf(w, "llamaswap_swap_total_bytes %d\n", int64(s.SwapTotalMB)*mbToBytes)

	fmt.Fprintf(w, "# HELP llamaswap_swap_used_bytes Used swap in bytes\n")
	fmt.Fprintf(w, "# TYPE llamaswap_swap_used_bytes gauge\n")
	fmt.Fprintf(w, "llamaswap_swap_used_bytes %d\n", int64(s.SwapUsedMB)*mbToBytes)

	fmt.Fprintf(w, "# HELP llamaswap_load_average Load average\n")
	fmt.Fprintf(w, "# TYPE llamaswap_load_average gauge\n")
	fmt.Fprintf(w, "llamaswap_load_average{interval=\"1m\"} %g\n", s.LoadAvg1)
	fmt.Fprintf(w, "llamaswap_load_average{interval=\"5m\"} %g\n", s.LoadAvg5)
	fmt.Fprintf(w, "llamaswap_load_average{interval=\"15m\"} %g\n", s.LoadAvg15)

	if len(s.NetIO) > 0 {
		fmt.Fprintf(w, "# HELP llamaswap_network_bytes_total Total network bytes transferred\n")
		fmt.Fprintf(w, "# TYPE llamaswap_network_bytes_total counter\n")
		for _, io := range s.NetIO {
			iface := sanitizeLabel(io.Name)
			fmt.Fprintf(w, "llamaswap_network_bytes_total{interface=\"%s\",direction=\"recv\"} %d\n", iface, io.BytesRecv)
			fmt.Fprintf(w, "llamaswap_network_bytes_total{interface=\"%s\",direction=\"sent\"} %d\n", iface, io.BytesSent)
		}
	}
}

func writeGpuMetrics(w http.ResponseWriter, gpus []GpuStat) {
	if len(gpus) == 0 {
		return
	}

	type gpuMetric struct {
		help  string
		name  string
		value func(GpuStat) float64
	}

	metrics := []gpuMetric{
		{"GPU temperature in Celsius", "llamaswap_gpu_temperature_celsius", func(g GpuStat) float64 { return float64(g.TempC) }},
		{"GPU VRAM temperature in Celsius", "llamaswap_gpu_vram_temperature_celsius", func(g GpuStat) float64 { return float64(g.VramTempC) }},
		{"GPU utilization percent (0-100)", "llamaswap_gpu_util_percent", func(g GpuStat) float64 { return g.GpuUtilPct }},
		{"GPU memory utilization percent (0-100)", "llamaswap_gpu_memory_util_percent", func(g GpuStat) float64 { return g.MemUtilPct }},
		{"GPU memory used in bytes", "llamaswap_gpu_memory_used_bytes", func(g GpuStat) float64 { return float64(g.MemUsedMB) * float64(mbToBytes) }},
		{"GPU memory total in bytes", "llamaswap_gpu_memory_total_bytes", func(g GpuStat) float64 { return float64(g.MemTotalMB) * float64(mbToBytes) }},
		{"GPU fan speed percent (0-100)", "llamaswap_gpu_fan_speed_percent", func(g GpuStat) float64 { return g.FanSpeedPct }},
		{"GPU power draw in watts", "llamaswap_gpu_power_draw_watts", func(g GpuStat) float64 { return g.PowerDrawW }},
	}

	for _, m := range metrics {
		fmt.Fprintf(w, "# HELP %s %s\n", m.name, m.help)
		fmt.Fprintf(w, "# TYPE %s gauge\n", m.name)
		for _, g := range gpus {
			if g.UUID != "" {
				fmt.Fprintf(w, "%s{id=\"%d\",name=\"%s\",uuid=\"%s\"} %g\n",
					m.name, g.ID, sanitizeLabel(g.Name), sanitizeLabel(g.UUID), m.value(g))
			} else {
				fmt.Fprintf(w, "%s{id=\"%d\",name=\"%s\"} %g\n",
					m.name, g.ID, sanitizeLabel(g.Name), m.value(g))
			}
		}
	}
}

// latestPerGPU returns the most recent GpuStat for each GPU ID, sorted by ID.
func latestPerGPU(stats []GpuStat) []GpuStat {
	latest := make(map[int]GpuStat)
	for _, g := range stats {
		if prev, ok := latest[g.ID]; !ok || g.Timestamp.After(prev.Timestamp) {
			latest[g.ID] = g
		}
	}
	result := make([]GpuStat, 0, len(latest))
	for _, g := range latest {
		result = append(result, g)
	}
	sort.Slice(result, func(i, j int) bool { return result[i].ID < result[j].ID })
	return result
}

// sanitizeLabel escapes characters that are invalid in Prometheus label values.
func sanitizeLabel(s string) string {
	return strings.NewReplacer(`"`, `\"`, `\`, `\\`, "\n", `\n`).Replace(s)
}