proxy: add swap matrix with solver-based model swapping (#646)
Add a new swap matrix to supersede groups for running concurrent models. The matrix uses a solver that picks the lowest cost evictions to make a requested model available. This simple approach along with a very basic DSL grammar can enable very complex swapping scenarios. - add DSL parser for set expressions with & (AND), | (OR), (), +ref - add MatrixConfig structs, validation, and topological sort for +ref - add MatrixSolver with cost-minimizing swap decisions - add Matrix runtime integrating solver with Process lifecycle - integrate matrix into ProxyManager with if-branches at all endpoints - update config.example.yaml and config-schema.json with matrix schema - config enforces groups XOR matrix (cannot use both) fixes #643
This commit is contained in:
Benson Wong
GitHub
+32
-13
@@ -129,6 +129,12 @@ type Config struct {
|
||||
Profiles map[string][]string `yaml:"profiles"`
|
||||
Groups map[string]GroupConfig `yaml:"groups"` /* key is group ID */
|
||||
|
||||
// swap matrix: solver-based alternative to groups
|
||||
Matrix *MatrixConfig `yaml:"matrix"`
|
||||
|
||||
// populated during validation when matrix is configured
|
||||
ExpandedSets []ExpandedSet `yaml:"-"`
|
||||
|
||||
// for key/value replacements in model's cmd, cmdStop, proxy, checkEndPoint
|
||||
Macros MacroList `yaml:"macros"`
|
||||
|
||||
@@ -438,22 +444,35 @@ func LoadConfigFromReader(r io.Reader) (Config, error) {
|
||||
config.Models[modelId] = modelConfig
|
||||
}
|
||||
|
||||
config = AddDefaultGroupToConfig(config)
|
||||
// groups XOR matrix
|
||||
if config.Matrix != nil && len(config.Groups) > 0 {
|
||||
return Config{}, fmt.Errorf("config cannot use both 'groups' and 'matrix'")
|
||||
}
|
||||
|
||||
// Validate group members
|
||||
memberUsage := make(map[string]string)
|
||||
for groupID, groupConfig := range config.Groups {
|
||||
prevSet := make(map[string]bool)
|
||||
for _, member := range groupConfig.Members {
|
||||
if _, found := prevSet[member]; found {
|
||||
return Config{}, fmt.Errorf("duplicate model member %s found in group: %s", member, groupID)
|
||||
}
|
||||
prevSet[member] = true
|
||||
if config.Matrix != nil {
|
||||
expandedSets, err := ValidateMatrix(*config.Matrix, config.Models)
|
||||
if err != nil {
|
||||
return Config{}, fmt.Errorf("matrix: %w", err)
|
||||
}
|
||||
config.ExpandedSets = expandedSets
|
||||
} else {
|
||||
config = AddDefaultGroupToConfig(config)
|
||||
|
||||
if existingGroup, exists := memberUsage[member]; exists {
|
||||
return Config{}, fmt.Errorf("model member %s is used in multiple groups: %s and %s", member, existingGroup, groupID)
|
||||
// Validate group members
|
||||
memberUsage := make(map[string]string)
|
||||
for groupID, groupConfig := range config.Groups {
|
||||
prevSet := make(map[string]bool)
|
||||
for _, member := range groupConfig.Members {
|
||||
if _, found := prevSet[member]; found {
|
||||
return Config{}, fmt.Errorf("duplicate model member %s found in group: %s", member, groupID)
|
||||
}
|
||||
prevSet[member] = true
|
||||
|
||||
if existingGroup, exists := memberUsage[member]; exists {
|
||||
return Config{}, fmt.Errorf("model member %s is used in multiple groups: %s and %s", member, existingGroup, groupID)
|
||||
}
|
||||
memberUsage[member] = groupID
|
||||
}
|
||||
memberUsage[member] = groupID
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,226 @@
|
||||
package config
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"regexp"
|
||||
"sort"
|
||||
|
||||
"gopkg.in/yaml.v3"
|
||||
)
|
||||
|
||||
var varKeyPattern = regexp.MustCompile(`^[a-zA-Z0-9]{1,8}$`)
|
||||
|
||||
// MatrixConfig represents the swap matrix configuration block.
|
||||
type MatrixConfig struct {
|
||||
Var map[string]string `yaml:"vars"`
|
||||
EvictCosts map[string]int `yaml:"evict_costs"`
|
||||
Sets OrderedSets `yaml:"sets"`
|
||||
}
|
||||
|
||||
// SetEntry is a single named set with its DSL expression.
|
||||
type SetEntry struct {
|
||||
Name string
|
||||
DSL string
|
||||
}
|
||||
|
||||
// OrderedSets preserves YAML definition order of sets (used for tie-breaking).
|
||||
type OrderedSets []SetEntry
|
||||
|
||||
func (os *OrderedSets) UnmarshalYAML(value *yaml.Node) error {
|
||||
if value.Kind != yaml.MappingNode {
|
||||
return fmt.Errorf("sets must be a mapping")
|
||||
}
|
||||
|
||||
entries := make([]SetEntry, 0, len(value.Content)/2)
|
||||
for i := 0; i < len(value.Content); i += 2 {
|
||||
keyNode := value.Content[i]
|
||||
valueNode := value.Content[i+1]
|
||||
|
||||
var name string
|
||||
if err := keyNode.Decode(&name); err != nil {
|
||||
return fmt.Errorf("failed to decode set name: %w", err)
|
||||
}
|
||||
|
||||
var dsl string
|
||||
if err := valueNode.Decode(&dsl); err != nil {
|
||||
return fmt.Errorf("failed to decode DSL for set %q: %w", name, err)
|
||||
}
|
||||
|
||||
entries = append(entries, SetEntry{Name: name, DSL: dsl})
|
||||
}
|
||||
|
||||
*os = entries
|
||||
return nil
|
||||
}
|
||||
|
||||
// ExpandedSet is one valid combination of concurrent models (real model names).
|
||||
type ExpandedSet struct {
|
||||
SetName string
|
||||
DSL string
|
||||
Models []string // real model names, sorted
|
||||
}
|
||||
|
||||
// ValidateMatrix validates the matrix config and returns all expanded sets.
|
||||
func ValidateMatrix(matrix MatrixConfig, models map[string]ModelConfig) ([]ExpandedSet, error) {
|
||||
if len(matrix.Sets) == 0 {
|
||||
return nil, fmt.Errorf("matrix must define at least one set")
|
||||
}
|
||||
|
||||
if len(matrix.Var) == 0 {
|
||||
return nil, fmt.Errorf("matrix must define at least one var")
|
||||
}
|
||||
|
||||
// Validate var entries
|
||||
if matrix.Var != nil {
|
||||
for id, modelName := range matrix.Var {
|
||||
if !varKeyPattern.MatchString(id) {
|
||||
return nil, fmt.Errorf("var key %q must be alphanumeric and 1-8 characters", id)
|
||||
}
|
||||
if _, exists := models[modelName]; !exists {
|
||||
return nil, fmt.Errorf("var key %q references unknown model %q", id, modelName)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Validate evict_costs
|
||||
if matrix.EvictCosts != nil {
|
||||
for key, cost := range matrix.EvictCosts {
|
||||
if cost <= 0 {
|
||||
return nil, fmt.Errorf("evict_cost for %q must be a positive integer, got %d", key, cost)
|
||||
}
|
||||
if _, ok := matrix.Var[key]; !ok {
|
||||
return nil, fmt.Errorf("evict_costs: unknown var ID %q", key)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Build dependency graph for +ref topological sort
|
||||
setNames := make(map[string]bool)
|
||||
for _, entry := range matrix.Sets {
|
||||
setNames[entry.Name] = true
|
||||
}
|
||||
|
||||
deps := make(map[string][]string) // setName -> set names it depends on
|
||||
for _, entry := range matrix.Sets {
|
||||
refs, err := extractRefs(entry.DSL)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("set %q: %w", entry.Name, err)
|
||||
}
|
||||
for _, ref := range refs {
|
||||
if !setNames[ref] {
|
||||
return nil, fmt.Errorf("set %q references undefined set %q", entry.Name, ref)
|
||||
}
|
||||
}
|
||||
deps[entry.Name] = refs
|
||||
}
|
||||
|
||||
// Topological sort with cycle detection
|
||||
order, err := topologicalSort(matrix.Sets, deps)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Expand sets in topological order
|
||||
resolvedRefs := make(map[string][][]string) // set name -> expanded alias-level combos
|
||||
var allExpanded []ExpandedSet
|
||||
totalCombinations := 0
|
||||
|
||||
// Build ordered map for efficient lookup
|
||||
setDSL := make(map[string]string)
|
||||
for _, entry := range matrix.Sets {
|
||||
setDSL[entry.Name] = entry.DSL
|
||||
}
|
||||
|
||||
for _, name := range order {
|
||||
dsl := setDSL[name]
|
||||
combos, err := ParseAndExpandDSL(dsl, resolvedRefs)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("set %q: %w", name, err)
|
||||
}
|
||||
|
||||
resolvedRefs[name] = combos
|
||||
|
||||
// Resolve var IDs to real model names
|
||||
for _, combo := range combos {
|
||||
resolved := make([]string, len(combo))
|
||||
for i, ident := range combo {
|
||||
realName, ok := matrix.Var[ident]
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("set %q: unknown var ID %q", name, ident)
|
||||
}
|
||||
resolved[i] = realName
|
||||
}
|
||||
sort.Strings(resolved)
|
||||
allExpanded = append(allExpanded, ExpandedSet{
|
||||
SetName: name,
|
||||
DSL: dsl,
|
||||
Models: resolved,
|
||||
})
|
||||
}
|
||||
|
||||
totalCombinations += len(combos)
|
||||
if totalCombinations > maxDSLExpansions {
|
||||
return nil, fmt.Errorf("total expanded combinations (%d) exceed limit of %d", totalCombinations, maxDSLExpansions)
|
||||
}
|
||||
}
|
||||
|
||||
return allExpanded, nil
|
||||
}
|
||||
|
||||
// topologicalSort returns set names in dependency order.
|
||||
// Returns an error if a cycle is detected.
|
||||
func topologicalSort(sets OrderedSets, deps map[string][]string) ([]string, error) {
|
||||
// States: 0 = unvisited, 1 = visiting, 2 = visited
|
||||
state := make(map[string]int)
|
||||
var order []string
|
||||
|
||||
var visit func(name string) error
|
||||
visit = func(name string) error {
|
||||
switch state[name] {
|
||||
case 1:
|
||||
return fmt.Errorf("circular reference detected involving set %q", name)
|
||||
case 2:
|
||||
return nil
|
||||
}
|
||||
state[name] = 1
|
||||
|
||||
for _, dep := range deps[name] {
|
||||
if err := visit(dep); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
state[name] = 2
|
||||
order = append(order, name)
|
||||
return nil
|
||||
}
|
||||
|
||||
// Visit in definition order for deterministic output
|
||||
for _, entry := range sets {
|
||||
if state[entry.Name] == 0 {
|
||||
if err := visit(entry.Name); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return order, nil
|
||||
}
|
||||
|
||||
// ResolvedEvictCosts returns a map of real model name -> evict cost,
|
||||
// resolving var IDs. Models not listed default to 1.
|
||||
func (m *MatrixConfig) ResolvedEvictCosts() map[string]int {
|
||||
costs := make(map[string]int)
|
||||
if m.EvictCosts == nil {
|
||||
return costs
|
||||
}
|
||||
for key, cost := range m.EvictCosts {
|
||||
// Resolve var ID if present
|
||||
if realName, ok := m.Var[key]; ok {
|
||||
costs[realName] = cost
|
||||
} else {
|
||||
costs[key] = cost
|
||||
}
|
||||
}
|
||||
return costs
|
||||
}
|
||||
@@ -0,0 +1,376 @@
|
||||
package config
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
"unicode"
|
||||
)
|
||||
|
||||
const maxDSLExpansions = 1000
|
||||
|
||||
// Token types for the DSL lexer
|
||||
type tokenType int
|
||||
|
||||
const (
|
||||
tokIdent tokenType = iota // model alias or name
|
||||
tokAnd // &
|
||||
tokOr // |
|
||||
tokLParen // (
|
||||
tokRParen // )
|
||||
tokRef // +setName
|
||||
tokEOF
|
||||
)
|
||||
|
||||
type token struct {
|
||||
typ tokenType
|
||||
val string
|
||||
}
|
||||
|
||||
// tokenize splits a DSL string into tokens.
|
||||
func tokenize(input string) ([]token, error) {
|
||||
var tokens []token
|
||||
i := 0
|
||||
runes := []rune(input)
|
||||
|
||||
for i < len(runes) {
|
||||
ch := runes[i]
|
||||
|
||||
// skip whitespace
|
||||
if unicode.IsSpace(ch) {
|
||||
i++
|
||||
continue
|
||||
}
|
||||
|
||||
switch ch {
|
||||
case '&':
|
||||
tokens = append(tokens, token{tokAnd, "&"})
|
||||
i++
|
||||
case '|':
|
||||
tokens = append(tokens, token{tokOr, "|"})
|
||||
i++
|
||||
case '(':
|
||||
tokens = append(tokens, token{tokLParen, "("})
|
||||
i++
|
||||
case ')':
|
||||
tokens = append(tokens, token{tokRParen, ")"})
|
||||
i++
|
||||
case '+':
|
||||
// +ref: read the identifier that follows
|
||||
i++
|
||||
start := i
|
||||
for i < len(runes) && isIdentChar(runes[i]) {
|
||||
i++
|
||||
}
|
||||
if i == start {
|
||||
return nil, fmt.Errorf("expected set name after '+' at position %d", start)
|
||||
}
|
||||
tokens = append(tokens, token{tokRef, string(runes[start:i])})
|
||||
default:
|
||||
if isIdentChar(ch) {
|
||||
start := i
|
||||
for i < len(runes) && isIdentChar(runes[i]) {
|
||||
i++
|
||||
}
|
||||
tokens = append(tokens, token{tokIdent, string(runes[start:i])})
|
||||
} else {
|
||||
return nil, fmt.Errorf("unexpected character %q at position %d", ch, i)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
tokens = append(tokens, token{tokEOF, ""})
|
||||
return tokens, nil
|
||||
}
|
||||
|
||||
func isIdentChar(ch rune) bool {
|
||||
return unicode.IsLetter(ch) || unicode.IsDigit(ch) || ch == '_' || ch == '-' || ch == '.'
|
||||
}
|
||||
|
||||
// AST node types
|
||||
type dslNode interface {
|
||||
dslNode()
|
||||
}
|
||||
|
||||
type andNode struct {
|
||||
children []dslNode
|
||||
}
|
||||
|
||||
type orNode struct {
|
||||
children []dslNode
|
||||
}
|
||||
|
||||
type leafNode struct {
|
||||
name string
|
||||
}
|
||||
|
||||
type refNode struct {
|
||||
setName string
|
||||
}
|
||||
|
||||
func (andNode) dslNode() {}
|
||||
func (orNode) dslNode() {}
|
||||
func (leafNode) dslNode() {}
|
||||
func (refNode) dslNode() {}
|
||||
|
||||
// parser holds state for recursive-descent parsing.
|
||||
type parser struct {
|
||||
tokens []token
|
||||
pos int
|
||||
}
|
||||
|
||||
func (p *parser) peek() token {
|
||||
if p.pos < len(p.tokens) {
|
||||
return p.tokens[p.pos]
|
||||
}
|
||||
return token{tokEOF, ""}
|
||||
}
|
||||
|
||||
func (p *parser) next() token {
|
||||
t := p.peek()
|
||||
if t.typ != tokEOF {
|
||||
p.pos++
|
||||
}
|
||||
return t
|
||||
}
|
||||
|
||||
func (p *parser) expect(typ tokenType) (token, error) {
|
||||
t := p.next()
|
||||
if t.typ != typ {
|
||||
return t, fmt.Errorf("expected token type %d, got %q", typ, t.val)
|
||||
}
|
||||
return t, nil
|
||||
}
|
||||
|
||||
// Grammar:
|
||||
//
|
||||
// expr = andExpr
|
||||
// andExpr = orExpr ('&' orExpr)*
|
||||
// orExpr = atom ('|' atom)*
|
||||
// atom = ident | '+' ident | '(' expr ')'
|
||||
//
|
||||
// & binds tighter than |, so "a | b & c" means "a | (b & c)"
|
||||
func parse(tokens []token) (dslNode, error) {
|
||||
p := &parser{tokens: tokens}
|
||||
node, err := p.parseExpr()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if p.peek().typ != tokEOF {
|
||||
return nil, fmt.Errorf("unexpected token %q after expression", p.peek().val)
|
||||
}
|
||||
return node, nil
|
||||
}
|
||||
|
||||
func (p *parser) parseExpr() (dslNode, error) {
|
||||
return p.parseOrExpr()
|
||||
}
|
||||
|
||||
func (p *parser) parseOrExpr() (dslNode, error) {
|
||||
left, err := p.parseAndExpr()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if p.peek().typ == tokOr {
|
||||
children := []dslNode{left}
|
||||
for p.peek().typ == tokOr {
|
||||
p.next() // consume |
|
||||
right, err := p.parseAndExpr()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
children = append(children, right)
|
||||
}
|
||||
return orNode{children: children}, nil
|
||||
}
|
||||
|
||||
return left, nil
|
||||
}
|
||||
|
||||
func (p *parser) parseAndExpr() (dslNode, error) {
|
||||
left, err := p.parseAtom()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if p.peek().typ == tokAnd {
|
||||
children := []dslNode{left}
|
||||
for p.peek().typ == tokAnd {
|
||||
p.next() // consume &
|
||||
right, err := p.parseAtom()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
children = append(children, right)
|
||||
}
|
||||
return andNode{children: children}, nil
|
||||
}
|
||||
|
||||
return left, nil
|
||||
}
|
||||
|
||||
func (p *parser) parseAtom() (dslNode, error) {
|
||||
t := p.peek()
|
||||
|
||||
switch t.typ {
|
||||
case tokIdent:
|
||||
p.next()
|
||||
return leafNode{name: t.val}, nil
|
||||
|
||||
case tokRef:
|
||||
p.next()
|
||||
return refNode{setName: t.val}, nil
|
||||
|
||||
case tokLParen:
|
||||
p.next() // consume (
|
||||
node, err := p.parseExpr()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if _, err := p.expect(tokRParen); err != nil {
|
||||
return nil, fmt.Errorf("missing closing parenthesis")
|
||||
}
|
||||
return node, nil
|
||||
|
||||
default:
|
||||
return nil, fmt.Errorf("unexpected token %q", t.val)
|
||||
}
|
||||
}
|
||||
|
||||
// expand walks the AST and produces all combinations.
|
||||
// resolvedRefs contains previously expanded sets for +ref resolution.
|
||||
func expand(node dslNode, resolvedRefs map[string][][]string) ([][]string, error) {
|
||||
switch n := node.(type) {
|
||||
case leafNode:
|
||||
return [][]string{{n.name}}, nil
|
||||
|
||||
case refNode:
|
||||
expanded, ok := resolvedRefs[n.setName]
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("unknown set reference +%s", n.setName)
|
||||
}
|
||||
// Return a copy
|
||||
result := make([][]string, len(expanded))
|
||||
for i, combo := range expanded {
|
||||
result[i] = make([]string, len(combo))
|
||||
copy(result[i], combo)
|
||||
}
|
||||
return result, nil
|
||||
|
||||
case orNode:
|
||||
// Union of all children's expansions
|
||||
var result [][]string
|
||||
for _, child := range n.children {
|
||||
childResult, err := expand(child, resolvedRefs)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
result = append(result, childResult...)
|
||||
if len(result) > maxDSLExpansions {
|
||||
return nil, fmt.Errorf("DSL expansion exceeded %d combinations", maxDSLExpansions)
|
||||
}
|
||||
}
|
||||
return result, nil
|
||||
|
||||
case andNode:
|
||||
// Cartesian product across children
|
||||
result := [][]string{{}} // start with one empty combo
|
||||
for _, child := range n.children {
|
||||
childResult, err := expand(child, resolvedRefs)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
result, err = cartesianProduct(result, childResult, maxDSLExpansions)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
return result, nil
|
||||
|
||||
default:
|
||||
return nil, fmt.Errorf("unknown node type %T", node)
|
||||
}
|
||||
}
|
||||
|
||||
// cartesianProduct computes the cartesian product of two sets of combinations.
|
||||
// It returns an error if the product would exceed cap.
|
||||
func cartesianProduct(left, right [][]string, cap int) ([][]string, error) {
|
||||
if int64(len(left))*int64(len(right)) > int64(cap) {
|
||||
return nil, fmt.Errorf("DSL expansion exceeded %d combinations", cap)
|
||||
}
|
||||
result := make([][]string, 0, len(left)*len(right))
|
||||
for _, l := range left {
|
||||
for _, r := range right {
|
||||
combo := make([]string, 0, len(l)+len(r))
|
||||
combo = append(combo, l...)
|
||||
combo = append(combo, r...)
|
||||
result = append(result, combo)
|
||||
}
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
// ParseAndExpandDSL tokenizes, parses, and expands a DSL string.
|
||||
// resolvedRefs contains previously expanded sets for +ref inlining.
|
||||
func ParseAndExpandDSL(dsl string, resolvedRefs map[string][][]string) ([][]string, error) {
|
||||
dsl = strings.TrimSpace(dsl)
|
||||
if dsl == "" {
|
||||
return nil, fmt.Errorf("empty DSL expression")
|
||||
}
|
||||
|
||||
tokens, err := tokenize(dsl)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("tokenize: %w", err)
|
||||
}
|
||||
|
||||
tree, err := parse(tokens)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("parse: %w", err)
|
||||
}
|
||||
|
||||
result, err := expand(tree, resolvedRefs)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Deduplicate models within each combination and sort for consistency
|
||||
for i, combo := range result {
|
||||
result[i] = dedupAndSort(combo)
|
||||
}
|
||||
|
||||
return result, nil
|
||||
}
|
||||
|
||||
// dedupAndSort removes duplicate entries and sorts alphabetically.
|
||||
func dedupAndSort(items []string) []string {
|
||||
seen := make(map[string]bool, len(items))
|
||||
var unique []string
|
||||
for _, item := range items {
|
||||
if !seen[item] {
|
||||
seen[item] = true
|
||||
unique = append(unique, item)
|
||||
}
|
||||
}
|
||||
sort.Strings(unique)
|
||||
return unique
|
||||
}
|
||||
|
||||
// extractRefs scans a DSL string for +ref tokens without full parsing.
|
||||
// Used for building the dependency graph for topological sorting.
|
||||
func extractRefs(dsl string) ([]string, error) {
|
||||
tokens, err := tokenize(dsl)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
var refs []string
|
||||
seen := make(map[string]bool)
|
||||
for _, t := range tokens {
|
||||
if t.typ == tokRef && !seen[t.val] {
|
||||
seen[t.val] = true
|
||||
refs = append(refs, t.val)
|
||||
}
|
||||
}
|
||||
return refs, nil
|
||||
}
|
||||
@@ -0,0 +1,300 @@
|
||||
package config
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestDSL_Tokenize(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
input string
|
||||
expect []token
|
||||
errMsg string
|
||||
}{
|
||||
{
|
||||
name: "single identifier",
|
||||
input: "abc",
|
||||
expect: []token{
|
||||
{tokIdent, "abc"},
|
||||
{tokEOF, ""},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "identifier with hyphens and dots",
|
||||
input: "model-name.v2",
|
||||
expect: []token{
|
||||
{tokIdent, "model-name.v2"},
|
||||
{tokEOF, ""},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "and expression",
|
||||
input: "a & b",
|
||||
expect: []token{
|
||||
{tokIdent, "a"},
|
||||
{tokAnd, "&"},
|
||||
{tokIdent, "b"},
|
||||
{tokEOF, ""},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "or expression",
|
||||
input: "a | b",
|
||||
expect: []token{
|
||||
{tokIdent, "a"},
|
||||
{tokOr, "|"},
|
||||
{tokIdent, "b"},
|
||||
{tokEOF, ""},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "parentheses",
|
||||
input: "(a | b) & c",
|
||||
expect: []token{
|
||||
{tokLParen, "("},
|
||||
{tokIdent, "a"},
|
||||
{tokOr, "|"},
|
||||
{tokIdent, "b"},
|
||||
{tokRParen, ")"},
|
||||
{tokAnd, "&"},
|
||||
{tokIdent, "c"},
|
||||
{tokEOF, ""},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "ref token",
|
||||
input: "+llms & v",
|
||||
expect: []token{
|
||||
{tokRef, "llms"},
|
||||
{tokAnd, "&"},
|
||||
{tokIdent, "v"},
|
||||
{tokEOF, ""},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "no whitespace",
|
||||
input: "(a|b)&c",
|
||||
expect: []token{
|
||||
{tokLParen, "("},
|
||||
{tokIdent, "a"},
|
||||
{tokOr, "|"},
|
||||
{tokIdent, "b"},
|
||||
{tokRParen, ")"},
|
||||
{tokAnd, "&"},
|
||||
{tokIdent, "c"},
|
||||
{tokEOF, ""},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "empty ref",
|
||||
input: "+",
|
||||
errMsg: "expected set name after '+'",
|
||||
},
|
||||
{
|
||||
name: "invalid character",
|
||||
input: "a @ b",
|
||||
errMsg: "unexpected character",
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
tokens, err := tokenize(tt.input)
|
||||
if tt.errMsg != "" {
|
||||
require.Error(t, err)
|
||||
assert.Contains(t, err.Error(), tt.errMsg)
|
||||
} else {
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, tt.expect, tokens)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestDSL_ParseAndExpand(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
dsl string
|
||||
refs map[string][][]string
|
||||
expect [][]string
|
||||
errMsg string
|
||||
}{
|
||||
{
|
||||
name: "single model",
|
||||
dsl: "L",
|
||||
expect: [][]string{{"L"}},
|
||||
},
|
||||
{
|
||||
name: "two models with AND",
|
||||
dsl: "a & b",
|
||||
expect: [][]string{{"a", "b"}},
|
||||
},
|
||||
{
|
||||
name: "two models with OR",
|
||||
dsl: "a | b",
|
||||
expect: [][]string{{"a"}, {"b"}},
|
||||
},
|
||||
{
|
||||
name: "three models with OR",
|
||||
dsl: "a | b | c",
|
||||
expect: [][]string{{"a"}, {"b"}, {"c"}},
|
||||
},
|
||||
{
|
||||
name: "cartesian product (a|b) & (c|d)",
|
||||
dsl: "(a | b) & (c | d)",
|
||||
expect: [][]string{
|
||||
{"a", "c"},
|
||||
{"a", "d"},
|
||||
{"b", "c"},
|
||||
{"b", "d"},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "three-way AND",
|
||||
dsl: "a & b & c",
|
||||
expect: [][]string{
|
||||
{"a", "b", "c"},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "(g | q | m) & v",
|
||||
dsl: "(g | q | m) & v",
|
||||
expect: [][]string{
|
||||
{"g", "v"},
|
||||
{"q", "v"},
|
||||
{"m", "v"},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "(g | q) & v & e",
|
||||
dsl: "(g | q) & v & e",
|
||||
expect: [][]string{
|
||||
{"e", "g", "v"},
|
||||
{"e", "q", "v"},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "precedence: a | b & c means a | (b & c)",
|
||||
dsl: "a | b & c",
|
||||
expect: [][]string{
|
||||
{"a"},
|
||||
{"b", "c"},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "+ref inlining",
|
||||
dsl: "+llms & v",
|
||||
refs: map[string][][]string{
|
||||
"llms": {{"g"}, {"q"}, {"m"}},
|
||||
},
|
||||
expect: [][]string{
|
||||
{"g", "v"},
|
||||
{"q", "v"},
|
||||
{"m", "v"},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "+ref chained",
|
||||
dsl: "+with_tts & e",
|
||||
refs: map[string][][]string{
|
||||
"with_tts": {{"g", "v"}, {"q", "v"}, {"m", "v"}},
|
||||
},
|
||||
expect: [][]string{
|
||||
{"e", "g", "v"},
|
||||
{"e", "q", "v"},
|
||||
{"e", "m", "v"},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "dedup within combination",
|
||||
dsl: "a & a",
|
||||
expect: [][]string{
|
||||
{"a"},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "empty expression",
|
||||
dsl: "",
|
||||
errMsg: "empty DSL expression",
|
||||
},
|
||||
{
|
||||
name: "unmatched open paren",
|
||||
dsl: "(a | b",
|
||||
errMsg: "missing closing parenthesis",
|
||||
},
|
||||
{
|
||||
name: "unmatched close paren",
|
||||
dsl: "a | b)",
|
||||
errMsg: "unexpected token",
|
||||
},
|
||||
{
|
||||
name: "unknown ref",
|
||||
dsl: "+unknown",
|
||||
errMsg: "unknown set reference +unknown",
|
||||
},
|
||||
{
|
||||
name: "empty parens",
|
||||
dsl: "()",
|
||||
errMsg: "unexpected token",
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
refs := tt.refs
|
||||
if refs == nil {
|
||||
refs = map[string][][]string{}
|
||||
}
|
||||
result, err := ParseAndExpandDSL(tt.dsl, refs)
|
||||
if tt.errMsg != "" {
|
||||
require.Error(t, err)
|
||||
assert.Contains(t, err.Error(), tt.errMsg)
|
||||
} else {
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, tt.expect, result)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestDSL_ExpansionCap(t *testing.T) {
|
||||
// Build an expression that would exceed 1000 combinations:
|
||||
// (a1|a2|...|a32) & (b1|b2|...|b32) = 1024 combos
|
||||
var aItems, bItems []string
|
||||
for i := 0; i < 32; i++ {
|
||||
aItems = append(aItems, fmt.Sprintf("a%d", i))
|
||||
bItems = append(bItems, fmt.Sprintf("b%d", i))
|
||||
}
|
||||
dsl := fmt.Sprintf("(%s) & (%s)",
|
||||
join(aItems, " | "),
|
||||
join(bItems, " | "),
|
||||
)
|
||||
_, err := ParseAndExpandDSL(dsl, map[string][][]string{})
|
||||
require.Error(t, err)
|
||||
assert.Contains(t, err.Error(), "exceeded")
|
||||
}
|
||||
|
||||
func TestDSL_ExtractRefs(t *testing.T) {
|
||||
refs, err := extractRefs("+llms & v & +other")
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, []string{"llms", "other"}, refs)
|
||||
|
||||
refs, err = extractRefs("a & b")
|
||||
require.NoError(t, err)
|
||||
assert.Empty(t, refs)
|
||||
}
|
||||
|
||||
func join(items []string, sep string) string {
|
||||
result := ""
|
||||
for i, item := range items {
|
||||
if i > 0 {
|
||||
result += sep
|
||||
}
|
||||
result += item
|
||||
}
|
||||
return result
|
||||
}
|
||||
@@ -0,0 +1,305 @@
|
||||
package config
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func makeModels(names ...string) map[string]ModelConfig {
|
||||
m := make(map[string]ModelConfig)
|
||||
for _, name := range names {
|
||||
m[name] = ModelConfig{Cmd: "echo " + name}
|
||||
}
|
||||
return m
|
||||
}
|
||||
|
||||
func TestValidateMatrix_Basic(t *testing.T) {
|
||||
models := makeModels("gemma", "qwen", "mistral", "voxtral", "llama70B")
|
||||
|
||||
matrix := MatrixConfig{
|
||||
Var: map[string]string{
|
||||
"g": "gemma",
|
||||
"q": "qwen",
|
||||
"m": "mistral",
|
||||
"v": "voxtral",
|
||||
"L": "llama70B",
|
||||
},
|
||||
EvictCosts: map[string]int{
|
||||
"L": 30,
|
||||
"v": 50,
|
||||
},
|
||||
Sets: OrderedSets{
|
||||
{Name: "standard", DSL: "(g | q | m) & v"},
|
||||
{Name: "full", DSL: "L"},
|
||||
},
|
||||
}
|
||||
|
||||
expanded, err := ValidateMatrix(matrix, models)
|
||||
require.NoError(t, err)
|
||||
|
||||
// standard expands to [gemma,voxtral], [qwen,voxtral], [mistral,voxtral]
|
||||
// full expands to [llama70B]
|
||||
assert.Len(t, expanded, 4)
|
||||
|
||||
assert.Equal(t, "standard", expanded[0].SetName)
|
||||
assert.Equal(t, []string{"gemma", "voxtral"}, expanded[0].Models)
|
||||
|
||||
assert.Equal(t, "standard", expanded[1].SetName)
|
||||
assert.Equal(t, []string{"qwen", "voxtral"}, expanded[1].Models)
|
||||
|
||||
assert.Equal(t, "standard", expanded[2].SetName)
|
||||
assert.Equal(t, []string{"mistral", "voxtral"}, expanded[2].Models)
|
||||
|
||||
assert.Equal(t, "full", expanded[3].SetName)
|
||||
assert.Equal(t, []string{"llama70B"}, expanded[3].Models)
|
||||
}
|
||||
|
||||
func TestValidateMatrix_WithRef(t *testing.T) {
|
||||
models := makeModels("gemma", "qwen", "mistral", "voxtral", "reranker")
|
||||
|
||||
matrix := MatrixConfig{
|
||||
Var: map[string]string{
|
||||
"g": "gemma",
|
||||
"q": "qwen",
|
||||
"m": "mistral",
|
||||
"v": "voxtral",
|
||||
"e": "reranker",
|
||||
},
|
||||
Sets: OrderedSets{
|
||||
{Name: "llms", DSL: "g | q | m"},
|
||||
{Name: "with_tts", DSL: "+llms & v"},
|
||||
{Name: "mega", DSL: "+with_tts & e"},
|
||||
},
|
||||
}
|
||||
|
||||
expanded, err := ValidateMatrix(matrix, models)
|
||||
require.NoError(t, err)
|
||||
|
||||
// llms: [gemma], [qwen], [mistral]
|
||||
// with_tts: [gemma,voxtral], [qwen,voxtral], [mistral,voxtral]
|
||||
// mega: [gemma,reranker,voxtral], [qwen,reranker,voxtral], [mistral,reranker,voxtral]
|
||||
assert.Len(t, expanded, 9)
|
||||
|
||||
// Check mega entries
|
||||
megaEntries := filterBySetName(expanded, "mega")
|
||||
assert.Len(t, megaEntries, 3)
|
||||
assert.Equal(t, []string{"gemma", "reranker", "voxtral"}, megaEntries[0].Models)
|
||||
}
|
||||
|
||||
func TestValidateMatrix_MapIDRequired(t *testing.T) {
|
||||
// DSL cannot use real model names directly — must use var IDs
|
||||
models := makeModels("gemma", "voxtral")
|
||||
|
||||
matrix := MatrixConfig{
|
||||
Var: map[string]string{"g": "gemma"},
|
||||
Sets: OrderedSets{
|
||||
{Name: "combo", DSL: "g & voxtral"},
|
||||
},
|
||||
}
|
||||
|
||||
_, err := ValidateMatrix(matrix, models)
|
||||
require.Error(t, err)
|
||||
assert.Contains(t, err.Error(), "unknown var ID")
|
||||
}
|
||||
|
||||
func TestValidateMatrix_InvalidAliasKey(t *testing.T) {
|
||||
models := makeModels("gemma")
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
alias string
|
||||
errMsg string
|
||||
}{
|
||||
{"too long", "abcdefghi", "alphanumeric and 1-8 characters"},
|
||||
{"has underscore", "a_b", "alphanumeric and 1-8 characters"},
|
||||
{"has hyphen", "a-b", "alphanumeric and 1-8 characters"},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
matrix := MatrixConfig{
|
||||
Var: map[string]string{tt.alias: "gemma"},
|
||||
Sets: OrderedSets{{Name: "s", DSL: tt.alias}},
|
||||
}
|
||||
_, err := ValidateMatrix(matrix, models)
|
||||
require.Error(t, err)
|
||||
assert.Contains(t, err.Error(), tt.errMsg)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateMatrix_AliasReferencesUnknownModel(t *testing.T) {
|
||||
models := makeModels("gemma")
|
||||
|
||||
matrix := MatrixConfig{
|
||||
Var: map[string]string{"x": "nonexistent"},
|
||||
Sets: OrderedSets{{Name: "s", DSL: "x"}},
|
||||
}
|
||||
|
||||
_, err := ValidateMatrix(matrix, models)
|
||||
require.Error(t, err)
|
||||
assert.Contains(t, err.Error(), "unknown model")
|
||||
}
|
||||
|
||||
func TestValidateMatrix_EvictCostInvalid(t *testing.T) {
|
||||
models := makeModels("gemma")
|
||||
|
||||
t.Run("zero cost", func(t *testing.T) {
|
||||
matrix := MatrixConfig{
|
||||
Var: map[string]string{"g": "gemma"},
|
||||
EvictCosts: map[string]int{"g": 0},
|
||||
Sets: OrderedSets{{Name: "s", DSL: "g"}},
|
||||
}
|
||||
_, err := ValidateMatrix(matrix, models)
|
||||
require.Error(t, err)
|
||||
assert.Contains(t, err.Error(), "positive integer")
|
||||
})
|
||||
|
||||
t.Run("negative cost", func(t *testing.T) {
|
||||
matrix := MatrixConfig{
|
||||
Var: map[string]string{"g": "gemma"},
|
||||
EvictCosts: map[string]int{"g": -1},
|
||||
Sets: OrderedSets{{Name: "s", DSL: "g"}},
|
||||
}
|
||||
_, err := ValidateMatrix(matrix, models)
|
||||
require.Error(t, err)
|
||||
assert.Contains(t, err.Error(), "positive integer")
|
||||
})
|
||||
|
||||
t.Run("unknown var ID in evict_costs", func(t *testing.T) {
|
||||
matrix := MatrixConfig{
|
||||
Var: map[string]string{"g": "gemma"},
|
||||
EvictCosts: map[string]int{"unknown": 5},
|
||||
Sets: OrderedSets{{Name: "s", DSL: "g"}},
|
||||
}
|
||||
_, err := ValidateMatrix(matrix, models)
|
||||
require.Error(t, err)
|
||||
assert.Contains(t, err.Error(), "unknown var ID")
|
||||
})
|
||||
}
|
||||
|
||||
func TestValidateMatrix_CycleDetection(t *testing.T) {
|
||||
models := makeModels("gemma")
|
||||
|
||||
matrix := MatrixConfig{
|
||||
Var: map[string]string{"g": "gemma"},
|
||||
Sets: OrderedSets{
|
||||
{Name: "a", DSL: "+b"},
|
||||
{Name: "b", DSL: "+a"},
|
||||
},
|
||||
}
|
||||
|
||||
_, err := ValidateMatrix(matrix, models)
|
||||
require.Error(t, err)
|
||||
assert.Contains(t, err.Error(), "circular reference")
|
||||
}
|
||||
|
||||
func TestValidateMatrix_UndefinedRefTarget(t *testing.T) {
|
||||
models := makeModels("gemma")
|
||||
|
||||
matrix := MatrixConfig{
|
||||
Var: map[string]string{"g": "gemma"},
|
||||
Sets: OrderedSets{
|
||||
{Name: "a", DSL: "+nonexistent"},
|
||||
},
|
||||
}
|
||||
|
||||
_, err := ValidateMatrix(matrix, models)
|
||||
require.Error(t, err)
|
||||
assert.Contains(t, err.Error(), "references undefined set")
|
||||
}
|
||||
|
||||
func TestValidateMatrix_NoSets(t *testing.T) {
|
||||
_, err := ValidateMatrix(MatrixConfig{}, makeModels("gemma"))
|
||||
require.Error(t, err)
|
||||
assert.Contains(t, err.Error(), "at least one set")
|
||||
}
|
||||
|
||||
func TestValidateMatrix_UnknownMapIDInDSL(t *testing.T) {
|
||||
models := makeModels("gemma")
|
||||
|
||||
matrix := MatrixConfig{
|
||||
Var: map[string]string{"g": "gemma"},
|
||||
Sets: OrderedSets{
|
||||
{Name: "s", DSL: "g & nonexistent"},
|
||||
},
|
||||
}
|
||||
|
||||
_, err := ValidateMatrix(matrix, models)
|
||||
require.Error(t, err)
|
||||
assert.Contains(t, err.Error(), "unknown var ID")
|
||||
}
|
||||
|
||||
func TestValidateMatrix_ResolvedEvictCosts(t *testing.T) {
|
||||
mc := &MatrixConfig{
|
||||
Var: map[string]string{
|
||||
"g": "gemma",
|
||||
"L": "llama70B",
|
||||
},
|
||||
EvictCosts: map[string]int{
|
||||
"L": 30,
|
||||
"g": 5,
|
||||
},
|
||||
}
|
||||
|
||||
costs := mc.ResolvedEvictCosts()
|
||||
assert.Equal(t, 30, costs["llama70B"])
|
||||
assert.Equal(t, 5, costs["gemma"])
|
||||
}
|
||||
|
||||
func TestValidateMatrix_ConfigXOR(t *testing.T) {
|
||||
// groups and matrix both defined
|
||||
yaml := `
|
||||
models:
|
||||
model1:
|
||||
cmd: echo model1
|
||||
proxy: http://localhost:8080
|
||||
groups:
|
||||
group1:
|
||||
members:
|
||||
- model1
|
||||
matrix:
|
||||
sets:
|
||||
s: "model1"
|
||||
`
|
||||
_, err := LoadConfigFromReader(strings.NewReader(yaml))
|
||||
require.Error(t, err)
|
||||
assert.Contains(t, err.Error(), "cannot use both")
|
||||
}
|
||||
|
||||
func TestValidateMatrix_ConfigMatrixOnly(t *testing.T) {
|
||||
yaml := `
|
||||
models:
|
||||
gemma:
|
||||
cmd: echo gemma
|
||||
proxy: http://localhost:8080
|
||||
qwen:
|
||||
cmd: echo qwen
|
||||
proxy: http://localhost:8081
|
||||
matrix:
|
||||
vars:
|
||||
g: gemma
|
||||
q: qwen
|
||||
sets:
|
||||
combo: "g | q"
|
||||
`
|
||||
cfg, err := LoadConfigFromReader(strings.NewReader(yaml))
|
||||
require.NoError(t, err)
|
||||
assert.NotNil(t, cfg.Matrix)
|
||||
assert.Len(t, cfg.ExpandedSets, 2)
|
||||
// Groups should be empty when matrix is used
|
||||
assert.Empty(t, cfg.Groups)
|
||||
}
|
||||
|
||||
func filterBySetName(sets []ExpandedSet, name string) []ExpandedSet {
|
||||
var result []ExpandedSet
|
||||
for _, s := range sets {
|
||||
if s.SetName == name {
|
||||
result = append(result, s)
|
||||
}
|
||||
}
|
||||
return result
|
||||
}
|
||||
Reference in New Issue
Block a user