c8a87f1733
executus CI / test (pull_request) Successful in 1m34s
All 3 cloud models converged on real concurrency bugs in the SQLite stores: - AppendVersion (HIGH): the seq key was `SELECT MAX(seq)+1` then INSERT in two un-transacted statements with a NON-unique index, AND the Scan error was swallowed (seq stayed 0 on failure). Concurrent appends could both land the same seq, silently breaking newest-first ordering. Now: one transaction, the Scan error is propagated, the (skill_id, seq) index is UNIQUE (the loser of a race fails loudly), and an empty SkillID is rejected. - MarkScheduledRun / MarkAgentScheduledRun (all 3): replaced the Get→mutate→Save read-modify-write (lost-update window) with a single atomic UPDATE using json_set, so a concurrent Mark/edit can't clobber it. json_set keeps the JSON blob's NextRunAt/LastScheduledRunAt consistent with the indexed column; RFC3339Nano matches Go's time encoding so the blob still round-trips (tested). - Open: actually applies PRAGMA busy_timeout=5000 (the doc advertised it but it was never set) — a contended writer waits instead of erroring SQLITE_BUSY. - budgetStore.Add: rejects NaN/Inf secondsUsed (would irrecoverably poison the column). Triaged-but-kept: plaintext webhook secret (documented design, high-entropy URL key, pre-existing); SQL()/free-form `where` helpers (no untrusted input reaches them — defense-in-depth notes only). Core go.sum still free of host/DB deps; contrib/store green (incl. a json_set blob-round-trip test). Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
281 lines
9.2 KiB
Go
281 lines
9.2 KiB
Go
package store
|
|
|
|
import (
|
|
"context"
|
|
"database/sql"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"time"
|
|
|
|
"gitea.stevedudenhoeffer.com/steve/executus/skill"
|
|
)
|
|
|
|
// skillStore is the SQLite-backed skill.SkillStore. Same JSON-blob + indexed
|
|
// columns approach as personaStore: the full Skill round-trips, lookups stay
|
|
// indexed. Versions live in their own table (each SkillVersion embeds a full
|
|
// Skill snapshot, stored as a JSON blob).
|
|
type skillStore struct{ db *sql.DB }
|
|
|
|
// Skills returns a durable skill.SkillStore backed by this database.
|
|
func (d *DB) Skills() skill.SkillStore { return &skillStore{db: d.sql} }
|
|
|
|
var _ skill.SkillStore = (*skillStore)(nil)
|
|
|
|
func (s *skillStore) Initialize(ctx context.Context) error {
|
|
_, err := s.db.ExecContext(ctx, `
|
|
CREATE TABLE IF NOT EXISTS skills (
|
|
id TEXT PRIMARY KEY,
|
|
owner_id TEXT NOT NULL DEFAULT '',
|
|
name TEXT NOT NULL DEFAULT '',
|
|
source TEXT NOT NULL DEFAULT '',
|
|
visibility TEXT NOT NULL DEFAULT '',
|
|
chatbot INTEGER NOT NULL DEFAULT 0, -- ExposeAsChatbotTool
|
|
schedule TEXT NOT NULL DEFAULT '',
|
|
next_run_at INTEGER NOT NULL DEFAULT 0,
|
|
data TEXT NOT NULL
|
|
);
|
|
CREATE INDEX IF NOT EXISTS idx_skills_owner ON skills(owner_id);
|
|
CREATE INDEX IF NOT EXISTS idx_skills_vis ON skills(visibility);
|
|
CREATE INDEX IF NOT EXISTS idx_skills_sched ON skills(schedule, next_run_at);
|
|
CREATE TABLE IF NOT EXISTS skill_versions (
|
|
id TEXT PRIMARY KEY,
|
|
skill_id TEXT NOT NULL,
|
|
version TEXT NOT NULL DEFAULT '',
|
|
seq INTEGER NOT NULL, -- append order, for newest-first
|
|
data TEXT NOT NULL
|
|
);
|
|
CREATE UNIQUE INDEX IF NOT EXISTS idx_skill_versions_skill ON skill_versions(skill_id, seq);`)
|
|
if err != nil {
|
|
return fmt.Errorf("skillStore.Initialize: %w", err)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (s *skillStore) Save(ctx context.Context, sk *skill.Skill) error {
|
|
blob, err := json.Marshal(sk)
|
|
if err != nil {
|
|
return fmt.Errorf("skillStore.Save: marshal: %w", err)
|
|
}
|
|
var next int64
|
|
if !sk.NextRunAt.IsZero() {
|
|
next = sk.NextRunAt.Unix()
|
|
}
|
|
chatbot := 0
|
|
if sk.ExposeAsChatbotTool {
|
|
chatbot = 1
|
|
}
|
|
_, err = s.db.ExecContext(ctx, `
|
|
INSERT INTO skills (id, owner_id, name, source, visibility, chatbot, schedule, next_run_at, data)
|
|
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
ON CONFLICT(id) DO UPDATE SET
|
|
owner_id=excluded.owner_id, name=excluded.name, source=excluded.source,
|
|
visibility=excluded.visibility, chatbot=excluded.chatbot, schedule=excluded.schedule,
|
|
next_run_at=excluded.next_run_at, data=excluded.data`,
|
|
sk.ID, sk.OwnerID, sk.Name, string(sk.Source), string(sk.Visibility), chatbot,
|
|
sk.Schedule, next, string(blob))
|
|
if err != nil {
|
|
return fmt.Errorf("skillStore.Save: %w", err)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func scanSkills(rows *sql.Rows) ([]skill.Skill, error) {
|
|
defer rows.Close()
|
|
var out []skill.Skill
|
|
for rows.Next() {
|
|
var blob string
|
|
if err := rows.Scan(&blob); err != nil {
|
|
return nil, err
|
|
}
|
|
var sk skill.Skill
|
|
if err := json.Unmarshal([]byte(blob), &sk); err != nil {
|
|
return nil, err
|
|
}
|
|
out = append(out, sk)
|
|
}
|
|
return out, rows.Err()
|
|
}
|
|
|
|
func (s *skillStore) getOne(ctx context.Context, where string, arg ...any) (*skill.Skill, error) {
|
|
var blob string
|
|
err := s.db.QueryRowContext(ctx, `SELECT data FROM skills WHERE `+where, arg...).Scan(&blob)
|
|
switch {
|
|
case errors.Is(err, sql.ErrNoRows):
|
|
return nil, skill.ErrNotFound
|
|
case err != nil:
|
|
return nil, err
|
|
}
|
|
var sk skill.Skill
|
|
if err := json.Unmarshal([]byte(blob), &sk); err != nil {
|
|
return nil, err
|
|
}
|
|
return &sk, nil
|
|
}
|
|
|
|
func (s *skillStore) Get(ctx context.Context, id string) (*skill.Skill, error) {
|
|
return s.getOne(ctx, "id = ?", id)
|
|
}
|
|
|
|
func (s *skillStore) GetByName(ctx context.Context, ownerID, name string) (*skill.Skill, error) {
|
|
return s.getOne(ctx, "owner_id = ? AND name = ?", ownerID, name)
|
|
}
|
|
|
|
func (s *skillStore) ListBuiltinByName(ctx context.Context, name string) (*skill.Skill, error) {
|
|
return s.getOne(ctx, "source = ? AND name = ?", string(skill.SourceBuiltin), name)
|
|
}
|
|
|
|
func (s *skillStore) Delete(ctx context.Context, id string) error {
|
|
if _, err := s.db.ExecContext(ctx, `DELETE FROM skills WHERE id = ?`, id); err != nil {
|
|
return fmt.Errorf("skillStore.Delete: %w", err)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (s *skillStore) query(ctx context.Context, where string, arg ...any) ([]skill.Skill, error) {
|
|
rows, err := s.db.QueryContext(ctx, `SELECT data FROM skills WHERE `+where+` ORDER BY name`, arg...)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return scanSkills(rows)
|
|
}
|
|
|
|
func (s *skillStore) ListByOwner(ctx context.Context, ownerID string) ([]skill.Skill, error) {
|
|
return s.query(ctx, "owner_id = ?", ownerID)
|
|
}
|
|
|
|
func (s *skillStore) ListPublic(ctx context.Context) ([]skill.Skill, error) {
|
|
return s.query(ctx, "visibility = ?", string(skill.VisibilityPublic))
|
|
}
|
|
|
|
func (s *skillStore) ListChatbotExposed(ctx context.Context) ([]skill.Skill, error) {
|
|
return s.query(ctx, "chatbot = 1")
|
|
}
|
|
|
|
// ListSharedWith loads visibility=shared rows and filters SharedWith in Go (the
|
|
// shared set per skill is small; avoids a JSON-array query).
|
|
func (s *skillStore) ListSharedWith(ctx context.Context, memberID string) ([]skill.Skill, error) {
|
|
shared, err := s.query(ctx, "visibility = ?", string(skill.VisibilityShared))
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
out := shared[:0]
|
|
for _, sk := range shared {
|
|
for _, id := range sk.SharedWith {
|
|
if id == memberID {
|
|
out = append(out, sk)
|
|
break
|
|
}
|
|
}
|
|
}
|
|
return out, nil
|
|
}
|
|
|
|
func (s *skillStore) ListDueScheduled(ctx context.Context, now time.Time) ([]skill.Skill, error) {
|
|
rows, err := s.db.QueryContext(ctx,
|
|
`SELECT data FROM skills WHERE schedule != '' AND next_run_at > 0 AND next_run_at <= ? ORDER BY next_run_at`,
|
|
now.Unix())
|
|
if err != nil {
|
|
return nil, fmt.Errorf("skillStore.ListDueScheduled: %w", err)
|
|
}
|
|
return scanSkills(rows)
|
|
}
|
|
|
|
func (s *skillStore) MarkScheduledRun(ctx context.Context, skillID string, ranAt, nextAt time.Time) error {
|
|
// Single atomic statement instead of Get→mutate→Save: a concurrent Mark or
|
|
// admin edit can't lose this update (no read-modify-write window). json_set
|
|
// keeps the JSON blob's NextRunAt/LastScheduledRunAt consistent with the
|
|
// indexed next_run_at column; RFC3339Nano matches Go's time JSON encoding so
|
|
// the blob still round-trips through Get.
|
|
var next int64
|
|
if !nextAt.IsZero() {
|
|
next = nextAt.Unix()
|
|
}
|
|
res, err := s.db.ExecContext(ctx,
|
|
`UPDATE skills SET next_run_at=?, data=json_set(data,'$.NextRunAt',?,'$.LastScheduledRunAt',?) WHERE id=?`,
|
|
next, nextAt.Format(time.RFC3339Nano), ranAt.Format(time.RFC3339Nano), skillID)
|
|
if err != nil {
|
|
return fmt.Errorf("skillStore.MarkScheduledRun: %w", err)
|
|
}
|
|
if n, _ := res.RowsAffected(); n == 0 {
|
|
return skill.ErrNotFound
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (s *skillStore) AppendVersion(ctx context.Context, sv skill.SkillVersion) error {
|
|
if sv.SkillID == "" {
|
|
return fmt.Errorf("skillStore.AppendVersion: skill_id is required")
|
|
}
|
|
blob, err := json.Marshal(sv)
|
|
if err != nil {
|
|
return fmt.Errorf("skillStore.AppendVersion: marshal: %w", err)
|
|
}
|
|
// seq = current max+1 for this skill (newest-first ordering key). The
|
|
// MAX-then-INSERT runs in ONE transaction and the (skill_id, seq) index is
|
|
// UNIQUE, so two concurrent appends can't both land the same seq: the loser
|
|
// fails loudly on commit instead of silently corrupting the ordering. The
|
|
// Scan error is propagated (was swallowed, leaving seq=0 on failure).
|
|
tx, err := s.db.BeginTx(ctx, nil)
|
|
if err != nil {
|
|
return fmt.Errorf("skillStore.AppendVersion: begin: %w", err)
|
|
}
|
|
defer tx.Rollback() //nolint:errcheck // no-op after Commit
|
|
var seq int64
|
|
if err := tx.QueryRowContext(ctx, `SELECT COALESCE(MAX(seq),0)+1 FROM skill_versions WHERE skill_id = ?`, sv.SkillID).Scan(&seq); err != nil {
|
|
return fmt.Errorf("skillStore.AppendVersion: seq: %w", err)
|
|
}
|
|
if _, err := tx.ExecContext(ctx,
|
|
`INSERT INTO skill_versions (id, skill_id, version, seq, data) VALUES (?, ?, ?, ?, ?)`,
|
|
sv.ID, sv.SkillID, sv.Version, seq, string(blob)); err != nil {
|
|
return fmt.Errorf("skillStore.AppendVersion: insert: %w", err)
|
|
}
|
|
if err := tx.Commit(); err != nil {
|
|
return fmt.Errorf("skillStore.AppendVersion: commit: %w", err)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (s *skillStore) ListVersionsBySkill(ctx context.Context, skillID string, limit int) ([]skill.SkillVersion, error) {
|
|
q := `SELECT data FROM skill_versions WHERE skill_id = ? ORDER BY seq DESC`
|
|
args := []any{skillID}
|
|
if limit > 0 {
|
|
q += ` LIMIT ?`
|
|
args = append(args, limit)
|
|
}
|
|
rows, err := s.db.QueryContext(ctx, q, args...)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("skillStore.ListVersionsBySkill: %w", err)
|
|
}
|
|
defer rows.Close()
|
|
var out []skill.SkillVersion
|
|
for rows.Next() {
|
|
var blob string
|
|
if err := rows.Scan(&blob); err != nil {
|
|
return nil, err
|
|
}
|
|
var sv skill.SkillVersion
|
|
if err := json.Unmarshal([]byte(blob), &sv); err != nil {
|
|
return nil, err
|
|
}
|
|
out = append(out, sv)
|
|
}
|
|
return out, rows.Err()
|
|
}
|
|
|
|
func (s *skillStore) GetVersionByID(ctx context.Context, versionID string) (*skill.SkillVersion, error) {
|
|
var blob string
|
|
err := s.db.QueryRowContext(ctx, `SELECT data FROM skill_versions WHERE id = ?`, versionID).Scan(&blob)
|
|
switch {
|
|
case errors.Is(err, sql.ErrNoRows):
|
|
return nil, skill.ErrNotFound
|
|
case err != nil:
|
|
return nil, err
|
|
}
|
|
var sv skill.SkillVersion
|
|
if err := json.Unmarshal([]byte(blob), &sv); err != nil {
|
|
return nil, err
|
|
}
|
|
return &sv, nil
|
|
}
|