daf07fd759
Add the async job submission API, webhook state notifications, and
artifact serving endpoints on top of the Phase 3 queue infrastructure.
Key changes:
- POST /jobs: async job submission with 202 + job_id ULID; optional
state_webhook_url for push notifications on state transitions
- GET /jobs/{id}: job status polling with result, error, and artifact
metadata; artifacts <= 256KB inlined, larger ones by URL reference
- GET /jobs/{id}/artifacts/{name}: raw artifact data serving
- Webhook dispatcher: at-least-once delivery with exponential backoff
(5 retries); optional HMAC-SHA256 signing (X-Foreman-Signature)
- ADR-0014: state_webhook_url only honored on POST /jobs, not sync
/api/chat (caller already blocks for result)
- Comprehensive tests for /jobs lifecycle, webhook delivery, HMAC
verification, artifact inline/URL threshold, and TTL pruning
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
287 lines
7.0 KiB
Go
287 lines
7.0 KiB
Go
package webhook
|
|
|
|
import (
|
|
"encoding/json"
|
|
"io"
|
|
"log/slog"
|
|
"net/http"
|
|
"net/http/httptest"
|
|
"sync/atomic"
|
|
"testing"
|
|
"time"
|
|
)
|
|
|
|
func TestDispatcher_Fire_Delivers(t *testing.T) {
|
|
ch := make(chan []byte, 1)
|
|
|
|
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
body, _ := io.ReadAll(r.Body)
|
|
ch <- body
|
|
w.WriteHeader(http.StatusOK)
|
|
}))
|
|
defer srv.Close()
|
|
|
|
logger := slog.New(slog.NewJSONHandler(io.Discard, nil))
|
|
d := NewDispatcher("", logger)
|
|
|
|
event := Event{
|
|
JobID: "01TEST001",
|
|
State: "done",
|
|
PreviousState: "working",
|
|
Timestamp: time.Now().UTC(),
|
|
Model: "qwen3:30b",
|
|
Attempt: 1,
|
|
}
|
|
|
|
d.Fire(srv.URL, event)
|
|
|
|
var receivedBody []byte
|
|
select {
|
|
case receivedBody = <-ch:
|
|
case <-time.After(3 * time.Second):
|
|
t.Fatal("timed out waiting for webhook delivery")
|
|
}
|
|
|
|
// Verify the body is valid JSON with the right fields.
|
|
var got Event
|
|
if err := json.Unmarshal(receivedBody, &got); err != nil {
|
|
t.Fatalf("unmarshal: %v", err)
|
|
}
|
|
if got.JobID != "01TEST001" {
|
|
t.Errorf("job_id = %q, want %q", got.JobID, "01TEST001")
|
|
}
|
|
if got.State != "done" {
|
|
t.Errorf("state = %q, want %q", got.State, "done")
|
|
}
|
|
if got.PreviousState != "working" {
|
|
t.Errorf("previous_state = %q, want %q", got.PreviousState, "working")
|
|
}
|
|
if got.Model != "qwen3:30b" {
|
|
t.Errorf("model = %q, want %q", got.Model, "qwen3:30b")
|
|
}
|
|
}
|
|
|
|
func TestDispatcher_Fire_RetriesOn500(t *testing.T) {
|
|
var attempts atomic.Int32
|
|
|
|
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
n := attempts.Add(1)
|
|
if n <= 2 {
|
|
w.WriteHeader(http.StatusInternalServerError)
|
|
return
|
|
}
|
|
w.WriteHeader(http.StatusOK)
|
|
}))
|
|
defer srv.Close()
|
|
|
|
logger := slog.New(slog.NewJSONHandler(io.Discard, nil))
|
|
d := NewDispatcher("", logger)
|
|
d.baseDelay = 10 * time.Millisecond // Fast retries for testing.
|
|
|
|
event := Event{
|
|
JobID: "01RETRY001",
|
|
State: "done",
|
|
Model: "qwen3:30b",
|
|
}
|
|
|
|
d.Fire(srv.URL, event)
|
|
|
|
// Wait for retries to complete.
|
|
deadline := time.Now().Add(5 * time.Second)
|
|
for time.Now().Before(deadline) {
|
|
if attempts.Load() >= 3 {
|
|
break
|
|
}
|
|
time.Sleep(10 * time.Millisecond)
|
|
}
|
|
|
|
if got := attempts.Load(); got < 3 {
|
|
t.Errorf("attempts = %d, want >= 3 (2 failures + 1 success)", got)
|
|
}
|
|
}
|
|
|
|
func TestDispatcher_Fire_DoesNotBlockCaller(t *testing.T) {
|
|
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
time.Sleep(100 * time.Millisecond)
|
|
w.WriteHeader(http.StatusOK)
|
|
}))
|
|
defer srv.Close()
|
|
|
|
logger := slog.New(slog.NewJSONHandler(io.Discard, nil))
|
|
d := NewDispatcher("", logger)
|
|
|
|
start := time.Now()
|
|
d.Fire(srv.URL, Event{JobID: "test", State: "done"})
|
|
elapsed := time.Since(start)
|
|
|
|
// Fire should return immediately (< 10ms), not wait for the HTTP call.
|
|
if elapsed > 50*time.Millisecond {
|
|
t.Errorf("Fire blocked for %v, should return immediately", elapsed)
|
|
}
|
|
}
|
|
|
|
func TestDispatcher_HMAC_Signing(t *testing.T) {
|
|
type capture struct {
|
|
signature string
|
|
body []byte
|
|
}
|
|
ch := make(chan capture, 1)
|
|
|
|
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
body, _ := io.ReadAll(r.Body)
|
|
ch <- capture{signature: r.Header.Get("X-Foreman-Signature"), body: body}
|
|
w.WriteHeader(http.StatusOK)
|
|
}))
|
|
defer srv.Close()
|
|
|
|
secret := "test-secret-key"
|
|
logger := slog.New(slog.NewJSONHandler(io.Discard, nil))
|
|
d := NewDispatcher(secret, logger)
|
|
|
|
event := Event{
|
|
JobID: "01HMAC001",
|
|
State: "done",
|
|
Model: "qwen3:30b",
|
|
}
|
|
|
|
d.Fire(srv.URL, event)
|
|
|
|
var got capture
|
|
select {
|
|
case got = <-ch:
|
|
case <-time.After(3 * time.Second):
|
|
t.Fatal("timed out waiting for webhook delivery")
|
|
}
|
|
|
|
if got.signature == "" {
|
|
t.Fatal("X-Foreman-Signature header should be set when secret is configured")
|
|
}
|
|
|
|
if len(got.signature) < 8 || got.signature[:7] != "sha256=" {
|
|
t.Fatalf("signature format wrong: %q", got.signature)
|
|
}
|
|
|
|
// Verify the signature against the received body.
|
|
if !VerifySignature(got.body, got.signature, secret) {
|
|
t.Error("HMAC verification failed with correct secret")
|
|
}
|
|
|
|
// Verify wrong secret fails.
|
|
if VerifySignature(got.body, got.signature, "wrong-secret") {
|
|
t.Error("HMAC verification should fail with wrong secret")
|
|
}
|
|
}
|
|
|
|
func TestDispatcher_NoHMAC_WhenNoSecret(t *testing.T) {
|
|
ch := make(chan string, 1)
|
|
|
|
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
ch <- r.Header.Get("X-Foreman-Signature")
|
|
w.WriteHeader(http.StatusOK)
|
|
}))
|
|
defer srv.Close()
|
|
|
|
logger := slog.New(slog.NewJSONHandler(io.Discard, nil))
|
|
d := NewDispatcher("", logger) // No secret.
|
|
|
|
d.Fire(srv.URL, Event{JobID: "test", State: "done"})
|
|
|
|
var gotSignature string
|
|
select {
|
|
case gotSignature = <-ch:
|
|
case <-time.After(3 * time.Second):
|
|
t.Fatal("timed out waiting for webhook delivery")
|
|
}
|
|
|
|
if gotSignature != "" {
|
|
t.Errorf("X-Foreman-Signature should not be set when no secret is configured, got %q", gotSignature)
|
|
}
|
|
}
|
|
|
|
func TestVerifySignature_InvalidFormat(t *testing.T) {
|
|
if VerifySignature([]byte("test"), "invalid", "secret") {
|
|
t.Error("should reject signatures without sha256= prefix")
|
|
}
|
|
if VerifySignature([]byte("test"), "sha256", "secret") {
|
|
t.Error("should reject too-short signatures")
|
|
}
|
|
if VerifySignature([]byte("test"), "", "secret") {
|
|
t.Error("should reject empty signature")
|
|
}
|
|
}
|
|
|
|
func TestFormatArtifacts_SmallInline(t *testing.T) {
|
|
metas := []ArtifactMeta{
|
|
{
|
|
Name: "completion",
|
|
ContentType: "application/json",
|
|
Size: 100,
|
|
Data: []byte(`{"done":true}`),
|
|
},
|
|
}
|
|
|
|
result := FormatArtifacts("01TEST", metas)
|
|
if result == nil {
|
|
t.Fatal("result should not be nil")
|
|
}
|
|
|
|
var parsed []struct {
|
|
Name string `json:"name"`
|
|
Data string `json:"data"`
|
|
URL string `json:"url"`
|
|
}
|
|
if err := json.Unmarshal(result, &parsed); err != nil {
|
|
t.Fatalf("unmarshal: %v", err)
|
|
}
|
|
|
|
if len(parsed) != 1 {
|
|
t.Fatalf("len = %d, want 1", len(parsed))
|
|
}
|
|
if parsed[0].Data == "" {
|
|
t.Error("small artifact should be inlined")
|
|
}
|
|
if parsed[0].URL != "" {
|
|
t.Error("small artifact should not have a URL")
|
|
}
|
|
}
|
|
|
|
func TestFormatArtifacts_LargeByURL(t *testing.T) {
|
|
largeData := make([]byte, 300*1024) // 300KB > 256KB threshold.
|
|
metas := []ArtifactMeta{
|
|
{
|
|
Name: "completion",
|
|
ContentType: "application/json",
|
|
Size: int64(len(largeData)),
|
|
Data: largeData,
|
|
},
|
|
}
|
|
|
|
result := FormatArtifacts("01LARGE", metas)
|
|
|
|
var parsed []struct {
|
|
Name string `json:"name"`
|
|
Data string `json:"data"`
|
|
URL string `json:"url"`
|
|
}
|
|
if err := json.Unmarshal(result, &parsed); err != nil {
|
|
t.Fatalf("unmarshal: %v", err)
|
|
}
|
|
|
|
if parsed[0].Data != "" {
|
|
t.Error("large artifact should not be inlined")
|
|
}
|
|
if parsed[0].URL == "" {
|
|
t.Error("large artifact should have a URL")
|
|
}
|
|
if parsed[0].URL != "/jobs/01LARGE/artifacts/completion" {
|
|
t.Errorf("URL = %q, want %q", parsed[0].URL, "/jobs/01LARGE/artifacts/completion")
|
|
}
|
|
}
|
|
|
|
func TestFormatArtifacts_Empty(t *testing.T) {
|
|
result := FormatArtifacts("01EMPTY", nil)
|
|
if result != nil {
|
|
t.Errorf("empty artifacts should return nil, got %s", result)
|
|
}
|
|
}
|