Seed-packet capture: vision model, extraction, catalog match, create (backend) (#81)

Photograph a seed packet → it fills in the plant and the purchase. This is the
backend; the scan UI is a follow-up PR.

Vision model config (mirrors the agent model from #79):
- Migration 0011 adds instance_settings.vision_model; PANSY_VISION_MODEL is the
  env default. Precedence Settings → env → empty; the KEY stays in the env.
- EffectiveVision resolves it; /capabilities advertises "vision" only when a
  model + key are configured, so the UI offers the scan button only when it works.

Extraction is one-shot, NOT an agent loop (internal/vision):
- majordomo.Generate[SeedPacket] derives a JSON schema from the struct tags and
  hands the image to the vision model; it can't call a tool, so it can't touch
  the garden — it only reads a picture and returns data. Numeric fields are
  pointers, so a field the packet doesn't print comes back nil, not a made-up 0.
- Hermetic test: majordomo's fake provider returns canned packet JSON and
  Generate unmarshals it, image + derived schema included. No live model.

The image is normalized to JPEG at the upload boundary (imagenorm from #80),
which is where an iPhone HEIC becomes readable — majordomo's media path can't
decode HEIC. imagenorm now links into the binary (~7 MB, the cost #80 deferred).

The hard part is catalog matching, not OCR (internal/service/seed_packet.go):
- A wrong auto-match splits a variety's seed-lot history across duplicate rows,
  so the service NEVER auto-creates. matchPlants surfaces RANKED candidates
  (exact name → variety-in-name → same species, conservative and name-based),
  the user confirms, and CreateFromPacket makes the plant (new or existing) + the
  lot. Exactly one of plantId/newPlant, refused otherwise.
- Plants/lots aren't in the undo history (catalog/inventory), so no change set.
- The extractor is injectable (service.WithPacketExtractor) so ExtractSeedPacket
  and the /scan endpoint test end to end against a fake, no live model.

Endpoints: POST /seed-lots/scan (multipart image → proposal, reads only; extends
the read deadline for a slow phone upload, caps the body, maps too-large/unreadable
to clear statuses) and POST /seed-lots/from-packet (confirmed proposal → 201).

Docs: README (PANSY_VISION_MODEL), DESIGN (routes + the decision and why the
model can't touch the garden).

Co-Authored-By: Claude Opus 4.8 (1M context) <[email protected]>
Claude-Session: https://claude.ai/code/session_01H3zbym8Doka2d7D48maSgZ

internal/api/api.go

@@ -185,6 +185,10 @@ func New(cfg *config.Config, svc *service.Service) *gin.Engine {
 	seedLots.GET("/:id", h.getSeedLot)
 	seedLots.PATCH("/:id", h.updateSeedLot)
 	seedLots.DELETE("/:id", h.deleteSeedLot)
+	// Seed-packet capture (#81): scan a photo into a proposal, then create the
+	// plant + lot from the confirmed proposal. scan reads only.
+	seedLots.POST("/scan", h.scanSeedPacket)
+	seedLots.POST("/from-packet", h.createFromPacket)
 
 	// Public, unauthenticated read of a garden by its share token. Deliberately
 	// NOT behind requireAuth: the token is the capability, so a logged-out visitor
@@ -204,7 +208,14 @@ func New(cfg *config.Config, svc *service.Service) *gin.Engine {
 // so offering the tab must track the live Runner. Reading agent.get() (an atomic
 // load) means this reflects a settings-driven swap on the very next poll.
 func (h *handlers) capabilities(c *gin.Context) {
-	c.JSON(http.StatusOK, gin.H{"agent": h.agent.get() != nil})
+	// vision advertises whether seed-packet scanning (#81) can be offered — a
+	// configured, resolvable vision model + a key. Read per-request so a settings
+	// change is reflected on the next poll, same as agent.
+	vision := false
+	if vis, err := h.svc.EffectiveVision(c.Request.Context()); err == nil {
+		vision = vis.Ready()
+	}
+	c.JSON(http.StatusOK, gin.H{"agent": h.agent.get() != nil, "vision": vision})
 }
 
 // healthz is a liveness probe: always returns {"ok": true} when the server is up.

internal/api/seed_packet.go

@@ -0,0 +1,136 @@
+package api
+
+import (
+	"errors"
+	"net/http"
+	"time"
+
+	"github.com/gin-gonic/gin"
+
+	"gitea.stevedudenhoeffer.com/steve/pansy/internal/domain"
+	"gitea.stevedudenhoeffer.com/steve/pansy/internal/imagenorm"
+	"gitea.stevedudenhoeffer.com/steve/pansy/internal/service"
+)
+
+// Seed-packet capture (#81). Two steps, deliberately separate:
+//   POST /seed-lots/scan          multipart image → a proposal (reads only)
+//   POST /seed-lots/from-packet   confirmed proposal → a plant + lot
+// The scan never writes; creation happens only from an explicit confirm, so a
+// misread can't add anything to the catalog on its own.
+
+// scanUploadLimit bounds the multipart body. imagenorm caps the decoded image at
+// 25 MiB; this is a little over that for the multipart envelope. A phone photo is
+// a few MB, so this is generous.
+const scanUploadLimit = 30 << 20
+
+// scanReadTimeout is how long we allow the image upload to take. The server's
+// default ReadTimeout (15s) is fine for JSON but tight for a multi-megabyte photo
+// on a slow phone connection, so this endpoint extends it — the same
+// ResponseController mechanism the SSE path uses for writes (#78).
+const scanReadTimeout = 60 * time.Second
+
+// scanSeedPacket reads an uploaded packet photo and returns a proposal.
+func (h *handlers) scanSeedPacket(c *gin.Context) {
+	// Extend the read deadline for the (potentially large, potentially slow)
+	// upload. Best-effort: if the writer doesn't support it, the default applies.
+	_ = http.NewResponseController(c.Writer).SetReadDeadline(time.Now().Add(scanReadTimeout))
+
+	c.Request.Body = http.MaxBytesReader(c.Writer, c.Request.Body, scanUploadLimit)
+	file, err := c.FormFile("image")
+	if err != nil {
+		writeAPIError(c, http.StatusBadRequest, "INVALID_INPUT", "attach an image as the \"image\" field")
+		return
+	}
+	f, err := file.Open()
+	if err != nil {
+		writeAPIError(c, http.StatusBadRequest, "INVALID_INPUT", "could not read the uploaded image")
+		return
+	}
+	defer f.Close()
+
+	// Normalize to JPEG (decodes HEIC/webp/png/jpeg, downscales, re-encodes) so
+	// everything downstream — including the vision model — only sees a format it
+	// can read. This is where an iPhone HEIC becomes usable.
+	jpeg, format, err := imagenorm.Normalize(f, imagenorm.Options{})
+	if err != nil {
+		if errors.Is(err, imagenorm.ErrTooLarge) {
+			writeAPIError(c, http.StatusRequestEntityTooLarge, "IMAGE_TOO_LARGE", "that image is too large — try a smaller photo")
+			return
+		}
+		writeAPIError(c, http.StatusBadRequest, "INVALID_INPUT", "that doesn't look like an image we can read (JPEG, PNG, HEIC or WebP)")
+		return
+	}
+	_ = format // available for logging if wanted
+
+	prop, err := h.svc.ExtractSeedPacket(c.Request.Context(), mustActor(c).ID, jpeg)
+	if err != nil {
+		// A missing vision model surfaces as ErrInvalidInput from the service; give
+		// it a clearer message than the generic 400, since the UI shouldn't have
+		// offered the button at all in that case.
+		if errors.Is(err, domain.ErrInvalidInput) {
+			writeAPIError(c, http.StatusServiceUnavailable, "VISION_DISABLED", "packet scanning isn't set up on this instance")
+			return
+		}
+		writeServiceError(c, err)
+		return
+	}
+	c.JSON(http.StatusOK, prop)
+}
+
+// packetLotRequest is the lot half of a confirm. It's seedLotCreateRequest
+// WITHOUT plantId — the plant comes from the confirm's plantId/newPlant choice,
+// not the lot body, and reusing seedLotCreateRequest would wrongly require one.
+type packetLotRequest struct {
+	Vendor         string   `json:"vendor"`
+	SourceURL      string   `json:"sourceUrl"`
+	SKU            string   `json:"sku"`
+	LotCode        string   `json:"lotCode"`
+	PurchasedAt    *string  `json:"purchasedAt"`
+	PackedForYear  *int     `json:"packedForYear"`
+	Quantity       float64  `json:"quantity"`
+	Unit           string   `json:"unit" binding:"required"`
+	CostCents      *int     `json:"costCents"`
+	GerminationPct *float64 `json:"germinationPct"`
+	Notes          string   `json:"notes"`
+}
+
+func (r packetLotRequest) toInput() service.SeedLotInput {
+	return service.SeedLotInput{
+		Vendor: r.Vendor, SourceURL: r.SourceURL, SKU: r.SKU, LotCode: r.LotCode,
+		PurchasedAt: r.PurchasedAt, PackedForYear: r.PackedForYear, Quantity: r.Quantity,
+		Unit: r.Unit, CostCents: r.CostCents, GerminationPct: r.GerminationPct, Notes: r.Notes,
+	}
+}
+
+// fromPacketRequest confirms a proposal: exactly one of plantId (attach to an
+// existing plant) or newPlant (create a variety), plus the lot to record.
+type fromPacketRequest struct {
+	PlantID  *int64              `json:"plantId"`
+	NewPlant *plantCreateRequest `json:"newPlant"`
+	Lot      packetLotRequest    `json:"lot"`
+}
+
+// createFromPacket turns a confirmed proposal into a plant + lot.
+func (h *handlers) createFromPacket(c *gin.Context) {
+	var req fromPacketRequest
+	if err := c.ShouldBindJSON(&req); err != nil {
+		writeAPIError(c, http.StatusBadRequest, "INVALID_INPUT", "a lot and exactly one of plantId or newPlant are required")
+		return
+	}
+
+	confirm := service.PacketConfirm{
+		PlantID: req.PlantID,
+		Lot:     req.Lot.toInput(), // no plantId in the lot body; the service attributes it
+	}
+	if req.NewPlant != nil {
+		in := req.NewPlant.toInput()
+		confirm.NewPlant = &in
+	}
+
+	res, err := h.svc.CreateFromPacket(c.Request.Context(), mustActor(c).ID, confirm)
+	if err != nil {
+		writeServiceError(c, err)
+		return
+	}
+	c.JSON(http.StatusCreated, res)
+}

internal/api/seed_packet_test.go

@@ -0,0 +1,219 @@
+package api
+
+import (
+	"bytes"
+	"context"
+	"image"
+	"image/png"
+	"mime/multipart"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+
+	"github.com/gin-gonic/gin"
+
+	"gitea.stevedudenhoeffer.com/steve/pansy/internal/config"
+	"gitea.stevedudenhoeffer.com/steve/pansy/internal/service"
+	"gitea.stevedudenhoeffer.com/steve/pansy/internal/store"
+	"gitea.stevedudenhoeffer.com/steve/pansy/internal/vision"
+)
+
+// packetEngine builds an engine whose service reads seed packets via the given
+// canned extractor, so the scan endpoint can be tested without a live model.
+func packetEngine(t *testing.T, cfg *config.Config, extract func() (vision.SeedPacket, error)) *gin.Engine {
+	t.Helper()
+	gin.SetMode(gin.TestMode)
+	db, err := store.Open(":memory:")
+	if err != nil {
+		t.Fatalf("store.Open: %v", err)
+	}
+	t.Cleanup(func() { db.Close() })
+	if err := db.Migrate(context.Background()); err != nil {
+		t.Fatalf("Migrate: %v", err)
+	}
+	svc := service.New(db, cfg, service.WithPacketExtractor(
+		func(context.Context, string, string, []byte) (vision.SeedPacket, error) { return extract() },
+	))
+	return New(cfg, svc)
+}
+
+// visionCfg is a config with a vision model + key configured, so packet scanning
+// is available.
+func visionCfg() *config.Config {
+	c := localCfg()
+	c.Agent = config.AgentConfig{OllamaCloudAPIKey: "k", VisionModel: "ollama-cloud/vision:cloud"}
+	return c
+}
+
+// pngUpload builds a multipart body with a real PNG under the "image" field.
+func pngUpload(t *testing.T) (body *bytes.Buffer, contentType string) {
+	t.Helper()
+	var img bytes.Buffer
+	if err := png.Encode(&img, image.NewRGBA(image.Rect(0, 0, 32, 24))); err != nil {
+		t.Fatalf("encode png: %v", err)
+	}
+	var buf bytes.Buffer
+	w := multipart.NewWriter(&buf)
+	part, err := w.CreateFormFile("image", "packet.png")
+	if err != nil {
+		t.Fatalf("form file: %v", err)
+	}
+	part.Write(img.Bytes())
+	w.Close()
+	return &buf, w.FormDataContentType()
+}
+
+func doMultipart(t *testing.T, r *gin.Engine, path, contentType string, body *bytes.Buffer, cookie *http.Cookie) *httptest.ResponseRecorder {
+	t.Helper()
+	req := httptest.NewRequest(http.MethodPost, path, body)
+	req.Header.Set("Content-Type", contentType)
+	if cookie != nil {
+		req.AddCookie(cookie)
+	}
+	w := httptest.NewRecorder()
+	r.ServeHTTP(w, req)
+	return w
+}
+
+// TestScanSeedPacketAPI: a multipart image → a proposal, end to end through the
+// router. The extractor is canned; imagenorm runs for real on the uploaded PNG.
+func TestScanSeedPacketAPI(t *testing.T) {
+	r := packetEngine(t, visionCfg(), func() (vision.SeedPacket, error) {
+		return vision.SeedPacket{Species: "garlic", Variety: "Music", Category: "vegetable"}, nil
+	})
+	cookie := registerAndCookie(t, r, "[email protected]")
+	// A matching plant so the proposal has a candidate.
+	createPlantAPI(t, r, cookie, "Music Garlic", 15)
+
+	body, ct := pngUpload(t)
+	w := doMultipart(t, r, "/api/v1/seed-lots/scan", ct, body, cookie)
+	if w.Code != http.StatusOK {
+		t.Fatalf("scan: status %d, body %s", w.Code, w.Body.String())
+	}
+	res := decodeMap(t, w.Body.Bytes())
+	pkt, _ := res["packet"].(map[string]any)
+	if pkt["variety"] != "Music" {
+		t.Errorf("packet variety = %v", pkt["variety"])
+	}
+	if cands, _ := res["candidates"].([]any); len(cands) == 0 {
+		t.Error("expected a candidate match for Music Garlic")
+	}
+	if res["suggestedName"] != "Music" {
+		t.Errorf("suggestedName = %v", res["suggestedName"])
+	}
+}
+
+// TestScanSeedPacketErrorsAPI: no image, unreadable bytes, and vision-not-
+// configured each get their own clear status.
+func TestScanSeedPacketErrorsAPI(t *testing.T) {
+	// Vision configured, so we reach imagenorm / the extractor.
+	r := packetEngine(t, visionCfg(), func() (vision.SeedPacket, error) {
+		return vision.SeedPacket{Variety: "X"}, nil
+	})
+	cookie := registerAndCookie(t, r, "[email protected]")
+
+	// No file field → 400.
+	if w := doMultipart(t, r, "/api/v1/seed-lots/scan", "multipart/form-data; boundary=x", bytes.NewBufferString(""), cookie); w.Code != http.StatusBadRequest {
+		t.Errorf("no image = %d, want 400", w.Code)
+	}
+	// A file that isn't an image → 400 (imagenorm rejects it).
+	var buf bytes.Buffer
+	mw := multipart.NewWriter(&buf)
+	part, _ := mw.CreateFormFile("image", "notes.txt")
+	part.Write([]byte("this is not an image"))
+	mw.Close()
+	if w := doMultipart(t, r, "/api/v1/seed-lots/scan", mw.FormDataContentType(), &buf, cookie); w.Code != http.StatusBadRequest {
+		t.Errorf("non-image = %d, want 400", w.Code)
+	}
+
+	// Vision NOT configured → 503, even with a valid image.
+	r2 := packetEngine(t, localCfg(), func() (vision.SeedPacket, error) { return vision.SeedPacket{}, nil })
+	cookie2 := registerAndCookie(t, r2, "[email protected]")
+	body, ct := pngUpload(t)
+	if w := doMultipart(t, r2, "/api/v1/seed-lots/scan", ct, body, cookie2); w.Code != http.StatusServiceUnavailable {
+		t.Errorf("no vision model = %d, want 503", w.Code)
+	}
+
+	// Unauthenticated → 401.
+	b3, ct3 := pngUpload(t)
+	if w := doMultipart(t, r, "/api/v1/seed-lots/scan", ct3, b3, nil); w.Code != http.StatusUnauthorized {
+		t.Errorf("anonymous scan = %d, want 401", w.Code)
+	}
+}
+
+// TestCreateFromPacketAPI: confirm → plant + lot. No model involved, so the full
+// path runs through the router.
+func TestCreateFromPacketAPI(t *testing.T) {
+	r := packetEngine(t, visionCfg(), func() (vision.SeedPacket, error) { return vision.SeedPacket{}, nil })
+	cookie := registerAndCookie(t, r, "[email protected]")
+
+	// New plant + lot.
+	w := doJSON(t, r, http.MethodPost, "/api/v1/seed-lots/from-packet", map[string]any{
+		"newPlant": map[string]any{"name": "Music Garlic", "category": "vegetable", "color": "#4a7c3f", "icon": "🧄", "spacingCm": 15},
+		"lot":      map[string]any{"vendor": "Johnny's", "quantity": 8, "unit": "bulbs"},
+	}, cookie)
+	if w.Code != http.StatusCreated {
+		t.Fatalf("new plant confirm: status %d, body %s", w.Code, w.Body.String())
+	}
+	res := decodeMap(t, w.Body.Bytes())
+	if res["plantIsNew"] != true {
+		t.Errorf("plantIsNew = %v, want true", res["plantIsNew"])
+	}
+	plantObj, _ := res["plant"].(map[string]any)
+	plantID := int64(plantObj["id"].(float64))
+	lotObj, _ := res["lot"].(map[string]any)
+	if int64(lotObj["plantId"].(float64)) != plantID {
+		t.Errorf("lot not attributed to the new plant")
+	}
+
+	// Existing plant + lot.
+	w = doJSON(t, r, http.MethodPost, "/api/v1/seed-lots/from-packet", map[string]any{
+		"plantId": plantID,
+		"lot":     map[string]any{"vendor": "Fedco", "quantity": 10, "unit": "bulbs"},
+	}, cookie)
+	if w.Code != http.StatusCreated {
+		t.Fatalf("existing plant confirm: status %d, body %s", w.Code, w.Body.String())
+	}
+	if decodeMap(t, w.Body.Bytes())["plantIsNew"] != false {
+		t.Error("plantIsNew should be false for an existing plant")
+	}
+
+	// Both plantId and newPlant → 400.
+	if w := doJSON(t, r, http.MethodPost, "/api/v1/seed-lots/from-packet", map[string]any{
+		"plantId":  plantID,
+		"newPlant": map[string]any{"name": "X", "category": "vegetable", "color": "#4a7c3f", "icon": "🌱"},
+		"lot":      map[string]any{"vendor": "V", "quantity": 1, "unit": "seeds"},
+	}, cookie); w.Code != http.StatusBadRequest {
+		t.Errorf("both plant choices = %d, want 400", w.Code)
+	}
+
+	// Neither → 400.
+	if w := doJSON(t, r, http.MethodPost, "/api/v1/seed-lots/from-packet", map[string]any{
+		"lot": map[string]any{"vendor": "V", "quantity": 1, "unit": "seeds"},
+	}, cookie); w.Code != http.StatusBadRequest {
+		t.Errorf("neither plant choice = %d, want 400", w.Code)
+	}
+
+	// Unauthenticated → 401.
+	if w := doJSON(t, r, http.MethodPost, "/api/v1/seed-lots/from-packet", nil, nil); w.Code != http.StatusUnauthorized {
+		t.Errorf("anonymous confirm = %d, want 401", w.Code)
+	}
+}
+
+// TestCapabilitiesReportsVision: /capabilities advertises vision only when a
+// vision model is configured.
+func TestCapabilitiesReportsVision(t *testing.T) {
+	on := packetEngine(t, visionCfg(), func() (vision.SeedPacket, error) { return vision.SeedPacket{}, nil })
+	cookie := registerAndCookie(t, on, "[email protected]")
+	w := doJSON(t, on, http.MethodGet, "/api/v1/capabilities", nil, cookie)
+	if decodeMap(t, w.Body.Bytes())["vision"] != true {
+		t.Errorf("vision should be true with a model configured: %s", w.Body.String())
+	}
+
+	off := packetEngine(t, localCfg(), func() (vision.SeedPacket, error) { return vision.SeedPacket{}, nil })
+	cookie2 := registerAndCookie(t, off, "[email protected]")
+	w = doJSON(t, off, http.MethodGet, "/api/v1/capabilities", nil, cookie2)
+	if decodeMap(t, w.Body.Bytes())["vision"] != false {
+		t.Errorf("vision should be false with no model: %s", w.Body.String())
+	}
+}

internal/api/settings.go

@@ -51,6 +51,10 @@ type effectiveView struct {
 	// can be false even when Enabled+HasApiKey are true (an unresolvable model),
 	// which is exactly the case the UI needs to surface.
 	AgentLive bool `json:"agentLive"`
+	// VisionModel is the resolved seed-packet model (DB-over-env). VisionReady is
+	// whether capture can actually be offered (a key and a model).
+	VisionModel string `json:"visionModel"`
+	VisionReady bool   `json:"visionReady"`
 }
 
 // settingsPayload builds the response, or an error. It does NOT swallow an
@@ -64,13 +68,19 @@ func (h *handlers) settingsPayload(c *gin.Context, st *domain.InstanceSettings)
 	if err != nil {
 		return settingsResponse{}, err
 	}
+	vis, err := h.svc.EffectiveVision(c.Request.Context())
+	if err != nil {
+		return settingsResponse{}, err
+	}
 	return settingsResponse{
 		Settings: st,
 		Effective: effectiveView{
-			Model:     eff.Model,
-			Enabled:   eff.Enabled,
-			HasApiKey: eff.APIKey != "",
-			AgentLive: h.agent.get() != nil,
+			Model:       eff.Model,
+			Enabled:     eff.Enabled,
+			HasApiKey:   eff.APIKey != "",
+			AgentLive:   h.agent.get() != nil,
+			VisionModel: vis.Model,
+			VisionReady: vis.Ready(),
 		},
 	}, nil
 }
@@ -95,6 +105,7 @@ func (h *handlers) getSettings(c *gin.Context) {
 type settingsUpdateRequest struct {
 	AgentModel   string          `json:"agentModel"`
 	AgentEnabled json.RawMessage `json:"agentEnabled"`
+	VisionModel  string          `json:"visionModel"`
 	Version      int64           `json:"version" binding:"required"`
 }
 
@@ -117,6 +128,7 @@ func (h *handlers) updateSettings(c *gin.Context) {
 	st, err := h.svc.UpdateInstanceSettings(c.Request.Context(), mustActor(c).ID, service.InstanceSettingsPatch{
 		AgentModel:   req.AgentModel,
 		AgentEnabled: enabled,
+		VisionModel:  req.VisionModel,
 		Version:      req.Version,
 	})
 	if err != nil {