pansy/internal/vision/vision.go

// Package vision reads a photographed seed packet into structured fields (#81).
//
// It is one-shot structured extraction, NOT an agent loop: majordomo.Generate[T]
// derives a JSON schema from the SeedPacket struct, hands the image to a vision
// model, and unmarshals the reply into SeedPacket. Because it can't call a tool,
// it can't touch the garden — it only reads a picture and returns data, which the
// service then turns into a plant + lot after the user confirms.
package vision

import (
	"context"
	"fmt"

	"gitea.stevedudenhoeffer.com/steve/majordomo"

	"gitea.stevedudenhoeffer.com/steve/pansy/internal/agentmodel"
)

// SeedPacket is what a vision model reads off a packet. The json/description/enum
// tags drive the schema majordomo.Generate derives; pointer fields are nullable,
// so a field the packet doesn't print comes back nil rather than a made-up zero.
//
// These are the packet's PRINTED facts. Mapping them onto a pansy Plant + SeedLot
// (and deciding whether the variety is one already in the catalog) is the
// service's job, not the model's.
type SeedPacket struct {
	Species        string   `json:"species" description:"the plant species in plain words, e.g. tomato, garlic, basil"`
	Variety        string   `json:"variety" description:"the cultivar or variety name, e.g. Cherokee Purple; empty if the packet only names a species"`
	Category       string   `json:"category" enum:"vegetable,herb,flower,fruit,tree_shrub,cover" description:"the single best-fit category"`
	Vendor         string   `json:"vendor" description:"the seed company, e.g. Johnny's Selected Seeds"`
	SKU            string   `json:"sku" description:"the vendor's item/product number, if printed"`
	LotCode        string   `json:"lotCode" description:"the lot or batch code, if printed"`
	PackedForYear  *int     `json:"packedForYear" description:"the 'packed for' or 'sell by' year, if printed"`
	DaysToMaturity *int     `json:"daysToMaturity" description:"days to maturity/harvest, if printed"`
	SpacingCM      *float64 `json:"spacingCm" description:"recommended in-row spacing in CENTIMETERS; convert if the packet uses inches"`
	SeedCount      *int     `json:"seedCount" description:"approximate seed count in the packet, if printed"`
}

// extractPrompt tells the model the conventions it can't guess: centimeters, and
// that a missing field must be left empty rather than invented.
const extractPrompt = `You are reading a photograph of a seed packet. Extract only what is actually printed on it.
Rules:
- Spacing must be in CENTIMETERS. If the packet gives inches, convert (1 in = 2.54 cm).
- If a field is not printed on the packet, leave it empty or null. Do not guess or fill from general knowledge.
- "variety" is the cultivar name (e.g. "Cherokee Purple"); "species" is the plain plant name (e.g. "tomato").`

// Extract runs one vision extraction: it resolves the model spec against pansy's
// registry, sends the JPEG with the prompt, and returns the parsed SeedPacket.
// The image bytes should already be normalized to JPEG (see internal/imagenorm).
//
// It makes a live model call, so callers give it a bounded context.
func Extract(ctx context.Context, apiKey, modelSpec string, jpeg []byte) (SeedPacket, error) {
	if len(jpeg) == 0 {
		return SeedPacket{}, fmt.Errorf("vision: empty image")
	}
	model, err := agentmodel.Resolve(apiKey, modelSpec)
	if err != nil {
		return SeedPacket{}, err
	}
	return majordomo.Generate[SeedPacket](ctx, model, majordomo.Request{
		Messages: []majordomo.Message{
			majordomo.UserParts(
				majordomo.Text(extractPrompt),
				majordomo.Image("image/jpeg", jpeg),
			),
		},
	})
}