// Package vision reads a photographed seed packet into structured fields (#81). // // It is one-shot structured extraction, NOT an agent loop: majordomo.Generate[T] // derives a JSON schema from the SeedPacket struct, hands the image to a vision // model, and unmarshals the reply into SeedPacket. Because it can't call a tool, // it can't touch the garden — it only reads a picture and returns data, which the // service then turns into a plant + lot after the user confirms. package vision import ( "context" "fmt" "gitea.stevedudenhoeffer.com/steve/majordomo" "gitea.stevedudenhoeffer.com/steve/majordomo/llm" "gitea.stevedudenhoeffer.com/steve/pansy/internal/agentmodel" ) // SeedPacket is what a vision model reads off a packet. The json/description/enum // tags drive the schema majordomo.Generate derives; pointer fields are nullable, // so a field the packet doesn't print comes back nil rather than a made-up zero. // // These are the packet's PRINTED facts. Mapping them onto a pansy Plant + SeedLot // (and deciding whether the variety is one already in the catalog) is the // service's job, not the model's. type SeedPacket struct { Species string `json:"species" description:"the plant species in plain words, e.g. tomato, garlic, basil"` Variety string `json:"variety" description:"the cultivar or variety name, e.g. Cherokee Purple; empty if the packet only names a species"` Category string `json:"category" enum:"vegetable,herb,flower,fruit,tree_shrub,cover" description:"the single best-fit category"` Vendor string `json:"vendor" description:"the seed company, e.g. Johnny's Selected Seeds"` SKU string `json:"sku" description:"the vendor's item/product number, if printed"` LotCode string `json:"lotCode" description:"the lot or batch code, if printed"` PackedForYear *int `json:"packedForYear" description:"the 'packed for' or 'sell by' year, if printed"` DaysToMaturity *int `json:"daysToMaturity" description:"days to maturity/harvest, if printed"` SpacingCM *float64 `json:"spacingCm" description:"recommended in-row spacing in CENTIMETERS; convert if the packet uses inches"` SeedCount *int `json:"seedCount" description:"approximate seed count in the packet, if printed"` } // extractPrompt tells the model the conventions it can't guess: centimeters, and // that a missing field must be left empty rather than invented. const extractPrompt = `You are reading a photograph of a seed packet. Extract only what is actually printed on it. Rules: - Spacing must be in CENTIMETERS. If the packet gives inches, convert (1 in = 2.54 cm). - If a field is not printed on the packet, leave it empty or null. Do not guess or fill from general knowledge. - "variety" is the cultivar name (e.g. "Cherokee Purple"); "species" is the plain plant name (e.g. "tomato").` // Extract runs one vision extraction: it resolves the model spec against pansy's // registry, sends the JPEG with the prompt, and returns the parsed SeedPacket. // The image bytes should already be normalized to JPEG (see internal/imagenorm). // // It makes a live model call, so callers give it a bounded context. func Extract(ctx context.Context, apiKey, modelSpec string, jpeg []byte) (SeedPacket, error) { if len(jpeg) == 0 { return SeedPacket{}, fmt.Errorf("vision: empty image") } model, err := agentmodel.Resolve(apiKey, modelSpec) if err != nil { return SeedPacket{}, err } return generate(ctx, model, jpeg) } // generate makes the actual one-shot call against an already-resolved model. It // is split from Extract on purpose: the hermetic test drives THIS function with a // fake model, so the prompt, the derived schema and the image part it builds are // the real ones the live path uses — not a hand-copied double that could drift. func generate(ctx context.Context, model llm.Model, jpeg []byte) (SeedPacket, error) { return majordomo.Generate[SeedPacket](ctx, model, majordomo.Request{ Messages: []majordomo.Message{ majordomo.UserParts( majordomo.Text(extractPrompt), majordomo.Image("image/jpeg", jpeg), ), }, }) }