Image normalization: decode HEIC/webp/png/jpeg → JPEG #91
@@ -5,9 +5,11 @@ go 1.26.2
|
||||
require (
|
||||
gitea.stevedudenhoeffer.com/steve/majordomo v0.0.0-20260718232210-a941f5ff4a3f
|
||||
github.com/coreos/go-oidc/v3 v3.20.0
|
||||
github.com/gen2brain/heic v0.7.1
|
||||
github.com/gin-gonic/gin v1.10.1
|
||||
github.com/samber/slog-gin v1.15.0
|
||||
golang.org/x/crypto v0.36.0
|
||||
golang.org/x/image v0.44.0
|
||||
golang.org/x/oauth2 v0.36.0
|
||||
modernc.org/sqlite v1.34.4
|
||||
)
|
||||
@@ -33,6 +35,7 @@ require (
|
||||
github.com/cloudwego/base64x v0.1.4 // indirect
|
||||
github.com/cloudwego/iasm v0.2.0 // indirect
|
||||
github.com/dustin/go-humanize v1.0.1 // indirect
|
||||
github.com/ebitengine/purego v0.10.1 // indirect
|
||||
github.com/gabriel-vasile/mimetype v1.4.4 // indirect
|
||||
github.com/gin-contrib/sse v0.1.0 // indirect
|
||||
github.com/go-jose/go-jose/v4 v4.1.4 // indirect
|
||||
@@ -51,14 +54,15 @@ require (
|
||||
github.com/ncruces/go-strftime v0.1.9 // indirect
|
||||
github.com/pelletier/go-toml/v2 v2.2.2 // indirect
|
||||
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
|
||||
github.com/tetratelabs/wazero v1.12.0 // indirect
|
||||
github.com/twitchyliquid64/golang-asm v0.15.1 // indirect
|
||||
github.com/ugorji/go/codec v1.2.12 // indirect
|
||||
go.opentelemetry.io/otel v1.29.0 // indirect
|
||||
go.opentelemetry.io/otel/trace v1.29.0 // indirect
|
||||
golang.org/x/arch v0.8.0 // indirect
|
||||
golang.org/x/net v0.38.0 // indirect
|
||||
golang.org/x/sys v0.31.0 // indirect
|
||||
golang.org/x/text v0.23.0 // indirect
|
||||
golang.org/x/sys v0.44.0 // indirect
|
||||
golang.org/x/text v0.40.0 // indirect
|
||||
google.golang.org/protobuf v1.34.2 // indirect
|
||||
gopkg.in/yaml.v3 v3.0.1 // indirect
|
||||
modernc.org/gc/v3 v3.0.0-20240107210532-573471604cb6 // indirect
|
||||
|
||||
@@ -26,12 +26,16 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c
|
||||
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
|
||||
github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
|
||||
github.com/ebitengine/purego v0.10.1 h1:dewVBCBT2GaMu1SrNTYxQhgQBethzfhiwvZiLGP/qyY=
|
||||
github.com/ebitengine/purego v0.10.1/go.mod h1:iIjxzd6CiRiOG0UyXP+V1+jWqUXVjPKLAI0mRfJZTmQ=
|
||||
github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
|
||||
github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
|
||||
github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98=
|
||||
github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
|
||||
github.com/gabriel-vasile/mimetype v1.4.4 h1:QjV6pZ7/XZ7ryI2KuyeEDE8wnh7fHP9YnQy+R0LnH8I=
|
||||
github.com/gabriel-vasile/mimetype v1.4.4/go.mod h1:JwLei5XPtWdGiMFB5Pjle1oEeoSeEuJfJE+TtfvdB/s=
|
||||
github.com/gen2brain/heic v0.7.1 h1:Aha1sZdKEeZeWl5o0xkSg7NBRhhkrlokGVCRri+2Qcc=
|
||||
github.com/gen2brain/heic v0.7.1/go.mod h1:ja42wMJc4fpnKsfdUJxeZa2YqqRnes1wS0xqs5+8o5w=
|
||||
github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE=
|
||||
github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI=
|
||||
github.com/gin-gonic/gin v1.10.1 h1:T0ujvqyCSqRopADpgPgiTT63DUQVSfojyME59Ei63pQ=
|
||||
@@ -126,6 +130,8 @@ github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o
|
||||
github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
|
||||
github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
|
||||
github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
|
||||
github.com/tetratelabs/wazero v1.12.0 h1:DuWcpNu/FzgEXgGBDp8J1Spc+CWOvvtvVyjKlaZopYU=
|
||||
github.com/tetratelabs/wazero v1.12.0/go.mod h1:LvKtzl2RqO4gyF27BiXU+nKAjcV8f38U+kP/q2vgxh0=
|
||||
github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS4MhqMhdFk5YI=
|
||||
github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08=
|
||||
github.com/ugorji/go/codec v1.2.12 h1:9LC83zGrHhuUA9l16C9AHXAqEV/2wBQ4nkvumAE65EE=
|
||||
@@ -144,11 +150,13 @@ golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPh
|
||||
golang.org/x/crypto v0.36.0 h1:AnAEvhDddvBdpY+uR+MyHmuZzzNqXSe/GvuDeob5L34=
|
||||
golang.org/x/crypto v0.36.0/go.mod h1:Y4J0ReaxCR1IMaabaSMugxJES1EpwhBHhv2bDHklZvc=
|
||||
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
|
||||
golang.org/x/image v0.44.0 h1:+tDekMZED9+LrtB3G5xzRggpVh9CARjZqROla3R3R+I=
|
||||
golang.org/x/image v0.44.0/go.mod h1:V8K3KE9KKKE+pLpQDOeN18w9oacNSvy1tDOirTu4xtY=
|
||||
golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
|
||||
golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
|
||||
golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
|
||||
golang.org/x/mod v0.17.0 h1:zY54UmvipHiNd+pm+m0x9KhZ9hl1/7QNMyxXbc6ICqA=
|
||||
golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
|
||||
golang.org/x/mod v0.37.0 h1:vF1DjpVEshcIqoEaauuHebaLk1O1forxjxBaVn884JQ=
|
||||
golang.org/x/mod v0.37.0/go.mod h1:m8S8VeM9r4dzDwjrKO0a1sZP3YjeMamRRlD+fmR2Q/0=
|
||||
golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
@@ -163,27 +171,27 @@ golang.org/x/oauth2 v0.36.0/go.mod h1:YDBUJMTkDnJS+A4BP4eZBjCqtokkg1hODuPjwiGPO7
|
||||
golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.12.0 h1:MHc5BpPuC30uJk597Ri8TV3CNZcTLu6B6z4lJy+g6Jw=
|
||||
golang.org/x/sync v0.12.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA=
|
||||
golang.org/x/sync v0.22.0 h1:SZjpbeLmrCk4xhRSZFNZW5gFUeCeFgjekvI/+gfScek=
|
||||
golang.org/x/sync v0.22.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0=
|
||||
golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.31.0 h1:ioabZlmFYtWhL+TRYpcnNlLwhyxaM9kWTDEmfnprqik=
|
||||
golang.org/x/sys v0.31.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k=
|
||||
golang.org/x/sys v0.44.0 h1:ildZl3J4uzeKP07r2F++Op7E9B29JRUy+a27EibtBTQ=
|
||||
golang.org/x/sys v0.44.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
|
||||
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
|
||||
golang.org/x/text v0.23.0 h1:D71I7dUrlY+VX0gQShAThNGHFxZ13dGLBHQLVl1mJlY=
|
||||
golang.org/x/text v0.23.0/go.mod h1:/BLNzu4aZCJ1+kcD0DNRotWKage4q2rGVAg4o22unh4=
|
||||
golang.org/x/text v0.40.0 h1:Ub2Z6/xjgF1WrYQz2nuITOEegKFtiIy+rieRJ5lHZKs=
|
||||
golang.org/x/text v0.40.0/go.mod h1:hpnzDAfGV753zIKo+wk3u1bVKCGPbrnF7+7LBF/UHVY=
|
||||
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||
golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||
golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
|
||||
golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
|
||||
golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
|
||||
golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d h1:vU5i/LfpvrRCpgM/VPfJLg5KjxD3E+hfT1SH+d9zLwg=
|
||||
golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk=
|
||||
golang.org/x/tools v0.47.0 h1:7Kn5x/d1svx/PzryTsqeoZN4TZwqeH5pGWjefhLi/1Q=
|
||||
golang.org/x/tools v0.47.0/go.mod h1:dFHnyTvFWY212G+h7ZY4Vsp/K3U4/7W9TyVaAul8uCA=
|
||||
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
|
||||
google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
|
||||
|
||||
@@ -0,0 +1,207 @@
|
||||
// Package imagenorm normalizes an uploaded image to a JPEG the rest of pansy
|
||||
// (and majordomo's vision path) can rely on, decoding the formats a phone
|
||||
// actually produces.
|
||||
//
|
||||
// # Why this exists
|
||||
//
|
||||
// majordomo's own media pipeline is stdlib-based, so it cannot decode HEIC or
|
||||
// WebP — and HEIC is the iPhone camera default. The seed-packet feature's very
|
||||
// first input is "a photo from my phone", so without this the feature fails on
|
||||
// the exact device that motivates it. Normalizing at the upload boundary means
|
||||
// everything downstream only ever sees JPEG.
|
||||
//
|
||||
// # The CGO constraint
|
||||
//
|
||||
// pansy is CGO_ENABLED=0 (a single static binary — the reason modernc/sqlite was
|
||||
// chosen over the C one), so a libheif *binding* is out. github.com/gen2brain/heic
|
||||
// runs libheif as WebAssembly via wazero: pure Go, no cgo, and it registers with
|
||||
// image.Decode like any other format. golang.org/x/image/webp is pure Go too.
|
||||
//
|
||||
// # Import-driven registry
|
||||
//
|
||||
// Go's image decoders register via blank imports, and the failure mode is
|
||||
// backwards from intuition: forget "image/png" and PNG uploads fail with
|
||||
// "unknown format" while the exotic HEIC still works. So the blank imports below
|
||||
// are load-bearing, and TestNormalizeAllFormats exercises all four formats to
|
||||
|
|
||||
// keep them so.
|
||||
package imagenorm
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"errors"
|
||||
"fmt"
|
||||
"image"
|
||||
"image/jpeg"
|
||||
"io"
|
||||
|
||||
"golang.org/x/image/draw"
|
||||
|
||||
// Decoders, registered with image.Decode by side effect. All four matter:
|
||||
// jpeg/png are the common cases, heic is the iPhone default, webp is common
|
||||
// on the web. Dropping any one silently breaks that format's uploads.
|
||||
_ "image/jpeg"
|
||||
_ "image/png"
|
||||
|
||||
_ "github.com/gen2brain/heic"
|
||||
_ "golang.org/x/image/webp"
|
||||
)
|
||||
|
||||
// Defaults chosen for the seed-packet path against ollama-cloud's limits (8
|
||||
// images, 20 MiB, 2048px, jpeg+png). We re-encode to JPEG well under all of them.
|
||||
const (
|
||||
// DefaultMaxDim is the longest-edge ceiling. 2048 matches ollama-cloud's
|
||||
// MaxDim; anything larger is downscaled. A packet photo has plenty of detail
|
||||
// left at 2048.
|
||||
DefaultMaxDim = 2048
|
||||
// DefaultMaxBytes caps the *input* we will read. A phone photo is 3–8 MB; 25
|
||||
// MiB leaves headroom for a large HEIC without inviting a decompression bomb
|
||||
// as an unbounded read. The re-encoded output is far smaller.
|
||||
DefaultMaxBytes = 25 << 20
|
||||
// maxDecodePixels bounds the DECODED bitmap regardless of input byte size, so
|
||||
// a small file claiming enormous dimensions (a decompression bomb) is refused
|
||||
gitea-actions
commented
🟠 maxDecodePixels (100 MP) permits ~400 MB peak allocation per upload — memory-exhaustion DoS surface, and the huge intermediate bitmap is discarded since MaxDim=2048 downscales everything larger maintainability, security · flagged by 1 model
🪰 Gadfly · advisory 🟠 **maxDecodePixels (100 MP) permits ~400 MB peak allocation per upload — memory-exhaustion DoS surface, and the huge intermediate bitmap is discarded since MaxDim=2048 downscales everything larger**
_maintainability, security · flagged by 1 model_
- **`internal/imagenorm/imagenorm.go:61` — `maxDecodePixels` (100 MP) permits ~400 MB peak allocation per upload.** The guard at line 120 rejects only *more* than 100 MP, so a decoded bitmap up to 100 MP is allowed through to `image.Decode` (line 124). At 4 bytes/px (RGBA) that is ~400 MB allocated before the downscale step even runs. A hostile, low-entropy JPEG/HEIC well under the 25 MiB *input* cap (line 57) can legitimately decode to that size — the byte cap does not bound decoded size for co…
<sub>🪰 Gadfly · advisory</sub>
|
||||
// before its ~4-bytes/px bitmap is allocated. 50 MP ≈ 200 MB peak — above any
|
||||
// current phone camera (a 48 MP sensor is 48 MP) while capping the
|
||||
// amplification a hostile header can force. maxDecodePixels and maxDimension
|
||||
// are internal safety floors, not knobs — unlike MaxDim/MaxBytes there's no
|
||||
// reason for a caller to raise them.
|
||||
maxDecodePixels = 50_000_000
|
||||
// maxDimension caps EACH side independently. It exists to make the pixel-count
|
||||
// check overflow-safe: without it, a header claiming ~2^32 on a side could
|
||||
// wrap int64(w)*int64(h) negative and slip past maxDecodePixels. No real image
|
||||
// is 50k px on a side.
|
||||
maxDimension = 50_000
|
||||
// jpegQuality for the normalized output. 85 is visually clean and keeps the
|
||||
// file small; the model reads text off it, not fine gradients.
|
||||
jpegQuality = 85
|
||||
)
|
||||
|
||||
// Options tunes Normalize. The zero value uses the Default* constants.
|
||||
type Options struct {
|
||||
MaxDim int // longest edge; 0 → DefaultMaxDim
|
||||
MaxBytes int // input read cap; 0 → DefaultMaxBytes
|
||||
}
|
||||
|
||||
func (o Options) maxDim() int {
|
||||
if o.MaxDim > 0 {
|
||||
return o.MaxDim
|
||||
}
|
||||
return DefaultMaxDim
|
||||
}
|
||||
|
||||
|
gitea-actions
commented
🟡 Sentinel errors use fmt.Errorf instead of errors.New, diverging from repo convention maintainability · flagged by 4 models
🪰 Gadfly · advisory 🟡 **Sentinel errors use fmt.Errorf instead of errors.New, diverging from repo convention**
_maintainability · flagged by 4 models_
- **`internal/imagenorm/imagenorm.go:90-91` — `fmt.Errorf` for sentinel errors instead of `errors.New`.** Every other sentinel error declaration in the repo uses `errors.New` (`internal/domain/domain.go:14-64`, `internal/service/password.go:29`, `internal/agentmodel/agentmodel.go:39`, `internal/agent/runtime.go:52`). `fmt.Errorf("imagenorm: image too large")` has no format verbs and pays a fmt-formatting cost for a constant string. `errors.Is` still works, but the package diverges from the estab…
<sub>🪰 Gadfly · advisory</sub>
|
||||
func (o Options) maxBytes() int {
|
||||
if o.MaxBytes > 0 {
|
||||
return o.MaxBytes
|
||||
}
|
||||
return DefaultMaxBytes
|
||||
}
|
||||
|
||||
// ErrTooLarge means the input exceeded the byte cap or decoded to an absurd
|
||||
// pixel count. ErrUnsupported means the bytes weren't a decodable image format —
|
||||
// or a decoder panicked on them (see the recover in Normalize).
|
||||
var (
|
||||
|
gitea-actions
commented
🟠 Doc contract overpromises: read/encode I/O errors don't actually map to the documented ErrUnsupported/ErrTooLarge sentinels error-handling, security · flagged by 3 models
🪰 Gadfly · advisory 🟠 **Doc contract overpromises: read/encode I/O errors don't actually map to the documented ErrUnsupported/ErrTooLarge sentinels**
_error-handling, security · flagged by 3 models_
- **`internal/imagenorm/imagenorm.go:103`** — `Normalize(r io.Reader, opts Options)` takes no `context.Context`, so once wired to an HTTP handler, decoding an untrusted, WASM-interpreted HEIC (or a large PNG/webp under the 100 MP cap) can't be cancelled. Verified in `cmd/pansy/main.go:60-67` that the server sets `WriteTimeout: 30s`, but `http.Server`'s `WriteTimeout` only closes the client connection — it does not cancel the in-flight handler goroutine, which will keep burning CPU/decoding to co…
<sub>🪰 Gadfly · advisory</sub>
|
||||
ErrTooLarge = errors.New("imagenorm: image too large")
|
||||
ErrUnsupported = errors.New("imagenorm: unsupported or corrupt image")
|
||||
)
|
||||
|
||||
// Normalize reads an image of any supported format (JPEG, PNG, HEIC, WebP),
|
||||
// downscales it to fit opts.MaxDim on its longest edge, and returns it re-encoded
|
||||
// as JPEG, plus the decoded format name (e.g. "heic") — handy for logging what a
|
||||
// phone actually sent. On any error the returned bytes are nil and format is "".
|
||||
//
|
||||
// Errors, by cause:
|
||||
// - input over opts.MaxBytes, or a decoded canvas over maxDecodePixels /
|
||||
// maxDimension → ErrTooLarge, refused before the bitmap is allocated;
|
||||
// - bytes that aren't a decodable image, or a decoder that panics on them →
|
||||
// ErrUnsupported;
|
||||
// - a genuine read or JPEG-encode I/O failure → a wrapped error (not a
|
||||
|
gitea-actions
commented
🟠 Missing panic recovery around image decoders exposes goroutine DoS from malicious uploads security · flagged by 2 models
🪰 Gadfly · advisory 🟠 **Missing panic recovery around image decoders exposes goroutine DoS from malicious uploads**
_security · flagged by 2 models_
- **`internal/imagenorm/imagenorm.go:116-127`** — `Normalize` does not recover from panics inside `image.DecodeConfig` or `image.Decode`. Third-party decoders (especially the WASM-backed HEIC path) are complex C-in-WASM parsers and historically have had crash-on-malformed-input bugs. A single hostile upload can crash the processing goroutine. Since the PR explicitly frames this as a hostile-upload boundary, add a `defer recover()` guard around the decode phase and map recovered panics to `ErrUns…
<sub>🪰 Gadfly · advisory</sub>
|
||||
// sentinel), since those are the caller's stream/environment, not the image.
|
||||
//
|
||||
// It bounds work against a hostile upload three ways: the byte cap, the
|
||||
// pre-decode pixel/dimension check, and a recover around the third-party decoders
|
||||
|
gitea-actions
commented
🔴 *Integer overflow in decompression-bomb guard: int64(w)int64(h) wraps negative for crafted ~2^32-dimension headers, bypassing the maxDecodePixels check and allowing an OOM/panic in image.Decode instead of ErrTooLarge error-handling, security · flagged by 3 models
🪰 Gadfly · advisory 🔴 **Integer overflow in decompression-bomb guard: int64(w)*int64(h) wraps negative for crafted ~2^32-dimension headers, bypassing the maxDecodePixels check and allowing an OOM/panic in image.Decode instead of ErrTooLarge**
_error-handling, security · flagged by 3 models_
- **`internal/imagenorm/imagenorm.go:120`** — Integer-overflow bypass of the decompression-bomb guard. `int64(cfg.Width)*int64(cfg.Height)` wraps negative for adversarial headers (e.g. a PNG IHDR claiming ~2³² on each edge: `(2³²−1)²` = `2⁶⁴ − 2³³ + 1`, which overflows `int64` to ≈ −8.59×10⁹), so the `> maxDecodePixels` check is `false` and `image.Decode` proceeds with those claimed dimensions. The guard's whole purpose (documented at lines 58–61, 114–115) is to stop exactly this; it is bypassab…
<sub>🪰 Gadfly · advisory</sub>
|
||||
// (a malformed HEIC/WebP shouldn't take the process down).
|
||||
//
|
||||
// Two known gaps, both deferred to the upload handler that wires this in (#81):
|
||||
// - EXIF ORIENTATION is not applied, so a portrait phone photo tagged
|
||||
|
gitea-actions
commented
🟠 EXIF orientation is discarded on decode/re-encode, causing portrait phone photos to be uploaded sideways correctness, security · flagged by 2 models EXIF orientation is dropped during normalization — 🪰 Gadfly · advisory 🟠 **EXIF orientation is discarded on decode/re-encode, causing portrait phone photos to be uploaded sideways**
_correctness, security · flagged by 2 models_
**EXIF orientation is dropped during normalization** — `internal/imagenorm/imagenorm.go:124-134`
<sub>🪰 Gadfly · advisory</sub>
|
||||
// "rotate 90°" comes out sideways. That's best fixed and tested with a real
|
||||
// oriented photo end-to-end, which the library has no consumer for yet.
|
||||
// - There is no context: image.Decode is CPU-bound and not cancellable
|
||||
// mid-decode, so a caller that needs a hard deadline should run Normalize
|
||||
// under its own timeout. The size guards keep the work finite regardless.
|
||||
func Normalize(r io.Reader, opts Options) (out []byte, format string, err error) {
|
||||
// Cap the read at MaxBytes+1 so we can tell "exactly at the cap" from "over".
|
||||
// maxBytes() is always a sane positive (default 25 MiB); guard the +1 anyway.
|
||||
byteCap := opts.maxBytes()
|
||||
limit := int64(byteCap) + 1
|
||||
if limit < 1 {
|
||||
limit = int64(DefaultMaxBytes) + 1
|
||||
}
|
||||
raw, err := io.ReadAll(io.LimitReader(r, limit))
|
||||
if err != nil {
|
||||
return nil, "", fmt.Errorf("imagenorm: read: %w", err)
|
||||
}
|
||||
if len(raw) > byteCap {
|
||||
return nil, "", ErrTooLarge
|
||||
}
|
||||
|
||||
// Check dimensions BEFORE a full decode, so a decompression bomb is refused
|
||||
// before it allocates its bitmap. The per-side maxDimension check runs first
|
||||
// so the pixel-count multiply below can't overflow.
|
||||
cfg, _, err := image.DecodeConfig(bytes.NewReader(raw))
|
||||
if err != nil {
|
||||
return nil, "", ErrUnsupported
|
||||
}
|
||||
if cfg.Width <= 0 || cfg.Height <= 0 ||
|
||||
cfg.Width > maxDimension || cfg.Height > maxDimension ||
|
||||
|
gitea-actions
commented
🟡 *downscale() always targets image.RGBA, forcing jpeg.Encode's slow generic per-pixel path instead of the YCbCr fast path (package has no caller yet in this diff) performance · flagged by 1 model
🪰 Gadfly · advisory 🟡 **downscale() always targets *image.RGBA, forcing jpeg.Encode's slow generic per-pixel path instead of the YCbCr fast path (package has no caller yet in this diff)**
_performance · flagged by 1 model_
- `internal/imagenorm/imagenorm.go:154` (`downscale`) — When an image needs resizing, `downscale` draws into a freshly allocated `*image.RGBA` (`dst := image.NewRGBA(...)`, `draw.CatmullRom.Scale(dst, ...)`, confirmed at lines 154–155), and that `*image.RGBA` is what `Normalize` hands to `jpeg.Encode` at line 132. Go's stdlib JPEG encoder only has a bulk-plane fast path for `*image.YCbCr` and `*image.Gray`; any other concrete type — including `*image.RGBA` — falls back to a generic per-pixel pat…
<sub>🪰 Gadfly · advisory</sub>
|
||||
int64(cfg.Width)*int64(cfg.Height) > maxDecodePixels {
|
||||
return nil, "", ErrTooLarge
|
||||
}
|
||||
|
||||
img, format, err := decodeSafely(raw)
|
||||
if err != nil {
|
||||
return nil, "", err
|
||||
}
|
||||
|
||||
img = downscale(img, opts.maxDim())
|
||||
|
||||
var buf bytes.Buffer
|
||||
if err := jpeg.Encode(&buf, img, &jpeg.Options{Quality: jpegQuality}); err != nil {
|
||||
return nil, "", fmt.Errorf("imagenorm: encode jpeg: %w", err)
|
||||
}
|
||||
return buf.Bytes(), format, nil
|
||||
}
|
||||
|
||||
// decodeSafely decodes raw, converting both a decode error and a decoder PANIC
|
||||
// into ErrUnsupported. The recover matters because the image comes from an
|
||||
// untrusted upload and the HEIC/WebP decoders are third-party (libheif via WASM,
|
||||
// x/image/webp): a malformed file that panics one of them must fail this one
|
||||
// request, not crash the process.
|
||||
func decodeSafely(raw []byte) (img image.Image, format string, err error) {
|
||||
defer func() {
|
||||
if r := recover(); r != nil {
|
||||
img, format, err = nil, "", ErrUnsupported
|
||||
}
|
||||
}()
|
||||
img, format, err = image.Decode(bytes.NewReader(raw))
|
||||
if err != nil {
|
||||
return nil, "", ErrUnsupported
|
||||
}
|
||||
return img, format, nil
|
||||
}
|
||||
|
||||
// downscale returns img shrunk so its longest edge is at most maxDim, preserving
|
||||
// aspect ratio. An image already within bounds is returned unchanged (no
|
||||
// re-sampling, no quality loss beyond the JPEG round-trip). Uses Catmull-Rom for
|
||||
// a sharp result on text, which is what a packet photo is mostly made of.
|
||||
func downscale(img image.Image, maxDim int) image.Image {
|
||||
b := img.Bounds()
|
||||
w, h := b.Dx(), b.Dy()
|
||||
longest := max(w, h)
|
||||
if longest <= maxDim || longest == 0 {
|
||||
return img
|
||||
}
|
||||
scale := float64(maxDim) / float64(longest)
|
||||
nw, nh := max(int(float64(w)*scale), 1), max(int(float64(h)*scale), 1)
|
||||
dst := image.NewRGBA(image.Rect(0, 0, nw, nh))
|
||||
draw.CatmullRom.Scale(dst, dst.Bounds(), img, b, draw.Over, nil)
|
||||
return dst
|
||||
}
|
||||
@@ -0,0 +1,202 @@
|
||||
package imagenorm
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/binary"
|
||||
"hash/crc32"
|
||||
"image"
|
||||
"image/jpeg"
|
||||
"image/png"
|
||||
"os"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// pngBytes and jpegBytes generate in-memory fixtures for the two formats Go can
|
||||
// encode; heic/webp come from testdata (Go has no encoder for them).
|
||||
func pngBytes(t *testing.T, w, h int) []byte {
|
||||
t.Helper()
|
||||
m := image.NewRGBA(image.Rect(0, 0, w, h))
|
||||
for y := range h {
|
||||
for x := range w {
|
||||
m.Pix[m.PixOffset(x, y)+0] = uint8(x)
|
||||
m.Pix[m.PixOffset(x, y)+3] = 255
|
||||
}
|
||||
}
|
||||
var b bytes.Buffer
|
||||
if err := png.Encode(&b, m); err != nil {
|
||||
t.Fatalf("encode png: %v", err)
|
||||
}
|
||||
return b.Bytes()
|
||||
}
|
||||
|
||||
func jpegBytes(t *testing.T, w, h int) []byte {
|
||||
t.Helper()
|
||||
m := image.NewRGBA(image.Rect(0, 0, w, h))
|
||||
var b bytes.Buffer
|
||||
if err := jpeg.Encode(&b, m, nil); err != nil {
|
||||
t.Fatalf("encode jpeg: %v", err)
|
||||
}
|
||||
return b.Bytes()
|
||||
}
|
||||
|
||||
func readTestdata(t *testing.T, name string) []byte {
|
||||
t.Helper()
|
||||
b, err := os.ReadFile("testdata/" + name)
|
||||
if err != nil {
|
||||
t.Fatalf("read %s: %v", name, err)
|
||||
}
|
||||
return b
|
||||
}
|
||||
|
||||
// TestNormalizeAllFormats is the load-bearing test: every format pansy claims to
|
||||
// accept must round-trip to a valid JPEG. It exists specifically to catch a
|
||||
// dropped blank import — the failure mode where the common format (PNG) breaks
|
||||
// while the exotic one (HEIC) works, because someone deleted `_ "image/png"`.
|
||||
func TestNormalizeAllFormats(t *testing.T) {
|
||||
cases := []struct {
|
||||
name string
|
||||
input []byte
|
||||
wantFormat string
|
||||
}{
|
||||
{"png", pngBytes(t, 120, 90), "png"},
|
||||
{"jpeg", jpegBytes(t, 120, 90), "jpeg"},
|
||||
{"heic", readTestdata(t, "sample.heic"), "heic"},
|
||||
{"webp", readTestdata(t, "sample.webp"), "webp"},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
out, format, err := Normalize(bytes.NewReader(tc.input), Options{})
|
||||
if err != nil {
|
||||
t.Fatalf("Normalize(%s): %v", tc.name, err)
|
||||
}
|
||||
if format != tc.wantFormat {
|
||||
t.Errorf("format = %q, want %q", format, tc.wantFormat)
|
||||
}
|
||||
// The output must itself be a decodable JPEG.
|
||||
_, outFormat, err := image.Decode(bytes.NewReader(out))
|
||||
if err != nil {
|
||||
t.Fatalf("output isn't a valid image: %v", err)
|
||||
}
|
||||
if outFormat != "jpeg" {
|
||||
t.Errorf("output format = %q, want jpeg", outFormat)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestNormalizeDownscales checks a large image is shrunk to fit MaxDim on its
|
||||
// longest edge with aspect ratio preserved, and a small one is left alone.
|
||||
func TestNormalizeDownscales(t *testing.T) {
|
||||
|
gitea-actions
commented
🟡 Test silently depends on DefaultMaxDim constant maintainability · flagged by 1 model
🪰 Gadfly · advisory 🟡 **Test silently depends on DefaultMaxDim constant**
_maintainability · flagged by 1 model_
- `internal/imagenorm/imagenorm_test.go:89` — `TestNormalizeDownscales` silently couples itself to `DefaultMaxDim`. It passes `Options{}` and then hardcodes expected dimensions (`2048`, `512`). If `DefaultMaxDim` is ever changed, this test will fail with opaque assertions that don't reveal the root cause. **Fix:** Make the dependency explicit by passing `Options{MaxDim: 2048}` (or compute expectations from `DefaultMaxDim` directly).
<sub>🪰 Gadfly · advisory</sub>
|
||||
// A 2:1 image twice as wide as DefaultMaxDim → clamped to DefaultMaxDim on the
|
||||
// long edge with aspect preserved. Derived from the constant, not hard-coded,
|
||||
// so the test tracks the default rather than silently asserting a magic number.
|
||||
longEdge := DefaultMaxDim * 2
|
||||
big := pngBytes(t, longEdge, longEdge/2)
|
||||
out, _, err := Normalize(bytes.NewReader(big), Options{})
|
||||
if err != nil {
|
||||
t.Fatalf("Normalize: %v", err)
|
||||
}
|
||||
cfg, _, err := image.DecodeConfig(bytes.NewReader(out))
|
||||
if err != nil {
|
||||
t.Fatalf("decode out: %v", err)
|
||||
}
|
||||
if cfg.Width != DefaultMaxDim {
|
||||
t.Errorf("width = %d, want %d (longest edge clamped)", cfg.Width, DefaultMaxDim)
|
||||
}
|
||||
if cfg.Height != DefaultMaxDim/2 {
|
||||
t.Errorf("height = %d, want %d (aspect preserved)", cfg.Height, DefaultMaxDim/2)
|
||||
}
|
||||
|
||||
// A small image within bounds keeps its dimensions.
|
||||
small := pngBytes(t, 100, 80)
|
||||
|
gitea-actions
commented
🟡 image.DecodeConfig error ignored in small-image downscale test error-handling · flagged by 1 model
🪰 Gadfly · advisory 🟡 **image.DecodeConfig error ignored in small-image downscale test**
_error-handling · flagged by 1 model_
* **`internal/imagenorm/imagenorm_test.go:111`** — `image.DecodeConfig` error discarded. In `TestNormalizeDownscales` the small-image branch does `cfg, _, _ = image.DecodeConfig(bytes.NewReader(out))`. If the normalized JPEG is somehow corrupt, the test reports a misleading dimension mismatch instead of the real decode failure. Check the error with `t.Fatalf` like the big-image branch does.
<sub>🪰 Gadfly · advisory</sub>
|
||||
out, _, err = Normalize(bytes.NewReader(small), Options{})
|
||||
if err != nil {
|
||||
t.Fatalf("Normalize small: %v", err)
|
||||
}
|
||||
cfg, _, err = image.DecodeConfig(bytes.NewReader(out))
|
||||
if err != nil {
|
||||
t.Fatalf("decode small out: %v", err)
|
||||
}
|
||||
if cfg.Width != 100 || cfg.Height != 80 {
|
||||
t.Errorf("small image resized to %dx%d, want 100x80", cfg.Width, cfg.Height)
|
||||
}
|
||||
}
|
||||
|
||||
// TestNormalizeRejectsOversizeInput: an input past the byte cap is ErrTooLarge,
|
||||
// refused without a full decode.
|
||||
func TestNormalizeRejectsOversizeInput(t *testing.T) {
|
||||
|
gitea-actions
commented
🟠 TestNormalizeRejectsGarbage has a stale comment header describing a non-existent TestNormalizeRejectsBombDimensions; the decompression-bomb (maxDecodePixels) path is untested correctness, error-handling, maintainability · flagged by 5 models
🪰 Gadfly · advisory 🟠 **TestNormalizeRejectsGarbage has a stale comment header describing a non-existent TestNormalizeRejectsBombDimensions; the decompression-bomb (maxDecodePixels) path is untested**
_correctness, error-handling, maintainability · flagged by 5 models_
- `internal/imagenorm/imagenorm_test.go:127-130` — The doc comment above `TestNormalizeRejectsGarbage` is stale: it describes a `TestNormalizeRejectsBombDimensions` scenario ("a small file claiming a huge canvas is refused before the bitmap is allocated... the guard is on the DECODED pixel count") but the function's body only exercises garbage bytes and a truncated PNG — it never constructs an oversized-`DecodeConfig` fixture. No test anywhere in the file (or package) actually drives the `maxDec…
<sub>🪰 Gadfly · advisory</sub>
|
||||
big := pngBytes(t, 500, 500)
|
||||
_, _, err := Normalize(bytes.NewReader(big), Options{MaxBytes: 100})
|
||||
if err != ErrTooLarge {
|
||||
t.Errorf("over-cap input err = %v, want ErrTooLarge", err)
|
||||
}
|
||||
}
|
||||
|
||||
// TestNormalizeRejectsGarbage: unreadable-as-image bytes and a truncated image
|
||||
// both fail cleanly with ErrUnsupported, not a panic.
|
||||
func TestNormalizeRejectsGarbage(t *testing.T) {
|
||||
_, _, err := Normalize(bytes.NewReader([]byte("not an image at all")), Options{})
|
||||
if err != ErrUnsupported {
|
||||
t.Errorf("garbage err = %v, want ErrUnsupported", err)
|
||||
}
|
||||
// A truncated image (valid header, cut body) also fails cleanly, not a panic.
|
||||
png := pngBytes(t, 100, 100)
|
||||
_, _, err = Normalize(bytes.NewReader(png[:len(png)/2]), Options{})
|
||||
if err != ErrUnsupported {
|
||||
t.Errorf("truncated image err = %v, want ErrUnsupported", err)
|
||||
}
|
||||
}
|
||||
|
||||
// pngHeader builds a valid PNG signature + IHDR chunk (with a correct CRC, which
|
||||
// DecodeConfig verifies) for the given dimensions, and nothing else. It's enough
|
||||
// for image.DecodeConfig to report width/height without a real bitmap — exactly
|
||||
// what's needed to exercise the pre-decode size guard with a tiny input.
|
||||
func pngHeader(w, h uint32) []byte {
|
||||
var buf bytes.Buffer
|
||||
buf.Write([]byte{0x89, 'P', 'N', 'G', 0x0d, 0x0a, 0x1a, 0x0a})
|
||||
ihdr := make([]byte, 13)
|
||||
binary.BigEndian.PutUint32(ihdr[0:], w)
|
||||
binary.BigEndian.PutUint32(ihdr[4:], h)
|
||||
ihdr[8] = 8 // bit depth
|
||||
ihdr[9] = 6 // colour type: RGBA
|
||||
// compression/filter/interlace already 0.
|
||||
binary.Write(&buf, binary.BigEndian, uint32(len(ihdr)))
|
||||
chunk := append([]byte("IHDR"), ihdr...)
|
||||
buf.Write(chunk)
|
||||
binary.Write(&buf, binary.BigEndian, crc32.ChecksumIEEE(chunk))
|
||||
return buf.Bytes()
|
||||
}
|
||||
|
||||
// TestNormalizeRejectsPixelBomb is the guard the review found untested: a small
|
||||
// input (a bare ~40-byte PNG header) claiming an enormous canvas is refused with
|
||||
// ErrTooLarge from DecodeConfig alone, before image.Decode allocates anything.
|
||||
// Covers both the per-side maxDimension trip and the pixel-count trip — and, via
|
||||
// the near-2^16-per-side case, that the count math doesn't overflow.
|
||||
func TestNormalizeRejectsPixelBomb(t *testing.T) {
|
||||
cases := []struct {
|
||||
name string
|
||||
w, h uint32
|
||||
}{
|
||||
{"huge single side", 60000, 10}, // > maxDimension on width
|
||||
{"huge area within side cap", 40000, 40000}, // sides < cap, area 1.6 GP > maxDecodePixels
|
||||
}
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
hdr := pngHeader(tc.w, tc.h)
|
||||
if len(hdr) > 100 {
|
||||
t.Fatalf("header unexpectedly large (%d bytes) — not a bomb test", len(hdr))
|
||||
}
|
||||
// Sanity: the header really does decode to those dimensions.
|
||||
cfg, _, err := image.DecodeConfig(bytes.NewReader(hdr))
|
||||
if err != nil {
|
||||
t.Fatalf("crafted PNG header didn't parse: %v", err)
|
||||
}
|
||||
if uint32(cfg.Width) != tc.w || uint32(cfg.Height) != tc.h {
|
||||
t.Fatalf("header reports %dx%d, want %dx%d", cfg.Width, cfg.Height, tc.w, tc.h)
|
||||
}
|
||||
if _, _, err := Normalize(bytes.NewReader(hdr), Options{}); err != ErrTooLarge {
|
||||
t.Errorf("pixel bomb %dx%d err = %v, want ErrTooLarge", tc.w, tc.h, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
+14
@@ -0,0 +1,14 @@
|
||||
# imagenorm test fixtures
|
||||
|
||||
Go has no encoder for HEIC or WebP, so these small samples are committed rather
|
||||
than generated at test time. They are used by `TestNormalizeAllFormats` to prove
|
||||
every accepted format round-trips to JPEG (and, mainly, to catch a dropped blank
|
||||
import — see the package doc).
|
||||
|
||||
- `sample.heic` — a 240×160 gradient, created from a Go-generated PNG with macOS
|
||||
`sips -s format heic`. HEVC still-image profile.
|
||||
- `sample.webp` — a 16×16 image copied from CPython's stdlib test corpus
|
||||
(`Lib/test/test_email/data/python.webp`), verified to decode with
|
||||
`golang.org/x/image/webp` before committing. Used only as a decode fixture.
|
||||
|
||||
Neither contains anything meaningful; they exist purely to be decoded.
|
||||
BIN
Binary file not shown.
BIN
Binary file not shown.
|
After Width: | Height: | Size: 432 B |
Reference in New Issue
Block a user
⚪ Package doc references non-existent TestNormalize; actual test is TestNormalizeAllFormats
maintainability · flagged by 1 model
internal/imagenorm/imagenorm.go:25— stale doc reference. The package doc saysTestNormalize exercises all four formats— there is noTestNormalize; the actual test isTestNormalizeAllFormats. Minor, but the CLAUDE.md rule "stale comments are worse than none" applies. Update the reference.🪰 Gadfly · advisory