Compare commits
18 Commits
ff1d6c491a
...
main
| Author | SHA1 | Date | |
|---|---|---|---|
| 3b6d864330 | |||
| adefaaef36 | |||
| d89031b20d | |||
| 84e811572b | |||
| 61b68adfd0 | |||
| 0447f1bdbe | |||
| ace6c1e0bf | |||
| 1b95d12890 | |||
| 035151d9fa | |||
| 00ff7ea830 | |||
| d35d144fa2 | |||
| e0da88b9b0 | |||
| 39371dc261 | |||
| debf0ee2ed | |||
| 01aea52533 | |||
| 4772b153b8 | |||
| 8eb69c1dee | |||
| 6647e4f63d |
@@ -52,6 +52,16 @@ func initBrowser(opt BrowserOptions) (*browserInitResult, error) {
|
||||
return nil, ErrInvalidBrowserSelection
|
||||
}
|
||||
|
||||
// Auto-select a User-Agent matching the browser engine when the caller hasn't set one.
|
||||
if opt.UserAgent == "" {
|
||||
switch opt.Browser {
|
||||
case BrowserChromium:
|
||||
opt.UserAgent = DefaultChromiumUserAgent
|
||||
default:
|
||||
opt.UserAgent = DefaultFirefoxUserAgent
|
||||
}
|
||||
}
|
||||
|
||||
// Collect launch args and init scripts, starting with any stealth-mode presets.
|
||||
stealth := opt.Stealth == nil || *opt.Stealth
|
||||
var launchArgs []string
|
||||
@@ -64,9 +74,9 @@ func initBrowser(opt BrowserOptions) (*browserInitResult, error) {
|
||||
initScripts = append(initScripts, stealthCommonScripts...)
|
||||
switch opt.Browser {
|
||||
case BrowserChromium:
|
||||
initScripts = append(initScripts, stealthChromiumScripts...)
|
||||
initScripts = append(initScripts, buildChromiumStealthScripts(randomChromiumProfile())...)
|
||||
case BrowserFirefox:
|
||||
initScripts = append(initScripts, stealthFirefoxScripts...)
|
||||
initScripts = append(initScripts, buildFirefoxStealthScripts(randomFirefoxProfile())...)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -142,12 +152,11 @@ func initBrowser(opt BrowserOptions) (*browserInitResult, error) {
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("error getting cookies from cookie jar: %w", err)
|
||||
}
|
||||
pwCookies := make([]playwright.OptionalCookie, len(cookies))
|
||||
for i, c := range cookies {
|
||||
pwCookies[i] = cookieToPlaywrightOptionalCookie(c)
|
||||
}
|
||||
if err := bctx.AddCookies(pwCookies); err != nil {
|
||||
return nil, fmt.Errorf("error adding cookies to browser: %w", err)
|
||||
for _, c := range cookies {
|
||||
oc := cookieToPlaywrightOptionalCookie(c)
|
||||
if err := bctx.AddCookies([]playwright.OptionalCookie{oc}); err != nil {
|
||||
slog.Warn("skipping invalid cookie", "name", c.Name, "host", c.Host, "error", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
10
document.go
10
document.go
@@ -22,9 +22,10 @@ type Document interface {
|
||||
|
||||
type document struct {
|
||||
node
|
||||
pw *playwright.Playwright
|
||||
browser playwright.Browser
|
||||
page playwright.Page
|
||||
pw *playwright.Playwright
|
||||
browser playwright.Browser
|
||||
page playwright.Page
|
||||
detached bool
|
||||
}
|
||||
|
||||
func newDocument(pw *playwright.Playwright, browser playwright.Browser, page playwright.Page) (Document, error) {
|
||||
@@ -44,6 +45,9 @@ func newDocument(pw *playwright.Playwright, browser playwright.Browser, page pla
|
||||
return res, nil
|
||||
}
|
||||
func (d *document) Close() error {
|
||||
if d.detached {
|
||||
return nil
|
||||
}
|
||||
return d.page.Close()
|
||||
}
|
||||
|
||||
|
||||
130
interactive.go
130
interactive.go
@@ -23,8 +23,13 @@ type InteractiveBrowser interface {
|
||||
|
||||
// MouseClick clicks at the given coordinates with the specified button ("left", "middle", "right").
|
||||
MouseClick(x, y float64, button string) error
|
||||
// MouseMove moves the mouse to the given coordinates.
|
||||
MouseMove(x, y float64) error
|
||||
// MouseDown presses the mouse button at the given coordinates without releasing.
|
||||
MouseDown(x, y float64, button string) error
|
||||
// MouseUp releases the mouse button at the given coordinates.
|
||||
MouseUp(x, y float64, button string) error
|
||||
// MouseMove moves the mouse to the given coordinates. An optional steps parameter
|
||||
// controls how many intermediate mousemove events are generated (default 1).
|
||||
MouseMove(x, y float64, steps ...int) error
|
||||
// MouseWheel scrolls by the given delta.
|
||||
MouseWheel(deltaX, deltaY float64) error
|
||||
|
||||
@@ -43,15 +48,27 @@ type InteractiveBrowser interface {
|
||||
// Cookies returns all cookies from the browser context.
|
||||
Cookies() ([]Cookie, error)
|
||||
|
||||
// SetDefaultTimeout sets the default timeout for all Playwright operations
|
||||
// (navigation, clicks, screenshots, cookie extraction, etc.). A value of 0
|
||||
// disables timeouts. By default, Playwright uses a 30-second timeout.
|
||||
//
|
||||
// This is the primary mechanism for preventing hung sessions: callers can
|
||||
// set a timeout so that any Playwright call returns an error instead of
|
||||
// blocking forever if the browser process crashes or the remote server
|
||||
// becomes unresponsive.
|
||||
SetDefaultTimeout(timeout time.Duration)
|
||||
|
||||
// Close tears down the browser.
|
||||
Close() error
|
||||
}
|
||||
|
||||
type interactiveBrowser struct {
|
||||
pw *playwright.Playwright
|
||||
browser playwright.Browser
|
||||
ctx playwright.BrowserContext
|
||||
page playwright.Page
|
||||
pw *playwright.Playwright
|
||||
browser playwright.Browser
|
||||
ctx playwright.BrowserContext
|
||||
page playwright.Page
|
||||
ownsInfrastructure bool
|
||||
detached bool
|
||||
}
|
||||
|
||||
// NewInteractiveBrowser creates a headless browser with a page ready for interactive control.
|
||||
@@ -59,10 +76,9 @@ type interactiveBrowser struct {
|
||||
func NewInteractiveBrowser(ctx context.Context, opts ...BrowserOptions) (InteractiveBrowser, error) {
|
||||
var thirtySeconds = 30 * time.Second
|
||||
opt := mergeOptions(BrowserOptions{
|
||||
UserAgent: DefaultUserAgent,
|
||||
Browser: BrowserFirefox,
|
||||
Timeout: &thirtySeconds,
|
||||
Stealth: Bool(true),
|
||||
Browser: BrowserFirefox,
|
||||
Timeout: &thirtySeconds,
|
||||
Stealth: Bool(true),
|
||||
Dimensions: Size{
|
||||
Width: 1280,
|
||||
Height: 720,
|
||||
@@ -89,22 +105,32 @@ func NewInteractiveBrowser(ctx context.Context, opts ...BrowserOptions) (Interac
|
||||
|
||||
page, err := res.bctx.NewPage()
|
||||
if err != nil {
|
||||
_ = res.bctx.Close()
|
||||
_ = res.browser.Close()
|
||||
_ = res.pw.Stop()
|
||||
ch <- result{nil, fmt.Errorf("failed to create page: %w", err)}
|
||||
return
|
||||
}
|
||||
|
||||
ch <- result{
|
||||
ib: &interactiveBrowser{
|
||||
pw: res.pw,
|
||||
browser: res.browser,
|
||||
ctx: res.bctx,
|
||||
page: page,
|
||||
pw: res.pw,
|
||||
browser: res.browser,
|
||||
ctx: res.bctx,
|
||||
page: page,
|
||||
ownsInfrastructure: true,
|
||||
},
|
||||
}
|
||||
}()
|
||||
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
go func() {
|
||||
r := <-ch
|
||||
if r.err == nil && r.ib != nil {
|
||||
_ = r.ib.Close()
|
||||
}
|
||||
}()
|
||||
return nil, ctx.Err()
|
||||
case r := <-ch:
|
||||
return r.ib, r.err
|
||||
@@ -154,8 +180,44 @@ func (ib *interactiveBrowser) MouseClick(x, y float64, button string) error {
|
||||
return ib.page.Mouse().Click(x, y, playwright.MouseClickOptions{Button: btn})
|
||||
}
|
||||
|
||||
func (ib *interactiveBrowser) MouseMove(x, y float64) error {
|
||||
return ib.page.Mouse().Move(x, y)
|
||||
func (ib *interactiveBrowser) MouseDown(x, y float64, button string) error {
|
||||
if err := ib.page.Mouse().Move(x, y); err != nil {
|
||||
return err
|
||||
}
|
||||
var btn *playwright.MouseButton
|
||||
switch button {
|
||||
case "right":
|
||||
btn = playwright.MouseButtonRight
|
||||
case "middle":
|
||||
btn = playwright.MouseButtonMiddle
|
||||
default:
|
||||
btn = playwright.MouseButtonLeft
|
||||
}
|
||||
return ib.page.Mouse().Down(playwright.MouseDownOptions{Button: btn})
|
||||
}
|
||||
|
||||
func (ib *interactiveBrowser) MouseUp(x, y float64, button string) error {
|
||||
if err := ib.page.Mouse().Move(x, y); err != nil {
|
||||
return err
|
||||
}
|
||||
var btn *playwright.MouseButton
|
||||
switch button {
|
||||
case "right":
|
||||
btn = playwright.MouseButtonRight
|
||||
case "middle":
|
||||
btn = playwright.MouseButtonMiddle
|
||||
default:
|
||||
btn = playwright.MouseButtonLeft
|
||||
}
|
||||
return ib.page.Mouse().Up(playwright.MouseUpOptions{Button: btn})
|
||||
}
|
||||
|
||||
func (ib *interactiveBrowser) MouseMove(x, y float64, steps ...int) error {
|
||||
var opts playwright.MouseMoveOptions
|
||||
if len(steps) > 0 && steps[0] > 1 {
|
||||
opts.Steps = playwright.Int(steps[0])
|
||||
}
|
||||
return ib.page.Mouse().Move(x, y, opts)
|
||||
}
|
||||
|
||||
func (ib *interactiveBrowser) MouseWheel(deltaX, deltaY float64) error {
|
||||
@@ -194,26 +256,40 @@ func (ib *interactiveBrowser) Cookies() ([]Cookie, error) {
|
||||
return cookies, nil
|
||||
}
|
||||
|
||||
func (ib *interactiveBrowser) SetDefaultTimeout(timeout time.Duration) {
|
||||
ms := float64(timeout.Milliseconds())
|
||||
ib.page.SetDefaultTimeout(ms)
|
||||
ib.page.SetDefaultNavigationTimeout(ms)
|
||||
ib.ctx.SetDefaultTimeout(ms)
|
||||
ib.ctx.SetDefaultNavigationTimeout(ms)
|
||||
}
|
||||
|
||||
func (ib *interactiveBrowser) Close() error {
|
||||
if ib.detached {
|
||||
return nil
|
||||
}
|
||||
|
||||
var errs []error
|
||||
if ib.page != nil {
|
||||
if err := ib.page.Close(); err != nil {
|
||||
errs = append(errs, err)
|
||||
}
|
||||
}
|
||||
if ib.ctx != nil {
|
||||
if err := ib.ctx.Close(); err != nil {
|
||||
errs = append(errs, err)
|
||||
if ib.ownsInfrastructure {
|
||||
if ib.ctx != nil {
|
||||
if err := ib.ctx.Close(); err != nil {
|
||||
errs = append(errs, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
if ib.browser != nil {
|
||||
if err := ib.browser.Close(); err != nil {
|
||||
errs = append(errs, err)
|
||||
if ib.browser != nil {
|
||||
if err := ib.browser.Close(); err != nil {
|
||||
errs = append(errs, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
if ib.pw != nil {
|
||||
if err := ib.pw.Stop(); err != nil {
|
||||
errs = append(errs, err)
|
||||
if ib.pw != nil {
|
||||
if err := ib.pw.Stop(); err != nil {
|
||||
errs = append(errs, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
if len(errs) > 0 {
|
||||
|
||||
2
node.go
2
node.go
@@ -29,7 +29,7 @@ type node struct {
|
||||
}
|
||||
|
||||
func (n node) Type(input string) error {
|
||||
return n.locator.Type(input)
|
||||
return n.locator.PressSequentially(input)
|
||||
}
|
||||
|
||||
func (n node) Click() error {
|
||||
|
||||
@@ -36,8 +36,14 @@ const (
|
||||
BrowserWebKit BrowserSelection = "webkit"
|
||||
)
|
||||
|
||||
// DefaultUserAgent is the user-agent string used by all browser instances.
|
||||
const DefaultUserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:147.0) Gecko/20100101 Firefox/147.0"
|
||||
// DefaultFirefoxUserAgent is the user-agent string used for Firefox browser instances.
|
||||
const DefaultFirefoxUserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:147.0) Gecko/20100101 Firefox/147.0"
|
||||
|
||||
// DefaultChromiumUserAgent is the user-agent string used for Chromium browser instances.
|
||||
const DefaultChromiumUserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36"
|
||||
|
||||
// DefaultUserAgent is an alias for DefaultFirefoxUserAgent, retained for backward compatibility.
|
||||
const DefaultUserAgent = DefaultFirefoxUserAgent
|
||||
|
||||
// Bool returns a pointer to the given bool value.
|
||||
func Bool(v bool) *bool { return &v }
|
||||
@@ -47,7 +53,7 @@ type Size struct {
|
||||
Height int
|
||||
}
|
||||
type BrowserOptions struct {
|
||||
UserAgent string // If empty, defaults to DefaultUserAgent
|
||||
UserAgent string // If empty, auto-selected based on Browser engine
|
||||
Browser BrowserSelection // If unset defaults to Firefox.
|
||||
Timeout *time.Duration // If unset defaults to 30 seconds timeout. If set to 0, no timeout
|
||||
|
||||
@@ -116,12 +122,18 @@ func playwrightSameSiteToSameSite(s *playwright.SameSiteAttribute) SameSite {
|
||||
}
|
||||
|
||||
func cookieToPlaywrightOptionalCookie(cookie Cookie) playwright.OptionalCookie {
|
||||
expires := float64(cookie.Expires.Unix())
|
||||
if cookie.Expires.IsZero() || expires <= 0 {
|
||||
expires = -1
|
||||
}
|
||||
|
||||
oc := playwright.OptionalCookie{
|
||||
Name: cookie.Name,
|
||||
Value: cookie.Value,
|
||||
Domain: playwright.String(cookie.Host),
|
||||
Path: playwright.String(cookie.Path),
|
||||
Expires: playwright.Float(float64(cookie.Expires.Unix())),
|
||||
Expires: playwright.Float(expires),
|
||||
Secure: playwright.Bool(cookie.Secure),
|
||||
HttpOnly: playwright.Bool(cookie.HttpOnly),
|
||||
}
|
||||
if cookie.SameSite != "" {
|
||||
@@ -137,6 +149,7 @@ func playwrightCookieToCookie(cookie playwright.Cookie) Cookie {
|
||||
Host: cookie.Domain,
|
||||
Path: cookie.Path,
|
||||
Expires: time.Unix(int64(cookie.Expires), 0),
|
||||
Secure: cookie.Secure,
|
||||
HttpOnly: cookie.HttpOnly,
|
||||
SameSite: playwrightSameSiteToSameSite(cookie.SameSite),
|
||||
}
|
||||
@@ -145,10 +158,13 @@ func playwrightCookieToCookie(cookie playwright.Cookie) Cookie {
|
||||
func NewBrowser(ctx context.Context, opts ...BrowserOptions) (Browser, error) {
|
||||
var thirtySeconds = 30 * time.Second
|
||||
opt := mergeOptions(BrowserOptions{
|
||||
UserAgent: DefaultUserAgent,
|
||||
Browser: BrowserFirefox,
|
||||
Timeout: &thirtySeconds,
|
||||
Stealth: Bool(true),
|
||||
Browser: BrowserFirefox,
|
||||
Timeout: &thirtySeconds,
|
||||
Stealth: Bool(true),
|
||||
Dimensions: Size{
|
||||
Width: 1920,
|
||||
Height: 1080,
|
||||
},
|
||||
}, opts)
|
||||
|
||||
if err := ctx.Err(); err != nil {
|
||||
@@ -184,6 +200,12 @@ func NewBrowser(ctx context.Context, opts ...BrowserOptions) (Browser, error) {
|
||||
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
go func() {
|
||||
r := <-resultCh
|
||||
if r.err == nil && r.browser != nil {
|
||||
_ = r.browser.Close()
|
||||
}
|
||||
}()
|
||||
return nil, ctx.Err()
|
||||
case result := <-resultCh:
|
||||
return result.browser, result.err
|
||||
@@ -197,11 +219,35 @@ func (b playWrightBrowser) updateCookies(_ context.Context, page playwright.Page
|
||||
return fmt.Errorf("error getting cookies from browser: %w", err)
|
||||
}
|
||||
|
||||
// Build a lookup of existing cookies so we can preserve their security
|
||||
// attributes. Chromium's Cookies() API can lose or normalize Secure,
|
||||
// SameSite, and HttpOnly during the AddCookies → navigate → Cookies()
|
||||
// round-trip, so we only update Value and Expires for cookies that
|
||||
// already exist in the jar.
|
||||
existing, err := b.cookieJar.Get(page.URL())
|
||||
if err != nil {
|
||||
return fmt.Errorf("error getting existing cookies from jar: %w", err)
|
||||
}
|
||||
type cookieKey struct{ Name, Path string }
|
||||
existingMap := make(map[cookieKey]Cookie, len(existing))
|
||||
for _, c := range existing {
|
||||
existingMap[cookieKey{c.Name, c.Path}] = c
|
||||
}
|
||||
|
||||
for _, cookie := range cookies {
|
||||
// TODO: add support for deleting cookies from the jar which are deleted in the browser
|
||||
err = b.cookieJar.Set(playwrightCookieToCookie(cookie))
|
||||
c := playwrightCookieToCookie(cookie)
|
||||
|
||||
if err != nil {
|
||||
if prev, ok := existingMap[cookieKey{c.Name, c.Path}]; ok {
|
||||
// Preserve the original security attributes; only update
|
||||
// Value and Expires which are the fields that legitimately
|
||||
// change during navigation.
|
||||
c.Secure = prev.Secure
|
||||
c.HttpOnly = prev.HttpOnly
|
||||
c.SameSite = prev.SameSite
|
||||
}
|
||||
|
||||
if err = b.cookieJar.Set(c); err != nil {
|
||||
return fmt.Errorf("error setting cookie in cookie jar: %w", err)
|
||||
}
|
||||
}
|
||||
@@ -231,6 +277,7 @@ func (b playWrightBrowser) openPage(_ context.Context, target string, opts OpenP
|
||||
|
||||
resp, err := page.Goto(target, pwOpts)
|
||||
if err != nil {
|
||||
_ = page.Close()
|
||||
return nil, err
|
||||
}
|
||||
|
||||
@@ -266,8 +313,8 @@ func (b playWrightBrowser) Open(ctx context.Context, url string, opts OpenPageOp
|
||||
|
||||
func (b playWrightBrowser) Close() error {
|
||||
return errors.Join(
|
||||
b.browser.Close(),
|
||||
b.ctx.Close(),
|
||||
b.browser.Close(),
|
||||
b.pw.Stop(),
|
||||
)
|
||||
}
|
||||
|
||||
138
playwright_test.go
Normal file
138
playwright_test.go
Normal file
@@ -0,0 +1,138 @@
|
||||
package extractor
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/playwright-community/playwright-go"
|
||||
)
|
||||
|
||||
func TestPlaywrightCookieToCookie_AllFields(t *testing.T) {
|
||||
pwCookie := playwright.Cookie{
|
||||
Name: "session",
|
||||
Value: "abc123",
|
||||
Domain: ".example.com",
|
||||
Path: "/app",
|
||||
Expires: 1700000000,
|
||||
Secure: true,
|
||||
HttpOnly: true,
|
||||
SameSite: playwright.SameSiteAttributeStrict,
|
||||
}
|
||||
|
||||
c := playwrightCookieToCookie(pwCookie)
|
||||
|
||||
if c.Name != "session" {
|
||||
t.Errorf("Name = %q, want %q", c.Name, "session")
|
||||
}
|
||||
if c.Value != "abc123" {
|
||||
t.Errorf("Value = %q, want %q", c.Value, "abc123")
|
||||
}
|
||||
if c.Host != ".example.com" {
|
||||
t.Errorf("Host = %q, want %q", c.Host, ".example.com")
|
||||
}
|
||||
if c.Path != "/app" {
|
||||
t.Errorf("Path = %q, want %q", c.Path, "/app")
|
||||
}
|
||||
if c.Expires != time.Unix(1700000000, 0) {
|
||||
t.Errorf("Expires = %v, want %v", c.Expires, time.Unix(1700000000, 0))
|
||||
}
|
||||
if !c.Secure {
|
||||
t.Error("Secure = false, want true")
|
||||
}
|
||||
if !c.HttpOnly {
|
||||
t.Error("HttpOnly = false, want true")
|
||||
}
|
||||
if c.SameSite != SameSiteStrict {
|
||||
t.Errorf("SameSite = %q, want %q", c.SameSite, SameSiteStrict)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPlaywrightCookieToCookie_SecureFalse(t *testing.T) {
|
||||
pwCookie := playwright.Cookie{
|
||||
Name: "tracking",
|
||||
Value: "xyz",
|
||||
Domain: "example.com",
|
||||
Path: "/",
|
||||
Secure: false,
|
||||
}
|
||||
|
||||
c := playwrightCookieToCookie(pwCookie)
|
||||
|
||||
if c.Secure {
|
||||
t.Error("Secure = true, want false")
|
||||
}
|
||||
}
|
||||
|
||||
func TestCookieToPlaywrightOptionalCookie_AllFields(t *testing.T) {
|
||||
c := Cookie{
|
||||
Name: "__Secure-ID",
|
||||
Value: "token123",
|
||||
Host: ".example.com",
|
||||
Path: "/secure",
|
||||
Expires: time.Unix(1700000000, 0),
|
||||
Secure: true,
|
||||
HttpOnly: true,
|
||||
SameSite: SameSiteLax,
|
||||
}
|
||||
|
||||
oc := cookieToPlaywrightOptionalCookie(c)
|
||||
|
||||
if oc.Name != "__Secure-ID" {
|
||||
t.Errorf("Name = %q, want %q", oc.Name, "__Secure-ID")
|
||||
}
|
||||
if oc.Value != "token123" {
|
||||
t.Errorf("Value = %q, want %q", oc.Value, "token123")
|
||||
}
|
||||
if oc.Domain == nil || *oc.Domain != ".example.com" {
|
||||
t.Errorf("Domain = %v, want %q", oc.Domain, ".example.com")
|
||||
}
|
||||
if oc.Path == nil || *oc.Path != "/secure" {
|
||||
t.Errorf("Path = %v, want %q", oc.Path, "/secure")
|
||||
}
|
||||
if oc.Expires == nil || *oc.Expires != 1700000000 {
|
||||
t.Errorf("Expires = %v, want %v", oc.Expires, 1700000000)
|
||||
}
|
||||
if oc.Secure == nil || !*oc.Secure {
|
||||
t.Error("Secure = nil or false, want *true")
|
||||
}
|
||||
if oc.HttpOnly == nil || !*oc.HttpOnly {
|
||||
t.Error("HttpOnly = nil or false, want *true")
|
||||
}
|
||||
if oc.SameSite == nil || *oc.SameSite != *playwright.SameSiteAttributeLax {
|
||||
t.Errorf("SameSite = %v, want Lax", oc.SameSite)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCookieToPlaywrightOptionalCookie_SecureFalse(t *testing.T) {
|
||||
c := Cookie{
|
||||
Name: "tracker",
|
||||
Value: "v",
|
||||
Host: "example.com",
|
||||
Path: "/",
|
||||
Secure: false,
|
||||
}
|
||||
|
||||
oc := cookieToPlaywrightOptionalCookie(c)
|
||||
|
||||
if oc.Secure == nil {
|
||||
t.Fatal("Secure = nil, want *false")
|
||||
}
|
||||
if *oc.Secure {
|
||||
t.Error("Secure = *true, want *false")
|
||||
}
|
||||
}
|
||||
|
||||
func TestCookieToPlaywrightOptionalCookie_NoSameSite(t *testing.T) {
|
||||
c := Cookie{
|
||||
Name: "basic",
|
||||
Value: "val",
|
||||
Host: "example.com",
|
||||
Path: "/",
|
||||
}
|
||||
|
||||
oc := cookieToPlaywrightOptionalCookie(c)
|
||||
|
||||
if oc.SameSite != nil {
|
||||
t.Errorf("SameSite = %v, want nil", oc.SameSite)
|
||||
}
|
||||
}
|
||||
65
promote.go
Normal file
65
promote.go
Normal file
@@ -0,0 +1,65 @@
|
||||
package extractor
|
||||
|
||||
import "errors"
|
||||
|
||||
// ErrNotPromotable is returned when a Document cannot be promoted to an InteractiveBrowser.
|
||||
// This happens when the Document is not backed by a Playwright page (e.g. a mock or custom implementation).
|
||||
var ErrNotPromotable = errors.New("document is not promotable to InteractiveBrowser")
|
||||
|
||||
// ErrNotDemotable is returned when an InteractiveBrowser cannot be demoted to a Document.
|
||||
// This happens when the InteractiveBrowser is not backed by a Playwright page.
|
||||
var ErrNotDemotable = errors.New("interactive browser is not demotable to Document")
|
||||
|
||||
// ErrAlreadyDetached is returned when attempting to promote or demote an object that has
|
||||
// already been transferred. Each Document or InteractiveBrowser can only be promoted/demoted once.
|
||||
var ErrAlreadyDetached = errors.New("already detached")
|
||||
|
||||
// PromoteToInteractive transfers ownership of the underlying Playwright page from a Document
|
||||
// to a new InteractiveBrowser. After promotion, the Document's Close method becomes a no-op
|
||||
// (the page is now owned by the returned InteractiveBrowser).
|
||||
//
|
||||
// The caller must keep the original Browser alive while the promoted InteractiveBrowser is in use,
|
||||
// since the Browser still owns the Playwright process and browser instance.
|
||||
//
|
||||
// Returns ErrNotPromotable if the Document is not backed by a Playwright page,
|
||||
// or ErrAlreadyDetached if the Document was already promoted.
|
||||
func PromoteToInteractive(doc Document) (InteractiveBrowser, error) {
|
||||
d, ok := doc.(*document)
|
||||
if !ok {
|
||||
return nil, ErrNotPromotable
|
||||
}
|
||||
|
||||
if d.detached {
|
||||
return nil, ErrAlreadyDetached
|
||||
}
|
||||
|
||||
d.detached = true
|
||||
|
||||
return &interactiveBrowser{
|
||||
pw: d.pw,
|
||||
browser: d.browser,
|
||||
ctx: d.page.Context(),
|
||||
page: d.page,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// DemoteToDocument transfers ownership of the underlying Playwright page from an
|
||||
// InteractiveBrowser back to a new Document. After demotion, the InteractiveBrowser's
|
||||
// Close method becomes a no-op (the page is now owned by the returned Document).
|
||||
//
|
||||
// Returns ErrNotDemotable if the InteractiveBrowser is not backed by a Playwright page,
|
||||
// or ErrAlreadyDetached if the InteractiveBrowser was already demoted.
|
||||
func DemoteToDocument(ib InteractiveBrowser) (Document, error) {
|
||||
b, ok := ib.(*interactiveBrowser)
|
||||
if !ok {
|
||||
return nil, ErrNotDemotable
|
||||
}
|
||||
|
||||
if b.detached {
|
||||
return nil, ErrAlreadyDetached
|
||||
}
|
||||
|
||||
b.detached = true
|
||||
|
||||
return newDocument(b.pw, b.browser, b.page)
|
||||
}
|
||||
59
promote_test.go
Normal file
59
promote_test.go
Normal file
@@ -0,0 +1,59 @@
|
||||
package extractor
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// mockInteractiveBrowser implements InteractiveBrowser for testing without Playwright.
|
||||
type mockInteractiveBrowser struct{}
|
||||
|
||||
func (m mockInteractiveBrowser) Navigate(string) (string, error) { return "", nil }
|
||||
func (m mockInteractiveBrowser) GoBack() (string, error) { return "", nil }
|
||||
func (m mockInteractiveBrowser) GoForward() (string, error) { return "", nil }
|
||||
func (m mockInteractiveBrowser) URL() string { return "" }
|
||||
func (m mockInteractiveBrowser) MouseClick(float64, float64, string) error { return nil }
|
||||
func (m mockInteractiveBrowser) MouseDown(float64, float64, string) error { return nil }
|
||||
func (m mockInteractiveBrowser) MouseUp(float64, float64, string) error { return nil }
|
||||
func (m mockInteractiveBrowser) MouseMove(float64, float64, ...int) error { return nil }
|
||||
func (m mockInteractiveBrowser) MouseWheel(float64, float64) error { return nil }
|
||||
func (m mockInteractiveBrowser) KeyboardType(string) error { return nil }
|
||||
func (m mockInteractiveBrowser) KeyboardPress(string) error { return nil }
|
||||
func (m mockInteractiveBrowser) KeyboardInsertText(string) error { return nil }
|
||||
func (m mockInteractiveBrowser) Screenshot(int) ([]byte, error) { return nil, nil }
|
||||
func (m mockInteractiveBrowser) Cookies() ([]Cookie, error) { return nil, nil }
|
||||
func (m mockInteractiveBrowser) SetDefaultTimeout(time.Duration) {}
|
||||
func (m mockInteractiveBrowser) Close() error { return nil }
|
||||
|
||||
func TestPromoteToInteractive_NonPromotable(t *testing.T) {
|
||||
doc := &mockDocument{}
|
||||
_, err := PromoteToInteractive(doc)
|
||||
if !errors.Is(err, ErrNotPromotable) {
|
||||
t.Fatalf("expected ErrNotPromotable, got: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPromoteToInteractive_AlreadyDetached(t *testing.T) {
|
||||
d := &document{detached: true}
|
||||
_, err := PromoteToInteractive(d)
|
||||
if !errors.Is(err, ErrAlreadyDetached) {
|
||||
t.Fatalf("expected ErrAlreadyDetached, got: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDemoteToDocument_NonDemotable(t *testing.T) {
|
||||
ib := &mockInteractiveBrowser{}
|
||||
_, err := DemoteToDocument(ib)
|
||||
if !errors.Is(err, ErrNotDemotable) {
|
||||
t.Fatalf("expected ErrNotDemotable, got: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDemoteToDocument_AlreadyDetached(t *testing.T) {
|
||||
ib := &interactiveBrowser{detached: true}
|
||||
_, err := DemoteToDocument(ib)
|
||||
if !errors.Is(err, ErrAlreadyDetached) {
|
||||
t.Fatalf("expected ErrAlreadyDetached, got: %v", err)
|
||||
}
|
||||
}
|
||||
165
stealth.go
165
stealth.go
@@ -1,5 +1,10 @@
|
||||
package extractor
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"math/rand/v2"
|
||||
)
|
||||
|
||||
// stealthChromiumArgs are launch arguments that reduce automation detection for Chromium-based browsers.
|
||||
var stealthChromiumArgs = []string{
|
||||
"--disable-blink-features=AutomationControlled",
|
||||
@@ -39,10 +44,47 @@ var stealthCommonScripts = []string{
|
||||
})()`,
|
||||
}
|
||||
|
||||
// stealthChromiumScripts are JavaScript snippets specific to Chromium-based browsers.
|
||||
var stealthChromiumScripts = []string{
|
||||
// Populate navigator.plugins with realistic Chromium entries so plugins.length > 0.
|
||||
`Object.defineProperty(navigator, 'plugins', {
|
||||
// chromiumHWProfile holds hardware fingerprint values for a Chromium browser session.
|
||||
type chromiumHWProfile struct {
|
||||
WebGLVendor string
|
||||
WebGLRenderer string
|
||||
ConnRTT int // base RTT in ms (jittered ±20 per session)
|
||||
ConnDownlink float64 // base downlink in Mbps (jittered ±2 per session)
|
||||
}
|
||||
|
||||
// chromiumHWProfiles is a pool of realistic Chromium hardware profiles.
|
||||
// Index 0 matches the original hardcoded values.
|
||||
var chromiumHWProfiles = []chromiumHWProfile{
|
||||
{"Google Inc. (Intel)", "ANGLE (Intel, Intel(R) UHD Graphics 630, OpenGL 4.5)", 50, 10},
|
||||
{"Google Inc. (NVIDIA)", "ANGLE (NVIDIA, NVIDIA GeForce GTX 1660 SUPER, D3D11)", 30, 25},
|
||||
{"Google Inc. (AMD)", "ANGLE (AMD, AMD Radeon RX 580, D3D11)", 100, 5},
|
||||
{"Google Inc. (Intel)", "ANGLE (Intel, Intel(R) UHD Graphics 770, OpenGL 4.5)", 50, 10},
|
||||
{"Google Inc. (NVIDIA)", "ANGLE (NVIDIA, NVIDIA GeForce RTX 3060, D3D11)", 25, 50},
|
||||
{"Google Inc. (Intel)", "ANGLE (Intel, Intel(R) Iris Xe Graphics, D3D11)", 75, 8},
|
||||
}
|
||||
|
||||
// randomChromiumProfile returns a randomly selected Chromium hardware profile.
|
||||
func randomChromiumProfile() chromiumHWProfile {
|
||||
return chromiumHWProfiles[rand.IntN(len(chromiumHWProfiles))]
|
||||
}
|
||||
|
||||
// buildChromiumStealthScripts returns Chromium stealth init scripts with the given hardware profile
|
||||
// values templated into the WebGL and connection spoofing scripts. Connection RTT and downlink
|
||||
// receive per-session jitter (±20ms RTT, ±2 Mbps downlink).
|
||||
func buildChromiumStealthScripts(p chromiumHWProfile) []string {
|
||||
// Apply jitter to connection stats.
|
||||
rtt := p.ConnRTT + rand.IntN(41) - 20 // ±20ms
|
||||
if rtt < 0 {
|
||||
rtt = 0
|
||||
}
|
||||
downlink := p.ConnDownlink + (rand.Float64()*4 - 2) // ±2 Mbps
|
||||
if downlink < 0.5 {
|
||||
downlink = 0.5
|
||||
}
|
||||
|
||||
return []string{
|
||||
// Populate navigator.plugins with realistic Chromium entries so plugins.length > 0.
|
||||
`Object.defineProperty(navigator, 'plugins', {
|
||||
get: () => {
|
||||
const arr = [
|
||||
{ name: 'PDF Viewer', filename: 'internal-pdf-viewer', description: 'Portable Document Format' },
|
||||
@@ -56,8 +98,8 @@ var stealthChromiumScripts = []string{
|
||||
},
|
||||
})`,
|
||||
|
||||
// Populate navigator.mimeTypes to match the fake Chromium plugins above.
|
||||
`Object.defineProperty(navigator, 'mimeTypes', {
|
||||
// Populate navigator.mimeTypes to match the fake Chromium plugins above.
|
||||
`Object.defineProperty(navigator, 'mimeTypes', {
|
||||
get: () => {
|
||||
const arr = [
|
||||
{ type: 'application/pdf', suffixes: 'pdf', description: 'Portable Document Format' },
|
||||
@@ -68,13 +110,13 @@ var stealthChromiumScripts = []string{
|
||||
},
|
||||
})`,
|
||||
|
||||
// Provide window.chrome runtime stub (Chromium-only signal).
|
||||
`if (!window.chrome) {
|
||||
// Provide window.chrome runtime stub (Chromium-only signal).
|
||||
`if (!window.chrome) {
|
||||
window.chrome = { runtime: {} };
|
||||
}`,
|
||||
|
||||
// Add chrome.app, chrome.csi, and chrome.loadTimes stubs missing in headless.
|
||||
`(function() {
|
||||
// Add chrome.app, chrome.csi, and chrome.loadTimes stubs missing in headless.
|
||||
`(function() {
|
||||
if (!window.chrome) window.chrome = {};
|
||||
if (!window.chrome.app) {
|
||||
window.chrome.app = { isInstalled: false, InstallState: { DISABLED: 'disabled', INSTALLED: 'installed', NOT_INSTALLED: 'not_installed' }, RunningState: { CANNOT_RUN: 'cannot_run', READY_TO_RUN: 'ready_to_run', RUNNING: 'running' } };
|
||||
@@ -87,37 +129,37 @@ var stealthChromiumScripts = []string{
|
||||
}
|
||||
})()`,
|
||||
|
||||
// Spoof WebGL renderer to hide SwiftShader (headless GPU) fingerprint with Chromium ANGLE strings.
|
||||
`(function() {
|
||||
// Spoof WebGL renderer to hide SwiftShader (headless GPU) fingerprint with Chromium ANGLE strings.
|
||||
fmt.Sprintf(`(function() {
|
||||
const getParam = WebGLRenderingContext.prototype.getParameter;
|
||||
WebGLRenderingContext.prototype.getParameter = function(param) {
|
||||
if (param === 37445) return 'Google Inc. (Intel)';
|
||||
if (param === 37446) return 'ANGLE (Intel, Intel(R) UHD Graphics 630, OpenGL 4.5)';
|
||||
if (param === 37445) return '%s';
|
||||
if (param === 37446) return '%s';
|
||||
return getParam.call(this, param);
|
||||
};
|
||||
if (typeof WebGL2RenderingContext !== 'undefined') {
|
||||
const getParam2 = WebGL2RenderingContext.prototype.getParameter;
|
||||
WebGL2RenderingContext.prototype.getParameter = function(param) {
|
||||
if (param === 37445) return 'Google Inc. (Intel)';
|
||||
if (param === 37446) return 'ANGLE (Intel, Intel(R) UHD Graphics 630, OpenGL 4.5)';
|
||||
if (param === 37445) return '%s';
|
||||
if (param === 37446) return '%s';
|
||||
return getParam2.call(this, param);
|
||||
};
|
||||
}
|
||||
})()`,
|
||||
})()`, p.WebGLVendor, p.WebGLRenderer, p.WebGLVendor, p.WebGLRenderer),
|
||||
|
||||
// Stub navigator.connection (Network Information API) if missing (Chrome-only API).
|
||||
`(function() {
|
||||
// Stub navigator.connection (Network Information API) if missing (Chrome-only API).
|
||||
fmt.Sprintf(`(function() {
|
||||
if (!navigator.connection) {
|
||||
Object.defineProperty(navigator, 'connection', {
|
||||
get: function() {
|
||||
return { effectiveType: '4g', rtt: 50, downlink: 10, saveData: false, onchange: null };
|
||||
return { effectiveType: '4g', rtt: %d, downlink: %.1f, saveData: false, onchange: null };
|
||||
},
|
||||
});
|
||||
}
|
||||
})()`,
|
||||
})()`, rtt, downlink),
|
||||
|
||||
// Remove CDP artifacts (window.cdc_* globals injected by Chrome DevTools Protocol).
|
||||
`(function() {
|
||||
// Remove CDP artifacts (window.cdc_* globals injected by Chrome DevTools Protocol).
|
||||
`(function() {
|
||||
for (var key in window) {
|
||||
if (key.match(/^cdc_/)) {
|
||||
delete window[key];
|
||||
@@ -125,8 +167,8 @@ var stealthChromiumScripts = []string{
|
||||
}
|
||||
})()`,
|
||||
|
||||
// Strip "HeadlessChrome" from navigator.userAgent if present.
|
||||
`(function() {
|
||||
// Strip "HeadlessChrome" from navigator.userAgent if present.
|
||||
`(function() {
|
||||
var ua = navigator.userAgent;
|
||||
if (ua.indexOf('HeadlessChrome') !== -1) {
|
||||
Object.defineProperty(navigator, 'userAgent', {
|
||||
@@ -134,12 +176,40 @@ var stealthChromiumScripts = []string{
|
||||
});
|
||||
}
|
||||
})()`,
|
||||
}
|
||||
}
|
||||
|
||||
// stealthFirefoxScripts are JavaScript snippets specific to Firefox.
|
||||
var stealthFirefoxScripts = []string{
|
||||
// Harden navigator.webdriver for Firefox: ensure Object.getOwnPropertyDescriptor also returns undefined.
|
||||
`(function() {
|
||||
// firefoxHWProfile holds hardware fingerprint values for a Firefox browser session.
|
||||
type firefoxHWProfile struct {
|
||||
WebGLVendor string
|
||||
WebGLRenderer string
|
||||
MozInnerScreenX int
|
||||
MozInnerScreenY int
|
||||
HardwareConcurrency int
|
||||
}
|
||||
|
||||
// firefoxHWProfiles is a pool of realistic Firefox hardware profiles.
|
||||
// Index 0 matches the original hardcoded values.
|
||||
var firefoxHWProfiles = []firefoxHWProfile{
|
||||
{"Intel Open Source Technology Center", "Mesa DRI Intel(R) UHD Graphics 630", 8, 51, 4},
|
||||
{"Intel Open Source Technology Center", "Mesa DRI Intel(R) HD Graphics 530", 0, 71, 8},
|
||||
{"X.Org", "AMD Radeon RX 580 (polaris10, LLVM 15.0.7, DRM 3.49, 6.1.0-18-amd64)", 8, 51, 8},
|
||||
{"Intel Open Source Technology Center", "Mesa DRI Intel(R) UHD Graphics 770", 0, 51, 16},
|
||||
{"nouveau", "NV167", 8, 71, 4},
|
||||
{"Intel", "Mesa Intel(R) Iris(R) Xe Graphics", 0, 51, 8},
|
||||
}
|
||||
|
||||
// randomFirefoxProfile returns a randomly selected Firefox hardware profile.
|
||||
func randomFirefoxProfile() firefoxHWProfile {
|
||||
return firefoxHWProfiles[rand.IntN(len(firefoxHWProfiles))]
|
||||
}
|
||||
|
||||
// buildFirefoxStealthScripts returns Firefox stealth init scripts with the given hardware profile
|
||||
// values templated into the WebGL, mozInnerScreen, and hardwareConcurrency spoofing scripts.
|
||||
func buildFirefoxStealthScripts(p firefoxHWProfile) []string {
|
||||
return []string{
|
||||
// Harden navigator.webdriver for Firefox: ensure Object.getOwnPropertyDescriptor also returns undefined.
|
||||
`(function() {
|
||||
const proto = Object.getPrototypeOf(navigator);
|
||||
const origGetOwnPropDesc = Object.getOwnPropertyDescriptor;
|
||||
Object.getOwnPropertyDescriptor = function(obj, prop) {
|
||||
@@ -150,43 +220,43 @@ var stealthFirefoxScripts = []string{
|
||||
};
|
||||
})()`,
|
||||
|
||||
// Spoof WebGL renderer with Firefox-appropriate Mesa/Intel strings.
|
||||
`(function() {
|
||||
// Spoof WebGL renderer with Firefox-appropriate Mesa/driver strings.
|
||||
fmt.Sprintf(`(function() {
|
||||
const getParam = WebGLRenderingContext.prototype.getParameter;
|
||||
WebGLRenderingContext.prototype.getParameter = function(param) {
|
||||
if (param === 37445) return 'Intel Open Source Technology Center';
|
||||
if (param === 37446) return 'Mesa DRI Intel(R) UHD Graphics 630';
|
||||
if (param === 37445) return '%s';
|
||||
if (param === 37446) return '%s';
|
||||
return getParam.call(this, param);
|
||||
};
|
||||
if (typeof WebGL2RenderingContext !== 'undefined') {
|
||||
const getParam2 = WebGL2RenderingContext.prototype.getParameter;
|
||||
WebGL2RenderingContext.prototype.getParameter = function(param) {
|
||||
if (param === 37445) return 'Intel Open Source Technology Center';
|
||||
if (param === 37446) return 'Mesa DRI Intel(R) UHD Graphics 630';
|
||||
if (param === 37445) return '%s';
|
||||
if (param === 37446) return '%s';
|
||||
return getParam2.call(this, param);
|
||||
};
|
||||
}
|
||||
})()`,
|
||||
})()`, p.WebGLVendor, p.WebGLRenderer, p.WebGLVendor, p.WebGLRenderer),
|
||||
|
||||
// Spoof mozInnerScreenX/mozInnerScreenY which are 0 in headless Firefox.
|
||||
`(function() {
|
||||
// Spoof mozInnerScreenX/mozInnerScreenY which are 0 in headless Firefox.
|
||||
fmt.Sprintf(`(function() {
|
||||
if (window.mozInnerScreenX === 0) {
|
||||
Object.defineProperty(window, 'mozInnerScreenX', { get: () => 8 });
|
||||
Object.defineProperty(window, 'mozInnerScreenX', { get: () => %d });
|
||||
}
|
||||
if (window.mozInnerScreenY === 0) {
|
||||
Object.defineProperty(window, 'mozInnerScreenY', { get: () => 51 });
|
||||
Object.defineProperty(window, 'mozInnerScreenY', { get: () => %d });
|
||||
}
|
||||
})()`,
|
||||
})()`, p.MozInnerScreenX, p.MozInnerScreenY),
|
||||
|
||||
// Normalize navigator.hardwareConcurrency (Firefox headless sometimes reports 2).
|
||||
`(function() {
|
||||
// Normalize navigator.hardwareConcurrency (Firefox headless sometimes reports 2).
|
||||
fmt.Sprintf(`(function() {
|
||||
if (navigator.hardwareConcurrency <= 2) {
|
||||
Object.defineProperty(navigator, 'hardwareConcurrency', { get: () => 4 });
|
||||
Object.defineProperty(navigator, 'hardwareConcurrency', { get: () => %d });
|
||||
}
|
||||
})()`,
|
||||
})()`, p.HardwareConcurrency),
|
||||
|
||||
// Override navigator.plugins with Firefox-appropriate PDF.js entry.
|
||||
`Object.defineProperty(navigator, 'plugins', {
|
||||
// Override navigator.plugins with Firefox-appropriate PDF.js entry.
|
||||
`Object.defineProperty(navigator, 'plugins', {
|
||||
get: () => {
|
||||
const arr = [
|
||||
{ name: 'PDF.js', filename: 'internal-pdf-viewer', description: 'Portable Document Format' },
|
||||
@@ -197,4 +267,5 @@ var stealthFirefoxScripts = []string{
|
||||
return arr;
|
||||
},
|
||||
})`,
|
||||
}
|
||||
}
|
||||
|
||||
199
stealth_test.go
199
stealth_test.go
@@ -129,14 +129,16 @@ func TestStealthCommonScripts_Notification(t *testing.T) {
|
||||
// --- Chromium scripts ---
|
||||
|
||||
func TestStealthChromiumScripts_Count(t *testing.T) {
|
||||
if len(stealthChromiumScripts) != 8 {
|
||||
t.Fatalf("expected 8 chromium stealth scripts, got %d", len(stealthChromiumScripts))
|
||||
scripts := buildChromiumStealthScripts(chromiumHWProfiles[0])
|
||||
if len(scripts) != 8 {
|
||||
t.Fatalf("expected 8 chromium stealth scripts, got %d", len(scripts))
|
||||
}
|
||||
}
|
||||
|
||||
func TestStealthChromiumScripts_Plugins(t *testing.T) {
|
||||
scripts := buildChromiumStealthScripts(chromiumHWProfiles[0])
|
||||
found := false
|
||||
for _, s := range stealthChromiumScripts {
|
||||
for _, s := range scripts {
|
||||
if strings.Contains(s, "Chrome PDF Plugin") && strings.Contains(s, "navigator") && strings.Contains(s, "plugins") {
|
||||
found = true
|
||||
break
|
||||
@@ -148,8 +150,9 @@ func TestStealthChromiumScripts_Plugins(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestStealthChromiumScripts_MimeTypes(t *testing.T) {
|
||||
scripts := buildChromiumStealthScripts(chromiumHWProfiles[0])
|
||||
found := false
|
||||
for _, s := range stealthChromiumScripts {
|
||||
for _, s := range scripts {
|
||||
if strings.Contains(s, "mimeTypes") && strings.Contains(s, "application/pdf") {
|
||||
found = true
|
||||
break
|
||||
@@ -161,8 +164,9 @@ func TestStealthChromiumScripts_MimeTypes(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestStealthChromiumScripts_WindowChrome(t *testing.T) {
|
||||
scripts := buildChromiumStealthScripts(chromiumHWProfiles[0])
|
||||
found := false
|
||||
for _, s := range stealthChromiumScripts {
|
||||
for _, s := range scripts {
|
||||
if strings.Contains(s, "window.chrome") && strings.Contains(s, "runtime") {
|
||||
found = true
|
||||
break
|
||||
@@ -174,8 +178,9 @@ func TestStealthChromiumScripts_WindowChrome(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestStealthChromiumScripts_ChromeApp(t *testing.T) {
|
||||
scripts := buildChromiumStealthScripts(chromiumHWProfiles[0])
|
||||
found := false
|
||||
for _, s := range stealthChromiumScripts {
|
||||
for _, s := range scripts {
|
||||
if strings.Contains(s, "chrome.app") && strings.Contains(s, "chrome.csi") && strings.Contains(s, "chrome.loadTimes") {
|
||||
found = true
|
||||
break
|
||||
@@ -187,8 +192,9 @@ func TestStealthChromiumScripts_ChromeApp(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestStealthChromiumScripts_WebGLSpoof(t *testing.T) {
|
||||
scripts := buildChromiumStealthScripts(chromiumHWProfiles[0])
|
||||
found := false
|
||||
for _, s := range stealthChromiumScripts {
|
||||
for _, s := range scripts {
|
||||
if strings.Contains(s, "37446") && strings.Contains(s, "ANGLE") {
|
||||
found = true
|
||||
break
|
||||
@@ -200,8 +206,9 @@ func TestStealthChromiumScripts_WebGLSpoof(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestStealthChromiumScripts_NavigatorConnection(t *testing.T) {
|
||||
scripts := buildChromiumStealthScripts(chromiumHWProfiles[0])
|
||||
found := false
|
||||
for _, s := range stealthChromiumScripts {
|
||||
for _, s := range scripts {
|
||||
if strings.Contains(s, "connection") && strings.Contains(s, "effectiveType") {
|
||||
found = true
|
||||
break
|
||||
@@ -213,8 +220,9 @@ func TestStealthChromiumScripts_NavigatorConnection(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestStealthChromiumScripts_CDPCleanup(t *testing.T) {
|
||||
scripts := buildChromiumStealthScripts(chromiumHWProfiles[0])
|
||||
found := false
|
||||
for _, s := range stealthChromiumScripts {
|
||||
for _, s := range scripts {
|
||||
if strings.Contains(s, "cdc_") && strings.Contains(s, "delete") {
|
||||
found = true
|
||||
break
|
||||
@@ -226,8 +234,9 @@ func TestStealthChromiumScripts_CDPCleanup(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestStealthChromiumScripts_UserAgentStrip(t *testing.T) {
|
||||
scripts := buildChromiumStealthScripts(chromiumHWProfiles[0])
|
||||
found := false
|
||||
for _, s := range stealthChromiumScripts {
|
||||
for _, s := range scripts {
|
||||
if strings.Contains(s, "HeadlessChrome") && strings.Contains(s, "userAgent") {
|
||||
found = true
|
||||
break
|
||||
@@ -241,14 +250,16 @@ func TestStealthChromiumScripts_UserAgentStrip(t *testing.T) {
|
||||
// --- Firefox scripts ---
|
||||
|
||||
func TestStealthFirefoxScripts_Count(t *testing.T) {
|
||||
if len(stealthFirefoxScripts) != 5 {
|
||||
t.Fatalf("expected 5 firefox stealth scripts, got %d", len(stealthFirefoxScripts))
|
||||
scripts := buildFirefoxStealthScripts(firefoxHWProfiles[0])
|
||||
if len(scripts) != 5 {
|
||||
t.Fatalf("expected 5 firefox stealth scripts, got %d", len(scripts))
|
||||
}
|
||||
}
|
||||
|
||||
func TestStealthFirefoxScripts_WebdriverHardening(t *testing.T) {
|
||||
scripts := buildFirefoxStealthScripts(firefoxHWProfiles[0])
|
||||
found := false
|
||||
for _, s := range stealthFirefoxScripts {
|
||||
for _, s := range scripts {
|
||||
if strings.Contains(s, "getOwnPropertyDescriptor") && strings.Contains(s, "webdriver") {
|
||||
found = true
|
||||
break
|
||||
@@ -260,8 +271,9 @@ func TestStealthFirefoxScripts_WebdriverHardening(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestStealthFirefoxScripts_WebGLSpoof(t *testing.T) {
|
||||
scripts := buildFirefoxStealthScripts(firefoxHWProfiles[0])
|
||||
found := false
|
||||
for _, s := range stealthFirefoxScripts {
|
||||
for _, s := range scripts {
|
||||
if strings.Contains(s, "37446") && strings.Contains(s, "Mesa DRI") {
|
||||
found = true
|
||||
break
|
||||
@@ -273,8 +285,9 @@ func TestStealthFirefoxScripts_WebGLSpoof(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestStealthFirefoxScripts_MozInnerScreen(t *testing.T) {
|
||||
scripts := buildFirefoxStealthScripts(firefoxHWProfiles[0])
|
||||
found := false
|
||||
for _, s := range stealthFirefoxScripts {
|
||||
for _, s := range scripts {
|
||||
if strings.Contains(s, "mozInnerScreenX") && strings.Contains(s, "mozInnerScreenY") {
|
||||
found = true
|
||||
break
|
||||
@@ -286,8 +299,9 @@ func TestStealthFirefoxScripts_MozInnerScreen(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestStealthFirefoxScripts_HardwareConcurrency(t *testing.T) {
|
||||
scripts := buildFirefoxStealthScripts(firefoxHWProfiles[0])
|
||||
found := false
|
||||
for _, s := range stealthFirefoxScripts {
|
||||
for _, s := range scripts {
|
||||
if strings.Contains(s, "hardwareConcurrency") {
|
||||
found = true
|
||||
break
|
||||
@@ -299,8 +313,9 @@ func TestStealthFirefoxScripts_HardwareConcurrency(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestStealthFirefoxScripts_PDFjsPlugins(t *testing.T) {
|
||||
scripts := buildFirefoxStealthScripts(firefoxHWProfiles[0])
|
||||
found := false
|
||||
for _, s := range stealthFirefoxScripts {
|
||||
for _, s := range scripts {
|
||||
if strings.Contains(s, "PDF.js") && strings.Contains(s, "plugins") {
|
||||
found = true
|
||||
break
|
||||
@@ -314,17 +329,19 @@ func TestStealthFirefoxScripts_PDFjsPlugins(t *testing.T) {
|
||||
// --- Cross-category validation ---
|
||||
|
||||
func TestStealthScripts_NoOverlap(t *testing.T) {
|
||||
chromiumScripts := buildChromiumStealthScripts(chromiumHWProfiles[0])
|
||||
firefoxScripts := buildFirefoxStealthScripts(firefoxHWProfiles[0])
|
||||
all := make(map[string]string) // script -> category
|
||||
for _, s := range stealthCommonScripts {
|
||||
all[s] = "common"
|
||||
}
|
||||
for _, s := range stealthChromiumScripts {
|
||||
for _, s := range chromiumScripts {
|
||||
if cat, ok := all[s]; ok {
|
||||
t.Fatalf("chromium script also appears in %s category", cat)
|
||||
}
|
||||
all[s] = "chromium"
|
||||
}
|
||||
for _, s := range stealthFirefoxScripts {
|
||||
for _, s := range firefoxScripts {
|
||||
if cat, ok := all[s]; ok {
|
||||
t.Fatalf("firefox script also appears in %s category", cat)
|
||||
}
|
||||
@@ -354,8 +371,9 @@ func TestStealthCommonScripts_NoFirefoxMarkers(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestStealthChromiumScripts_NoFirefoxMarkers(t *testing.T) {
|
||||
scripts := buildChromiumStealthScripts(chromiumHWProfiles[0])
|
||||
firefoxMarkers := []string{"mozInnerScreen", "Mesa DRI", "PDF.js"}
|
||||
for _, s := range stealthChromiumScripts {
|
||||
for _, s := range scripts {
|
||||
for _, marker := range firefoxMarkers {
|
||||
if strings.Contains(s, marker) {
|
||||
t.Fatalf("chromium script contains Firefox-specific marker %q", marker)
|
||||
@@ -365,8 +383,9 @@ func TestStealthChromiumScripts_NoFirefoxMarkers(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestStealthFirefoxScripts_NoChromiumMarkers(t *testing.T) {
|
||||
scripts := buildFirefoxStealthScripts(firefoxHWProfiles[0])
|
||||
chromiumMarkers := []string{"window.chrome", "chrome.app", "chrome.csi", "chrome.loadTimes", "HeadlessChrome", "cdc_", "Chrome PDF Plugin", "ANGLE"}
|
||||
for _, s := range stealthFirefoxScripts {
|
||||
for _, s := range scripts {
|
||||
for _, marker := range chromiumMarkers {
|
||||
if strings.Contains(s, marker) {
|
||||
t.Fatalf("firefox script contains Chromium-specific marker %q", marker)
|
||||
@@ -374,3 +393,141 @@ func TestStealthFirefoxScripts_NoChromiumMarkers(t *testing.T) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// --- User-Agent constants ---
|
||||
|
||||
func TestDefaultUserAgent_BackwardCompat(t *testing.T) {
|
||||
if DefaultUserAgent != DefaultFirefoxUserAgent {
|
||||
t.Fatal("DefaultUserAgent must equal DefaultFirefoxUserAgent for backward compatibility")
|
||||
}
|
||||
}
|
||||
|
||||
func TestDefaultFirefoxUserAgent_Content(t *testing.T) {
|
||||
if !strings.Contains(DefaultFirefoxUserAgent, "Firefox") {
|
||||
t.Fatal("DefaultFirefoxUserAgent must contain 'Firefox'")
|
||||
}
|
||||
if strings.Contains(DefaultFirefoxUserAgent, "Chrome") {
|
||||
t.Fatal("DefaultFirefoxUserAgent must not contain 'Chrome'")
|
||||
}
|
||||
}
|
||||
|
||||
func TestDefaultChromiumUserAgent_Content(t *testing.T) {
|
||||
if !strings.Contains(DefaultChromiumUserAgent, "Chrome") {
|
||||
t.Fatal("DefaultChromiumUserAgent must contain 'Chrome'")
|
||||
}
|
||||
if strings.Contains(DefaultChromiumUserAgent, "Firefox") {
|
||||
t.Fatal("DefaultChromiumUserAgent must not contain 'Firefox'")
|
||||
}
|
||||
}
|
||||
|
||||
// --- Viewport and UA defaults via mergeOptions ---
|
||||
|
||||
func TestMergeOptions_DefaultViewport(t *testing.T) {
|
||||
base := BrowserOptions{
|
||||
Dimensions: Size{Width: 1920, Height: 1080},
|
||||
}
|
||||
got := mergeOptions(base, nil)
|
||||
if got.Dimensions.Width != 1920 || got.Dimensions.Height != 1080 {
|
||||
t.Fatalf("expected default viewport 1920x1080, got %dx%d", got.Dimensions.Width, got.Dimensions.Height)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMergeOptions_ViewportOverride(t *testing.T) {
|
||||
base := BrowserOptions{
|
||||
Dimensions: Size{Width: 1920, Height: 1080},
|
||||
}
|
||||
got := mergeOptions(base, []BrowserOptions{{Dimensions: Size{Width: 1280, Height: 720}}})
|
||||
if got.Dimensions.Width != 1280 || got.Dimensions.Height != 720 {
|
||||
t.Fatalf("expected overridden viewport 1280x720, got %dx%d", got.Dimensions.Width, got.Dimensions.Height)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMergeOptions_EmptyUANotOverridden(t *testing.T) {
|
||||
base := BrowserOptions{}
|
||||
got := mergeOptions(base, []BrowserOptions{{Browser: BrowserChromium}})
|
||||
if got.UserAgent != "" {
|
||||
t.Fatalf("expected empty UserAgent after merge with no explicit UA, got %q", got.UserAgent)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMergeOptions_ExplicitUAPreserved(t *testing.T) {
|
||||
base := BrowserOptions{}
|
||||
customUA := "MyCustomAgent/1.0"
|
||||
got := mergeOptions(base, []BrowserOptions{{UserAgent: customUA}})
|
||||
if got.UserAgent != customUA {
|
||||
t.Fatalf("expected explicit UA %q preserved, got %q", customUA, got.UserAgent)
|
||||
}
|
||||
}
|
||||
|
||||
// --- Hardware profile pools ---
|
||||
|
||||
func TestChromiumHWProfiles_NotEmpty(t *testing.T) {
|
||||
if len(chromiumHWProfiles) < 2 {
|
||||
t.Fatalf("expected at least 2 chromium hardware profiles, got %d", len(chromiumHWProfiles))
|
||||
}
|
||||
}
|
||||
|
||||
func TestFirefoxHWProfiles_NotEmpty(t *testing.T) {
|
||||
if len(firefoxHWProfiles) < 2 {
|
||||
t.Fatalf("expected at least 2 firefox hardware profiles, got %d", len(firefoxHWProfiles))
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildChromiumStealthScripts_ProfileValues(t *testing.T) {
|
||||
p := chromiumHWProfiles[1] // NVIDIA profile
|
||||
scripts := buildChromiumStealthScripts(p)
|
||||
joined := strings.Join(scripts, "\n")
|
||||
if !strings.Contains(joined, p.WebGLVendor) {
|
||||
t.Fatalf("expected chromium scripts to contain vendor %q", p.WebGLVendor)
|
||||
}
|
||||
if !strings.Contains(joined, p.WebGLRenderer) {
|
||||
t.Fatalf("expected chromium scripts to contain renderer %q", p.WebGLRenderer)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildFirefoxStealthScripts_ProfileValues(t *testing.T) {
|
||||
p := firefoxHWProfiles[2] // AMD profile
|
||||
scripts := buildFirefoxStealthScripts(p)
|
||||
joined := strings.Join(scripts, "\n")
|
||||
if !strings.Contains(joined, p.WebGLVendor) {
|
||||
t.Fatalf("expected firefox scripts to contain vendor %q", p.WebGLVendor)
|
||||
}
|
||||
if !strings.Contains(joined, p.WebGLRenderer) {
|
||||
t.Fatalf("expected firefox scripts to contain renderer %q", p.WebGLRenderer)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildChromiumStealthScripts_ConnectionJitter(t *testing.T) {
|
||||
p := chromiumHWProfiles[0]
|
||||
seen := make(map[string]bool)
|
||||
for range 50 {
|
||||
scripts := buildChromiumStealthScripts(p)
|
||||
// The connection script is at index 5.
|
||||
seen[scripts[5]] = true
|
||||
}
|
||||
if len(seen) < 2 {
|
||||
t.Fatal("expected connection script to vary across calls due to jitter, but all 50 were identical")
|
||||
}
|
||||
}
|
||||
|
||||
func TestChromiumHWProfiles_NoSingleQuotes(t *testing.T) {
|
||||
for i, p := range chromiumHWProfiles {
|
||||
if strings.Contains(p.WebGLVendor, "'") {
|
||||
t.Fatalf("chromium profile %d vendor contains single quote (breaks JS)", i)
|
||||
}
|
||||
if strings.Contains(p.WebGLRenderer, "'") {
|
||||
t.Fatalf("chromium profile %d renderer contains single quote (breaks JS)", i)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestFirefoxHWProfiles_NoSingleQuotes(t *testing.T) {
|
||||
for i, p := range firefoxHWProfiles {
|
||||
if strings.Contains(p.WebGLVendor, "'") {
|
||||
t.Fatalf("firefox profile %d vendor contains single quote (breaks JS)", i)
|
||||
}
|
||||
if strings.Contains(p.WebGLRenderer, "'") {
|
||||
t.Fatalf("firefox profile %d renderer contains single quote (breaks JS)", i)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user