enhance: thread-safe CookieJar, SameSite cookie attr, dynamic Google countries
- Wrap staticCookieJar in struct with sync.RWMutex for thread safety - Add SameSite field to Cookie struct with Strict/Lax/None constants - Update Playwright cookie conversion functions for SameSite - Replace hardcoded 4-country switch with dynamic country code generation Closes #20, #22, #23 Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
10
cookiejar.go
10
cookiejar.go
@@ -6,6 +6,15 @@ import (
|
|||||||
"time"
|
"time"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
// SameSite represents the SameSite attribute of a cookie.
|
||||||
|
type SameSite string
|
||||||
|
|
||||||
|
const (
|
||||||
|
SameSiteStrict SameSite = "Strict"
|
||||||
|
SameSiteLax SameSite = "Lax"
|
||||||
|
SameSiteNone SameSite = "None"
|
||||||
|
)
|
||||||
|
|
||||||
type Cookie struct {
|
type Cookie struct {
|
||||||
Host string
|
Host string
|
||||||
Path string
|
Path string
|
||||||
@@ -14,6 +23,7 @@ type Cookie struct {
|
|||||||
HttpOnly bool
|
HttpOnly bool
|
||||||
Name string
|
Name string
|
||||||
Value string
|
Value string
|
||||||
|
SameSite SameSite
|
||||||
}
|
}
|
||||||
|
|
||||||
func (c Cookie) IsTargetMatch(target string) (bool, error) {
|
func (c Cookie) IsTargetMatch(target string) (bool, error) {
|
||||||
|
|||||||
@@ -101,10 +101,10 @@ func TestCookie_IsTargetMatch_InvalidURL(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestStaticCookieJar_GetAll(t *testing.T) {
|
func TestStaticCookieJar_GetAll(t *testing.T) {
|
||||||
jar := &staticCookieJar{
|
jar := &staticCookieJar{cookies: []Cookie{
|
||||||
Cookie{Host: "a.com", Name: "a", Value: "1"},
|
{Host: "a.com", Name: "a", Value: "1"},
|
||||||
Cookie{Host: "b.com", Name: "b", Value: "2"},
|
{Host: "b.com", Name: "b", Value: "2"},
|
||||||
}
|
}}
|
||||||
|
|
||||||
cookies, err := jar.GetAll()
|
cookies, err := jar.GetAll()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -116,10 +116,10 @@ func TestStaticCookieJar_GetAll(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestStaticCookieJar_Get(t *testing.T) {
|
func TestStaticCookieJar_Get(t *testing.T) {
|
||||||
jar := &staticCookieJar{
|
jar := &staticCookieJar{cookies: []Cookie{
|
||||||
Cookie{Host: "example.com", Path: "/", Name: "a", Value: "1"},
|
{Host: "example.com", Path: "/", Name: "a", Value: "1"},
|
||||||
Cookie{Host: "other.com", Path: "/", Name: "b", Value: "2"},
|
{Host: "other.com", Path: "/", Name: "b", Value: "2"},
|
||||||
}
|
}}
|
||||||
|
|
||||||
cookies, err := jar.Get("https://example.com/page")
|
cookies, err := jar.Get("https://example.com/page")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -150,9 +150,9 @@ func TestStaticCookieJar_Set_New(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestStaticCookieJar_Set_Update(t *testing.T) {
|
func TestStaticCookieJar_Set_Update(t *testing.T) {
|
||||||
jar := &staticCookieJar{
|
jar := &staticCookieJar{cookies: []Cookie{
|
||||||
Cookie{Host: "example.com", Path: "/", Name: "a", Value: "1"},
|
{Host: "example.com", Path: "/", Name: "a", Value: "1"},
|
||||||
}
|
}}
|
||||||
err := jar.Set(Cookie{Host: "example.com", Path: "/", Name: "a", Value: "2"})
|
err := jar.Set(Cookie{Host: "example.com", Path: "/", Name: "a", Value: "2"})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("Set() error: %v", err)
|
t.Fatalf("Set() error: %v", err)
|
||||||
@@ -168,10 +168,10 @@ func TestStaticCookieJar_Set_Update(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestStaticCookieJar_Delete(t *testing.T) {
|
func TestStaticCookieJar_Delete(t *testing.T) {
|
||||||
jar := &staticCookieJar{
|
jar := &staticCookieJar{cookies: []Cookie{
|
||||||
Cookie{Host: "example.com", Path: "/", Name: "a", Value: "1"},
|
{Host: "example.com", Path: "/", Name: "a", Value: "1"},
|
||||||
Cookie{Host: "other.com", Path: "/", Name: "b", Value: "2"},
|
{Host: "other.com", Path: "/", Name: "b", Value: "2"},
|
||||||
}
|
}}
|
||||||
err := jar.Delete(Cookie{Host: "example.com", Path: "/", Name: "a"})
|
err := jar.Delete(Cookie{Host: "example.com", Path: "/", Name: "a"})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("Delete() error: %v", err)
|
t.Fatalf("Delete() error: %v", err)
|
||||||
@@ -187,9 +187,9 @@ func TestStaticCookieJar_Delete(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestStaticCookieJar_Delete_NotFound(t *testing.T) {
|
func TestStaticCookieJar_Delete_NotFound(t *testing.T) {
|
||||||
jar := &staticCookieJar{
|
jar := &staticCookieJar{cookies: []Cookie{
|
||||||
Cookie{Host: "example.com", Path: "/", Name: "a", Value: "1"},
|
{Host: "example.com", Path: "/", Name: "a", Value: "1"},
|
||||||
}
|
}}
|
||||||
err := jar.Delete(Cookie{Host: "nonexistent.com", Path: "/", Name: "x"})
|
err := jar.Delete(Cookie{Host: "nonexistent.com", Path: "/", Name: "x"})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("Delete() error: %v", err)
|
t.Fatalf("Delete() error: %v", err)
|
||||||
@@ -202,9 +202,9 @@ func TestStaticCookieJar_Delete_NotFound(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestReadOnlyCookieJar_SetIsNoop(t *testing.T) {
|
func TestReadOnlyCookieJar_SetIsNoop(t *testing.T) {
|
||||||
inner := &staticCookieJar{
|
inner := &staticCookieJar{cookies: []Cookie{
|
||||||
Cookie{Host: "example.com", Path: "/", Name: "a", Value: "1"},
|
{Host: "example.com", Path: "/", Name: "a", Value: "1"},
|
||||||
}
|
}}
|
||||||
ro := ReadOnlyCookieJar{Jar: inner}
|
ro := ReadOnlyCookieJar{Jar: inner}
|
||||||
|
|
||||||
err := ro.Set(Cookie{Host: "example.com", Path: "/", Name: "new", Value: "val"})
|
err := ro.Set(Cookie{Host: "example.com", Path: "/", Name: "new", Value: "val"})
|
||||||
@@ -219,9 +219,9 @@ func TestReadOnlyCookieJar_SetIsNoop(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestReadOnlyCookieJar_DeleteIsNoop(t *testing.T) {
|
func TestReadOnlyCookieJar_DeleteIsNoop(t *testing.T) {
|
||||||
inner := &staticCookieJar{
|
inner := &staticCookieJar{cookies: []Cookie{
|
||||||
Cookie{Host: "example.com", Path: "/", Name: "a", Value: "1"},
|
{Host: "example.com", Path: "/", Name: "a", Value: "1"},
|
||||||
}
|
}}
|
||||||
ro := ReadOnlyCookieJar{Jar: inner}
|
ro := ReadOnlyCookieJar{Jar: inner}
|
||||||
|
|
||||||
err := ro.Delete(Cookie{Host: "example.com", Path: "/", Name: "a"})
|
err := ro.Delete(Cookie{Host: "example.com", Path: "/", Name: "a"})
|
||||||
@@ -236,9 +236,9 @@ func TestReadOnlyCookieJar_DeleteIsNoop(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestReadOnlyCookieJar_GetAll(t *testing.T) {
|
func TestReadOnlyCookieJar_GetAll(t *testing.T) {
|
||||||
inner := &staticCookieJar{
|
inner := &staticCookieJar{cookies: []Cookie{
|
||||||
Cookie{Host: "example.com", Path: "/", Name: "a", Value: "1"},
|
{Host: "example.com", Path: "/", Name: "a", Value: "1"},
|
||||||
}
|
}}
|
||||||
ro := ReadOnlyCookieJar{Jar: inner}
|
ro := ReadOnlyCookieJar{Jar: inner}
|
||||||
|
|
||||||
cookies, err := ro.GetAll()
|
cookies, err := ro.GetAll()
|
||||||
@@ -251,9 +251,9 @@ func TestReadOnlyCookieJar_GetAll(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestReadOnlyCookieJar_Get(t *testing.T) {
|
func TestReadOnlyCookieJar_Get(t *testing.T) {
|
||||||
inner := &staticCookieJar{
|
inner := &staticCookieJar{cookies: []Cookie{
|
||||||
Cookie{Host: "example.com", Path: "/", Name: "a", Value: "1"},
|
{Host: "example.com", Path: "/", Name: "a", Value: "1"},
|
||||||
}
|
}}
|
||||||
ro := ReadOnlyCookieJar{Jar: inner}
|
ro := ReadOnlyCookieJar{Jar: inner}
|
||||||
|
|
||||||
cookies, err := ro.Get("https://example.com/page")
|
cookies, err := ro.Get("https://example.com/page")
|
||||||
|
|||||||
@@ -6,21 +6,32 @@ import (
|
|||||||
"os"
|
"os"
|
||||||
"strconv"
|
"strconv"
|
||||||
"strings"
|
"strings"
|
||||||
|
"sync"
|
||||||
"time"
|
"time"
|
||||||
)
|
)
|
||||||
|
|
||||||
type staticCookieJar []Cookie
|
type staticCookieJar struct {
|
||||||
|
mu sync.RWMutex
|
||||||
|
cookies []Cookie
|
||||||
|
}
|
||||||
|
|
||||||
// GetAll will return all cookies in the jar.
|
// GetAll will return all cookies in the jar.
|
||||||
func (s *staticCookieJar) GetAll() ([]Cookie, error) {
|
func (s *staticCookieJar) GetAll() ([]Cookie, error) {
|
||||||
return *s, nil
|
s.mu.RLock()
|
||||||
|
defer s.mu.RUnlock()
|
||||||
|
out := make([]Cookie, len(s.cookies))
|
||||||
|
copy(out, s.cookies)
|
||||||
|
return out, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// Get will, given a URL, return all cookies that are valid for that URL.
|
// Get will, given a URL, return all cookies that are valid for that URL.
|
||||||
func (s *staticCookieJar) Get(target string) ([]Cookie, error) {
|
func (s *staticCookieJar) Get(target string) ([]Cookie, error) {
|
||||||
|
s.mu.RLock()
|
||||||
|
defer s.mu.RUnlock()
|
||||||
|
|
||||||
var validCookies []Cookie
|
var validCookies []Cookie
|
||||||
|
|
||||||
for _, cookie := range *s {
|
for _, cookie := range s.cookies {
|
||||||
if match, err := cookie.IsTargetMatch(target); err != nil {
|
if match, err := cookie.IsTargetMatch(target); err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
} else if match {
|
} else if match {
|
||||||
@@ -32,22 +43,28 @@ func (s *staticCookieJar) Get(target string) ([]Cookie, error) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func (s *staticCookieJar) Set(cookie Cookie) error {
|
func (s *staticCookieJar) Set(cookie Cookie) error {
|
||||||
|
s.mu.Lock()
|
||||||
|
defer s.mu.Unlock()
|
||||||
|
|
||||||
// see if the cookie already exists
|
// see if the cookie already exists
|
||||||
for i, c := range *s {
|
for i, c := range s.cookies {
|
||||||
if c.Name == cookie.Name && c.Host == cookie.Host && c.Path == cookie.Path {
|
if c.Name == cookie.Name && c.Host == cookie.Host && c.Path == cookie.Path {
|
||||||
(*s)[i] = cookie
|
s.cookies[i] = cookie
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
*s = append(*s, cookie)
|
s.cookies = append(s.cookies, cookie)
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (s *staticCookieJar) Delete(cookie Cookie) error {
|
func (s *staticCookieJar) Delete(cookie Cookie) error {
|
||||||
for i, c := range *s {
|
s.mu.Lock()
|
||||||
|
defer s.mu.Unlock()
|
||||||
|
|
||||||
|
for i, c := range s.cookies {
|
||||||
if c.Name == cookie.Name && c.Host == cookie.Host && c.Path == cookie.Path {
|
if c.Name == cookie.Name && c.Host == cookie.Host && c.Path == cookie.Path {
|
||||||
*s = append((*s)[:i], (*s)[i+1:]...)
|
s.cookies = append(s.cookies[:i], s.cookies[i+1:]...)
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -66,7 +83,7 @@ func LoadCookiesFile(path string) (CookieJar, error) {
|
|||||||
_ = cl.Close()
|
_ = cl.Close()
|
||||||
}(fp)
|
}(fp)
|
||||||
|
|
||||||
var cookies staticCookieJar
|
var cookies []Cookie
|
||||||
|
|
||||||
scanner := bufio.NewScanner(fp)
|
scanner := bufio.NewScanner(fp)
|
||||||
|
|
||||||
@@ -102,5 +119,5 @@ func LoadCookiesFile(path string) (CookieJar, error) {
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
return &cookies, nil
|
return &staticCookieJar{cookies: cookies}, nil
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -72,8 +72,37 @@ type BrowserOptions struct {
|
|||||||
UseLocalOnly bool
|
UseLocalOnly bool
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func sameSiteToPlaywright(s SameSite) *playwright.SameSiteAttribute {
|
||||||
|
switch s {
|
||||||
|
case SameSiteStrict:
|
||||||
|
return playwright.SameSiteAttributeStrict
|
||||||
|
case SameSiteLax:
|
||||||
|
return playwright.SameSiteAttributeLax
|
||||||
|
case SameSiteNone:
|
||||||
|
return playwright.SameSiteAttributeNone
|
||||||
|
default:
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func playwrightSameSiteToSameSite(s *playwright.SameSiteAttribute) SameSite {
|
||||||
|
if s == nil {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
switch *s {
|
||||||
|
case *playwright.SameSiteAttributeStrict:
|
||||||
|
return SameSiteStrict
|
||||||
|
case *playwright.SameSiteAttributeLax:
|
||||||
|
return SameSiteLax
|
||||||
|
case *playwright.SameSiteAttributeNone:
|
||||||
|
return SameSiteNone
|
||||||
|
default:
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func cookieToPlaywrightOptionalCookie(cookie Cookie) playwright.OptionalCookie {
|
func cookieToPlaywrightOptionalCookie(cookie Cookie) playwright.OptionalCookie {
|
||||||
return playwright.OptionalCookie{
|
oc := playwright.OptionalCookie{
|
||||||
Name: cookie.Name,
|
Name: cookie.Name,
|
||||||
Value: cookie.Value,
|
Value: cookie.Value,
|
||||||
Domain: playwright.String(cookie.Host),
|
Domain: playwright.String(cookie.Host),
|
||||||
@@ -81,6 +110,10 @@ func cookieToPlaywrightOptionalCookie(cookie Cookie) playwright.OptionalCookie {
|
|||||||
Expires: playwright.Float(float64(cookie.Expires.Unix())),
|
Expires: playwright.Float(float64(cookie.Expires.Unix())),
|
||||||
HttpOnly: playwright.Bool(cookie.HttpOnly),
|
HttpOnly: playwright.Bool(cookie.HttpOnly),
|
||||||
}
|
}
|
||||||
|
if cookie.SameSite != "" {
|
||||||
|
oc.SameSite = sameSiteToPlaywright(cookie.SameSite)
|
||||||
|
}
|
||||||
|
return oc
|
||||||
}
|
}
|
||||||
|
|
||||||
func playwrightCookieToCookie(cookie playwright.Cookie) Cookie {
|
func playwrightCookieToCookie(cookie playwright.Cookie) Cookie {
|
||||||
@@ -91,6 +124,7 @@ func playwrightCookieToCookie(cookie playwright.Cookie) Cookie {
|
|||||||
Path: cookie.Path,
|
Path: cookie.Path,
|
||||||
Expires: time.Unix(int64(cookie.Expires), 0),
|
Expires: time.Unix(int64(cookie.Expires), 0),
|
||||||
HttpOnly: cookie.HttpOnly,
|
HttpOnly: cookie.HttpOnly,
|
||||||
|
SameSite: playwrightSameSiteToSameSite(cookie.SameSite),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -5,6 +5,7 @@ import (
|
|||||||
"fmt"
|
"fmt"
|
||||||
"log/slog"
|
"log/slog"
|
||||||
"net/url"
|
"net/url"
|
||||||
|
"strings"
|
||||||
|
|
||||||
"gitea.stevedudenhoeffer.com/steve/go-extractor"
|
"gitea.stevedudenhoeffer.com/steve/go-extractor"
|
||||||
)
|
)
|
||||||
@@ -65,24 +66,7 @@ func (c Config) Search(ctx context.Context, b extractor.Browser, query string) (
|
|||||||
}
|
}
|
||||||
|
|
||||||
if c.Country != "" {
|
if c.Country != "" {
|
||||||
country := ""
|
vals.Set("cr", "country"+strings.ToUpper(c.Country))
|
||||||
switch c.Country {
|
|
||||||
case "us":
|
|
||||||
country = "countryUS"
|
|
||||||
|
|
||||||
case "uk":
|
|
||||||
country = "countryUK"
|
|
||||||
|
|
||||||
case "au":
|
|
||||||
country = "countryAU"
|
|
||||||
|
|
||||||
case "ca":
|
|
||||||
country = "countryCA"
|
|
||||||
}
|
|
||||||
|
|
||||||
if country != "" {
|
|
||||||
vals.Set("cr", country)
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
u.RawQuery = vals.Encode()
|
u.RawQuery = vals.Encode()
|
||||||
|
|||||||
Reference in New Issue
Block a user