steve/go-extractor
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request 'Thread-safe CookieJar, SameSite, Google countries' (#42) from enhance/cookies-and-google into main
All checks were successful
CI / vet (push) Successful in 41s
Details
CI / build (push) Successful in 1m23s
Details
CI / test (push) Successful in 1m24s
Details

Browse Source
This commit was merged in pull request #42.
This commit is contained in:
Claude
2026-02-15 16:35:10 +00:00
parent 0ba9cc9b98 963696cd62
commit ddb701fca0
5 changed files with 104 additions and 59 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
cookiejar.go
View File

@@ -6,6 +6,15 @@ import (
"time"
)
// SameSite represents the SameSite attribute of a cookie.
type SameSite string
const (
SameSiteStrict SameSite = "Strict"
SameSiteLax SameSite = "Lax"
SameSiteNone SameSite = "None"
)
type Cookie struct {
Host string
Path string
@@ -14,6 +23,7 @@ type Cookie struct {
HttpOnly bool
Name string
Value string
SameSite SameSite
}
func (c Cookie) IsTargetMatch(target string) (bool, error) {

60
cookiejar_test.go
View File

@@ -101,10 +101,10 @@ func TestCookie_IsTargetMatch_InvalidURL(t *testing.T) {
}
func TestStaticCookieJar_GetAll(t *testing.T) {
jar := &staticCookieJar{
Cookie{Host: "a.com", Name: "a", Value: "1"},
Cookie{Host: "b.com", Name: "b", Value: "2"},
}
jar := &staticCookieJar{cookies: []Cookie{
{Host: "a.com", Name: "a", Value: "1"},
{Host: "b.com", Name: "b", Value: "2"},
}}
cookies, err := jar.GetAll()
if err != nil {
@@ -116,10 +116,10 @@ func TestStaticCookieJar_GetAll(t *testing.T) {
}
func TestStaticCookieJar_Get(t *testing.T) {
jar := &staticCookieJar{
Cookie{Host: "example.com", Path: "/", Name: "a", Value: "1"},
Cookie{Host: "other.com", Path: "/", Name: "b", Value: "2"},
}
jar := &staticCookieJar{cookies: []Cookie{
{Host: "example.com", Path: "/", Name: "a", Value: "1"},
{Host: "other.com", Path: "/", Name: "b", Value: "2"},
}}
cookies, err := jar.Get("https://example.com/page")
if err != nil {
@@ -150,9 +150,9 @@ func TestStaticCookieJar_Set_New(t *testing.T) {
}
func TestStaticCookieJar_Set_Update(t *testing.T) {
jar := &staticCookieJar{
Cookie{Host: "example.com", Path: "/", Name: "a", Value: "1"},
}
jar := &staticCookieJar{cookies: []Cookie{
{Host: "example.com", Path: "/", Name: "a", Value: "1"},
}}
err := jar.Set(Cookie{Host: "example.com", Path: "/", Name: "a", Value: "2"})
if err != nil {
t.Fatalf("Set() error: %v", err)
@@ -168,10 +168,10 @@ func TestStaticCookieJar_Set_Update(t *testing.T) {
}
func TestStaticCookieJar_Delete(t *testing.T) {
jar := &staticCookieJar{
Cookie{Host: "example.com", Path: "/", Name: "a", Value: "1"},
Cookie{Host: "other.com", Path: "/", Name: "b", Value: "2"},
}
jar := &staticCookieJar{cookies: []Cookie{
{Host: "example.com", Path: "/", Name: "a", Value: "1"},
{Host: "other.com", Path: "/", Name: "b", Value: "2"},
}}
err := jar.Delete(Cookie{Host: "example.com", Path: "/", Name: "a"})
if err != nil {
t.Fatalf("Delete() error: %v", err)
@@ -187,9 +187,9 @@ func TestStaticCookieJar_Delete(t *testing.T) {
}
func TestStaticCookieJar_Delete_NotFound(t *testing.T) {
jar := &staticCookieJar{
Cookie{Host: "example.com", Path: "/", Name: "a", Value: "1"},
}
jar := &staticCookieJar{cookies: []Cookie{
{Host: "example.com", Path: "/", Name: "a", Value: "1"},
}}
err := jar.Delete(Cookie{Host: "nonexistent.com", Path: "/", Name: "x"})
if err != nil {
t.Fatalf("Delete() error: %v", err)
@@ -202,9 +202,9 @@ func TestStaticCookieJar_Delete_NotFound(t *testing.T) {
}
func TestReadOnlyCookieJar_SetIsNoop(t *testing.T) {
inner := &staticCookieJar{
Cookie{Host: "example.com", Path: "/", Name: "a", Value: "1"},
}
inner := &staticCookieJar{cookies: []Cookie{
{Host: "example.com", Path: "/", Name: "a", Value: "1"},
}}
ro := ReadOnlyCookieJar{Jar: inner}
err := ro.Set(Cookie{Host: "example.com", Path: "/", Name: "new", Value: "val"})
@@ -219,9 +219,9 @@ func TestReadOnlyCookieJar_SetIsNoop(t *testing.T) {
}
func TestReadOnlyCookieJar_DeleteIsNoop(t *testing.T) {
inner := &staticCookieJar{
Cookie{Host: "example.com", Path: "/", Name: "a", Value: "1"},
}
inner := &staticCookieJar{cookies: []Cookie{
{Host: "example.com", Path: "/", Name: "a", Value: "1"},
}}
ro := ReadOnlyCookieJar{Jar: inner}
err := ro.Delete(Cookie{Host: "example.com", Path: "/", Name: "a"})
@@ -236,9 +236,9 @@ func TestReadOnlyCookieJar_DeleteIsNoop(t *testing.T) {
}
func TestReadOnlyCookieJar_GetAll(t *testing.T) {
inner := &staticCookieJar{
Cookie{Host: "example.com", Path: "/", Name: "a", Value: "1"},
}
inner := &staticCookieJar{cookies: []Cookie{
{Host: "example.com", Path: "/", Name: "a", Value: "1"},
}}
ro := ReadOnlyCookieJar{Jar: inner}
cookies, err := ro.GetAll()
@@ -251,9 +251,9 @@ func TestReadOnlyCookieJar_GetAll(t *testing.T) {
}
func TestReadOnlyCookieJar_Get(t *testing.T) {
inner := &staticCookieJar{
Cookie{Host: "example.com", Path: "/", Name: "a", Value: "1"},
}
inner := &staticCookieJar{cookies: []Cookie{
{Host: "example.com", Path: "/", Name: "a", Value: "1"},
}}
ro := ReadOnlyCookieJar{Jar: inner}
cookies, err := ro.Get("https://example.com/page")

37
cookies_txt.go
View File

@@ -6,21 +6,32 @@ import (
"os"
"strconv"
"strings"
"sync"
"time"
)
type staticCookieJar []Cookie
type staticCookieJar struct {
mu sync.RWMutex
cookies []Cookie
}
// GetAll will return all cookies in the jar.
func (s *staticCookieJar) GetAll() ([]Cookie, error) {
return *s, nil
s.mu.RLock()
defer s.mu.RUnlock()
out := make([]Cookie, len(s.cookies))
copy(out, s.cookies)
return out, nil
}
// Get will, given a URL, return all cookies that are valid for that URL.
func (s *staticCookieJar) Get(target string) ([]Cookie, error) {
s.mu.RLock()
defer s.mu.RUnlock()
var validCookies []Cookie
for _, cookie := range *s {
for _, cookie := range s.cookies {
if match, err := cookie.IsTargetMatch(target); err != nil {
return nil, err
} else if match {
@@ -32,22 +43,28 @@ func (s *staticCookieJar) Get(target string) ([]Cookie, error) {
}
func (s *staticCookieJar) Set(cookie Cookie) error {
s.mu.Lock()
defer s.mu.Unlock()
// see if the cookie already exists
for i, c := range *s {
for i, c := range s.cookies {
if c.Name == cookie.Name && c.Host == cookie.Host && c.Path == cookie.Path {
(*s)[i] = cookie
s.cookies[i] = cookie
return nil
}
}
*s = append(*s, cookie)
s.cookies = append(s.cookies, cookie)
return nil
}
func (s *staticCookieJar) Delete(cookie Cookie) error {
for i, c := range *s {
s.mu.Lock()
defer s.mu.Unlock()
for i, c := range s.cookies {
if c.Name == cookie.Name && c.Host == cookie.Host && c.Path == cookie.Path {
*s = append((*s)[:i], (*s)[i+1:]...)
s.cookies = append(s.cookies[:i], s.cookies[i+1:]...)
return nil
}
}
@@ -66,7 +83,7 @@ func LoadCookiesFile(path string) (CookieJar, error) {
_ = cl.Close()
}(fp)
var cookies staticCookieJar
var cookies []Cookie
scanner := bufio.NewScanner(fp)
@@ -102,5 +119,5 @@ func LoadCookiesFile(path string) (CookieJar, error) {
})
}
return &cookies, nil
return &staticCookieJar{cookies: cookies}, nil
}

36
playwright.go
View File

@@ -72,8 +72,37 @@ type BrowserOptions struct {
UseLocalOnly bool
}
func sameSiteToPlaywright(s SameSite) *playwright.SameSiteAttribute {
switch s {
case SameSiteStrict:
return playwright.SameSiteAttributeStrict
case SameSiteLax:
return playwright.SameSiteAttributeLax
case SameSiteNone:
return playwright.SameSiteAttributeNone
default:
return nil
}
}
func playwrightSameSiteToSameSite(s *playwright.SameSiteAttribute) SameSite {
if s == nil {
return ""
}
switch *s {
case *playwright.SameSiteAttributeStrict:
return SameSiteStrict
case *playwright.SameSiteAttributeLax:
return SameSiteLax
case *playwright.SameSiteAttributeNone:
return SameSiteNone
default:
return ""
}
}
func cookieToPlaywrightOptionalCookie(cookie Cookie) playwright.OptionalCookie {
return playwright.OptionalCookie{
oc := playwright.OptionalCookie{
Name: cookie.Name,
Value: cookie.Value,
Domain: playwright.String(cookie.Host),
@@ -81,6 +110,10 @@ func cookieToPlaywrightOptionalCookie(cookie Cookie) playwright.OptionalCookie {
Expires: playwright.Float(float64(cookie.Expires.Unix())),
HttpOnly: playwright.Bool(cookie.HttpOnly),
}
if cookie.SameSite != "" {
oc.SameSite = sameSiteToPlaywright(cookie.SameSite)
}
return oc
}
func playwrightCookieToCookie(cookie playwright.Cookie) Cookie {
@@ -91,6 +124,7 @@ func playwrightCookieToCookie(cookie playwright.Cookie) Cookie {
Path: cookie.Path,
Expires: time.Unix(int64(cookie.Expires), 0),
HttpOnly: cookie.HttpOnly,
SameSite: playwrightSameSiteToSameSite(cookie.SameSite),
}
}

20
sites/google/google.go
View File

@@ -5,6 +5,7 @@ import (
"fmt"
"log/slog"
"net/url"
"strings"
"gitea.stevedudenhoeffer.com/steve/go-extractor"
)
@@ -65,24 +66,7 @@ func (c Config) Search(ctx context.Context, b extractor.Browser, query string) (
}
if c.Country != "" {
country := ""
switch c.Country {
case "us":
country = "countryUS"
case "uk":
country = "countryUK"
case "au":
country = "countryAU"
case "ca":
country = "countryCA"
}
if country != "" {
vals.Set("cr", country)
}
vals.Set("cr", "country"+strings.ToUpper(c.Country))
}
u.RawQuery = vals.Encode()