From 0fb3287cef3346f7442a26667264904c9d1b613e Mon Sep 17 00:00:00 2001 From: sxueck Date: Wed, 30 Jun 2021 23:30:32 +0800 Subject: [PATCH 1/2] feat. colly execute add proxy --- search.go | 13 +++++++++++++ search_test.go | 3 ++- 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/search.go b/search.go index 7a3ddac..7b533d4 100644 --- a/search.go +++ b/search.go @@ -8,6 +8,7 @@ import ( "strings" "github.com/gocolly/colly/v2" + "github.com/gocolly/colly/v2/proxy" ) // Result represents a single result from Google Search. @@ -254,6 +255,9 @@ type SearchOptions struct { // OverLimit searches for more results than that specified by Limit. // It then reduces the returned results to match Limit. OverLimit bool + + // set up proxy address to avoid ip blocking + ProxyAddr string } // Search returns a list of search results from Google. @@ -329,6 +333,15 @@ func Search(ctx context.Context, searchTerm string, opts ...SearchOptions) ([]Re } url := url(searchTerm, opts[0].CountryCode, lc, limit, opts[0].Start) + + if opts[0].ProxyAddr != "" { + rp, err := proxy.RoundRobinProxySwitcher(opts[0].ProxyAddr) + if err != nil { + return nil, err + } + c.SetProxyFunc(rp) + } + c.Visit(url) if rErr != nil { diff --git a/search_test.go b/search_test.go index 8dc52cb..90d17f5 100644 --- a/search_test.go +++ b/search_test.go @@ -16,7 +16,8 @@ func TestSearch(t *testing.T) { q := "Hello World" opts := googlesearch.SearchOptions{ - Limit: 20, + Limit: 20, + ProxyAddr: "socks://127.0.0.1:7890", } returnLinks, err := googlesearch.Search(ctx, q, opts) From 8238e33edd13f8c8e86752a31ea2cc368479e2d6 Mon Sep 17 00:00:00 2001 From: rocketlaunchr-cto <38447948+rocketlaunchr-cto@users.noreply.github.com> Date: Thu, 1 Jul 2021 11:58:03 +1000 Subject: [PATCH 2/2] - update comment for docs --- search.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/search.go b/search.go index 7b533d4..13c1d2f 100644 --- a/search.go +++ b/search.go @@ -256,7 +256,7 @@ type SearchOptions struct { // It then reduces the returned results to match Limit. OverLimit bool - // set up proxy address to avoid ip blocking + // ProxyAddr sets a proxy address to avoid IP blocking. ProxyAddr string }