From 0fb3287cef3346f7442a26667264904c9d1b613e Mon Sep 17 00:00:00 2001 From: sxueck Date: Wed, 30 Jun 2021 23:30:32 +0800 Subject: [PATCH] feat. colly execute add proxy --- search.go | 13 +++++++++++++ search_test.go | 3 ++- 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/search.go b/search.go index 7a3ddac..7b533d4 100644 --- a/search.go +++ b/search.go @@ -8,6 +8,7 @@ import ( "strings" "github.com/gocolly/colly/v2" + "github.com/gocolly/colly/v2/proxy" ) // Result represents a single result from Google Search. @@ -254,6 +255,9 @@ type SearchOptions struct { // OverLimit searches for more results than that specified by Limit. // It then reduces the returned results to match Limit. OverLimit bool + + // set up proxy address to avoid ip blocking + ProxyAddr string } // Search returns a list of search results from Google. @@ -329,6 +333,15 @@ func Search(ctx context.Context, searchTerm string, opts ...SearchOptions) ([]Re } url := url(searchTerm, opts[0].CountryCode, lc, limit, opts[0].Start) + + if opts[0].ProxyAddr != "" { + rp, err := proxy.RoundRobinProxySwitcher(opts[0].ProxyAddr) + if err != nil { + return nil, err + } + c.SetProxyFunc(rp) + } + c.Visit(url) if rErr != nil { diff --git a/search_test.go b/search_test.go index 8dc52cb..90d17f5 100644 --- a/search_test.go +++ b/search_test.go @@ -16,7 +16,8 @@ func TestSearch(t *testing.T) { q := "Hello World" opts := googlesearch.SearchOptions{ - Limit: 20, + Limit: 20, + ProxyAddr: "socks://127.0.0.1:7890", } returnLinks, err := googlesearch.Search(ctx, q, opts)