- Google DOM changed. Update scraping code.

This commit is contained in:
rocketlaunchr-cto 2020-10-04 10:25:55 +11:00
parent b9085ebbb0
commit d52bae27ed
2 changed files with 31 additions and 4 deletions

View File

@ -299,15 +299,13 @@ func Search(ctx context.Context, searchTerm string, opts ...SearchOptions) ([]Re
item := sel.Eq(i)
rDiv := item.Find("div.r")
rDiv := item.Find("div.rc")
linkHref, _ := rDiv.Find("a").Attr("href")
linkText := strings.TrimSpace(linkHref)
titleText := strings.TrimSpace(rDiv.Find("h3").Text())
sDiv := item.Find("div.s")
descText := strings.TrimSpace(sDiv.Find("span.st").Text())
descText := strings.TrimSpace(rDiv.Find("div > div > span > span").Text())
if linkText != "" && linkText != "#" {
result := Result{

29
search_test.go Normal file
View File

@ -0,0 +1,29 @@
package googlesearch_test
import (
"context"
"testing"
"github.com/rocketlaunchr/google-search"
)
var ctx = context.Background()
func TestSearch(t *testing.T) {
q := "Hello World"
opts := googlesearch.SearchOptions{
Limit: 20,
}
returnLinks, err := googlesearch.Search(ctx, q, opts)
if err != nil {
t.Errorf("something went wrong: %v", err)
return
}
if len(returnLinks) == 0 {
t.Errorf("no results returned: %v", returnLinks)
}
}