go-extractor/sites/google/google.go

package google

import (
	"context"
	"fmt"
	"io"
	"net/url"

	"gitea.stevedudenhoeffer.com/steve/go-extractor"
)

type Config struct {
	// BaseURL is the base URL for the search engine, if empty "google.com" is used
	BaseURL string

	// Language is the language to use for the search engine, if empty "en" is used
	Language string

	// Country is the country to use for the search engine, if empty "us" is used
	Country string
}

var DefaultConfig = Config{
	BaseURL:  "google.com",
	Language: "en",
	Country:  "us",
}

func (c Config) validate() Config {
	if c.BaseURL == "" {
		c.BaseURL = "google.com"
	}

	if c.Language == "" {
		c.Language = "en"
	}

	if c.Country == "" {
		c.Country = "us"
	}

	return c
}

type Result struct {
	URL         string
	Title       string
	Description string
}

func deferClose(cl io.Closer) {
	if cl != nil {
		_ = cl.Close()
	}
}

func (c Config) Search(ctx context.Context, b extractor.Browser, query string) ([]Result, error) {
	c = c.validate()

	u, err := url.Parse(fmt.Sprintf("https://%s/search?q=%s", c.BaseURL, query))

	if err != nil {
		return nil, fmt.Errorf("invalid url: %w", err)
	}

	if c.Language != "" {
		u.Query().Set("hl", c.Language)
	}

	if c.Country != "" {
		country := ""
		switch c.Country {
		case "us":
			country = "countryUS"

		case "uk":
			country = "countryUK"

		case "au":
			country = "countryAU"

		case "ca":
			country = "countryCA"
		}

		if country != "" {
			u.Query().Set("cr", country)
		}
	}

	doc, err := b.Open(ctx, u.String(), extractor.OpenPageOptions{})

	if err != nil {
		return nil, fmt.Errorf("failed to open url: %w", err)
	}

	defer deferClose(doc)

	var res []Result

	err = doc.ForEach("div.g", func(s extractor.Node) error {
		var u string
		var title string
		var desc string

		// get the first link in the div
		link := s.Select("a")

		if len(link) == 0 {
			return nil
		}

		u, err := link[0].Attr("href")

		if err != nil {
			return fmt.Errorf("failed to get link: %w", err)
		}

		titles := s.Select("div > div > div a > h3")

		if len(titles) != 0 {
			title, _ = titles[0].Text()
		}

		descs := s.Select("div:nth-child(1) > div:nth-child(2) > div:nth-child(1) > span:not([class])")

		if len(descs) != 0 {
			desc, _ = descs[0].Text()
		}

		res = append(res, Result{
			URL:         u,
			Title:       title,
			Description: desc,
		})
		return nil
	})

	return res, err
}

func Search(ctx context.Context, b extractor.Browser, query string) ([]Result, error) {
	return DefaultConfig.Search(ctx, b, query)
}