steve
841f1ec2bf
DuckDuckGo intermittently serves a CAPTCHA modal ("Unfortunately, bots
use DuckDuckGo too...") instead of search results. The result selector
matches zero elements on that page, so callers used to get
([]Result{}, nil) — silent empty results that look like "no matches."
Detect the challenge via the BEM class .anomaly-modal__title and return
a typed ErrBlocked so callers can distinguish blocked from no-results
and react (retry, fallback to another engine, surface to user).
Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
105 lines
2.2 KiB
Go
105 lines
2.2 KiB
Go
package duckduckgo
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"fmt"
|
|
"log/slog"
|
|
"net/url"
|
|
|
|
"gitea.stevedudenhoeffer.com/steve/go-extractor"
|
|
)
|
|
|
|
// ErrBlocked is returned when DuckDuckGo serves an anti-bot challenge
|
|
// page instead of search results.
|
|
var ErrBlocked = errors.New("duckduckgo: blocked by anti-bot challenge")
|
|
|
|
type SafeSearch int
|
|
|
|
const (
|
|
SafeSearchOn SafeSearch = 1
|
|
SafeSearchModerate SafeSearch = -1
|
|
SafeSearchOff SafeSearch = -2
|
|
)
|
|
|
|
type Config struct {
|
|
// SafeSearch is the safe-search level to use. If empty, SafeSearchOff will be used.
|
|
SafeSearch SafeSearch
|
|
|
|
// Region is the region to use for the search engine.
|
|
// See: https://duckduckgo.com/duckduckgo-help-pages/settings/params/ for more values
|
|
Region string
|
|
}
|
|
|
|
func (c Config) validate() Config {
|
|
if c.SafeSearch == 0 {
|
|
c.SafeSearch = SafeSearchOff
|
|
}
|
|
|
|
return c
|
|
}
|
|
func (c Config) ToSearchURL(query string) *url.URL {
|
|
c = c.validate()
|
|
|
|
res, _ := url.Parse("https://duckduckgo.com/")
|
|
|
|
var vals = res.Query()
|
|
|
|
switch c.SafeSearch {
|
|
case SafeSearchOn:
|
|
vals.Set("kp", "1")
|
|
case SafeSearchModerate:
|
|
vals.Set("kp", "-1")
|
|
case SafeSearchOff:
|
|
vals.Set("kp", "-2")
|
|
}
|
|
|
|
if c.Region != "" {
|
|
vals.Set("kl", c.Region)
|
|
}
|
|
|
|
vals.Set("q", query)
|
|
|
|
res.RawQuery = vals.Encode()
|
|
|
|
return res
|
|
}
|
|
|
|
var DefaultConfig = Config{
|
|
SafeSearch: SafeSearchOff,
|
|
}
|
|
|
|
type Result struct {
|
|
URL string
|
|
Title string
|
|
Description string
|
|
}
|
|
|
|
func (c Config) OpenSearch(ctx context.Context, b extractor.Browser, query string) (SearchPage, error) {
|
|
u := c.ToSearchURL(query)
|
|
|
|
slog.Info("searching", "url", u, "query", query, "config", c, "browser", b)
|
|
doc, err := b.Open(ctx, u.String(), extractor.OpenPageOptions{})
|
|
if err != nil {
|
|
if doc != nil {
|
|
_ = doc.Close()
|
|
}
|
|
return nil, fmt.Errorf("failed to open url: %w", err)
|
|
}
|
|
|
|
return searchPage{doc}, nil
|
|
}
|
|
|
|
func (c Config) Search(ctx context.Context, b extractor.Browser, query string) ([]Result, error) {
|
|
u := c.ToSearchURL(query)
|
|
|
|
slog.Info("searching", "url", u, "query", query, "config", c, "browser", b)
|
|
doc, err := b.Open(ctx, u.String(), extractor.OpenPageOptions{})
|
|
if err != nil {
|
|
return nil, fmt.Errorf("failed to open url: %w", err)
|
|
}
|
|
defer extractor.DeferClose(doc)
|
|
|
|
return extractResults(doc)
|
|
}
|