feature: add DuckDuckGo weather and stock widget extractors
Add weather.go with GetWeather() for extracting structured weather data (location, temp, conditions, forecast) and stock.go with GetStockQuote() and GetStockChart() for stock data extraction and chart screenshots. Both include mock-based tests. CSS selectors may need tuning against the live site since DuckDuckGo's React-rendered widgets use dynamic class names. Closes #25, #26 Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
144
sites/duckduckgo/stock.go
Normal file
144
sites/duckduckgo/stock.go
Normal file
@@ -0,0 +1,144 @@
|
||||
package duckduckgo
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"time"
|
||||
|
||||
"gitea.stevedudenhoeffer.com/steve/go-extractor"
|
||||
"gitea.stevedudenhoeffer.com/steve/go-extractor/sites/internal/parse"
|
||||
)
|
||||
|
||||
// StockData holds structured stock quote information from DuckDuckGo.
|
||||
type StockData struct {
|
||||
Symbol string
|
||||
Name string
|
||||
Price float64
|
||||
Change float64
|
||||
ChangePct float64
|
||||
}
|
||||
|
||||
// StockPeriod represents a time period for stock charts.
|
||||
type StockPeriod string
|
||||
|
||||
const (
|
||||
Period1D StockPeriod = "1D"
|
||||
Period5D StockPeriod = "5D"
|
||||
Period1M StockPeriod = "1M"
|
||||
PeriodYTD StockPeriod = "YTD"
|
||||
Period1Y StockPeriod = "1Y"
|
||||
Period5Y StockPeriod = "5Y"
|
||||
PeriodAll StockPeriod = "All"
|
||||
)
|
||||
|
||||
// GetStockQuote extracts structured stock data from DuckDuckGo's stock widget.
|
||||
func (c Config) GetStockQuote(ctx context.Context, b extractor.Browser, symbol string) (*StockData, error) {
|
||||
c = c.validate()
|
||||
|
||||
u := c.ToSearchURL(symbol + " stock")
|
||||
|
||||
slog.Info("fetching stock", "url", u, "symbol", symbol)
|
||||
doc, err := b.Open(ctx, u.String(), extractor.OpenPageOptions{})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to open stock page: %w", err)
|
||||
}
|
||||
defer extractor.DeferClose(doc)
|
||||
|
||||
timeout := 10 * time.Second
|
||||
if err := doc.WaitForNetworkIdle(&timeout); err != nil {
|
||||
slog.Warn("WaitForNetworkIdle failed", "err", err)
|
||||
}
|
||||
|
||||
return extractStock(doc)
|
||||
}
|
||||
|
||||
// GetStockQuote is a convenience function using DefaultConfig.
|
||||
func GetStockQuote(ctx context.Context, b extractor.Browser, symbol string) (*StockData, error) {
|
||||
return DefaultConfig.GetStockQuote(ctx, b, symbol)
|
||||
}
|
||||
|
||||
// GetStockChart screenshots the stock chart widget for a given period.
|
||||
func (c Config) GetStockChart(ctx context.Context, b extractor.Browser, symbol string, period StockPeriod) ([]byte, error) {
|
||||
c = c.validate()
|
||||
|
||||
u := c.ToSearchURL(symbol + " stock")
|
||||
|
||||
slog.Info("fetching stock chart", "url", u, "symbol", symbol, "period", period)
|
||||
doc, err := b.Open(ctx, u.String(), extractor.OpenPageOptions{})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to open stock page: %w", err)
|
||||
}
|
||||
defer extractor.DeferClose(doc)
|
||||
|
||||
timeout := 10 * time.Second
|
||||
if err := doc.WaitForNetworkIdle(&timeout); err != nil {
|
||||
slog.Warn("WaitForNetworkIdle failed", "err", err)
|
||||
}
|
||||
|
||||
// Click the period selector if not 1D (default)
|
||||
if period != Period1D {
|
||||
selector := fmt.Sprintf(`div.module__content button[data-period="%s"]`, string(period))
|
||||
_ = doc.ForEach(selector, func(n extractor.Node) error {
|
||||
return n.Click()
|
||||
})
|
||||
|
||||
// Wait for chart to update
|
||||
chartTimeout := 5 * time.Second
|
||||
if err := doc.WaitForNetworkIdle(&chartTimeout); err != nil {
|
||||
slog.Warn("WaitForNetworkIdle after period click failed", "err", err)
|
||||
}
|
||||
}
|
||||
|
||||
// Screenshot the stock module
|
||||
modules := doc.Select("div.module__content")
|
||||
if len(modules) == 0 {
|
||||
return nil, fmt.Errorf("stock module not found")
|
||||
}
|
||||
|
||||
return modules[0].Screenshot()
|
||||
}
|
||||
|
||||
// GetStockChart is a convenience function using DefaultConfig.
|
||||
func GetStockChart(ctx context.Context, b extractor.Browser, symbol string, period StockPeriod) ([]byte, error) {
|
||||
return DefaultConfig.GetStockChart(ctx, b, symbol, period)
|
||||
}
|
||||
|
||||
func extractStock(doc extractor.Node) (*StockData, error) {
|
||||
var data StockData
|
||||
|
||||
// Symbol
|
||||
syms := doc.Select("div.module--stocks .module__title__link")
|
||||
if len(syms) > 0 {
|
||||
data.Symbol, _ = syms[0].Text()
|
||||
}
|
||||
|
||||
// Name
|
||||
names := doc.Select("div.module--stocks .module__subtitle")
|
||||
if len(names) > 0 {
|
||||
data.Name, _ = names[0].Text()
|
||||
}
|
||||
|
||||
// Price
|
||||
prices := doc.Select("div.module--stocks .module__price")
|
||||
if len(prices) > 0 {
|
||||
txt, _ := prices[0].Text()
|
||||
data.Price = parse.NumericOnly(txt)
|
||||
}
|
||||
|
||||
// Change
|
||||
changes := doc.Select("div.module--stocks .module__price-change")
|
||||
if len(changes) > 0 {
|
||||
txt, _ := changes[0].Text()
|
||||
data.Change = parse.NumericOnly(txt)
|
||||
}
|
||||
|
||||
// Change percentage
|
||||
pcts := doc.Select("div.module--stocks .module__price-change-pct")
|
||||
if len(pcts) > 0 {
|
||||
txt, _ := pcts[0].Text()
|
||||
data.ChangePct = parse.NumericOnly(txt)
|
||||
}
|
||||
|
||||
return &data, nil
|
||||
}
|
||||
Reference in New Issue
Block a user