feature: add DuckDuckGo weather and stock widget extractors
Add weather.go with GetWeather() for extracting structured weather data (location, temp, conditions, forecast) and stock.go with GetStockQuote() and GetStockChart() for stock data extraction and chart screenshots. Both include mock-based tests. CSS selectors may need tuning against the live site since DuckDuckGo's React-rendered widgets use dynamic class names. Closes #25, #26 Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
144
sites/duckduckgo/stock.go
Normal file
144
sites/duckduckgo/stock.go
Normal file
@@ -0,0 +1,144 @@
|
||||
package duckduckgo
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"time"
|
||||
|
||||
"gitea.stevedudenhoeffer.com/steve/go-extractor"
|
||||
"gitea.stevedudenhoeffer.com/steve/go-extractor/sites/internal/parse"
|
||||
)
|
||||
|
||||
// StockData holds structured stock quote information from DuckDuckGo.
|
||||
type StockData struct {
|
||||
Symbol string
|
||||
Name string
|
||||
Price float64
|
||||
Change float64
|
||||
ChangePct float64
|
||||
}
|
||||
|
||||
// StockPeriod represents a time period for stock charts.
|
||||
type StockPeriod string
|
||||
|
||||
const (
|
||||
Period1D StockPeriod = "1D"
|
||||
Period5D StockPeriod = "5D"
|
||||
Period1M StockPeriod = "1M"
|
||||
PeriodYTD StockPeriod = "YTD"
|
||||
Period1Y StockPeriod = "1Y"
|
||||
Period5Y StockPeriod = "5Y"
|
||||
PeriodAll StockPeriod = "All"
|
||||
)
|
||||
|
||||
// GetStockQuote extracts structured stock data from DuckDuckGo's stock widget.
|
||||
func (c Config) GetStockQuote(ctx context.Context, b extractor.Browser, symbol string) (*StockData, error) {
|
||||
c = c.validate()
|
||||
|
||||
u := c.ToSearchURL(symbol + " stock")
|
||||
|
||||
slog.Info("fetching stock", "url", u, "symbol", symbol)
|
||||
doc, err := b.Open(ctx, u.String(), extractor.OpenPageOptions{})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to open stock page: %w", err)
|
||||
}
|
||||
defer extractor.DeferClose(doc)
|
||||
|
||||
timeout := 10 * time.Second
|
||||
if err := doc.WaitForNetworkIdle(&timeout); err != nil {
|
||||
slog.Warn("WaitForNetworkIdle failed", "err", err)
|
||||
}
|
||||
|
||||
return extractStock(doc)
|
||||
}
|
||||
|
||||
// GetStockQuote is a convenience function using DefaultConfig.
|
||||
func GetStockQuote(ctx context.Context, b extractor.Browser, symbol string) (*StockData, error) {
|
||||
return DefaultConfig.GetStockQuote(ctx, b, symbol)
|
||||
}
|
||||
|
||||
// GetStockChart screenshots the stock chart widget for a given period.
|
||||
func (c Config) GetStockChart(ctx context.Context, b extractor.Browser, symbol string, period StockPeriod) ([]byte, error) {
|
||||
c = c.validate()
|
||||
|
||||
u := c.ToSearchURL(symbol + " stock")
|
||||
|
||||
slog.Info("fetching stock chart", "url", u, "symbol", symbol, "period", period)
|
||||
doc, err := b.Open(ctx, u.String(), extractor.OpenPageOptions{})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to open stock page: %w", err)
|
||||
}
|
||||
defer extractor.DeferClose(doc)
|
||||
|
||||
timeout := 10 * time.Second
|
||||
if err := doc.WaitForNetworkIdle(&timeout); err != nil {
|
||||
slog.Warn("WaitForNetworkIdle failed", "err", err)
|
||||
}
|
||||
|
||||
// Click the period selector if not 1D (default)
|
||||
if period != Period1D {
|
||||
selector := fmt.Sprintf(`div.module__content button[data-period="%s"]`, string(period))
|
||||
_ = doc.ForEach(selector, func(n extractor.Node) error {
|
||||
return n.Click()
|
||||
})
|
||||
|
||||
// Wait for chart to update
|
||||
chartTimeout := 5 * time.Second
|
||||
if err := doc.WaitForNetworkIdle(&chartTimeout); err != nil {
|
||||
slog.Warn("WaitForNetworkIdle after period click failed", "err", err)
|
||||
}
|
||||
}
|
||||
|
||||
// Screenshot the stock module
|
||||
modules := doc.Select("div.module__content")
|
||||
if len(modules) == 0 {
|
||||
return nil, fmt.Errorf("stock module not found")
|
||||
}
|
||||
|
||||
return modules[0].Screenshot()
|
||||
}
|
||||
|
||||
// GetStockChart is a convenience function using DefaultConfig.
|
||||
func GetStockChart(ctx context.Context, b extractor.Browser, symbol string, period StockPeriod) ([]byte, error) {
|
||||
return DefaultConfig.GetStockChart(ctx, b, symbol, period)
|
||||
}
|
||||
|
||||
func extractStock(doc extractor.Node) (*StockData, error) {
|
||||
var data StockData
|
||||
|
||||
// Symbol
|
||||
syms := doc.Select("div.module--stocks .module__title__link")
|
||||
if len(syms) > 0 {
|
||||
data.Symbol, _ = syms[0].Text()
|
||||
}
|
||||
|
||||
// Name
|
||||
names := doc.Select("div.module--stocks .module__subtitle")
|
||||
if len(names) > 0 {
|
||||
data.Name, _ = names[0].Text()
|
||||
}
|
||||
|
||||
// Price
|
||||
prices := doc.Select("div.module--stocks .module__price")
|
||||
if len(prices) > 0 {
|
||||
txt, _ := prices[0].Text()
|
||||
data.Price = parse.NumericOnly(txt)
|
||||
}
|
||||
|
||||
// Change
|
||||
changes := doc.Select("div.module--stocks .module__price-change")
|
||||
if len(changes) > 0 {
|
||||
txt, _ := changes[0].Text()
|
||||
data.Change = parse.NumericOnly(txt)
|
||||
}
|
||||
|
||||
// Change percentage
|
||||
pcts := doc.Select("div.module--stocks .module__price-change-pct")
|
||||
if len(pcts) > 0 {
|
||||
txt, _ := pcts[0].Text()
|
||||
data.ChangePct = parse.NumericOnly(txt)
|
||||
}
|
||||
|
||||
return &data, nil
|
||||
}
|
||||
98
sites/duckduckgo/stock_test.go
Normal file
98
sites/duckduckgo/stock_test.go
Normal file
@@ -0,0 +1,98 @@
|
||||
package duckduckgo
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
|
||||
"gitea.stevedudenhoeffer.com/steve/go-extractor"
|
||||
"gitea.stevedudenhoeffer.com/steve/go-extractor/extractortest"
|
||||
)
|
||||
|
||||
func makeStockDoc() *extractortest.MockDocument {
|
||||
return &extractortest.MockDocument{
|
||||
URLValue: "https://duckduckgo.com/?q=aapl+stock",
|
||||
MockNode: extractortest.MockNode{
|
||||
Children: map[string]extractor.Nodes{
|
||||
"div.module--stocks .module__title__link": {
|
||||
&extractortest.MockNode{TextValue: "AAPL"},
|
||||
},
|
||||
"div.module--stocks .module__subtitle": {
|
||||
&extractortest.MockNode{TextValue: "Apple Inc."},
|
||||
},
|
||||
"div.module--stocks .module__price": {
|
||||
&extractortest.MockNode{TextValue: "$189.84"},
|
||||
},
|
||||
"div.module--stocks .module__price-change": {
|
||||
&extractortest.MockNode{TextValue: "+2.45"},
|
||||
},
|
||||
"div.module--stocks .module__price-change-pct": {
|
||||
&extractortest.MockNode{TextValue: "(1.31%)"},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func TestExtractStock(t *testing.T) {
|
||||
doc := makeStockDoc()
|
||||
|
||||
data, err := extractStock(doc)
|
||||
if err != nil {
|
||||
t.Fatalf("extractStock() error: %v", err)
|
||||
}
|
||||
|
||||
if data.Symbol != "AAPL" {
|
||||
t.Errorf("Symbol = %q, want %q", data.Symbol, "AAPL")
|
||||
}
|
||||
if data.Name != "Apple Inc." {
|
||||
t.Errorf("Name = %q, want %q", data.Name, "Apple Inc.")
|
||||
}
|
||||
if data.Price != 189.84 {
|
||||
t.Errorf("Price = %v, want 189.84", data.Price)
|
||||
}
|
||||
if data.Change != 2.45 {
|
||||
t.Errorf("Change = %v, want 2.45", data.Change)
|
||||
}
|
||||
if data.ChangePct != 1.31 {
|
||||
t.Errorf("ChangePct = %v, want 1.31", data.ChangePct)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGetStockQuote_MockBrowser(t *testing.T) {
|
||||
doc := makeStockDoc()
|
||||
|
||||
browser := &extractortest.MockBrowser{
|
||||
Documents: map[string]*extractortest.MockDocument{
|
||||
"https://duckduckgo.com/?kp=-2&q=aapl+stock": doc,
|
||||
},
|
||||
}
|
||||
|
||||
data, err := DefaultConfig.GetStockQuote(context.Background(), browser, "aapl")
|
||||
if err != nil {
|
||||
t.Fatalf("GetStockQuote() error: %v", err)
|
||||
}
|
||||
|
||||
if data.Symbol != "AAPL" {
|
||||
t.Errorf("Symbol = %q, want %q", data.Symbol, "AAPL")
|
||||
}
|
||||
if data.Price != 189.84 {
|
||||
t.Errorf("Price = %v, want 189.84", data.Price)
|
||||
}
|
||||
}
|
||||
|
||||
func TestExtractStock_Empty(t *testing.T) {
|
||||
doc := &extractortest.MockDocument{
|
||||
MockNode: extractortest.MockNode{
|
||||
Children: map[string]extractor.Nodes{},
|
||||
},
|
||||
}
|
||||
|
||||
data, err := extractStock(doc)
|
||||
if err != nil {
|
||||
t.Fatalf("extractStock() error: %v", err)
|
||||
}
|
||||
|
||||
if data.Symbol != "" || data.Price != 0 {
|
||||
t.Error("expected zero values for empty doc")
|
||||
}
|
||||
}
|
||||
137
sites/duckduckgo/weather.go
Normal file
137
sites/duckduckgo/weather.go
Normal file
@@ -0,0 +1,137 @@
|
||||
package duckduckgo
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"time"
|
||||
|
||||
"gitea.stevedudenhoeffer.com/steve/go-extractor"
|
||||
"gitea.stevedudenhoeffer.com/steve/go-extractor/sites/internal/parse"
|
||||
)
|
||||
|
||||
// WeatherData holds structured weather information extracted from DuckDuckGo.
|
||||
type WeatherData struct {
|
||||
Location string
|
||||
CurrentTemp float64
|
||||
Condition string
|
||||
HighTemp float64
|
||||
LowTemp float64
|
||||
Humidity string
|
||||
Wind string
|
||||
Forecast []DayForecast
|
||||
}
|
||||
|
||||
// DayForecast holds a single day's forecast.
|
||||
type DayForecast struct {
|
||||
Day string
|
||||
HighTemp float64
|
||||
LowTemp float64
|
||||
Condition string
|
||||
}
|
||||
|
||||
// GetWeather extracts weather data from DuckDuckGo's weather widget.
|
||||
func (c Config) GetWeather(ctx context.Context, b extractor.Browser, city string) (*WeatherData, error) {
|
||||
c = c.validate()
|
||||
|
||||
u := c.ToSearchURL("weather " + city)
|
||||
|
||||
slog.Info("fetching weather", "url", u, "city", city)
|
||||
doc, err := b.Open(ctx, u.String(), extractor.OpenPageOptions{})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to open weather page: %w", err)
|
||||
}
|
||||
defer extractor.DeferClose(doc)
|
||||
|
||||
timeout := 10 * time.Second
|
||||
if err := doc.WaitForNetworkIdle(&timeout); err != nil {
|
||||
slog.Warn("WaitForNetworkIdle failed", "err", err)
|
||||
}
|
||||
|
||||
return extractWeather(doc)
|
||||
}
|
||||
|
||||
// GetWeather is a convenience function using DefaultConfig.
|
||||
func GetWeather(ctx context.Context, b extractor.Browser, city string) (*WeatherData, error) {
|
||||
return DefaultConfig.GetWeather(ctx, b, city)
|
||||
}
|
||||
|
||||
func extractWeather(doc extractor.Node) (*WeatherData, error) {
|
||||
var data WeatherData
|
||||
|
||||
// Location
|
||||
locs := doc.Select("div.module--weather span.module__title__link")
|
||||
if len(locs) > 0 {
|
||||
data.Location, _ = locs[0].Text()
|
||||
}
|
||||
|
||||
// Current temperature
|
||||
temps := doc.Select("div.module--weather .module__current-temp")
|
||||
if len(temps) > 0 {
|
||||
txt, _ := temps[0].Text()
|
||||
data.CurrentTemp = parse.NumericOnly(txt)
|
||||
}
|
||||
|
||||
// Condition
|
||||
conds := doc.Select("div.module--weather .module__weather-summary")
|
||||
if len(conds) > 0 {
|
||||
data.Condition, _ = conds[0].Text()
|
||||
}
|
||||
|
||||
// High/low
|
||||
highs := doc.Select("div.module--weather .module__high-temp")
|
||||
if len(highs) > 0 {
|
||||
txt, _ := highs[0].Text()
|
||||
data.HighTemp = parse.NumericOnly(txt)
|
||||
}
|
||||
|
||||
lows := doc.Select("div.module--weather .module__low-temp")
|
||||
if len(lows) > 0 {
|
||||
txt, _ := lows[0].Text()
|
||||
data.LowTemp = parse.NumericOnly(txt)
|
||||
}
|
||||
|
||||
// Humidity
|
||||
humids := doc.Select("div.module--weather .module__humidity")
|
||||
if len(humids) > 0 {
|
||||
data.Humidity, _ = humids[0].Text()
|
||||
}
|
||||
|
||||
// Wind
|
||||
winds := doc.Select("div.module--weather .module__wind")
|
||||
if len(winds) > 0 {
|
||||
data.Wind, _ = winds[0].Text()
|
||||
}
|
||||
|
||||
// Daily forecast
|
||||
_ = doc.ForEach("div.module--weather .module__forecast-day", func(n extractor.Node) error {
|
||||
var day DayForecast
|
||||
|
||||
days := n.Select(".forecast-day__name")
|
||||
if len(days) > 0 {
|
||||
day.Day, _ = days[0].Text()
|
||||
}
|
||||
|
||||
dayHighs := n.Select(".forecast-day__high")
|
||||
if len(dayHighs) > 0 {
|
||||
txt, _ := dayHighs[0].Text()
|
||||
day.HighTemp = parse.NumericOnly(txt)
|
||||
}
|
||||
|
||||
dayLows := n.Select(".forecast-day__low")
|
||||
if len(dayLows) > 0 {
|
||||
txt, _ := dayLows[0].Text()
|
||||
day.LowTemp = parse.NumericOnly(txt)
|
||||
}
|
||||
|
||||
dayConds := n.Select(".forecast-day__condition")
|
||||
if len(dayConds) > 0 {
|
||||
day.Condition, _ = dayConds[0].Text()
|
||||
}
|
||||
|
||||
data.Forecast = append(data.Forecast, day)
|
||||
return nil
|
||||
})
|
||||
|
||||
return &data, nil
|
||||
}
|
||||
141
sites/duckduckgo/weather_test.go
Normal file
141
sites/duckduckgo/weather_test.go
Normal file
@@ -0,0 +1,141 @@
|
||||
package duckduckgo
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
|
||||
"gitea.stevedudenhoeffer.com/steve/go-extractor"
|
||||
"gitea.stevedudenhoeffer.com/steve/go-extractor/extractortest"
|
||||
)
|
||||
|
||||
func makeWeatherDoc() *extractortest.MockDocument {
|
||||
return &extractortest.MockDocument{
|
||||
URLValue: "https://duckduckgo.com/?q=weather+new+york",
|
||||
MockNode: extractortest.MockNode{
|
||||
Children: map[string]extractor.Nodes{
|
||||
"div.module--weather span.module__title__link": {
|
||||
&extractortest.MockNode{TextValue: "New York, NY"},
|
||||
},
|
||||
"div.module--weather .module__current-temp": {
|
||||
&extractortest.MockNode{TextValue: "72°F"},
|
||||
},
|
||||
"div.module--weather .module__weather-summary": {
|
||||
&extractortest.MockNode{TextValue: "Partly Cloudy"},
|
||||
},
|
||||
"div.module--weather .module__high-temp": {
|
||||
&extractortest.MockNode{TextValue: "78°"},
|
||||
},
|
||||
"div.module--weather .module__low-temp": {
|
||||
&extractortest.MockNode{TextValue: "65°"},
|
||||
},
|
||||
"div.module--weather .module__humidity": {
|
||||
&extractortest.MockNode{TextValue: "55%"},
|
||||
},
|
||||
"div.module--weather .module__wind": {
|
||||
&extractortest.MockNode{TextValue: "SW 10 mph"},
|
||||
},
|
||||
"div.module--weather .module__forecast-day": {
|
||||
&extractortest.MockNode{
|
||||
Children: map[string]extractor.Nodes{
|
||||
".forecast-day__name": {&extractortest.MockNode{TextValue: "Mon"}},
|
||||
".forecast-day__high": {&extractortest.MockNode{TextValue: "80°"}},
|
||||
".forecast-day__low": {&extractortest.MockNode{TextValue: "66°"}},
|
||||
".forecast-day__condition": {&extractortest.MockNode{TextValue: "Sunny"}},
|
||||
},
|
||||
},
|
||||
&extractortest.MockNode{
|
||||
Children: map[string]extractor.Nodes{
|
||||
".forecast-day__name": {&extractortest.MockNode{TextValue: "Tue"}},
|
||||
".forecast-day__high": {&extractortest.MockNode{TextValue: "75°"}},
|
||||
".forecast-day__low": {&extractortest.MockNode{TextValue: "62°"}},
|
||||
".forecast-day__condition": {&extractortest.MockNode{TextValue: "Rain"}},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func TestExtractWeather(t *testing.T) {
|
||||
doc := makeWeatherDoc()
|
||||
|
||||
data, err := extractWeather(doc)
|
||||
if err != nil {
|
||||
t.Fatalf("extractWeather() error: %v", err)
|
||||
}
|
||||
|
||||
if data.Location != "New York, NY" {
|
||||
t.Errorf("Location = %q, want %q", data.Location, "New York, NY")
|
||||
}
|
||||
if data.CurrentTemp != 72 {
|
||||
t.Errorf("CurrentTemp = %v, want 72", data.CurrentTemp)
|
||||
}
|
||||
if data.Condition != "Partly Cloudy" {
|
||||
t.Errorf("Condition = %q, want %q", data.Condition, "Partly Cloudy")
|
||||
}
|
||||
if data.HighTemp != 78 {
|
||||
t.Errorf("HighTemp = %v, want 78", data.HighTemp)
|
||||
}
|
||||
if data.LowTemp != 65 {
|
||||
t.Errorf("LowTemp = %v, want 65", data.LowTemp)
|
||||
}
|
||||
if data.Humidity != "55%" {
|
||||
t.Errorf("Humidity = %q, want %q", data.Humidity, "55%")
|
||||
}
|
||||
if data.Wind != "SW 10 mph" {
|
||||
t.Errorf("Wind = %q, want %q", data.Wind, "SW 10 mph")
|
||||
}
|
||||
|
||||
if len(data.Forecast) != 2 {
|
||||
t.Fatalf("Forecast len = %d, want 2", len(data.Forecast))
|
||||
}
|
||||
if data.Forecast[0].Day != "Mon" {
|
||||
t.Errorf("Forecast[0].Day = %q, want %q", data.Forecast[0].Day, "Mon")
|
||||
}
|
||||
if data.Forecast[0].HighTemp != 80 {
|
||||
t.Errorf("Forecast[0].HighTemp = %v, want 80", data.Forecast[0].HighTemp)
|
||||
}
|
||||
if data.Forecast[1].Condition != "Rain" {
|
||||
t.Errorf("Forecast[1].Condition = %q, want %q", data.Forecast[1].Condition, "Rain")
|
||||
}
|
||||
}
|
||||
|
||||
func TestGetWeather_MockBrowser(t *testing.T) {
|
||||
doc := makeWeatherDoc()
|
||||
|
||||
browser := &extractortest.MockBrowser{
|
||||
Documents: map[string]*extractortest.MockDocument{
|
||||
"https://duckduckgo.com/?kp=-2&q=weather+new+york": doc,
|
||||
},
|
||||
}
|
||||
|
||||
data, err := DefaultConfig.GetWeather(context.Background(), browser, "new york")
|
||||
if err != nil {
|
||||
t.Fatalf("GetWeather() error: %v", err)
|
||||
}
|
||||
|
||||
if data.Location != "New York, NY" {
|
||||
t.Errorf("Location = %q, want %q", data.Location, "New York, NY")
|
||||
}
|
||||
if data.CurrentTemp != 72 {
|
||||
t.Errorf("CurrentTemp = %v, want 72", data.CurrentTemp)
|
||||
}
|
||||
}
|
||||
|
||||
func TestExtractWeather_Empty(t *testing.T) {
|
||||
doc := &extractortest.MockDocument{
|
||||
MockNode: extractortest.MockNode{
|
||||
Children: map[string]extractor.Nodes{},
|
||||
},
|
||||
}
|
||||
|
||||
data, err := extractWeather(doc)
|
||||
if err != nil {
|
||||
t.Fatalf("extractWeather() error: %v", err)
|
||||
}
|
||||
|
||||
if data.Location != "" || data.CurrentTemp != 0 {
|
||||
t.Error("expected zero values for empty doc")
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user