feature: add DuckDuckGo weather and stock widget extractors
Add weather.go with GetWeather() for extracting structured weather data (location, temp, conditions, forecast) and stock.go with GetStockQuote() and GetStockChart() for stock data extraction and chart screenshots. Both include mock-based tests. CSS selectors may need tuning against the live site since DuckDuckGo's React-rendered widgets use dynamic class names. Closes #25, #26 Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
137
sites/duckduckgo/weather.go
Normal file
137
sites/duckduckgo/weather.go
Normal file
@@ -0,0 +1,137 @@
|
||||
package duckduckgo
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"time"
|
||||
|
||||
"gitea.stevedudenhoeffer.com/steve/go-extractor"
|
||||
"gitea.stevedudenhoeffer.com/steve/go-extractor/sites/internal/parse"
|
||||
)
|
||||
|
||||
// WeatherData holds structured weather information extracted from DuckDuckGo.
|
||||
type WeatherData struct {
|
||||
Location string
|
||||
CurrentTemp float64
|
||||
Condition string
|
||||
HighTemp float64
|
||||
LowTemp float64
|
||||
Humidity string
|
||||
Wind string
|
||||
Forecast []DayForecast
|
||||
}
|
||||
|
||||
// DayForecast holds a single day's forecast.
|
||||
type DayForecast struct {
|
||||
Day string
|
||||
HighTemp float64
|
||||
LowTemp float64
|
||||
Condition string
|
||||
}
|
||||
|
||||
// GetWeather extracts weather data from DuckDuckGo's weather widget.
|
||||
func (c Config) GetWeather(ctx context.Context, b extractor.Browser, city string) (*WeatherData, error) {
|
||||
c = c.validate()
|
||||
|
||||
u := c.ToSearchURL("weather " + city)
|
||||
|
||||
slog.Info("fetching weather", "url", u, "city", city)
|
||||
doc, err := b.Open(ctx, u.String(), extractor.OpenPageOptions{})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to open weather page: %w", err)
|
||||
}
|
||||
defer extractor.DeferClose(doc)
|
||||
|
||||
timeout := 10 * time.Second
|
||||
if err := doc.WaitForNetworkIdle(&timeout); err != nil {
|
||||
slog.Warn("WaitForNetworkIdle failed", "err", err)
|
||||
}
|
||||
|
||||
return extractWeather(doc)
|
||||
}
|
||||
|
||||
// GetWeather is a convenience function using DefaultConfig.
|
||||
func GetWeather(ctx context.Context, b extractor.Browser, city string) (*WeatherData, error) {
|
||||
return DefaultConfig.GetWeather(ctx, b, city)
|
||||
}
|
||||
|
||||
func extractWeather(doc extractor.Node) (*WeatherData, error) {
|
||||
var data WeatherData
|
||||
|
||||
// Location
|
||||
locs := doc.Select("div.module--weather span.module__title__link")
|
||||
if len(locs) > 0 {
|
||||
data.Location, _ = locs[0].Text()
|
||||
}
|
||||
|
||||
// Current temperature
|
||||
temps := doc.Select("div.module--weather .module__current-temp")
|
||||
if len(temps) > 0 {
|
||||
txt, _ := temps[0].Text()
|
||||
data.CurrentTemp = parse.NumericOnly(txt)
|
||||
}
|
||||
|
||||
// Condition
|
||||
conds := doc.Select("div.module--weather .module__weather-summary")
|
||||
if len(conds) > 0 {
|
||||
data.Condition, _ = conds[0].Text()
|
||||
}
|
||||
|
||||
// High/low
|
||||
highs := doc.Select("div.module--weather .module__high-temp")
|
||||
if len(highs) > 0 {
|
||||
txt, _ := highs[0].Text()
|
||||
data.HighTemp = parse.NumericOnly(txt)
|
||||
}
|
||||
|
||||
lows := doc.Select("div.module--weather .module__low-temp")
|
||||
if len(lows) > 0 {
|
||||
txt, _ := lows[0].Text()
|
||||
data.LowTemp = parse.NumericOnly(txt)
|
||||
}
|
||||
|
||||
// Humidity
|
||||
humids := doc.Select("div.module--weather .module__humidity")
|
||||
if len(humids) > 0 {
|
||||
data.Humidity, _ = humids[0].Text()
|
||||
}
|
||||
|
||||
// Wind
|
||||
winds := doc.Select("div.module--weather .module__wind")
|
||||
if len(winds) > 0 {
|
||||
data.Wind, _ = winds[0].Text()
|
||||
}
|
||||
|
||||
// Daily forecast
|
||||
_ = doc.ForEach("div.module--weather .module__forecast-day", func(n extractor.Node) error {
|
||||
var day DayForecast
|
||||
|
||||
days := n.Select(".forecast-day__name")
|
||||
if len(days) > 0 {
|
||||
day.Day, _ = days[0].Text()
|
||||
}
|
||||
|
||||
dayHighs := n.Select(".forecast-day__high")
|
||||
if len(dayHighs) > 0 {
|
||||
txt, _ := dayHighs[0].Text()
|
||||
day.HighTemp = parse.NumericOnly(txt)
|
||||
}
|
||||
|
||||
dayLows := n.Select(".forecast-day__low")
|
||||
if len(dayLows) > 0 {
|
||||
txt, _ := dayLows[0].Text()
|
||||
day.LowTemp = parse.NumericOnly(txt)
|
||||
}
|
||||
|
||||
dayConds := n.Select(".forecast-day__condition")
|
||||
if len(dayConds) > 0 {
|
||||
day.Condition, _ = dayConds[0].Text()
|
||||
}
|
||||
|
||||
data.Forecast = append(data.Forecast, day)
|
||||
return nil
|
||||
})
|
||||
|
||||
return &data, nil
|
||||
}
|
||||
Reference in New Issue
Block a user