feature: add DuckDuckGo weather and stock widget extractors
All checks were successful
CI / vet (pull_request) Successful in 29s
CI / build (pull_request) Successful in 46s
CI / test (pull_request) Successful in 48s

Add weather.go with GetWeather() for extracting structured weather data
(location, temp, conditions, forecast) and stock.go with GetStockQuote()
and GetStockChart() for stock data extraction and chart screenshots.

Both include mock-based tests. CSS selectors may need tuning against
the live site since DuckDuckGo's React-rendered widgets use dynamic
class names.

Closes #25, #26
Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
2026-02-15 16:40:53 +00:00
parent dcc977c0cc
commit 461b704792
4 changed files with 520 additions and 0 deletions

137
sites/duckduckgo/weather.go Normal file
View File

@@ -0,0 +1,137 @@
package duckduckgo
import (
"context"
"fmt"
"log/slog"
"time"
"gitea.stevedudenhoeffer.com/steve/go-extractor"
"gitea.stevedudenhoeffer.com/steve/go-extractor/sites/internal/parse"
)
// WeatherData holds structured weather information extracted from DuckDuckGo.
type WeatherData struct {
Location string
CurrentTemp float64
Condition string
HighTemp float64
LowTemp float64
Humidity string
Wind string
Forecast []DayForecast
}
// DayForecast holds a single day's forecast.
type DayForecast struct {
Day string
HighTemp float64
LowTemp float64
Condition string
}
// GetWeather extracts weather data from DuckDuckGo's weather widget.
func (c Config) GetWeather(ctx context.Context, b extractor.Browser, city string) (*WeatherData, error) {
c = c.validate()
u := c.ToSearchURL("weather " + city)
slog.Info("fetching weather", "url", u, "city", city)
doc, err := b.Open(ctx, u.String(), extractor.OpenPageOptions{})
if err != nil {
return nil, fmt.Errorf("failed to open weather page: %w", err)
}
defer extractor.DeferClose(doc)
timeout := 10 * time.Second
if err := doc.WaitForNetworkIdle(&timeout); err != nil {
slog.Warn("WaitForNetworkIdle failed", "err", err)
}
return extractWeather(doc)
}
// GetWeather is a convenience function using DefaultConfig.
func GetWeather(ctx context.Context, b extractor.Browser, city string) (*WeatherData, error) {
return DefaultConfig.GetWeather(ctx, b, city)
}
func extractWeather(doc extractor.Node) (*WeatherData, error) {
var data WeatherData
// Location
locs := doc.Select("div.module--weather span.module__title__link")
if len(locs) > 0 {
data.Location, _ = locs[0].Text()
}
// Current temperature
temps := doc.Select("div.module--weather .module__current-temp")
if len(temps) > 0 {
txt, _ := temps[0].Text()
data.CurrentTemp = parse.NumericOnly(txt)
}
// Condition
conds := doc.Select("div.module--weather .module__weather-summary")
if len(conds) > 0 {
data.Condition, _ = conds[0].Text()
}
// High/low
highs := doc.Select("div.module--weather .module__high-temp")
if len(highs) > 0 {
txt, _ := highs[0].Text()
data.HighTemp = parse.NumericOnly(txt)
}
lows := doc.Select("div.module--weather .module__low-temp")
if len(lows) > 0 {
txt, _ := lows[0].Text()
data.LowTemp = parse.NumericOnly(txt)
}
// Humidity
humids := doc.Select("div.module--weather .module__humidity")
if len(humids) > 0 {
data.Humidity, _ = humids[0].Text()
}
// Wind
winds := doc.Select("div.module--weather .module__wind")
if len(winds) > 0 {
data.Wind, _ = winds[0].Text()
}
// Daily forecast
_ = doc.ForEach("div.module--weather .module__forecast-day", func(n extractor.Node) error {
var day DayForecast
days := n.Select(".forecast-day__name")
if len(days) > 0 {
day.Day, _ = days[0].Text()
}
dayHighs := n.Select(".forecast-day__high")
if len(dayHighs) > 0 {
txt, _ := dayHighs[0].Text()
day.HighTemp = parse.NumericOnly(txt)
}
dayLows := n.Select(".forecast-day__low")
if len(dayLows) > 0 {
txt, _ := dayLows[0].Text()
day.LowTemp = parse.NumericOnly(txt)
}
dayConds := n.Select(".forecast-day__condition")
if len(dayConds) > 0 {
day.Condition, _ = dayConds[0].Text()
}
data.Forecast = append(data.Forecast, day)
return nil
})
return &data, nil
}