steve/go-extractor
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Compare commits

base: steve:4f8b6fb0246c22e52521ac732e78173d573b2104
Branches Tags
steve:main
...
compare: steve:cff4713282cb35065d43fe9e7451e0fdd37a4363
Branches Tags
steve:main

2 Commits
4f8b6fb024 ... cff4713282

Author SHA1 Message Date
steve
cff4713282 Merge pull request 'feature: add hourly forecast, precipitation, and icon hints to weather extractor' (#52) from feature/weather-hourly-precip-icons into main
All checks were successful
CI / vet (push) Successful in 1m5s
Details
CI / build (push) Successful in 1m6s
Details
CI / test (push) Successful in 1m8s
Details 
Reviewed-on: #52
 2026-02-15 21:23:35 +00:00
Steve Dudenhoeffer
469171da9c feature: add hourly forecast, precipitation, and icon hints to weather extractor
All checks were successful
CI / build (pull_request) Successful in 30s
Details
CI / vet (pull_request) Successful in 46s
Details
CI / test (pull_request) Successful in 49s
Details 
Add HourlyForecast struct and Hourly field to WeatherData for hourly
temperature/condition data. Add Precipitation (int, -1 if unavailable)
and IconHint (from aria-label/title/alt attributes) to both DayForecast
and HourlyForecast. This enables downstream consumers like mort to
replace inline DuckDuckGo scraping with a single GetWeather() call.

Closes #51

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
 2026-02-15 21:22:04 +00:00
2 changed files with 259 additions and 4 deletions
Expand all files Collapse all files

77
sites/duckduckgo/weather.go
View File

@@ -20,14 +20,26 @@ type WeatherData struct {
Humidity string
Wind string
Forecast []DayForecast
Hourly []HourlyForecast
}
// DayForecast holds a single day's forecast.
type DayForecast struct {
Day string
HighTemp float64
LowTemp float64
Condition string
Day string
HighTemp float64
LowTemp float64
Condition string
Precipitation int // percentage 0-100, -1 if unavailable
IconHint string // icon type from element attributes (e.g. "PartlyCloudy", "Snow")
}
// HourlyForecast holds a single hour's forecast.
type HourlyForecast struct {
Time string
Temp float64
Condition string
Precipitation int // percentage 0-100, -1 if unavailable
IconHint string // icon type from element attributes (e.g. "MostlyCloudy", "Rain")
}
// GetWeather extracts weather data from DuckDuckGo's weather widget.
@@ -106,6 +118,7 @@ func extractWeather(doc extractor.Node) (*WeatherData, error) {
// Daily forecast
_ = doc.ForEach("div.module--weather .module__forecast-day", func(n extractor.Node) error {
var day DayForecast
day.Precipitation = -1
days := n.Select(".forecast-day__name")
if len(days) > 0 {
@@ -129,9 +142,65 @@ func extractWeather(doc extractor.Node) (*WeatherData, error) {
day.Condition, _ = dayConds[0].Text()
}
precips := n.Select(".forecast-day__precip")
if len(precips) > 0 {
txt, _ := precips[0].Text()
day.Precipitation = int(parse.NumericOnly(txt))
}
day.IconHint = extractIconHint(n.Select(".forecast-day__icon"))
data.Forecast = append(data.Forecast, day)
return nil
})
// Hourly forecast
_ = doc.ForEach("div.module--weather .module__hourly-item", func(n extractor.Node) error {
var hour HourlyForecast
hour.Precipitation = -1
times := n.Select(".hourly-item__time")
if len(times) > 0 {
hour.Time, _ = times[0].Text()
}
temps := n.Select(".hourly-item__temp")
if len(temps) > 0 {
txt, _ := temps[0].Text()
hour.Temp = parse.NumericOnly(txt)
}
conds := n.Select(".hourly-item__condition")
if len(conds) > 0 {
hour.Condition, _ = conds[0].Text()
}
precips := n.Select(".hourly-item__precip")
if len(precips) > 0 {
txt, _ := precips[0].Text()
hour.Precipitation = int(parse.NumericOnly(txt))
}
hour.IconHint = extractIconHint(n.Select(".hourly-item__icon"))
data.Hourly = append(data.Hourly, hour)
return nil
})
return &data, nil
}
// extractIconHint reads the icon type from an element's aria-label, title, or alt attribute.
func extractIconHint(nodes extractor.Nodes) string {
if len(nodes) == 0 {
return ""
}
n := nodes[0]
for _, attr := range []string{"aria-label", "title", "alt"} {
v, _ := n.Attr(attr)
if v != "" {
return v
}
}
return ""
}

186
sites/duckduckgo/weather_test.go
View File

@@ -41,6 +41,8 @@ func makeWeatherDoc() *extractortest.MockDocument {
".forecast-day__high": {&extractortest.MockNode{TextValue: "80°"}},
".forecast-day__low": {&extractortest.MockNode{TextValue: "66°"}},
".forecast-day__condition": {&extractortest.MockNode{TextValue: "Sunny"}},
".forecast-day__precip": {&extractortest.MockNode{TextValue: "10%"}},
".forecast-day__icon": {&extractortest.MockNode{Attrs: map[string]string{"alt": "PartlyCloudy"}}},
},
},
&extractortest.MockNode{
@@ -49,6 +51,36 @@ func makeWeatherDoc() *extractortest.MockDocument {
".forecast-day__high": {&extractortest.MockNode{TextValue: "75°"}},
".forecast-day__low": {&extractortest.MockNode{TextValue: "62°"}},
".forecast-day__condition": {&extractortest.MockNode{TextValue: "Rain"}},
".forecast-day__precip": {&extractortest.MockNode{TextValue: "80%"}},
".forecast-day__icon": {&extractortest.MockNode{Attrs: map[string]string{"alt": "Rain"}}},
},
},
},
"div.module--weather .module__hourly-item": {
&extractortest.MockNode{
Children: map[string]extractor.Nodes{
".hourly-item__time": {&extractortest.MockNode{TextValue: "3 PM"}},
".hourly-item__temp": {&extractortest.MockNode{TextValue: "74°"}},
".hourly-item__condition": {&extractortest.MockNode{TextValue: "Partly Cloudy"}},
".hourly-item__precip": {&extractortest.MockNode{TextValue: "5%"}},
".hourly-item__icon": {&extractortest.MockNode{Attrs: map[string]string{"alt": "MostlyCloudy"}}},
},
},
&extractortest.MockNode{
Children: map[string]extractor.Nodes{
".hourly-item__time": {&extractortest.MockNode{TextValue: "4 PM"}},
".hourly-item__temp": {&extractortest.MockNode{TextValue: "73°"}},
".hourly-item__condition": {&extractortest.MockNode{TextValue: "Cloudy"}},
".hourly-item__icon": {&extractortest.MockNode{Attrs: map[string]string{"alt": "Cloudy"}}},
},
},
&extractortest.MockNode{
Children: map[string]extractor.Nodes{
".hourly-item__time": {&extractortest.MockNode{TextValue: "5 PM"}},
".hourly-item__temp": {&extractortest.MockNode{TextValue: "70°"}},
".hourly-item__condition": {&extractortest.MockNode{TextValue: "Rain"}},
".hourly-item__precip": {&extractortest.MockNode{TextValue: "60%"}},
".hourly-item__icon": {&extractortest.MockNode{Attrs: map[string]string{"aria-label": "HeavyRain"}}},
},
},
},
@@ -87,6 +119,7 @@ func TestExtractWeather(t *testing.T) {
t.Errorf("Wind = %q, want %q", data.Wind, "SW 10 mph")
}
// Daily forecast
if len(data.Forecast) != 2 {
t.Fatalf("Forecast len = %d, want 2", len(data.Forecast))
}
@@ -96,9 +129,60 @@ func TestExtractWeather(t *testing.T) {
if data.Forecast[0].HighTemp != 80 {
t.Errorf("Forecast[0].HighTemp = %v, want 80", data.Forecast[0].HighTemp)
}
if data.Forecast[0].Precipitation != 10 {
t.Errorf("Forecast[0].Precipitation = %d, want 10", data.Forecast[0].Precipitation)
}
if data.Forecast[0].IconHint != "PartlyCloudy" {
t.Errorf("Forecast[0].IconHint = %q, want %q", data.Forecast[0].IconHint, "PartlyCloudy")
}
if data.Forecast[1].Condition != "Rain" {
t.Errorf("Forecast[1].Condition = %q, want %q", data.Forecast[1].Condition, "Rain")
}
if data.Forecast[1].Precipitation != 80 {
t.Errorf("Forecast[1].Precipitation = %d, want 80", data.Forecast[1].Precipitation)
}
if data.Forecast[1].IconHint != "Rain" {
t.Errorf("Forecast[1].IconHint = %q, want %q", data.Forecast[1].IconHint, "Rain")
}
// Hourly forecast
if len(data.Hourly) != 3 {
t.Fatalf("Hourly len = %d, want 3", len(data.Hourly))
}
if data.Hourly[0].Time != "3 PM" {
t.Errorf("Hourly[0].Time = %q, want %q", data.Hourly[0].Time, "3 PM")
}
if data.Hourly[0].Temp != 74 {
t.Errorf("Hourly[0].Temp = %v, want 74", data.Hourly[0].Temp)
}
if data.Hourly[0].Condition != "Partly Cloudy" {
t.Errorf("Hourly[0].Condition = %q, want %q", data.Hourly[0].Condition, "Partly Cloudy")
}
if data.Hourly[0].Precipitation != 5 {
t.Errorf("Hourly[0].Precipitation = %d, want 5", data.Hourly[0].Precipitation)
}
if data.Hourly[0].IconHint != "MostlyCloudy" {
t.Errorf("Hourly[0].IconHint = %q, want %q", data.Hourly[0].IconHint, "MostlyCloudy")
}
// Second hourly item has no precipitation
if data.Hourly[1].Time != "4 PM" {
t.Errorf("Hourly[1].Time = %q, want %q", data.Hourly[1].Time, "4 PM")
}
if data.Hourly[1].Precipitation != -1 {
t.Errorf("Hourly[1].Precipitation = %d, want -1 (unavailable)", data.Hourly[1].Precipitation)
}
if data.Hourly[1].IconHint != "Cloudy" {
t.Errorf("Hourly[1].IconHint = %q, want %q", data.Hourly[1].IconHint, "Cloudy")
}
// Third hourly item uses aria-label for icon hint
if data.Hourly[2].Precipitation != 60 {
t.Errorf("Hourly[2].Precipitation = %d, want 60", data.Hourly[2].Precipitation)
}
if data.Hourly[2].IconHint != "HeavyRain" {
t.Errorf("Hourly[2].IconHint = %q, want %q", data.Hourly[2].IconHint, "HeavyRain")
}
}
func TestGetWeather_MockBrowser(t *testing.T) {
@@ -121,6 +205,9 @@ func TestGetWeather_MockBrowser(t *testing.T) {
if data.CurrentTemp != 72 {
t.Errorf("CurrentTemp = %v, want 72", data.CurrentTemp)
}
if len(data.Hourly) != 3 {
t.Errorf("Hourly len = %d, want 3", len(data.Hourly))
}
}
func TestExtractWeather_Empty(t *testing.T) {
@@ -138,4 +225,103 @@ func TestExtractWeather_Empty(t *testing.T) {
if data.Location != "" || data.CurrentTemp != 0 {
t.Error("expected zero values for empty doc")
}
if len(data.Hourly) != 0 {
t.Errorf("expected no hourly data for empty doc, got %d", len(data.Hourly))
}
}
func TestExtractWeather_NoPrecipitation(t *testing.T) {
doc := &extractortest.MockDocument{
MockNode: extractortest.MockNode{
Children: map[string]extractor.Nodes{
"div.module--weather .module__forecast-day": {
&extractortest.MockNode{
Children: map[string]extractor.Nodes{
".forecast-day__name": {&extractortest.MockNode{TextValue: "Wed"}},
".forecast-day__high": {&extractortest.MockNode{TextValue: "85°"}},
".forecast-day__low": {&extractortest.MockNode{TextValue: "70°"}},
".forecast-day__condition": {&extractortest.MockNode{TextValue: "Clear"}},
},
},
},
"div.module--weather .module__hourly-item": {
&extractortest.MockNode{
Children: map[string]extractor.Nodes{
".hourly-item__time": {&extractortest.MockNode{TextValue: "12 PM"}},
".hourly-item__temp": {&extractortest.MockNode{TextValue: "82°"}},
},
},
},
},
},
}
data, err := extractWeather(doc)
if err != nil {
t.Fatalf("extractWeather() error: %v", err)
}
if len(data.Forecast) != 1 {
t.Fatalf("Forecast len = %d, want 1", len(data.Forecast))
}
if data.Forecast[0].Precipitation != -1 {
t.Errorf("Forecast[0].Precipitation = %d, want -1 (unavailable)", data.Forecast[0].Precipitation)
}
if data.Forecast[0].IconHint != "" {
t.Errorf("Forecast[0].IconHint = %q, want empty", data.Forecast[0].IconHint)
}
if len(data.Hourly) != 1 {
t.Fatalf("Hourly len = %d, want 1", len(data.Hourly))
}
if data.Hourly[0].Precipitation != -1 {
t.Errorf("Hourly[0].Precipitation = %d, want -1 (unavailable)", data.Hourly[0].Precipitation)
}
if data.Hourly[0].IconHint != "" {
t.Errorf("Hourly[0].IconHint = %q, want empty", data.Hourly[0].IconHint)
}
}
func TestExtractIconHint_Priority(t *testing.T) {
// aria-label takes priority over title and alt
nodes := extractor.Nodes{
&extractortest.MockNode{
Attrs: map[string]string{
"aria-label": "Snow",
"title": "SnowTitle",
"alt": "SnowAlt",
},
},
}
if got := extractIconHint(nodes); got != "Snow" {
t.Errorf("extractIconHint() = %q, want %q (aria-label priority)", got, "Snow")
}
// title used when aria-label absent
nodes = extractor.Nodes{
&extractortest.MockNode{
Attrs: map[string]string{
"title": "Drizzle",
"alt": "DrizzleAlt",
},
},
}
if got := extractIconHint(nodes); got != "Drizzle" {
t.Errorf("extractIconHint() = %q, want %q (title fallback)", got, "Drizzle")
}
// alt used as last fallback
nodes = extractor.Nodes{
&extractortest.MockNode{
Attrs: map[string]string{"alt": "MostlyClear"},
},
}
if got := extractIconHint(nodes); got != "MostlyClear" {
t.Errorf("extractIconHint() = %q, want %q (alt fallback)", got, "MostlyClear")
}
// empty when no nodes
if got := extractIconHint(nil); got != "" {
t.Errorf("extractIconHint(nil) = %q, want empty", got)
}
}