Merge pull request 'feature: add DuckDuckGo weather and stock widget extractors' (#44) from feature/duckduckgo-widgets into main

sites/duckduckgo/stock.go

@@ -0,0 +1,144 @@
+package duckduckgo
+
+import (
+	"context"
+	"fmt"
+	"log/slog"
+	"time"
+
+	"gitea.stevedudenhoeffer.com/steve/go-extractor"
+	"gitea.stevedudenhoeffer.com/steve/go-extractor/sites/internal/parse"
+)
+
+// StockData holds structured stock quote information from DuckDuckGo.
+type StockData struct {
+	Symbol    string
+	Name      string
+	Price     float64
+	Change    float64
+	ChangePct float64
+}
+
+// StockPeriod represents a time period for stock charts.
+type StockPeriod string
+
+const (
+	Period1D  StockPeriod = "1D"
+	Period5D  StockPeriod = "5D"
+	Period1M  StockPeriod = "1M"
+	PeriodYTD StockPeriod = "YTD"
+	Period1Y  StockPeriod = "1Y"
+	Period5Y  StockPeriod = "5Y"
+	PeriodAll StockPeriod = "All"
+)
+
+// GetStockQuote extracts structured stock data from DuckDuckGo's stock widget.
+func (c Config) GetStockQuote(ctx context.Context, b extractor.Browser, symbol string) (*StockData, error) {
+	c = c.validate()
+
+	u := c.ToSearchURL(symbol + " stock")
+
+	slog.Info("fetching stock", "url", u, "symbol", symbol)
+	doc, err := b.Open(ctx, u.String(), extractor.OpenPageOptions{})
+	if err != nil {
+		return nil, fmt.Errorf("failed to open stock page: %w", err)
+	}
+	defer extractor.DeferClose(doc)
+
+	timeout := 10 * time.Second
+	if err := doc.WaitForNetworkIdle(&timeout); err != nil {
+		slog.Warn("WaitForNetworkIdle failed", "err", err)
+	}
+
+	return extractStock(doc)
+}
+
+// GetStockQuote is a convenience function using DefaultConfig.
+func GetStockQuote(ctx context.Context, b extractor.Browser, symbol string) (*StockData, error) {
+	return DefaultConfig.GetStockQuote(ctx, b, symbol)
+}
+
+// GetStockChart screenshots the stock chart widget for a given period.
+func (c Config) GetStockChart(ctx context.Context, b extractor.Browser, symbol string, period StockPeriod) ([]byte, error) {
+	c = c.validate()
+
+	u := c.ToSearchURL(symbol + " stock")
+
+	slog.Info("fetching stock chart", "url", u, "symbol", symbol, "period", period)
+	doc, err := b.Open(ctx, u.String(), extractor.OpenPageOptions{})
+	if err != nil {
+		return nil, fmt.Errorf("failed to open stock page: %w", err)
+	}
+	defer extractor.DeferClose(doc)
+
+	timeout := 10 * time.Second
+	if err := doc.WaitForNetworkIdle(&timeout); err != nil {
+		slog.Warn("WaitForNetworkIdle failed", "err", err)
+	}
+
+	// Click the period selector if not 1D (default)
+	if period != Period1D {
+		selector := fmt.Sprintf(`div.module__content button[data-period="%s"]`, string(period))
+		_ = doc.ForEach(selector, func(n extractor.Node) error {
+			return n.Click()
+		})
+
+		// Wait for chart to update
+		chartTimeout := 5 * time.Second
+		if err := doc.WaitForNetworkIdle(&chartTimeout); err != nil {
+			slog.Warn("WaitForNetworkIdle after period click failed", "err", err)
+		}
+	}
+
+	// Screenshot the stock module
+	modules := doc.Select("div.module__content")
+	if len(modules) == 0 {
+		return nil, fmt.Errorf("stock module not found")
+	}
+
+	return modules[0].Screenshot()
+}
+
+// GetStockChart is a convenience function using DefaultConfig.
+func GetStockChart(ctx context.Context, b extractor.Browser, symbol string, period StockPeriod) ([]byte, error) {
+	return DefaultConfig.GetStockChart(ctx, b, symbol, period)
+}
+
+func extractStock(doc extractor.Node) (*StockData, error) {
+	var data StockData
+
+	// Symbol
+	syms := doc.Select("div.module--stocks .module__title__link")
+	if len(syms) > 0 {
+		data.Symbol, _ = syms[0].Text()
+	}
+
+	// Name
+	names := doc.Select("div.module--stocks .module__subtitle")
+	if len(names) > 0 {
+		data.Name, _ = names[0].Text()
+	}
+
+	// Price
+	prices := doc.Select("div.module--stocks .module__price")
+	if len(prices) > 0 {
+		txt, _ := prices[0].Text()
+		data.Price = parse.NumericOnly(txt)
+	}
+
+	// Change
+	changes := doc.Select("div.module--stocks .module__price-change")
+	if len(changes) > 0 {
+		txt, _ := changes[0].Text()
+		data.Change = parse.NumericOnly(txt)
+	}
+
+	// Change percentage
+	pcts := doc.Select("div.module--stocks .module__price-change-pct")
+	if len(pcts) > 0 {
+		txt, _ := pcts[0].Text()
+		data.ChangePct = parse.NumericOnly(txt)
+	}
+
+	return &data, nil
+}

sites/duckduckgo/stock_test.go

@@ -0,0 +1,98 @@
+package duckduckgo
+
+import (
+	"context"
+	"testing"
+
+	"gitea.stevedudenhoeffer.com/steve/go-extractor"
+	"gitea.stevedudenhoeffer.com/steve/go-extractor/extractortest"
+)
+
+func makeStockDoc() *extractortest.MockDocument {
+	return &extractortest.MockDocument{
+		URLValue: "https://duckduckgo.com/?q=aapl+stock",
+		MockNode: extractortest.MockNode{
+			Children: map[string]extractor.Nodes{
+				"div.module--stocks .module__title__link": {
+					&extractortest.MockNode{TextValue: "AAPL"},
+				},
+				"div.module--stocks .module__subtitle": {
+					&extractortest.MockNode{TextValue: "Apple Inc."},
+				},
+				"div.module--stocks .module__price": {
+					&extractortest.MockNode{TextValue: "$189.84"},
+				},
+				"div.module--stocks .module__price-change": {
+					&extractortest.MockNode{TextValue: "+2.45"},
+				},
+				"div.module--stocks .module__price-change-pct": {
+					&extractortest.MockNode{TextValue: "(1.31%)"},
+				},
+			},
+		},
+	}
+}
+
+func TestExtractStock(t *testing.T) {
+	doc := makeStockDoc()
+
+	data, err := extractStock(doc)
+	if err != nil {
+		t.Fatalf("extractStock() error: %v", err)
+	}
+
+	if data.Symbol != "AAPL" {
+		t.Errorf("Symbol = %q, want %q", data.Symbol, "AAPL")
+	}
+	if data.Name != "Apple Inc." {
+		t.Errorf("Name = %q, want %q", data.Name, "Apple Inc.")
+	}
+	if data.Price != 189.84 {
+		t.Errorf("Price = %v, want 189.84", data.Price)
+	}
+	if data.Change != 2.45 {
+		t.Errorf("Change = %v, want 2.45", data.Change)
+	}
+	if data.ChangePct != 1.31 {
+		t.Errorf("ChangePct = %v, want 1.31", data.ChangePct)
+	}
+}
+
+func TestGetStockQuote_MockBrowser(t *testing.T) {
+	doc := makeStockDoc()
+
+	browser := &extractortest.MockBrowser{
+		Documents: map[string]*extractortest.MockDocument{
+			"https://duckduckgo.com/?kp=-2&q=aapl+stock": doc,
+		},
+	}
+
+	data, err := DefaultConfig.GetStockQuote(context.Background(), browser, "aapl")
+	if err != nil {
+		t.Fatalf("GetStockQuote() error: %v", err)
+	}
+
+	if data.Symbol != "AAPL" {
+		t.Errorf("Symbol = %q, want %q", data.Symbol, "AAPL")
+	}
+	if data.Price != 189.84 {
+		t.Errorf("Price = %v, want 189.84", data.Price)
+	}
+}
+
+func TestExtractStock_Empty(t *testing.T) {
+	doc := &extractortest.MockDocument{
+		MockNode: extractortest.MockNode{
+			Children: map[string]extractor.Nodes{},
+		},
+	}
+
+	data, err := extractStock(doc)
+	if err != nil {
+		t.Fatalf("extractStock() error: %v", err)
+	}
+
+	if data.Symbol != "" || data.Price != 0 {
+		t.Error("expected zero values for empty doc")
+	}
+}

sites/duckduckgo/weather.go

@@ -0,0 +1,137 @@
+package duckduckgo
+
+import (
+	"context"
+	"fmt"
+	"log/slog"
+	"time"
+
+	"gitea.stevedudenhoeffer.com/steve/go-extractor"
+	"gitea.stevedudenhoeffer.com/steve/go-extractor/sites/internal/parse"
+)
+
+// WeatherData holds structured weather information extracted from DuckDuckGo.
+type WeatherData struct {
+	Location    string
+	CurrentTemp float64
+	Condition   string
+	HighTemp    float64
+	LowTemp     float64
+	Humidity    string
+	Wind        string
+	Forecast    []DayForecast
+}
+
+// DayForecast holds a single day's forecast.
+type DayForecast struct {
+	Day      string
+	HighTemp float64
+	LowTemp  float64
+	Condition string
+}
+
+// GetWeather extracts weather data from DuckDuckGo's weather widget.
+func (c Config) GetWeather(ctx context.Context, b extractor.Browser, city string) (*WeatherData, error) {
+	c = c.validate()
+
+	u := c.ToSearchURL("weather " + city)
+
+	slog.Info("fetching weather", "url", u, "city", city)
+	doc, err := b.Open(ctx, u.String(), extractor.OpenPageOptions{})
+	if err != nil {
+		return nil, fmt.Errorf("failed to open weather page: %w", err)
+	}
+	defer extractor.DeferClose(doc)
+
+	timeout := 10 * time.Second
+	if err := doc.WaitForNetworkIdle(&timeout); err != nil {
+		slog.Warn("WaitForNetworkIdle failed", "err", err)
+	}
+
+	return extractWeather(doc)
+}
+
+// GetWeather is a convenience function using DefaultConfig.
+func GetWeather(ctx context.Context, b extractor.Browser, city string) (*WeatherData, error) {
+	return DefaultConfig.GetWeather(ctx, b, city)
+}
+
+func extractWeather(doc extractor.Node) (*WeatherData, error) {
+	var data WeatherData
+
+	// Location
+	locs := doc.Select("div.module--weather span.module__title__link")
+	if len(locs) > 0 {
+		data.Location, _ = locs[0].Text()
+	}
+
+	// Current temperature
+	temps := doc.Select("div.module--weather .module__current-temp")
+	if len(temps) > 0 {
+		txt, _ := temps[0].Text()
+		data.CurrentTemp = parse.NumericOnly(txt)
+	}
+
+	// Condition
+	conds := doc.Select("div.module--weather .module__weather-summary")
+	if len(conds) > 0 {
+		data.Condition, _ = conds[0].Text()
+	}
+
+	// High/low
+	highs := doc.Select("div.module--weather .module__high-temp")
+	if len(highs) > 0 {
+		txt, _ := highs[0].Text()
+		data.HighTemp = parse.NumericOnly(txt)
+	}
+
+	lows := doc.Select("div.module--weather .module__low-temp")
+	if len(lows) > 0 {
+		txt, _ := lows[0].Text()
+		data.LowTemp = parse.NumericOnly(txt)
+	}
+
+	// Humidity
+	humids := doc.Select("div.module--weather .module__humidity")
+	if len(humids) > 0 {
+		data.Humidity, _ = humids[0].Text()
+	}
+
+	// Wind
+	winds := doc.Select("div.module--weather .module__wind")
+	if len(winds) > 0 {
+		data.Wind, _ = winds[0].Text()
+	}
+
+	// Daily forecast
+	_ = doc.ForEach("div.module--weather .module__forecast-day", func(n extractor.Node) error {
+		var day DayForecast
+
+		days := n.Select(".forecast-day__name")
+		if len(days) > 0 {
+			day.Day, _ = days[0].Text()
+		}
+
+		dayHighs := n.Select(".forecast-day__high")
+		if len(dayHighs) > 0 {
+			txt, _ := dayHighs[0].Text()
+			day.HighTemp = parse.NumericOnly(txt)
+		}
+
+		dayLows := n.Select(".forecast-day__low")
+		if len(dayLows) > 0 {
+			txt, _ := dayLows[0].Text()
+			day.LowTemp = parse.NumericOnly(txt)
+		}
+
+		dayConds := n.Select(".forecast-day__condition")
+		if len(dayConds) > 0 {
+			day.Condition, _ = dayConds[0].Text()
+		}
+
+		data.Forecast = append(data.Forecast, day)
+		return nil
+	})
+
+	return &data, nil
+}

sites/duckduckgo/weather_test.go

@@ -0,0 +1,141 @@
+package duckduckgo
+
+import (
+	"context"
+	"testing"
+
+	"gitea.stevedudenhoeffer.com/steve/go-extractor"
+	"gitea.stevedudenhoeffer.com/steve/go-extractor/extractortest"
+)
+
+func makeWeatherDoc() *extractortest.MockDocument {
+	return &extractortest.MockDocument{
+		URLValue: "https://duckduckgo.com/?q=weather+new+york",
+		MockNode: extractortest.MockNode{
+			Children: map[string]extractor.Nodes{
+				"div.module--weather span.module__title__link": {
+					&extractortest.MockNode{TextValue: "New York, NY"},
+				},
+				"div.module--weather .module__current-temp": {
+					&extractortest.MockNode{TextValue: "72°F"},
+				},
+				"div.module--weather .module__weather-summary": {
+					&extractortest.MockNode{TextValue: "Partly Cloudy"},
+				},
+				"div.module--weather .module__high-temp": {
+					&extractortest.MockNode{TextValue: "78°"},
+				},
+				"div.module--weather .module__low-temp": {
+					&extractortest.MockNode{TextValue: "65°"},
+				},
+				"div.module--weather .module__humidity": {
+					&extractortest.MockNode{TextValue: "55%"},
+				},
+				"div.module--weather .module__wind": {
+					&extractortest.MockNode{TextValue: "SW 10 mph"},
+				},
+				"div.module--weather .module__forecast-day": {
+					&extractortest.MockNode{
+						Children: map[string]extractor.Nodes{
+							".forecast-day__name":      {&extractortest.MockNode{TextValue: "Mon"}},
+							".forecast-day__high":      {&extractortest.MockNode{TextValue: "80°"}},
+							".forecast-day__low":       {&extractortest.MockNode{TextValue: "66°"}},
+							".forecast-day__condition": {&extractortest.MockNode{TextValue: "Sunny"}},
+						},
+					},
+					&extractortest.MockNode{
+						Children: map[string]extractor.Nodes{
+							".forecast-day__name":      {&extractortest.MockNode{TextValue: "Tue"}},
+							".forecast-day__high":      {&extractortest.MockNode{TextValue: "75°"}},
+							".forecast-day__low":       {&extractortest.MockNode{TextValue: "62°"}},
+							".forecast-day__condition": {&extractortest.MockNode{TextValue: "Rain"}},
+						},
+					},
+				},
+			},
+		},
+	}
+}
+
+func TestExtractWeather(t *testing.T) {
+	doc := makeWeatherDoc()
+
+	data, err := extractWeather(doc)
+	if err != nil {
+		t.Fatalf("extractWeather() error: %v", err)
+	}
+
+	if data.Location != "New York, NY" {
+		t.Errorf("Location = %q, want %q", data.Location, "New York, NY")
+	}
+	if data.CurrentTemp != 72 {
+		t.Errorf("CurrentTemp = %v, want 72", data.CurrentTemp)
+	}
+	if data.Condition != "Partly Cloudy" {
+		t.Errorf("Condition = %q, want %q", data.Condition, "Partly Cloudy")
+	}
+	if data.HighTemp != 78 {
+		t.Errorf("HighTemp = %v, want 78", data.HighTemp)
+	}
+	if data.LowTemp != 65 {
+		t.Errorf("LowTemp = %v, want 65", data.LowTemp)
+	}
+	if data.Humidity != "55%" {
+		t.Errorf("Humidity = %q, want %q", data.Humidity, "55%")
+	}
+	if data.Wind != "SW 10 mph" {
+		t.Errorf("Wind = %q, want %q", data.Wind, "SW 10 mph")
+	}
+
+	if len(data.Forecast) != 2 {
+		t.Fatalf("Forecast len = %d, want 2", len(data.Forecast))
+	}
+	if data.Forecast[0].Day != "Mon" {
+		t.Errorf("Forecast[0].Day = %q, want %q", data.Forecast[0].Day, "Mon")
+	}
+	if data.Forecast[0].HighTemp != 80 {
+		t.Errorf("Forecast[0].HighTemp = %v, want 80", data.Forecast[0].HighTemp)
+	}
+	if data.Forecast[1].Condition != "Rain" {
+		t.Errorf("Forecast[1].Condition = %q, want %q", data.Forecast[1].Condition, "Rain")
+	}
+}
+
+func TestGetWeather_MockBrowser(t *testing.T) {
+	doc := makeWeatherDoc()
+
+	browser := &extractortest.MockBrowser{
+		Documents: map[string]*extractortest.MockDocument{
+			"https://duckduckgo.com/?kp=-2&q=weather+new+york": doc,
+		},
+	}
+
+	data, err := DefaultConfig.GetWeather(context.Background(), browser, "new york")
+	if err != nil {
+		t.Fatalf("GetWeather() error: %v", err)
+	}
+
+	if data.Location != "New York, NY" {
+		t.Errorf("Location = %q, want %q", data.Location, "New York, NY")
+	}
+	if data.CurrentTemp != 72 {
+		t.Errorf("CurrentTemp = %v, want 72", data.CurrentTemp)
+	}
+}
+
+func TestExtractWeather_Empty(t *testing.T) {
+	doc := &extractortest.MockDocument{
+		MockNode: extractortest.MockNode{
+			Children: map[string]extractor.Nodes{},
+		},
+	}
+
+	data, err := extractWeather(doc)
+	if err != nil {
+		t.Fatalf("extractWeather() error: %v", err)
+	}
+
+	if data.Location != "" || data.CurrentTemp != 0 {
+		t.Error("expected zero values for empty doc")
+	}
+}