diff --git a/sites/duckduckgo/stock.go b/sites/duckduckgo/stock.go new file mode 100644 index 0000000..74ed59a --- /dev/null +++ b/sites/duckduckgo/stock.go @@ -0,0 +1,144 @@ +package duckduckgo + +import ( + "context" + "fmt" + "log/slog" + "time" + + "gitea.stevedudenhoeffer.com/steve/go-extractor" + "gitea.stevedudenhoeffer.com/steve/go-extractor/sites/internal/parse" +) + +// StockData holds structured stock quote information from DuckDuckGo. +type StockData struct { + Symbol string + Name string + Price float64 + Change float64 + ChangePct float64 +} + +// StockPeriod represents a time period for stock charts. +type StockPeriod string + +const ( + Period1D StockPeriod = "1D" + Period5D StockPeriod = "5D" + Period1M StockPeriod = "1M" + PeriodYTD StockPeriod = "YTD" + Period1Y StockPeriod = "1Y" + Period5Y StockPeriod = "5Y" + PeriodAll StockPeriod = "All" +) + +// GetStockQuote extracts structured stock data from DuckDuckGo's stock widget. +func (c Config) GetStockQuote(ctx context.Context, b extractor.Browser, symbol string) (*StockData, error) { + c = c.validate() + + u := c.ToSearchURL(symbol + " stock") + + slog.Info("fetching stock", "url", u, "symbol", symbol) + doc, err := b.Open(ctx, u.String(), extractor.OpenPageOptions{}) + if err != nil { + return nil, fmt.Errorf("failed to open stock page: %w", err) + } + defer extractor.DeferClose(doc) + + timeout := 10 * time.Second + if err := doc.WaitForNetworkIdle(&timeout); err != nil { + slog.Warn("WaitForNetworkIdle failed", "err", err) + } + + return extractStock(doc) +} + +// GetStockQuote is a convenience function using DefaultConfig. +func GetStockQuote(ctx context.Context, b extractor.Browser, symbol string) (*StockData, error) { + return DefaultConfig.GetStockQuote(ctx, b, symbol) +} + +// GetStockChart screenshots the stock chart widget for a given period. +func (c Config) GetStockChart(ctx context.Context, b extractor.Browser, symbol string, period StockPeriod) ([]byte, error) { + c = c.validate() + + u := c.ToSearchURL(symbol + " stock") + + slog.Info("fetching stock chart", "url", u, "symbol", symbol, "period", period) + doc, err := b.Open(ctx, u.String(), extractor.OpenPageOptions{}) + if err != nil { + return nil, fmt.Errorf("failed to open stock page: %w", err) + } + defer extractor.DeferClose(doc) + + timeout := 10 * time.Second + if err := doc.WaitForNetworkIdle(&timeout); err != nil { + slog.Warn("WaitForNetworkIdle failed", "err", err) + } + + // Click the period selector if not 1D (default) + if period != Period1D { + selector := fmt.Sprintf(`div.module__content button[data-period="%s"]`, string(period)) + _ = doc.ForEach(selector, func(n extractor.Node) error { + return n.Click() + }) + + // Wait for chart to update + chartTimeout := 5 * time.Second + if err := doc.WaitForNetworkIdle(&chartTimeout); err != nil { + slog.Warn("WaitForNetworkIdle after period click failed", "err", err) + } + } + + // Screenshot the stock module + modules := doc.Select("div.module__content") + if len(modules) == 0 { + return nil, fmt.Errorf("stock module not found") + } + + return modules[0].Screenshot() +} + +// GetStockChart is a convenience function using DefaultConfig. +func GetStockChart(ctx context.Context, b extractor.Browser, symbol string, period StockPeriod) ([]byte, error) { + return DefaultConfig.GetStockChart(ctx, b, symbol, period) +} + +func extractStock(doc extractor.Node) (*StockData, error) { + var data StockData + + // Symbol + syms := doc.Select("div.module--stocks .module__title__link") + if len(syms) > 0 { + data.Symbol, _ = syms[0].Text() + } + + // Name + names := doc.Select("div.module--stocks .module__subtitle") + if len(names) > 0 { + data.Name, _ = names[0].Text() + } + + // Price + prices := doc.Select("div.module--stocks .module__price") + if len(prices) > 0 { + txt, _ := prices[0].Text() + data.Price = parse.NumericOnly(txt) + } + + // Change + changes := doc.Select("div.module--stocks .module__price-change") + if len(changes) > 0 { + txt, _ := changes[0].Text() + data.Change = parse.NumericOnly(txt) + } + + // Change percentage + pcts := doc.Select("div.module--stocks .module__price-change-pct") + if len(pcts) > 0 { + txt, _ := pcts[0].Text() + data.ChangePct = parse.NumericOnly(txt) + } + + return &data, nil +} diff --git a/sites/duckduckgo/stock_test.go b/sites/duckduckgo/stock_test.go new file mode 100644 index 0000000..4316cf4 --- /dev/null +++ b/sites/duckduckgo/stock_test.go @@ -0,0 +1,98 @@ +package duckduckgo + +import ( + "context" + "testing" + + "gitea.stevedudenhoeffer.com/steve/go-extractor" + "gitea.stevedudenhoeffer.com/steve/go-extractor/extractortest" +) + +func makeStockDoc() *extractortest.MockDocument { + return &extractortest.MockDocument{ + URLValue: "https://duckduckgo.com/?q=aapl+stock", + MockNode: extractortest.MockNode{ + Children: map[string]extractor.Nodes{ + "div.module--stocks .module__title__link": { + &extractortest.MockNode{TextValue: "AAPL"}, + }, + "div.module--stocks .module__subtitle": { + &extractortest.MockNode{TextValue: "Apple Inc."}, + }, + "div.module--stocks .module__price": { + &extractortest.MockNode{TextValue: "$189.84"}, + }, + "div.module--stocks .module__price-change": { + &extractortest.MockNode{TextValue: "+2.45"}, + }, + "div.module--stocks .module__price-change-pct": { + &extractortest.MockNode{TextValue: "(1.31%)"}, + }, + }, + }, + } +} + +func TestExtractStock(t *testing.T) { + doc := makeStockDoc() + + data, err := extractStock(doc) + if err != nil { + t.Fatalf("extractStock() error: %v", err) + } + + if data.Symbol != "AAPL" { + t.Errorf("Symbol = %q, want %q", data.Symbol, "AAPL") + } + if data.Name != "Apple Inc." { + t.Errorf("Name = %q, want %q", data.Name, "Apple Inc.") + } + if data.Price != 189.84 { + t.Errorf("Price = %v, want 189.84", data.Price) + } + if data.Change != 2.45 { + t.Errorf("Change = %v, want 2.45", data.Change) + } + if data.ChangePct != 1.31 { + t.Errorf("ChangePct = %v, want 1.31", data.ChangePct) + } +} + +func TestGetStockQuote_MockBrowser(t *testing.T) { + doc := makeStockDoc() + + browser := &extractortest.MockBrowser{ + Documents: map[string]*extractortest.MockDocument{ + "https://duckduckgo.com/?kp=-2&q=aapl+stock": doc, + }, + } + + data, err := DefaultConfig.GetStockQuote(context.Background(), browser, "aapl") + if err != nil { + t.Fatalf("GetStockQuote() error: %v", err) + } + + if data.Symbol != "AAPL" { + t.Errorf("Symbol = %q, want %q", data.Symbol, "AAPL") + } + if data.Price != 189.84 { + t.Errorf("Price = %v, want 189.84", data.Price) + } +} + +func TestExtractStock_Empty(t *testing.T) { + doc := &extractortest.MockDocument{ + MockNode: extractortest.MockNode{ + Children: map[string]extractor.Nodes{}, + }, + } + + data, err := extractStock(doc) + if err != nil { + t.Fatalf("extractStock() error: %v", err) + } + + if data.Symbol != "" || data.Price != 0 { + t.Error("expected zero values for empty doc") + } +} diff --git a/sites/duckduckgo/weather.go b/sites/duckduckgo/weather.go new file mode 100644 index 0000000..8924445 --- /dev/null +++ b/sites/duckduckgo/weather.go @@ -0,0 +1,137 @@ +package duckduckgo + +import ( + "context" + "fmt" + "log/slog" + "time" + + "gitea.stevedudenhoeffer.com/steve/go-extractor" + "gitea.stevedudenhoeffer.com/steve/go-extractor/sites/internal/parse" +) + +// WeatherData holds structured weather information extracted from DuckDuckGo. +type WeatherData struct { + Location string + CurrentTemp float64 + Condition string + HighTemp float64 + LowTemp float64 + Humidity string + Wind string + Forecast []DayForecast +} + +// DayForecast holds a single day's forecast. +type DayForecast struct { + Day string + HighTemp float64 + LowTemp float64 + Condition string +} + +// GetWeather extracts weather data from DuckDuckGo's weather widget. +func (c Config) GetWeather(ctx context.Context, b extractor.Browser, city string) (*WeatherData, error) { + c = c.validate() + + u := c.ToSearchURL("weather " + city) + + slog.Info("fetching weather", "url", u, "city", city) + doc, err := b.Open(ctx, u.String(), extractor.OpenPageOptions{}) + if err != nil { + return nil, fmt.Errorf("failed to open weather page: %w", err) + } + defer extractor.DeferClose(doc) + + timeout := 10 * time.Second + if err := doc.WaitForNetworkIdle(&timeout); err != nil { + slog.Warn("WaitForNetworkIdle failed", "err", err) + } + + return extractWeather(doc) +} + +// GetWeather is a convenience function using DefaultConfig. +func GetWeather(ctx context.Context, b extractor.Browser, city string) (*WeatherData, error) { + return DefaultConfig.GetWeather(ctx, b, city) +} + +func extractWeather(doc extractor.Node) (*WeatherData, error) { + var data WeatherData + + // Location + locs := doc.Select("div.module--weather span.module__title__link") + if len(locs) > 0 { + data.Location, _ = locs[0].Text() + } + + // Current temperature + temps := doc.Select("div.module--weather .module__current-temp") + if len(temps) > 0 { + txt, _ := temps[0].Text() + data.CurrentTemp = parse.NumericOnly(txt) + } + + // Condition + conds := doc.Select("div.module--weather .module__weather-summary") + if len(conds) > 0 { + data.Condition, _ = conds[0].Text() + } + + // High/low + highs := doc.Select("div.module--weather .module__high-temp") + if len(highs) > 0 { + txt, _ := highs[0].Text() + data.HighTemp = parse.NumericOnly(txt) + } + + lows := doc.Select("div.module--weather .module__low-temp") + if len(lows) > 0 { + txt, _ := lows[0].Text() + data.LowTemp = parse.NumericOnly(txt) + } + + // Humidity + humids := doc.Select("div.module--weather .module__humidity") + if len(humids) > 0 { + data.Humidity, _ = humids[0].Text() + } + + // Wind + winds := doc.Select("div.module--weather .module__wind") + if len(winds) > 0 { + data.Wind, _ = winds[0].Text() + } + + // Daily forecast + _ = doc.ForEach("div.module--weather .module__forecast-day", func(n extractor.Node) error { + var day DayForecast + + days := n.Select(".forecast-day__name") + if len(days) > 0 { + day.Day, _ = days[0].Text() + } + + dayHighs := n.Select(".forecast-day__high") + if len(dayHighs) > 0 { + txt, _ := dayHighs[0].Text() + day.HighTemp = parse.NumericOnly(txt) + } + + dayLows := n.Select(".forecast-day__low") + if len(dayLows) > 0 { + txt, _ := dayLows[0].Text() + day.LowTemp = parse.NumericOnly(txt) + } + + dayConds := n.Select(".forecast-day__condition") + if len(dayConds) > 0 { + day.Condition, _ = dayConds[0].Text() + } + + data.Forecast = append(data.Forecast, day) + return nil + }) + + return &data, nil +} diff --git a/sites/duckduckgo/weather_test.go b/sites/duckduckgo/weather_test.go new file mode 100644 index 0000000..4fa1610 --- /dev/null +++ b/sites/duckduckgo/weather_test.go @@ -0,0 +1,141 @@ +package duckduckgo + +import ( + "context" + "testing" + + "gitea.stevedudenhoeffer.com/steve/go-extractor" + "gitea.stevedudenhoeffer.com/steve/go-extractor/extractortest" +) + +func makeWeatherDoc() *extractortest.MockDocument { + return &extractortest.MockDocument{ + URLValue: "https://duckduckgo.com/?q=weather+new+york", + MockNode: extractortest.MockNode{ + Children: map[string]extractor.Nodes{ + "div.module--weather span.module__title__link": { + &extractortest.MockNode{TextValue: "New York, NY"}, + }, + "div.module--weather .module__current-temp": { + &extractortest.MockNode{TextValue: "72°F"}, + }, + "div.module--weather .module__weather-summary": { + &extractortest.MockNode{TextValue: "Partly Cloudy"}, + }, + "div.module--weather .module__high-temp": { + &extractortest.MockNode{TextValue: "78°"}, + }, + "div.module--weather .module__low-temp": { + &extractortest.MockNode{TextValue: "65°"}, + }, + "div.module--weather .module__humidity": { + &extractortest.MockNode{TextValue: "55%"}, + }, + "div.module--weather .module__wind": { + &extractortest.MockNode{TextValue: "SW 10 mph"}, + }, + "div.module--weather .module__forecast-day": { + &extractortest.MockNode{ + Children: map[string]extractor.Nodes{ + ".forecast-day__name": {&extractortest.MockNode{TextValue: "Mon"}}, + ".forecast-day__high": {&extractortest.MockNode{TextValue: "80°"}}, + ".forecast-day__low": {&extractortest.MockNode{TextValue: "66°"}}, + ".forecast-day__condition": {&extractortest.MockNode{TextValue: "Sunny"}}, + }, + }, + &extractortest.MockNode{ + Children: map[string]extractor.Nodes{ + ".forecast-day__name": {&extractortest.MockNode{TextValue: "Tue"}}, + ".forecast-day__high": {&extractortest.MockNode{TextValue: "75°"}}, + ".forecast-day__low": {&extractortest.MockNode{TextValue: "62°"}}, + ".forecast-day__condition": {&extractortest.MockNode{TextValue: "Rain"}}, + }, + }, + }, + }, + }, + } +} + +func TestExtractWeather(t *testing.T) { + doc := makeWeatherDoc() + + data, err := extractWeather(doc) + if err != nil { + t.Fatalf("extractWeather() error: %v", err) + } + + if data.Location != "New York, NY" { + t.Errorf("Location = %q, want %q", data.Location, "New York, NY") + } + if data.CurrentTemp != 72 { + t.Errorf("CurrentTemp = %v, want 72", data.CurrentTemp) + } + if data.Condition != "Partly Cloudy" { + t.Errorf("Condition = %q, want %q", data.Condition, "Partly Cloudy") + } + if data.HighTemp != 78 { + t.Errorf("HighTemp = %v, want 78", data.HighTemp) + } + if data.LowTemp != 65 { + t.Errorf("LowTemp = %v, want 65", data.LowTemp) + } + if data.Humidity != "55%" { + t.Errorf("Humidity = %q, want %q", data.Humidity, "55%") + } + if data.Wind != "SW 10 mph" { + t.Errorf("Wind = %q, want %q", data.Wind, "SW 10 mph") + } + + if len(data.Forecast) != 2 { + t.Fatalf("Forecast len = %d, want 2", len(data.Forecast)) + } + if data.Forecast[0].Day != "Mon" { + t.Errorf("Forecast[0].Day = %q, want %q", data.Forecast[0].Day, "Mon") + } + if data.Forecast[0].HighTemp != 80 { + t.Errorf("Forecast[0].HighTemp = %v, want 80", data.Forecast[0].HighTemp) + } + if data.Forecast[1].Condition != "Rain" { + t.Errorf("Forecast[1].Condition = %q, want %q", data.Forecast[1].Condition, "Rain") + } +} + +func TestGetWeather_MockBrowser(t *testing.T) { + doc := makeWeatherDoc() + + browser := &extractortest.MockBrowser{ + Documents: map[string]*extractortest.MockDocument{ + "https://duckduckgo.com/?kp=-2&q=weather+new+york": doc, + }, + } + + data, err := DefaultConfig.GetWeather(context.Background(), browser, "new york") + if err != nil { + t.Fatalf("GetWeather() error: %v", err) + } + + if data.Location != "New York, NY" { + t.Errorf("Location = %q, want %q", data.Location, "New York, NY") + } + if data.CurrentTemp != 72 { + t.Errorf("CurrentTemp = %v, want 72", data.CurrentTemp) + } +} + +func TestExtractWeather_Empty(t *testing.T) { + doc := &extractortest.MockDocument{ + MockNode: extractortest.MockNode{ + Children: map[string]extractor.Nodes{}, + }, + } + + data, err := extractWeather(doc) + if err != nil { + t.Fatalf("extractWeather() error: %v", err) + } + + if data.Location != "" || data.CurrentTemp != 0 { + t.Error("expected zero values for empty doc") + } +}