Merge pull request 'feature: add Steam Store game price extractor' (#47) from feature/steam-extractor into main
This commit was merged in pull request #47.
This commit is contained in:
256
sites/steam/steam.go
Normal file
256
sites/steam/steam.go
Normal file
@@ -0,0 +1,256 @@
|
||||
package steam
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"gitea.stevedudenhoeffer.com/steve/go-extractor"
|
||||
"gitea.stevedudenhoeffer.com/steve/go-extractor/sites/internal/parse"
|
||||
)
|
||||
|
||||
// GamePrice holds structured game price information from the Steam Store.
|
||||
type GamePrice struct {
|
||||
AppID int
|
||||
Name string
|
||||
Price float64
|
||||
OrigPrice float64 // 0 if not on sale
|
||||
DiscountPct int // 0 if not on sale
|
||||
OnSale bool
|
||||
FreeToPlay bool
|
||||
Currency string
|
||||
}
|
||||
|
||||
// Config holds configuration for the Steam Store extractor.
|
||||
type Config struct{}
|
||||
|
||||
// DefaultConfig is the default Steam Store configuration.
|
||||
var DefaultConfig = Config{}
|
||||
|
||||
func (c Config) validate() Config {
|
||||
return c
|
||||
}
|
||||
|
||||
// appURL returns the Steam store page URL for a given app ID.
|
||||
func appURL(appID int) string {
|
||||
return fmt.Sprintf("https://store.steampowered.com/app/%d", appID)
|
||||
}
|
||||
|
||||
// searchURL returns the Steam search URL for a given query.
|
||||
func searchURL(query string) string {
|
||||
return fmt.Sprintf("https://store.steampowered.com/search/?term=%s", strings.ReplaceAll(query, " ", "+"))
|
||||
}
|
||||
|
||||
// handleAgeGate attempts to bypass Steam's age verification if present.
|
||||
func handleAgeGate(doc extractor.Document) {
|
||||
// Set the year dropdown to bypass the age gate
|
||||
years := doc.Select("#ageYear")
|
||||
if len(years) > 0 {
|
||||
_ = years[0].SetAttribute("value", "1990")
|
||||
|
||||
// Click the View Page button
|
||||
btns := doc.Select("#view_product_page_btn")
|
||||
if len(btns) > 0 {
|
||||
_ = btns[0].Click()
|
||||
|
||||
timeout := 5 * time.Second
|
||||
_ = doc.WaitForNetworkIdle(&timeout)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// GetGamePrice extracts price info for a Steam app by its ID.
|
||||
func (c Config) GetGamePrice(ctx context.Context, b extractor.Browser, appID int) (*GamePrice, error) {
|
||||
c = c.validate()
|
||||
|
||||
u := appURL(appID)
|
||||
|
||||
slog.Info("fetching steam game", "url", u, "appID", appID)
|
||||
doc, err := b.Open(ctx, u, extractor.OpenPageOptions{})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to open steam page: %w", err)
|
||||
}
|
||||
defer extractor.DeferClose(doc)
|
||||
|
||||
timeout := 10 * time.Second
|
||||
if err := doc.WaitForNetworkIdle(&timeout); err != nil {
|
||||
slog.Warn("WaitForNetworkIdle failed", "err", err)
|
||||
}
|
||||
|
||||
handleAgeGate(doc)
|
||||
|
||||
return extractGamePrice(doc, appID)
|
||||
}
|
||||
|
||||
// GetGamePrice is a convenience function using DefaultConfig.
|
||||
func GetGamePrice(ctx context.Context, b extractor.Browser, appID int) (*GamePrice, error) {
|
||||
return DefaultConfig.GetGamePrice(ctx, b, appID)
|
||||
}
|
||||
|
||||
// SearchGames searches the Steam store and returns results.
|
||||
func (c Config) SearchGames(ctx context.Context, b extractor.Browser, query string) ([]GamePrice, error) {
|
||||
c = c.validate()
|
||||
|
||||
u := searchURL(query)
|
||||
|
||||
slog.Info("searching steam", "url", u, "query", query)
|
||||
doc, err := b.Open(ctx, u, extractor.OpenPageOptions{})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to open steam search: %w", err)
|
||||
}
|
||||
defer extractor.DeferClose(doc)
|
||||
|
||||
timeout := 10 * time.Second
|
||||
if err := doc.WaitForNetworkIdle(&timeout); err != nil {
|
||||
slog.Warn("WaitForNetworkIdle failed", "err", err)
|
||||
}
|
||||
|
||||
return extractSearchResults(doc)
|
||||
}
|
||||
|
||||
// SearchGames is a convenience function using DefaultConfig.
|
||||
func SearchGames(ctx context.Context, b extractor.Browser, query string) ([]GamePrice, error) {
|
||||
return DefaultConfig.SearchGames(ctx, b, query)
|
||||
}
|
||||
|
||||
func extractGamePrice(doc extractor.Node, appID int) (*GamePrice, error) {
|
||||
var data GamePrice
|
||||
data.AppID = appID
|
||||
|
||||
// Game name
|
||||
names := doc.Select("#appHubAppName")
|
||||
if len(names) > 0 {
|
||||
data.Name, _ = names[0].Text()
|
||||
data.Name = strings.TrimSpace(data.Name)
|
||||
}
|
||||
|
||||
// Check for free to play
|
||||
freeNodes := doc.Select("div.game_area_purchase_game_wrapper .game_purchase_price")
|
||||
if len(freeNodes) > 0 {
|
||||
txt, _ := freeNodes[0].Text()
|
||||
if strings.Contains(strings.ToLower(txt), "free") {
|
||||
data.FreeToPlay = true
|
||||
return &data, nil
|
||||
}
|
||||
}
|
||||
|
||||
// Check for discount
|
||||
discounts := doc.Select("div.game_area_purchase_game div.discount_pct")
|
||||
if len(discounts) > 0 {
|
||||
data.OnSale = true
|
||||
|
||||
txt, _ := discounts[0].Text()
|
||||
data.DiscountPct = int(parse.NumericOnly(txt))
|
||||
|
||||
// Original price
|
||||
origPrices := doc.Select("div.game_area_purchase_game div.discount_original_price")
|
||||
if len(origPrices) > 0 {
|
||||
txt, _ := origPrices[0].Text()
|
||||
data.OrigPrice = parse.NumericOnly(txt)
|
||||
data.Currency = extractCurrency(txt)
|
||||
}
|
||||
|
||||
// Final/sale price
|
||||
finalPrices := doc.Select("div.game_area_purchase_game div.discount_final_price")
|
||||
if len(finalPrices) > 0 {
|
||||
txt, _ := finalPrices[0].Text()
|
||||
data.Price = parse.NumericOnly(txt)
|
||||
}
|
||||
} else {
|
||||
// No discount — regular price
|
||||
prices := doc.Select("div.game_area_purchase_game div.game_purchase_price")
|
||||
if len(prices) > 0 {
|
||||
txt, _ := prices[0].Text()
|
||||
data.Price = parse.NumericOnly(txt)
|
||||
data.Currency = extractCurrency(txt)
|
||||
}
|
||||
}
|
||||
|
||||
return &data, nil
|
||||
}
|
||||
|
||||
func extractSearchResults(doc extractor.Node) ([]GamePrice, error) {
|
||||
var results []GamePrice
|
||||
|
||||
_ = doc.ForEach("a.search_result_row", func(n extractor.Node) error {
|
||||
var gp GamePrice
|
||||
|
||||
// App ID from data-ds-appid attribute
|
||||
appIDStr, _ := n.Attr("data-ds-appid")
|
||||
if appIDStr != "" {
|
||||
id, err := strconv.Atoi(appIDStr)
|
||||
if err == nil {
|
||||
gp.AppID = id
|
||||
}
|
||||
}
|
||||
|
||||
// Name
|
||||
titles := n.Select("span.title")
|
||||
if len(titles) > 0 {
|
||||
gp.Name, _ = titles[0].Text()
|
||||
gp.Name = strings.TrimSpace(gp.Name)
|
||||
}
|
||||
|
||||
// Discount percentage
|
||||
pcts := n.Select("div.discount_pct")
|
||||
if len(pcts) > 0 {
|
||||
gp.OnSale = true
|
||||
txt, _ := pcts[0].Text()
|
||||
gp.DiscountPct = int(parse.NumericOnly(txt))
|
||||
}
|
||||
|
||||
// Original price (when on sale)
|
||||
origPrices := n.Select("div.discount_original_price")
|
||||
if len(origPrices) > 0 {
|
||||
txt, _ := origPrices[0].Text()
|
||||
gp.OrigPrice = parse.NumericOnly(txt)
|
||||
gp.Currency = extractCurrency(txt)
|
||||
}
|
||||
|
||||
// Final price
|
||||
finalPrices := n.Select("div.discount_final_price")
|
||||
if len(finalPrices) > 0 {
|
||||
txt, _ := finalPrices[0].Text()
|
||||
gp.Price = parse.NumericOnly(txt)
|
||||
if gp.Currency == "" {
|
||||
gp.Currency = extractCurrency(txt)
|
||||
}
|
||||
} else {
|
||||
// Regular price (no sale)
|
||||
prices := n.Select("div.search_price")
|
||||
if len(prices) > 0 {
|
||||
txt, _ := prices[0].Text()
|
||||
txt = strings.TrimSpace(txt)
|
||||
if strings.Contains(strings.ToLower(txt), "free") {
|
||||
gp.FreeToPlay = true
|
||||
} else {
|
||||
gp.Price = parse.NumericOnly(txt)
|
||||
gp.Currency = extractCurrency(txt)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
results = append(results, gp)
|
||||
return nil
|
||||
})
|
||||
|
||||
return results, nil
|
||||
}
|
||||
|
||||
// extractCurrency returns the currency symbol from a price string.
|
||||
func extractCurrency(s string) string {
|
||||
s = strings.TrimSpace(s)
|
||||
if strings.HasPrefix(s, "$") {
|
||||
return "USD"
|
||||
}
|
||||
if strings.HasPrefix(s, "€") || strings.HasSuffix(s, "€") {
|
||||
return "EUR"
|
||||
}
|
||||
if strings.HasPrefix(s, "£") {
|
||||
return "GBP"
|
||||
}
|
||||
return ""
|
||||
}
|
||||
319
sites/steam/steam_test.go
Normal file
319
sites/steam/steam_test.go
Normal file
@@ -0,0 +1,319 @@
|
||||
package steam
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
|
||||
"gitea.stevedudenhoeffer.com/steve/go-extractor"
|
||||
"gitea.stevedudenhoeffer.com/steve/go-extractor/extractortest"
|
||||
)
|
||||
|
||||
func makeGameDoc(appID int) *extractortest.MockDocument {
|
||||
return &extractortest.MockDocument{
|
||||
URLValue: "https://store.steampowered.com/app/1245620",
|
||||
MockNode: extractortest.MockNode{
|
||||
Children: map[string]extractor.Nodes{
|
||||
"#appHubAppName": {
|
||||
&extractortest.MockNode{TextValue: "ELDEN RING"},
|
||||
},
|
||||
"div.game_area_purchase_game div.game_purchase_price": {
|
||||
&extractortest.MockNode{TextValue: "$59.99"},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func makeDiscountedGameDoc() *extractortest.MockDocument {
|
||||
return &extractortest.MockDocument{
|
||||
URLValue: "https://store.steampowered.com/app/292030",
|
||||
MockNode: extractortest.MockNode{
|
||||
Children: map[string]extractor.Nodes{
|
||||
"#appHubAppName": {
|
||||
&extractortest.MockNode{TextValue: "The Witcher 3: Wild Hunt"},
|
||||
},
|
||||
"div.game_area_purchase_game div.discount_pct": {
|
||||
&extractortest.MockNode{TextValue: "-80%"},
|
||||
},
|
||||
"div.game_area_purchase_game div.discount_original_price": {
|
||||
&extractortest.MockNode{TextValue: "$39.99"},
|
||||
},
|
||||
"div.game_area_purchase_game div.discount_final_price": {
|
||||
&extractortest.MockNode{TextValue: "$7.99"},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func makeFreeGameDoc() *extractortest.MockDocument {
|
||||
return &extractortest.MockDocument{
|
||||
URLValue: "https://store.steampowered.com/app/440",
|
||||
MockNode: extractortest.MockNode{
|
||||
Children: map[string]extractor.Nodes{
|
||||
"#appHubAppName": {
|
||||
&extractortest.MockNode{TextValue: "Team Fortress 2"},
|
||||
},
|
||||
"div.game_area_purchase_game_wrapper .game_purchase_price": {
|
||||
&extractortest.MockNode{TextValue: "Free To Play"},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func TestExtractGamePrice_Regular(t *testing.T) {
|
||||
doc := makeGameDoc(1245620)
|
||||
|
||||
data, err := extractGamePrice(doc, 1245620)
|
||||
if err != nil {
|
||||
t.Fatalf("extractGamePrice() error: %v", err)
|
||||
}
|
||||
|
||||
if data.AppID != 1245620 {
|
||||
t.Errorf("AppID = %d, want 1245620", data.AppID)
|
||||
}
|
||||
if data.Name != "ELDEN RING" {
|
||||
t.Errorf("Name = %q, want %q", data.Name, "ELDEN RING")
|
||||
}
|
||||
if data.Price != 59.99 {
|
||||
t.Errorf("Price = %v, want 59.99", data.Price)
|
||||
}
|
||||
if data.OnSale {
|
||||
t.Error("OnSale = true, want false")
|
||||
}
|
||||
if data.Currency != "USD" {
|
||||
t.Errorf("Currency = %q, want %q", data.Currency, "USD")
|
||||
}
|
||||
}
|
||||
|
||||
func TestExtractGamePrice_Discounted(t *testing.T) {
|
||||
doc := makeDiscountedGameDoc()
|
||||
|
||||
data, err := extractGamePrice(doc, 292030)
|
||||
if err != nil {
|
||||
t.Fatalf("extractGamePrice() error: %v", err)
|
||||
}
|
||||
|
||||
if data.Name != "The Witcher 3: Wild Hunt" {
|
||||
t.Errorf("Name = %q, want %q", data.Name, "The Witcher 3: Wild Hunt")
|
||||
}
|
||||
if !data.OnSale {
|
||||
t.Error("OnSale = false, want true")
|
||||
}
|
||||
if data.DiscountPct != 80 {
|
||||
t.Errorf("DiscountPct = %d, want 80", data.DiscountPct)
|
||||
}
|
||||
if data.OrigPrice != 39.99 {
|
||||
t.Errorf("OrigPrice = %v, want 39.99", data.OrigPrice)
|
||||
}
|
||||
if data.Price != 7.99 {
|
||||
t.Errorf("Price = %v, want 7.99", data.Price)
|
||||
}
|
||||
}
|
||||
|
||||
func TestExtractGamePrice_FreeToPlay(t *testing.T) {
|
||||
doc := makeFreeGameDoc()
|
||||
|
||||
data, err := extractGamePrice(doc, 440)
|
||||
if err != nil {
|
||||
t.Fatalf("extractGamePrice() error: %v", err)
|
||||
}
|
||||
|
||||
if data.Name != "Team Fortress 2" {
|
||||
t.Errorf("Name = %q, want %q", data.Name, "Team Fortress 2")
|
||||
}
|
||||
if !data.FreeToPlay {
|
||||
t.Error("FreeToPlay = false, want true")
|
||||
}
|
||||
if data.Price != 0 {
|
||||
t.Errorf("Price = %v, want 0", data.Price)
|
||||
}
|
||||
}
|
||||
|
||||
func TestExtractSearchResults(t *testing.T) {
|
||||
doc := &extractortest.MockDocument{
|
||||
MockNode: extractortest.MockNode{
|
||||
Children: map[string]extractor.Nodes{
|
||||
"a.search_result_row": {
|
||||
&extractortest.MockNode{
|
||||
Attrs: map[string]string{"data-ds-appid": "1245620"},
|
||||
Children: map[string]extractor.Nodes{
|
||||
"span.title": {&extractortest.MockNode{TextValue: "ELDEN RING"}},
|
||||
"div.discount_final_price": {&extractortest.MockNode{TextValue: "$59.99"}},
|
||||
},
|
||||
},
|
||||
&extractortest.MockNode{
|
||||
Attrs: map[string]string{"data-ds-appid": "292030"},
|
||||
Children: map[string]extractor.Nodes{
|
||||
"span.title": {&extractortest.MockNode{TextValue: "The Witcher 3: Wild Hunt"}},
|
||||
"div.discount_pct": {&extractortest.MockNode{TextValue: "-80%"}},
|
||||
"div.discount_original_price": {&extractortest.MockNode{TextValue: "$39.99"}},
|
||||
"div.discount_final_price": {&extractortest.MockNode{TextValue: "$7.99"}},
|
||||
},
|
||||
},
|
||||
&extractortest.MockNode{
|
||||
Attrs: map[string]string{"data-ds-appid": "440"},
|
||||
Children: map[string]extractor.Nodes{
|
||||
"span.title": {&extractortest.MockNode{TextValue: "Team Fortress 2"}},
|
||||
"div.search_price": {&extractortest.MockNode{TextValue: "Free To Play"}},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
results, err := extractSearchResults(doc)
|
||||
if err != nil {
|
||||
t.Fatalf("extractSearchResults() error: %v", err)
|
||||
}
|
||||
|
||||
if len(results) != 3 {
|
||||
t.Fatalf("len(results) = %d, want 3", len(results))
|
||||
}
|
||||
|
||||
// Regular price game
|
||||
if results[0].AppID != 1245620 {
|
||||
t.Errorf("results[0].AppID = %d, want 1245620", results[0].AppID)
|
||||
}
|
||||
if results[0].Name != "ELDEN RING" {
|
||||
t.Errorf("results[0].Name = %q, want %q", results[0].Name, "ELDEN RING")
|
||||
}
|
||||
if results[0].Price != 59.99 {
|
||||
t.Errorf("results[0].Price = %v, want 59.99", results[0].Price)
|
||||
}
|
||||
|
||||
// Discounted game
|
||||
if results[1].AppID != 292030 {
|
||||
t.Errorf("results[1].AppID = %d, want 292030", results[1].AppID)
|
||||
}
|
||||
if !results[1].OnSale {
|
||||
t.Error("results[1].OnSale = false, want true")
|
||||
}
|
||||
if results[1].DiscountPct != 80 {
|
||||
t.Errorf("results[1].DiscountPct = %d, want 80", results[1].DiscountPct)
|
||||
}
|
||||
if results[1].Price != 7.99 {
|
||||
t.Errorf("results[1].Price = %v, want 7.99", results[1].Price)
|
||||
}
|
||||
|
||||
// Free to play game
|
||||
if results[2].AppID != 440 {
|
||||
t.Errorf("results[2].AppID = %d, want 440", results[2].AppID)
|
||||
}
|
||||
if !results[2].FreeToPlay {
|
||||
t.Error("results[2].FreeToPlay = false, want true")
|
||||
}
|
||||
}
|
||||
|
||||
func TestGetGamePrice_MockBrowser(t *testing.T) {
|
||||
doc := makeGameDoc(1245620)
|
||||
|
||||
browser := &extractortest.MockBrowser{
|
||||
Documents: map[string]*extractortest.MockDocument{
|
||||
"https://store.steampowered.com/app/1245620": doc,
|
||||
},
|
||||
}
|
||||
|
||||
data, err := DefaultConfig.GetGamePrice(context.Background(), browser, 1245620)
|
||||
if err != nil {
|
||||
t.Fatalf("GetGamePrice() error: %v", err)
|
||||
}
|
||||
|
||||
if data.Name != "ELDEN RING" {
|
||||
t.Errorf("Name = %q, want %q", data.Name, "ELDEN RING")
|
||||
}
|
||||
if data.Price != 59.99 {
|
||||
t.Errorf("Price = %v, want 59.99", data.Price)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSearchGames_MockBrowser(t *testing.T) {
|
||||
doc := &extractortest.MockDocument{
|
||||
URLValue: "https://store.steampowered.com/search/?term=elden+ring",
|
||||
MockNode: extractortest.MockNode{
|
||||
Children: map[string]extractor.Nodes{
|
||||
"a.search_result_row": {
|
||||
&extractortest.MockNode{
|
||||
Attrs: map[string]string{"data-ds-appid": "1245620"},
|
||||
Children: map[string]extractor.Nodes{
|
||||
"span.title": {&extractortest.MockNode{TextValue: "ELDEN RING"}},
|
||||
"div.discount_final_price": {&extractortest.MockNode{TextValue: "$59.99"}},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
browser := &extractortest.MockBrowser{
|
||||
Documents: map[string]*extractortest.MockDocument{
|
||||
"https://store.steampowered.com/search/?term=elden+ring": doc,
|
||||
},
|
||||
}
|
||||
|
||||
results, err := DefaultConfig.SearchGames(context.Background(), browser, "elden ring")
|
||||
if err != nil {
|
||||
t.Fatalf("SearchGames() error: %v", err)
|
||||
}
|
||||
|
||||
if len(results) != 1 {
|
||||
t.Fatalf("len(results) = %d, want 1", len(results))
|
||||
}
|
||||
if results[0].Name != "ELDEN RING" {
|
||||
t.Errorf("Name = %q, want %q", results[0].Name, "ELDEN RING")
|
||||
}
|
||||
}
|
||||
|
||||
func TestExtractGamePrice_Empty(t *testing.T) {
|
||||
doc := &extractortest.MockDocument{
|
||||
MockNode: extractortest.MockNode{
|
||||
Children: map[string]extractor.Nodes{},
|
||||
},
|
||||
}
|
||||
|
||||
data, err := extractGamePrice(doc, 0)
|
||||
if err != nil {
|
||||
t.Fatalf("extractGamePrice() error: %v", err)
|
||||
}
|
||||
|
||||
if data.Name != "" || data.Price != 0 {
|
||||
t.Error("expected zero values for empty doc")
|
||||
}
|
||||
}
|
||||
|
||||
func TestExtractCurrency(t *testing.T) {
|
||||
tests := []struct {
|
||||
input string
|
||||
want string
|
||||
}{
|
||||
{"$59.99", "USD"},
|
||||
{"€49.99", "EUR"},
|
||||
{"£39.99", "GBP"},
|
||||
{"59.99", ""},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
got := extractCurrency(tt.input)
|
||||
if got != tt.want {
|
||||
t.Errorf("extractCurrency(%q) = %q, want %q", tt.input, got, tt.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestAppURL(t *testing.T) {
|
||||
got := appURL(1245620)
|
||||
want := "https://store.steampowered.com/app/1245620"
|
||||
if got != want {
|
||||
t.Errorf("appURL(1245620) = %q, want %q", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSearchURL(t *testing.T) {
|
||||
got := searchURL("elden ring")
|
||||
want := "https://store.steampowered.com/search/?term=elden+ring"
|
||||
if got != want {
|
||||
t.Errorf("searchURL(\"elden ring\") = %q, want %q", got, want)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user