5 Commits

Author SHA1 Message Date
964a98a5a8 Handle commands without automatic reaction responses
Introduce `ErrCommandNoReactions` to allow commands to opt out of success reactions. Adjust bot behavior to respect this error and prevent reactions when applicable, ensuring cleaner and more controlled responses. Add error handling and safeguard workers against panics.
2025-01-22 21:06:07 -05:00
81ea656332 Add unit price and unit parsing for items
This update enhances the `Item` structure to include `UnitPrice` and `Unit` fields. Additional logic is implemented to extract and parse unit pricing details from the HTML, improving data accuracy and granularity.
2025-01-21 19:42:25 -05:00
6de455b1bd Add price extraction and validate URL structure in parsers
Added price field to Item struct in AisleGopher and implemented logic to extract price data. Updated Wegmans parser to validate URL structure by ensuring the second segment is "product". These changes improve data accuracy and error handling.
2025-01-20 13:00:59 -05:00
f37e60dddc Add Wegmans module to fetch item details and prices
Introduce functionality to retrieve item details, including name and price, from Wegmans using a browser-based scraper. This includes a CLI tool to execute searches and robust error handling for URL validation and browser interactions.
2025-01-20 12:28:29 -05:00
654976de82 Add AisleGopher integration for data extraction
Introduced a new package and command for extracting data from aislegopher.com, including URL parsing and item retrieval. Updated dependencies in go.mod to support the new functionality. Additionally, refined import structure in the DuckDuckGo integration.
2025-01-20 02:16:32 -05:00
6 changed files with 360 additions and 4 deletions

4
go.mod
View File

@@ -5,6 +5,8 @@ go 1.23.2
require ( require (
github.com/go-shiori/go-readability v0.0.0-20241012063810-92284fa8a71f github.com/go-shiori/go-readability v0.0.0-20241012063810-92284fa8a71f
github.com/playwright-community/playwright-go v0.4802.0 github.com/playwright-community/playwright-go v0.4802.0
github.com/urfave/cli/v3 v3.0.0-beta1
golang.org/x/text v0.21.0
) )
require ( require (
@@ -15,7 +17,5 @@ require (
github.com/go-shiori/dom v0.0.0-20230515143342-73569d674e1c // indirect github.com/go-shiori/dom v0.0.0-20230515143342-73569d674e1c // indirect
github.com/go-stack/stack v1.8.1 // indirect github.com/go-stack/stack v1.8.1 // indirect
github.com/gogs/chardet v0.0.0-20211120154057-b7413eaefb8f // indirect github.com/gogs/chardet v0.0.0-20211120154057-b7413eaefb8f // indirect
github.com/urfave/cli/v3 v3.0.0-beta1 // indirect
golang.org/x/net v0.32.0 // indirect golang.org/x/net v0.32.0 // indirect
golang.org/x/text v0.21.0 // indirect
) )

View File

@@ -0,0 +1,81 @@
package aislegopher
import (
"context"
"errors"
"fmt"
"io"
"net/url"
"strconv"
"strings"
"gitea.stevedudenhoeffer.com/steve/go-extractor"
)
type Config struct {
}
var DefaultConfig = Config{}
var (
ErrInvalidURL = errors.New("invalid url")
)
type Item struct {
ID int
Name string
Price float64
}
func deferClose(cl io.Closer) {
if cl != nil {
_ = cl.Close()
}
}
func GetItemFromURL(ctx context.Context, b extractor.Browser, u *url.URL) (Item, error) {
return DefaultConfig.GetItemFromURL(ctx, b, u)
}
func (c Config) GetItemFromURL(ctx context.Context, b extractor.Browser, u *url.URL) (Item, error) {
res := Item{}
// the url will be in the format of aislegopher.com/p/slug/id
// we need to parse the slug and id from the url
a := strings.Split(u.Path, "/")
if len(a) != 4 {
return res, ErrInvalidURL
}
if a[1] != "p" {
return res, ErrInvalidURL
}
if u.Host != "aislegopher.com" && u.Host != "www.aislegopher.com" {
return res, ErrInvalidURL
}
res.ID, _ = strconv.Atoi(a[3])
doc, err := b.Open(ctx, u.String(), extractor.OpenPageOptions{})
defer deferClose(doc)
if err != nil {
return res, fmt.Errorf("failed to open page: %w", err)
}
names := doc.Select("h2.h4")
if len(names) > 0 {
res.Name, _ = names[0].Text()
}
prices := doc.Select("h4.h2")
if len(prices) > 0 {
priceStr, _ := prices[0].Text()
priceStr = strings.ReplaceAll(priceStr, "$", "")
priceStr = strings.TrimSpace(priceStr)
res.Price, _ = strconv.ParseFloat(priceStr, 64)
}
return res, nil
}

View File

@@ -0,0 +1,77 @@
package main
import (
"context"
"fmt"
"io"
"net/url"
"os"
"gitea.stevedudenhoeffer.com/steve/go-extractor/cmd/browser/pkg/browser"
"gitea.stevedudenhoeffer.com/steve/go-extractor/sites/aislegopher"
"github.com/urfave/cli/v3"
)
type AisleGopherFlags []cli.Flag
var Flags = AisleGopherFlags{}
func (f AisleGopherFlags) ToConfig(_ *cli.Command) aislegopher.Config {
res := aislegopher.DefaultConfig
return res
}
func deferClose(cl io.Closer) {
if cl != nil {
_ = cl.Close()
}
}
func main() {
var flags []cli.Flag
flags = append(flags, browser.Flags...)
flags = append(flags, Flags...)
cli := &cli.Command{
Name: "aislegopher",
Usage: "AisleGopher is a tool for extracting data from aislegopher.com",
Flags: flags,
Action: func(ctx context.Context, c *cli.Command) error {
cfg := Flags.ToConfig(c)
b, err := browser.FromCommand(ctx, c)
if err != nil {
return fmt.Errorf("failed to create browser: %w", err)
}
defer deferClose(b)
arg := c.Args().First()
if arg == "" {
return fmt.Errorf("url is required")
}
u, err := url.Parse(arg)
if err != nil {
return fmt.Errorf("failed to parse url: %w", err)
}
data, err := cfg.GetItemFromURL(ctx, b, u)
if err != nil {
return fmt.Errorf("failed to get item from url: %w", err)
}
fmt.Printf("Item: %+v\n", data)
return nil
},
}
err := cli.Run(context.Background(), os.Args)
if err != nil {
panic(err)
}
}

View File

@@ -7,10 +7,9 @@ import (
"os" "os"
"strings" "strings"
"gitea.stevedudenhoeffer.com/steve/go-extractor/cmd/browser/pkg/browser"
"github.com/urfave/cli/v3" "github.com/urfave/cli/v3"
"gitea.stevedudenhoeffer.com/steve/go-extractor/cmd/browser/pkg/browser"
"gitea.stevedudenhoeffer.com/steve/go-extractor/sites/duckduckgo" "gitea.stevedudenhoeffer.com/steve/go-extractor/sites/duckduckgo"
) )

View File

@@ -0,0 +1,81 @@
package main
import (
"context"
"fmt"
"io"
"net/url"
"os"
"gitea.stevedudenhoeffer.com/steve/go-extractor/cmd/browser/pkg/browser"
"github.com/urfave/cli/v3"
"gitea.stevedudenhoeffer.com/steve/go-extractor/sites/wegmans"
)
func deferClose(cl io.Closer) {
if cl != nil {
_ = cl.Close()
}
}
type WegmansFlags []cli.Flag
var Flags = WegmansFlags{}
func (f WegmansFlags) ToConfig(_ *cli.Command) wegmans.Config {
var res = wegmans.DefaultConfig
return res
}
func main() {
var flags []cli.Flag
flags = append(flags, browser.Flags...)
flags = append(flags, Flags...)
app := &cli.Command{
Name: "wegmans",
Usage: "Search Wegmans",
Flags: flags,
Action: func(ctx context.Context, cmd *cli.Command) error {
cfg := Flags.ToConfig(cmd)
b, err := browser.FromCommand(ctx, cmd)
defer deferClose(b)
if err != nil {
return fmt.Errorf("error creating browser: %w", err)
}
arg := cmd.Args().First()
if arg == "" {
return fmt.Errorf("url is required")
}
u, err := url.Parse(arg)
if err != nil {
return fmt.Errorf("failed to parse url: %w", err)
}
item, err := cfg.GetItemPrice(ctx, b, u)
if err != nil {
return fmt.Errorf("failed to get item price: %w", err)
}
fmt.Println(item)
return nil
},
}
err := app.Run(context.Background(), os.Args)
if err != nil {
panic(err)
}
}

118
sites/wegmans/wegmans.go Normal file
View File

@@ -0,0 +1,118 @@
package wegmans
import (
"context"
"errors"
"io"
"net/url"
"strconv"
"strings"
"time"
"gitea.stevedudenhoeffer.com/steve/go-extractor"
)
type Config struct {
}
var DefaultConfig = Config{}
var ErrNilBrowser = errors.New("browser is nil")
var ErrNilURL = errors.New("url is nil")
var ErrInvalidURL = errors.New("invalid url")
type Item struct {
ID int
Name string
Price float64
UnitPrice float64
Unit string
}
func deferClose(c io.Closer) {
if c != nil {
_ = c.Close()
}
}
func (c Config) GetItemPrice(ctx context.Context, b extractor.Browser, u *url.URL) (Item, error) {
if b == nil {
return Item{}, ErrNilBrowser
}
if u == nil {
return Item{}, ErrNilURL
}
// urls in the format of:
// https://shop.wegmans.com/product/24921[/wegmans-frozen-thin-crust-uncured-pepperoni-pizza]
// (the slug is optional)
// get the product ID
a := strings.Split(u.Path, "/")
if len(a) < 3 {
return Item{}, ErrInvalidURL
}
if a[1] != "product" {
return Item{}, ErrInvalidURL
}
id, _ := strconv.Atoi(a[2])
if id == 0 {
return Item{}, ErrInvalidURL
}
doc, err := b.Open(ctx, u.String(), extractor.OpenPageOptions{})
defer deferClose(doc)
if err != nil {
return Item{}, err
}
timeout := 15 * time.Second
_ = doc.WaitForNetworkIdle(&timeout)
res := Item{
ID: id,
}
titles := doc.Select("h1[data-test]")
if len(titles) != 0 {
res.Name, _ = titles[0].Text()
}
prices := doc.Select("span[data-test=\"amount\"] span:nth-child(1)")
if len(prices) != 0 {
priceStr, _ := prices[0].Text()
priceStr = strings.ReplaceAll(priceStr, "$", "")
priceStr = strings.ReplaceAll(priceStr, ",", "")
price, _ := strconv.ParseFloat(priceStr, 64)
res.Price = price
}
unitPrices := doc.Select(`span[data-test="per-unit-price"]`)
if len(unitPrices) != 0 {
unitPriceStr, _ := unitPrices[0].Text()
unitPriceStr = strings.TrimSpace(unitPriceStr)
unitPriceStr = strings.ReplaceAll(unitPriceStr, "(", "")
unitPriceStr = strings.ReplaceAll(unitPriceStr, ")", "")
unitPriceStr = strings.ReplaceAll(unitPriceStr, "$", "")
unitPriceStr = strings.ReplaceAll(unitPriceStr, ",", "")
units := strings.Split(unitPriceStr, "/")
if len(units) > 1 {
res.Unit = strings.TrimSpace(units[1])
res.UnitPrice, _ = strconv.ParseFloat(units[0], 64)
}
}
return res, nil
}