fix: bug fixes, test coverage, and CI workflow
- Fix Nodes.First() panic on empty slice (return nil) - Fix ticker leak in archive.go (create once, defer Stop) - Fix cookie path matching for empty and root paths - Fix lost query params in google.go (u.Query().Set was discarded) - Fix type assertion panic in useragents.go - Fix dropped date parse error in powerball.go - Remove unreachable dead code in megamillions.go and powerball.go - Simplify document.go WaitForNetworkIdle, remove unused root field - Remove debug fmt.Println calls across codebase - Replace panic(err) with stderr+exit in all cmd/ programs - Fix duckduckgo cmd: remove useless defer, return error on bad safesearch - Fix archive cmd: ToConfig returns error instead of panicking - Add 39+ unit tests across 6 new test files - Add Gitea Actions CI workflow (build, test, vet in parallel) Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
@@ -72,6 +72,7 @@ func main() {
|
||||
err := cli.Run(context.Background(), os.Args)
|
||||
|
||||
if err != nil {
|
||||
panic(err)
|
||||
fmt.Fprintf(os.Stderr, "error: %v\n", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -130,10 +130,9 @@ func (c Config) Archive(ctx context.Context, b extractor.Browser, target string)
|
||||
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
fmt.Println("context already done before entering the loop:", ctx.Err())
|
||||
slog.Debug("context already done before entering the loop", "err", ctx.Err())
|
||||
return nil, ctx.Err()
|
||||
default:
|
||||
fmt.Println("context not done yet")
|
||||
// Proceed with the loop
|
||||
}
|
||||
// now we are waiting for archive.ph to archive the page and redirect us to the archived page
|
||||
@@ -141,6 +140,9 @@ func (c Config) Archive(ctx context.Context, b extractor.Browser, target string)
|
||||
// if the page path starts with /wip/ then we are still waiting
|
||||
// also periodically refresh the page just in case
|
||||
|
||||
ticker := time.NewTicker(5 * time.Second)
|
||||
defer ticker.Stop()
|
||||
|
||||
keepGoing := true
|
||||
for keepGoing {
|
||||
select {
|
||||
@@ -148,14 +150,14 @@ func (c Config) Archive(ctx context.Context, b extractor.Browser, target string)
|
||||
slog.Info("context done")
|
||||
keepGoing = false
|
||||
|
||||
case <-time.NewTicker(5 * time.Second).C:
|
||||
case <-ticker.C:
|
||||
archivedUrl, err := url.Parse(doc.URL())
|
||||
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
|
||||
fmt.Println("checking url:", archivedUrl.String())
|
||||
slog.Debug("checking url", "url", archivedUrl.String())
|
||||
// if the url is not the same as the endpoint, or the path does not start with /wip/ or /submit then we are done
|
||||
if archivedUrl.Hostname() != endpoint.Hostname() || (!strings.HasPrefix(archivedUrl.Path, "/wip/") && !strings.HasPrefix(archivedUrl.Path, "/submit")) {
|
||||
keepGoing = false
|
||||
|
||||
@@ -28,7 +28,7 @@ var Flags = ArchiveFlags{
|
||||
},
|
||||
}
|
||||
|
||||
func (f ArchiveFlags) ToConfig(_ context.Context, cmd *cli.Command) archive.Config {
|
||||
func (f ArchiveFlags) ToConfig(_ context.Context, cmd *cli.Command) (archive.Config, error) {
|
||||
c := archive.DefaultConfig
|
||||
|
||||
if e := cmd.String("endpoint"); e != "" {
|
||||
@@ -38,12 +38,12 @@ func (f ArchiveFlags) ToConfig(_ context.Context, cmd *cli.Command) archive.Conf
|
||||
if t := cmd.String("timeout"); t != "" {
|
||||
d, err := time.ParseDuration(t)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
return c, fmt.Errorf("invalid timeout duration: %w", err)
|
||||
}
|
||||
c.Timeout = &d
|
||||
}
|
||||
|
||||
return c
|
||||
return c, nil
|
||||
}
|
||||
|
||||
func main() {
|
||||
@@ -122,7 +122,8 @@ func main() {
|
||||
err := cli.Run(context.Background(), os.Args)
|
||||
|
||||
if err != nil {
|
||||
panic(err)
|
||||
fmt.Fprintf(os.Stderr, "error: %v\n", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@@ -26,7 +26,7 @@ var Flags = DuckDuckGoFlags{
|
||||
},
|
||||
}
|
||||
|
||||
func (f DuckDuckGoFlags) ToConfig(cmd *cli.Command) duckduckgo.Config {
|
||||
func (f DuckDuckGoFlags) ToConfig(cmd *cli.Command) (duckduckgo.Config, error) {
|
||||
var res = duckduckgo.DefaultConfig
|
||||
|
||||
if r := cmd.String("region"); r != "" {
|
||||
@@ -42,11 +42,11 @@ func (f DuckDuckGoFlags) ToConfig(cmd *cli.Command) duckduckgo.Config {
|
||||
case "off":
|
||||
res.SafeSearch = duckduckgo.SafeSearchOff
|
||||
default:
|
||||
panic("invalid safe search value")
|
||||
return res, fmt.Errorf("invalid safe search value: %s", s)
|
||||
}
|
||||
}
|
||||
|
||||
return res
|
||||
return res, nil
|
||||
}
|
||||
|
||||
func deferClose(cl io.Closer) {
|
||||
@@ -66,8 +66,10 @@ func main() {
|
||||
Usage: "Search DuckDuckGo",
|
||||
Flags: flags,
|
||||
Action: func(ctx context.Context, command *cli.Command) error {
|
||||
c := Flags.ToConfig(command)
|
||||
defer deferClose(nil)
|
||||
c, err := Flags.ToConfig(command)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
query := strings.TrimSpace(strings.Join(command.Args().Slice(), " "))
|
||||
|
||||
@@ -105,9 +107,8 @@ func main() {
|
||||
},
|
||||
}
|
||||
|
||||
err := cli.Run(context.Background(), os.Args)
|
||||
|
||||
if err != nil {
|
||||
panic(err)
|
||||
if err := cli.Run(context.Background(), os.Args); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "error: %v\n", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
85
sites/duckduckgo/duckduckgo_test.go
Normal file
85
sites/duckduckgo/duckduckgo_test.go
Normal file
@@ -0,0 +1,85 @@
|
||||
package duckduckgo
|
||||
|
||||
import (
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestConfig_ToSearchURL_Default(t *testing.T) {
|
||||
c := Config{SafeSearch: SafeSearchOff}
|
||||
u := c.ToSearchURL("test query")
|
||||
|
||||
if u.Host != "duckduckgo.com" {
|
||||
t.Errorf("Host = %q, want %q", u.Host, "duckduckgo.com")
|
||||
}
|
||||
|
||||
if u.Query().Get("q") != "test query" {
|
||||
t.Errorf("q = %q, want %q", u.Query().Get("q"), "test query")
|
||||
}
|
||||
|
||||
if u.Query().Get("kp") != "-2" {
|
||||
t.Errorf("kp = %q, want %q", u.Query().Get("kp"), "-2")
|
||||
}
|
||||
}
|
||||
|
||||
func TestConfig_ToSearchURL_SafeSearchOn(t *testing.T) {
|
||||
c := Config{SafeSearch: SafeSearchOn}
|
||||
u := c.ToSearchURL("test")
|
||||
|
||||
if u.Query().Get("kp") != "1" {
|
||||
t.Errorf("kp = %q, want %q", u.Query().Get("kp"), "1")
|
||||
}
|
||||
}
|
||||
|
||||
func TestConfig_ToSearchURL_SafeSearchModerate(t *testing.T) {
|
||||
c := Config{SafeSearch: SafeSearchModerate}
|
||||
u := c.ToSearchURL("test")
|
||||
|
||||
if u.Query().Get("kp") != "-1" {
|
||||
t.Errorf("kp = %q, want %q", u.Query().Get("kp"), "-1")
|
||||
}
|
||||
}
|
||||
|
||||
func TestConfig_ToSearchURL_SafeSearchOff(t *testing.T) {
|
||||
c := Config{SafeSearch: SafeSearchOff}
|
||||
u := c.ToSearchURL("test")
|
||||
|
||||
if u.Query().Get("kp") != "-2" {
|
||||
t.Errorf("kp = %q, want %q", u.Query().Get("kp"), "-2")
|
||||
}
|
||||
}
|
||||
|
||||
func TestConfig_ToSearchURL_WithRegion(t *testing.T) {
|
||||
c := Config{SafeSearch: SafeSearchOff, Region: "us-en"}
|
||||
u := c.ToSearchURL("test")
|
||||
|
||||
if u.Query().Get("kl") != "us-en" {
|
||||
t.Errorf("kl = %q, want %q", u.Query().Get("kl"), "us-en")
|
||||
}
|
||||
}
|
||||
|
||||
func TestConfig_ToSearchURL_WithQuery(t *testing.T) {
|
||||
c := Config{SafeSearch: SafeSearchOff}
|
||||
u := c.ToSearchURL("golang testing")
|
||||
|
||||
if u.Query().Get("q") != "golang testing" {
|
||||
t.Errorf("q = %q, want %q", u.Query().Get("q"), "golang testing")
|
||||
}
|
||||
}
|
||||
|
||||
func TestConfig_Validate_DefaultsSafeSearch(t *testing.T) {
|
||||
c := Config{SafeSearch: 0}
|
||||
c = c.validate()
|
||||
|
||||
if c.SafeSearch != SafeSearchOff {
|
||||
t.Errorf("validate() SafeSearch = %d, want %d (SafeSearchOff)", c.SafeSearch, SafeSearchOff)
|
||||
}
|
||||
}
|
||||
|
||||
func TestConfig_ToSearchURL_NoRegion(t *testing.T) {
|
||||
c := Config{SafeSearch: SafeSearchOff}
|
||||
u := c.ToSearchURL("test")
|
||||
|
||||
if u.Query().Get("kl") != "" {
|
||||
t.Errorf("kl should be empty when no region, got %q", u.Query().Get("kl"))
|
||||
}
|
||||
}
|
||||
@@ -87,9 +87,8 @@ func main() {
|
||||
},
|
||||
}
|
||||
|
||||
err := cli.Run(context.Background(), os.Args)
|
||||
|
||||
if err != nil {
|
||||
panic(err)
|
||||
if err := cli.Run(context.Background(), os.Args); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "error: %v\n", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -57,14 +57,17 @@ func deferClose(cl io.Closer) {
|
||||
func (c Config) Search(ctx context.Context, b extractor.Browser, query string) ([]Result, error) {
|
||||
c = c.validate()
|
||||
|
||||
u, err := url.Parse(fmt.Sprintf("https://%s/search?q=%s", c.BaseURL, query))
|
||||
u, err := url.Parse(fmt.Sprintf("https://%s/search", c.BaseURL))
|
||||
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("invalid url: %w", err)
|
||||
}
|
||||
|
||||
vals := u.Query()
|
||||
vals.Set("q", query)
|
||||
|
||||
if c.Language != "" {
|
||||
u.Query().Set("hl", c.Language)
|
||||
vals.Set("hl", c.Language)
|
||||
}
|
||||
|
||||
if c.Country != "" {
|
||||
@@ -84,10 +87,12 @@ func (c Config) Search(ctx context.Context, b extractor.Browser, query string) (
|
||||
}
|
||||
|
||||
if country != "" {
|
||||
u.Query().Set("cr", country)
|
||||
vals.Set("cr", country)
|
||||
}
|
||||
}
|
||||
|
||||
u.RawQuery = vals.Encode()
|
||||
|
||||
doc, err := b.Open(ctx, u.String(), extractor.OpenPageOptions{})
|
||||
|
||||
if err != nil {
|
||||
|
||||
39
sites/google/google_test.go
Normal file
39
sites/google/google_test.go
Normal file
@@ -0,0 +1,39 @@
|
||||
package google
|
||||
|
||||
import (
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestConfig_Validate_Defaults(t *testing.T) {
|
||||
c := Config{}
|
||||
c = c.validate()
|
||||
|
||||
if c.BaseURL != "google.com" {
|
||||
t.Errorf("BaseURL = %q, want %q", c.BaseURL, "google.com")
|
||||
}
|
||||
if c.Language != "en" {
|
||||
t.Errorf("Language = %q, want %q", c.Language, "en")
|
||||
}
|
||||
if c.Country != "us" {
|
||||
t.Errorf("Country = %q, want %q", c.Country, "us")
|
||||
}
|
||||
}
|
||||
|
||||
func TestConfig_Validate_Preserves(t *testing.T) {
|
||||
c := Config{
|
||||
BaseURL: "google.co.uk",
|
||||
Language: "fr",
|
||||
Country: "uk",
|
||||
}
|
||||
c = c.validate()
|
||||
|
||||
if c.BaseURL != "google.co.uk" {
|
||||
t.Errorf("BaseURL = %q, want %q", c.BaseURL, "google.co.uk")
|
||||
}
|
||||
if c.Language != "fr" {
|
||||
t.Errorf("Language = %q, want %q", c.Language, "fr")
|
||||
}
|
||||
if c.Country != "uk" {
|
||||
t.Errorf("Country = %q, want %q", c.Country, "uk")
|
||||
}
|
||||
}
|
||||
@@ -51,10 +51,8 @@ func main() {
|
||||
},
|
||||
}
|
||||
|
||||
err := cli.Run(context.Background(), os.Args)
|
||||
|
||||
if err != nil {
|
||||
panic(err)
|
||||
if err := cli.Run(context.Background(), os.Args); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "error: %v\n", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@@ -64,7 +64,6 @@ func getDrawing(_ context.Context, doc extractor.Document) (*Drawing, error) {
|
||||
return nil, fmt.Errorf("failed to parse date: %w", err)
|
||||
}
|
||||
|
||||
fmt.Println("ticks", ticks)
|
||||
drawing.Date = netTicksToTime(ticks)
|
||||
|
||||
err = doc.ForEach("ul.numbers li.ball", func(n extractor.Node) error {
|
||||
@@ -199,23 +198,12 @@ func getNextDrawing(_ context.Context, doc extractor.Document) (*NextDrawing, er
|
||||
|
||||
numeric := numericOnly(txt)
|
||||
|
||||
set := false
|
||||
if strings.Contains(txt, "Billion") {
|
||||
amt := currency.USD.Amount(numeric * 1000000000)
|
||||
nextDrawing.Jackpot = amt
|
||||
set = true
|
||||
nextDrawing.Jackpot = currency.USD.Amount(numeric * 1000000000)
|
||||
} else if strings.Contains(txt, "Million") {
|
||||
amt := currency.USD.Amount(numeric * 1000000)
|
||||
nextDrawing.Jackpot = amt
|
||||
set = true
|
||||
nextDrawing.Jackpot = currency.USD.Amount(numeric * 1000000)
|
||||
} else {
|
||||
amt := currency.USD.Amount(numeric)
|
||||
nextDrawing.Jackpot = amt
|
||||
set = true
|
||||
}
|
||||
|
||||
if !set {
|
||||
return nil, fmt.Errorf("failed to convert jackpot to currency: %w", err)
|
||||
nextDrawing.Jackpot = currency.USD.Amount(numeric)
|
||||
}
|
||||
|
||||
return &nextDrawing, nil
|
||||
|
||||
43
sites/megamillions/megamillions_test.go
Normal file
43
sites/megamillions/megamillions_test.go
Normal file
@@ -0,0 +1,43 @@
|
||||
package megamillions
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestNetTicksToTime_Consistency(t *testing.T) {
|
||||
// netTicksToTime converts .NET ticks to Go time.
|
||||
// Verify it produces consistent results for the same input.
|
||||
ticks := int64(638396256000000000)
|
||||
t1 := netTicksToTime(ticks)
|
||||
t2 := netTicksToTime(ticks)
|
||||
|
||||
if !t1.Equal(t2) {
|
||||
t.Errorf("netTicksToTime is not consistent: %v != %v", t1, t2)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNetTicksToTime_Ordering(t *testing.T) {
|
||||
// A larger ticks value should produce a later time.
|
||||
earlier := netTicksToTime(638396256000000000)
|
||||
later := netTicksToTime(638396256100000000) // 10 seconds later in ticks
|
||||
|
||||
if !later.After(earlier) {
|
||||
t.Errorf("expected later ticks to produce later time: %v vs %v", earlier, later)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNetTicksToTime_DifferenceIsCorrect(t *testing.T) {
|
||||
// .NET ticks are 100-nanosecond intervals.
|
||||
// 10,000,000 ticks = 1 second.
|
||||
ticks1 := int64(638396256000000000)
|
||||
ticks2 := ticks1 + 10000000 // 1 second later
|
||||
|
||||
t1 := netTicksToTime(ticks1)
|
||||
t2 := netTicksToTime(ticks2)
|
||||
|
||||
diff := t2.Sub(t1)
|
||||
if diff != time.Second {
|
||||
t.Errorf("expected 1 second difference, got %v", diff)
|
||||
}
|
||||
}
|
||||
@@ -51,10 +51,8 @@ func main() {
|
||||
},
|
||||
}
|
||||
|
||||
err := cli.Run(context.Background(), os.Args)
|
||||
|
||||
if err != nil {
|
||||
panic(err)
|
||||
if err := cli.Run(context.Background(), os.Args); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "error: %v\n", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@@ -41,9 +41,20 @@ func deferClose(cl io.Closer) {
|
||||
func getDrawing(_ context.Context, doc extractor.Document) (*Drawing, error) {
|
||||
var drawing Drawing
|
||||
|
||||
dateStr, err := doc.SelectFirst("#numbers .title-date").Text()
|
||||
dateNode := doc.SelectFirst("#numbers .title-date")
|
||||
if dateNode == nil {
|
||||
return nil, fmt.Errorf("failed to find date element")
|
||||
}
|
||||
|
||||
dateStr, err := dateNode.Text()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to get date text: %w", err)
|
||||
}
|
||||
|
||||
drawing.Date, err = time.Parse("Mon, Jan 2, 2006", dateStr)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to parse date %q: %w", dateStr, err)
|
||||
}
|
||||
|
||||
nums := doc.Select("div.game-ball-group div.white-balls")
|
||||
|
||||
@@ -165,22 +176,12 @@ func getNextDrawing(_ context.Context, doc extractor.Document) (*NextDrawing, er
|
||||
|
||||
numeric := numericOnly(txt)
|
||||
|
||||
set := false
|
||||
if strings.Contains(txt, "Billion") {
|
||||
amt := numeric * 1000000000
|
||||
nextDrawing.JackpotDollars = int(amt)
|
||||
set = true
|
||||
nextDrawing.JackpotDollars = int(numeric * 1000000000)
|
||||
} else if strings.Contains(txt, "Million") {
|
||||
amt := numeric * 1000000
|
||||
nextDrawing.JackpotDollars = int(amt)
|
||||
set = true
|
||||
nextDrawing.JackpotDollars = int(numeric * 1000000)
|
||||
} else {
|
||||
nextDrawing.JackpotDollars = int(numeric)
|
||||
set = true
|
||||
}
|
||||
|
||||
if !set {
|
||||
return nil, fmt.Errorf("failed to convert jackpot to currency: %w", err)
|
||||
}
|
||||
|
||||
return &nextDrawing, nil
|
||||
|
||||
@@ -49,10 +49,8 @@ func main() {
|
||||
},
|
||||
}
|
||||
|
||||
err := cli.Run(context.Background(), os.Args)
|
||||
|
||||
if err != nil {
|
||||
panic(err)
|
||||
if err := cli.Run(context.Background(), os.Args); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "error: %v\n", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@@ -44,8 +44,6 @@ func (c Config) GetMostCommonDesktopUserAgent(ctx context.Context, b extractor.B
|
||||
}
|
||||
data := []map[string]any{}
|
||||
|
||||
fmt.Println("text", text)
|
||||
|
||||
err = json.Unmarshal([]byte(text), &data)
|
||||
|
||||
if err != nil {
|
||||
@@ -63,8 +61,12 @@ func (c Config) GetMostCommonDesktopUserAgent(ctx context.Context, b extractor.B
|
||||
}
|
||||
|
||||
if pct > highestPct {
|
||||
ua, ok := agent["ua"].(string)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
highestPct = pct
|
||||
highestAgent = agent["ua"].(string)
|
||||
highestAgent = ua
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -73,9 +73,8 @@ func main() {
|
||||
},
|
||||
}
|
||||
|
||||
err := app.Run(context.Background(), os.Args)
|
||||
|
||||
if err != nil {
|
||||
panic(err)
|
||||
if err := app.Run(context.Background(), os.Args); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "error: %v\n", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user