steve/go-extractor
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

changed browser api to return pages that can be acted on, not strictly contents

Browse Source
This commit is contained in:
Steve Dudenhoeffer
2024-12-17 23:16:13 -05:00
parent 23334991b1
commit 5e924eb3f9
5 changed files with 154 additions and 93 deletions
Download Patch File Download Diff File Expand all files Collapse all files

82
playwright.go
View File

@@ -11,6 +11,7 @@ import (
)
type playWrightBrowser struct {
pw *playwright.Playwright
browser playwright.Browser
ctx playwright.BrowserContext
userAgent string
@@ -148,6 +149,7 @@ func NewPlayWrightBrowser(opts ...PlayWrightBrowserOptions) (Browser, error) {
}
return playWrightBrowser{
pw: pw,
browser: browser,
userAgent: opt.UserAgent,
timeout: *opt.Timeout,
@@ -192,7 +194,7 @@ func (b playWrightBrowser) openPage(_ context.Context, target string) (playwrigh
return nil, err
}
slog.Info("opened page", "url", target, "status", resp.Status(), "request", resp.Request())
slog.Info("opened document", "url", target, "status", resp.Status(), "request", resp.Request())
if resp.Status() != 200 {
slog.Info("invalid status code", "status", resp.Status(), "request", resp.Request())
@@ -202,7 +204,7 @@ func (b playWrightBrowser) openPage(_ context.Context, target string) (playwrigh
return page, nil
}
func (b playWrightBrowser) Open(ctx context.Context, url string) (Source, error) {
func (b playWrightBrowser) Open(ctx context.Context, url string) (Document, error) {
page, err := b.openPage(ctx, url)
if err != nil {
@@ -210,86 +212,12 @@ func (b playWrightBrowser) Open(ctx context.Context, url string) (Source, error)
}
defer page.Close()
text, err := page.Content()
if err != nil {
return nil, err
}
err = b.updateCookies(ctx, page)
if err != nil {
return nil, err
}
return source{
sourceUrl: url,
content: text,
}, nil
}
func (b playWrightBrowser) getScreenshot(_ context.Context, page playwright.Page, opts ScreenshotOptions) ([]byte, error) {
var pwOpts playwright.PageScreenshotOptions
if opts.Style == "" {
opts.Style = ScreenshotStyleFullPage
}
if opts.Style == ScreenshotStyleFullPage {
pwOpts.FullPage = playwright.Bool(true)
} else if opts.Style == ScreenshotStyleViewport {
pwOpts.FullPage = playwright.Bool(false)
if opts.Width > 0 || opts.Height > 0 {
pwOpts.Clip = &playwright.Rect{
Width: float64(opts.Width),
Height: float64(opts.Height),
}
}
}
return page.Screenshot(pwOpts)
}
func (b playWrightBrowser) Screenshot(ctx context.Context, url string, opts ScreenshotOptions) ([]byte, error) {
page, err := b.openPage(ctx, url)
if err != nil {
return nil, err
}
defer page.Close()
err = b.updateCookies(ctx, page)
if err != nil {
return nil, err
}
return b.getScreenshot(ctx, page, opts)
}
func (b playWrightBrowser) OpenAndScreenshot(ctx context.Context, url string, opts ScreenshotOptions) (Source, []byte, error) {
page, err := b.openPage(ctx, url)
if err != nil {
return nil, nil, err
}
defer page.Close()
text, err := page.Content()
if err != nil {
return nil, nil, err
}
screenshot, err := b.getScreenshot(ctx, page, opts)
if err != nil {
return nil, nil, err
}
err = b.updateCookies(ctx, page)
if err != nil {
return nil, nil, err
}
return source{
sourceUrl: url,
content: text,
}, screenshot, nil
return newDocument(b.pw, b.browser, page)
}
func (b playWrightBrowser) Close() error {