Skip to content

Commit 3d6e2e7

Browse files
ProximystProximyst
authored
feat(go): add tonnes of knobs (#808)
1 parent c83fbaa commit 3d6e2e7

File tree

2 files changed

+168
-79
lines changed

2 files changed

+168
-79
lines changed

‎pkg/config/config.go‎

Lines changed: 91 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -314,10 +314,18 @@ type BrowserConfig struct {
314314
// That means for a viewport that is 500px high, and a webpage that is 2500px high, we will scroll 5 times, meaning a total wait duration of 6 * duration (as we have to wait on the first & last scrolls as well).
315315
TimeBetweenScrolls time.Duration
316316
// ReadinessTimeout is the maximum time to wait for the web-page to become ready (i.e. no longer loading anything).
317-
ReadinessTimeout time.Duration
318-
// LoadWait is the time to wait before checking for how ready the page is.
317+
ReadinessTimeout time.Duration
318+
ReadinessIterationInterval time.Duration
319+
// ReadinessPriorWait is the time to wait before checking for how ready the page is.
319320
// This lets you force the webpage to take a beat and just do its thing before the service starts looking for whether it's time to render anything.
320-
LoadWait time.Duration
321+
ReadinessPriorWait time.Duration
322+
ReadinessDisableQueryWait bool
323+
ReadinessFirstQueryTimeout time.Duration
324+
ReadinessQueriesTimeout time.Duration
325+
ReadinessDisableNetworkWait bool
326+
ReadinessNetworkIdleTimeout time.Duration
327+
ReadinessDisableDOMHashCodeWait bool
328+
ReadinessDOMHashCodeTimeout time.Duration
321329

322330
// MinWidth is the minimum width of the browser viewport.
323331
// If larger than MaxWidth, MaxWidth is used instead.
@@ -400,16 +408,66 @@ func BrowserFlags() []cli.Flag {
400408
Sources: FromConfig("browser.time-between-scrolls", "BROWSER_TIME_BETWEEN_SCROLLS"),
401409
},
402410
&cli.DurationFlag{
403-
Name: "browser.readiness-timeout",
404-
Usage: "The maximum time to wait for a web-page to become ready (i.e. no longer loading anything).",
411+
Name: "browser.readiness.timeout",
412+
Usage: "The maximum time to wait for a web-page to become ready (i.e. no longer loading anything). If <= 0, the timeout is disabled.",
405413
Value: time.Second * 30,
406-
Sources: FromConfig("browser.readiness-timeout", "BROWSER_READINESS_TIMEOUT"),
414+
Sources: FromConfig("browser.readiness.timeout", "BROWSER_READINESS_TIMEOUT"),
407415
},
408416
&cli.DurationFlag{
409-
Name: "browser.load-wait",
410-
Usage: "The time to wait before checking for how ready the page is. This lets you force the webpage to take a beat and just do its thing before the service starts looking for whether it's time to render anything.",
417+
Name: "browser.readiness.iteration-interval",
418+
Usage: "How long to wait between each iteration of checking whether the page is ready. Must be positive.",
419+
Value: time.Millisecond * 100,
420+
Validator: func(d time.Duration) error {
421+
if d <= 0 {
422+
return fmt.Errorf("browser readiness iteration-interval must be positive (got %v)", d)
423+
}
424+
return nil
425+
},
426+
},
427+
&cli.DurationFlag{
428+
Name: "browser.readiness.prior-wait",
429+
Usage: "The time to wait before checking for how ready the page is. This lets you force the webpage to take a beat and just do its thing before the service starts looking for whether it's time to render anything. If <= 0, this is disabled.",
411430
Value: time.Second,
412-
Sources: FromConfig("browser.load-wait", "BROWSER_LOAD_WAIT"),
431+
Sources: FromConfig("browser.readiness.prior-wait", "BROWSER_READINESS_PRIOR_WAIT"),
432+
},
433+
&cli.BoolFlag{
434+
Name: "browser.readiness.disable-query-wait",
435+
Usage: "Disable waiting for queries to finish before capturing.",
436+
Sources: FromConfig("browser.readiness.disable-query-wait", "BROWSER_READINESS_DISABLE_QUERY_WAIT"),
437+
},
438+
&cli.DurationFlag{
439+
Name: "browser.readiness.give-up-on-first-query",
440+
Usage: "How long to wait before giving up on a first query being registered. If <= 0, the give-up is disabled.",
441+
Value: time.Second * 3,
442+
Sources: FromConfig("browser.readiness.give-up-on-first-query", "BROWSER_READINESS_GIVE_UP_ON_FIRST_QUERY"),
443+
},
444+
&cli.DurationFlag{
445+
Name: "browser.readiness.give-up-on-all-queries",
446+
Usage: "How long to wait before giving up on all running queries. If <= 0, the give-up is disabled.",
447+
Value: 0,
448+
Sources: FromConfig("browser.readiness.give-up-on-all-queries", "BROWSER_READINESS_GIVE_UP_ON_ALL_QUERIES"),
449+
},
450+
&cli.BoolFlag{
451+
Name: "browser.readiness.disable-network-wait",
452+
Usage: "Disable waiting for network requests to finish before capturing.",
453+
Sources: FromConfig("browser.readiness.disable-network-wait", "BROWSER_READINESS_DISABLE_NETWORK_WAIT"),
454+
},
455+
&cli.DurationFlag{
456+
Name: "browser.readiness.network-idle-timeout",
457+
Usage: "How long to wait before giving up on the network being idle. If <= 0, the timeout is disabled.",
458+
Value: 0,
459+
Sources: FromConfig("browser.readiness.network-idle-timeout", "BROWSER_READINESS_NETWORK_IDLE_TIMEOUT"),
460+
},
461+
&cli.BoolFlag{
462+
Name: "browser.readiness.disable-dom-hashcode-wait",
463+
Usage: "Disable waiting for the DOM to stabilize (i.e. not change) before capturing.",
464+
Sources: FromConfig("browser.readiness.disable-dom-hashcode-wait", "BROWSER_READINESS_DISABLE_DOM_HASHCODE_WAIT"),
465+
},
466+
&cli.DurationFlag{
467+
Name: "browser.readiness.dom-hashcode-timeout",
468+
Usage: "How long to wait before giving up on the DOM stabilizing (i.e. not changing). If <= 0, the timeout is disabled.",
469+
Value: 0,
470+
Sources: FromConfig("browser.readiness.dom-hashcode-timeout", "BROWSER_READINESS_DOM_HASHCODE_TIMEOUT"),
413471
},
414472
&cli.IntFlag{
415473
Name: "browser.min-width",
@@ -504,21 +562,29 @@ func BrowserConfigFromCommand(c *cli.Command) (BrowserConfig, error) {
504562
}
505563

506564
return BrowserConfig{
507-
Path: c.String("browser.path"),
508-
Flags: c.StringSlice("browser.flag"),
509-
GPU: c.Bool("browser.gpu"),
510-
Sandbox: c.Bool("browser.sandbox"),
511-
TimeZone: timeZone,
512-
Cookies: nil,
513-
Headers: headers,
514-
TimeBetweenScrolls: c.Duration("browser.time-between-scrolls"),
515-
ReadinessTimeout: c.Duration("browser.readiness-timeout"),
516-
LoadWait: c.Duration("browser.load-wait"),
517-
MinWidth: minWidth,
518-
MinHeight: minHeight,
519-
MaxWidth: maxWidth,
520-
MaxHeight: maxHeight,
521-
PageScaleFactor: c.Float64("browser.page-scale-factor"),
522-
Landscape: !c.Bool("browser.portrait"),
565+
Path: c.String("browser.path"),
566+
Flags: c.StringSlice("browser.flag"),
567+
GPU: c.Bool("browser.gpu"),
568+
Sandbox: c.Bool("browser.sandbox"),
569+
TimeZone: timeZone,
570+
Cookies: nil,
571+
Headers: headers,
572+
TimeBetweenScrolls: c.Duration("browser.time-between-scrolls"),
573+
ReadinessTimeout: c.Duration("browser.readiness.timeout"),
574+
ReadinessIterationInterval: c.Duration("browser.readiness.iteration-interval"),
575+
ReadinessPriorWait: c.Duration("browser.readiness.prior-wait"),
576+
ReadinessDisableQueryWait: c.Bool("browser.readiness.disable-query-wait"),
577+
ReadinessFirstQueryTimeout: c.Duration("browser.readiness.give-up-on-first-query"),
578+
ReadinessQueriesTimeout: c.Duration("browser.readiness.give-up-on-all-queries"),
579+
ReadinessDisableNetworkWait: c.Bool("browser.readiness.disable-network-wait"),
580+
ReadinessNetworkIdleTimeout: c.Duration("browser.readiness.network-idle-timeout"),
581+
ReadinessDisableDOMHashCodeWait: c.Bool("browser.readiness.disable-dom-hashcode-wait"),
582+
ReadinessDOMHashCodeTimeout: c.Duration("browser.readiness.dom-hashcode-timeout"),
583+
MinWidth: minWidth,
584+
MinHeight: minHeight,
585+
MaxWidth: maxWidth,
586+
MaxHeight: maxHeight,
587+
PageScaleFactor: c.Float64("browser.page-scale-factor"),
588+
Landscape: !c.Bool("browser.portrait"),
523589
}, nil
524590
}

‎pkg/service/browser.go‎

Lines changed: 77 additions & 54 deletions
Original file line numberDiff line numberDiff line change
@@ -235,17 +235,17 @@ func (s *BrowserService) Render(ctx context.Context, url string, printer Printer
235235

236236
fileChan := make(chan []byte, 1) // buffered: we don't want the browser to stick around while we try to export this value.
237237
actions := []chromedp.Action{
238-
tracingAction("network.Enable", network.Enable()),
239-
tracingAction("fetch.Enable", fetch.Enable()), // required by handleNetworkEvents
238+
tracingAction("network.Enable", network.Enable()), // required by waitForReady
239+
tracingAction("fetch.Enable", fetch.Enable()), // required by handleNetworkEvents
240240
tracingAction("SetPageScaleFactor", emulation.SetPageScaleFactor(cfg.PageScaleFactor)),
241241
tracingAction("EmulateViewport", chromedp.EmulateViewport(int64(cfg.MinWidth), int64(cfg.MinHeight), orientation)),
242242
setHeaders(browserCtx, cfg.Headers),
243243
setCookies(cfg.Cookies),
244244
tracingAction("Navigate", chromedp.Navigate(url)),
245245
tracingAction("WaitReady(body)", chromedp.WaitReady("body", chromedp.ByQuery)), // wait for a body to exist; this is when the page has started to actually render
246246
scrollForElements(cfg.TimeBetweenScrolls),
247-
waitForDuration(cfg.LoadWait),
248-
waitForReady(browserCtx, cfg.ReadinessTimeout),
247+
waitForDuration(cfg.ReadinessPriorWait),
248+
waitForReady(browserCtx, cfg),
249249
printer.prepare(cfg),
250250
printer.action(fileChan, cfg),
251251
}
@@ -725,7 +725,7 @@ func (p *pngPrinter) prepare(cfg config.BrowserConfig) chromedp.Action {
725725
}
726726

727727
span.SetStatus(codes.Ok, "viewport resized successfully")
728-
if err := waitForReady(ctx, cfg.ReadinessTimeout).Do(ctx); err != nil {
728+
if err := waitForReady(ctx, cfg).Do(ctx); err != nil {
729729
return fmt.Errorf("failed to wait for readiness after resizing viewport: %w", err)
730730
}
731731
} else {
@@ -870,7 +870,7 @@ func scrollForElements(timeBetweenScrolls time.Duration) chromedp.Action {
870870
})
871871
}
872872

873-
func waitForReady(browserCtx context.Context, timeout time.Duration) chromedp.Action {
873+
func waitForReady(browserCtx context.Context, cfg config.BrowserConfig) chromedp.Action {
874874
getRunningQueries := func(ctx context.Context) (bool, error) {
875875
var running bool
876876
err := chromedp.Evaluate(`!!(window.__grafanaSceneContext && window.__grafanaRunningQueryCount > 0)`, &running).Do(ctx)
@@ -891,26 +891,35 @@ func waitForReady(browserCtx context.Context, timeout time.Duration) chromedp.Ac
891891
requests := &atomic.Int64{}
892892
lastRequest := &atomicTime{} // TODO: use this to wait for network stabilisation.
893893
lastRequest.Store(time.Now())
894-
chromedp.ListenTarget(browserCtx, func(ev any) {
895-
switch ev.(type) {
896-
case *network.EventRequestWillBeSent:
897-
requests.Add(1)
898-
lastRequest.Store(time.Now())
899-
case *network.EventLoadingFinished, *network.EventLoadingFailed:
900-
requests.Add(-1)
901-
}
902-
})
894+
networkListenerCtx, cancelNetworkListener := context.WithCancel(browserCtx)
895+
if !cfg.ReadinessDisableNetworkWait {
896+
chromedp.ListenTarget(networkListenerCtx, func(ev any) {
897+
switch ev.(type) {
898+
case *network.EventRequestWillBeSent:
899+
requests.Add(1)
900+
lastRequest.Store(time.Now())
901+
case *network.EventLoadingFinished, *network.EventLoadingFailed:
902+
requests.Add(-1)
903+
}
904+
})
905+
}
903906

904907
return chromedp.ActionFunc(func(ctx context.Context) error {
908+
defer cancelNetworkListener()
909+
905910
tracer := tracer(ctx)
906911
ctx, span := tracer.Start(ctx, "waitForReady",
907-
trace.WithAttributes(attribute.Float64("timeout_seconds", timeout.Seconds())))
912+
trace.WithAttributes(attribute.String("timeout", cfg.ReadinessTimeout.String())))
908913
defer span.End()
909914

910-
timeout := time.After(timeout)
915+
start := time.Now()
916+
917+
var readinessTimeout <-chan time.Time
918+
if cfg.ReadinessTimeout > 0 {
919+
readinessTimeout = time.After(cfg.ReadinessTimeout)
920+
}
911921

912-
hasHadQueries := false
913-
giveUpFirstQuery := time.Now().Add(time.Second * 3)
922+
hasSeenAnyQuery := false
914923

915924
var domHashCode int
916925
initialDOMPass := true
@@ -920,57 +929,70 @@ func waitForReady(browserCtx context.Context, timeout time.Duration) chromedp.Ac
920929
case <-ctx.Done():
921930
span.SetStatus(codes.Error, "context completed before readiness detected")
922931
return ctx.Err()
923-
case <-timeout:
932+
case <-readinessTimeout:
924933
span.SetStatus(codes.Error, "timed out waiting for readiness")
925934
return fmt.Errorf("timed out waiting for readiness")
926-
case <-time.After(100 * time.Millisecond):
935+
936+
case <-time.After(cfg.ReadinessIterationInterval):
937+
// Continue with the rest of the code; this is waiting for the next time we can do work.
927938
}
928939

929-
if requests.Load() > 0 {
940+
if !cfg.ReadinessDisableNetworkWait &&
941+
(cfg.ReadinessNetworkIdleTimeout <= 0 || time.Since(start) < cfg.ReadinessNetworkIdleTimeout) &&
942+
requests.Load() > 0 {
930943
initialDOMPass = true
931-
span.AddEvent("network requests still ongoing", trace.WithAttributes(attribute.Int64("inflightRequests", requests.Load())))
944+
span.AddEvent("network requests still ongoing", trace.WithAttributes(attribute.Int64("inflight_requests", requests.Load())))
932945
continue // still waiting on network requests to complete
933946
}
934947

935-
running, err := getRunningQueries(ctx)
936-
if err != nil {
937-
span.SetStatus(codes.Error, err.Error())
938-
return fmt.Errorf("failed to get running queries: %w", err)
939-
}
940-
span.AddEvent("queried running queries", trace.WithAttributes(attribute.Bool("running", running)))
941-
if running {
942-
initialDOMPass = true
943-
hasHadQueries = true
944-
continue // still waiting on queries to complete
945-
} else if !hasHadQueries && time.Now().Before(giveUpFirstQuery) {
946-
span.AddEvent("no first query detected yet; giving it more time")
947-
continue
948+
if !cfg.ReadinessDisableQueryWait && (cfg.ReadinessQueriesTimeout <= 0 || time.Since(start) < cfg.ReadinessQueriesTimeout) {
949+
running, err := getRunningQueries(ctx)
950+
if err != nil {
951+
span.SetStatus(codes.Error, err.Error())
952+
span.RecordError(err)
953+
return fmt.Errorf("failed to get running queries: %w", err)
954+
}
955+
span.AddEvent("queried running queries", trace.WithAttributes(attribute.Bool("running", running)))
956+
if running {
957+
initialDOMPass = true
958+
hasSeenAnyQuery = true
959+
continue // still waiting on queries to complete
960+
} else if !hasSeenAnyQuery && (cfg.ReadinessFirstQueryTimeout <= 0 || time.Since(start) < cfg.ReadinessFirstQueryTimeout) {
961+
span.AddEvent("no first query detected yet; giving it more time")
962+
continue
963+
}
948964
}
949965

950-
if initialDOMPass {
951-
domHashCode, err = getDOMHashCode(ctx)
966+
if !cfg.ReadinessDisableDOMHashCodeWait && (cfg.ReadinessDOMHashCodeTimeout <= 0 || time.Since(start) < cfg.ReadinessDOMHashCodeTimeout) {
967+
if initialDOMPass {
968+
var err error
969+
domHashCode, err = getDOMHashCode(ctx)
970+
if err != nil {
971+
span.SetStatus(codes.Error, err.Error())
972+
span.RecordError(err)
973+
return fmt.Errorf("failed to get DOM hash code: %w", err)
974+
}
975+
span.AddEvent("initial DOM hash code recorded", trace.WithAttributes(attribute.Int("hashCode", domHashCode)))
976+
initialDOMPass = false
977+
continue // not stable yet
978+
}
979+
980+
newHashCode, err := getDOMHashCode(ctx)
952981
if err != nil {
953982
span.SetStatus(codes.Error, err.Error())
983+
span.RecordError(err)
954984
return fmt.Errorf("failed to get DOM hash code: %w", err)
955985
}
956-
span.AddEvent("initial DOM hash code recorded", trace.WithAttributes(attribute.Int("hashCode", domHashCode)))
957-
initialDOMPass = false
958-
continue // not stable yet
986+
span.AddEvent("subsequent DOM hash code recorded", trace.WithAttributes(attribute.Int("hashCode", newHashCode)))
987+
if newHashCode != domHashCode {
988+
span.AddEvent("DOM hash code changed", trace.WithAttributes(attribute.Int("oldHashCode", domHashCode), attribute.Int("newHashCode", newHashCode)))
989+
domHashCode = newHashCode
990+
initialDOMPass = true
991+
continue // not stable yet
992+
}
993+
span.AddEvent("DOM hash code stable", trace.WithAttributes(attribute.Int("hashCode", domHashCode)))
959994
}
960995

961-
newHashCode, err := getDOMHashCode(ctx)
962-
if err != nil {
963-
span.SetStatus(codes.Error, err.Error())
964-
return fmt.Errorf("failed to get DOM hash code: %w", err)
965-
}
966-
span.AddEvent("subsequent DOM hash code recorded", trace.WithAttributes(attribute.Int("hashCode", newHashCode)))
967-
if newHashCode != domHashCode {
968-
span.AddEvent("DOM hash code changed", trace.WithAttributes(attribute.Int("oldHashCode", domHashCode), attribute.Int("newHashCode", newHashCode)))
969-
domHashCode = newHashCode
970-
initialDOMPass = true
971-
continue // not stable yet
972-
}
973-
span.AddEvent("DOM hash code stable", trace.WithAttributes(attribute.Int("hashCode", domHashCode)))
974996
break // we're done!!
975997
}
976998

@@ -1006,6 +1028,7 @@ func tracingAction(name string, action chromedp.Action) chromedp.Action {
10061028
err := action.Do(ctx)
10071029
if err != nil {
10081030
span.SetStatus(codes.Error, err.Error())
1031+
span.RecordError(err)
10091032
return err
10101033
}
10111034
span.SetStatus(codes.Ok, "action completed successfully")

0 commit comments

Comments
 (0)