Playwright+BrowserSteps - Fetch changes - Fetch simply after page starts rendering + delay seconds, disable service workers

changedetectionio/blueprint/browser_steps/browser_steps.py

@@ -75,15 +75,13 @@ class steppable_browser_interface():
     def action_goto_url(self, url, optional_value):
         # self.page.set_viewport_size({"width": 1280, "height": 5000})
         now = time.time()
-        response = self.page.goto(url, timeout=0, wait_until='domcontentloaded')
-        print("Time to goto URL", time.time() - now)
+        response = self.page.goto(url, timeout=0, wait_until='commit')
 
         # Wait_until = commit
         # - `'commit'` - consider operation to be finished when network response is received and the document started loading.
         # Better to not use any smarts from Playwright and just wait an arbitrary number of seconds
         # This seemed to solve nearly all 'TimeoutErrors'
-        extra_wait = int(os.getenv("WEBDRIVER_DELAY_BEFORE_CONTENT_READY", 5))
-        self.page.wait_for_timeout(extra_wait * 1000)
+        print("Time to goto URL ", time.time() - now)
 
     def action_click_element_containing_text(self, selector=None, value=''):
         if not len(value.strip()):

changedetectionio/content_fetcher.py

@@ -286,6 +286,8 @@ class base_html_playwright(Fetcher):
                 proxy=self.proxy,
                 # This is needed to enable JavaScript execution on GitHub and others
                 bypass_csp=True,
+                # Can't think why we need the service workers for our use case?
+                service_workers='block',
                 # Should never be needed
                 accept_downloads=False
             )
@@ -306,8 +308,7 @@ class base_html_playwright(Fetcher):
 
                 # Waits for the next navigation. Using Python context manager
                 # prevents a race condition between clicking and waiting for a navigation.
-                with self.page.expect_navigation():
-                    response = self.page.goto(url, wait_until='load')
+                response = self.page.goto(url, wait_until='commit')
                 # Wait_until = commit
                 # - `'commit'` - consider operation to be finished when network response is received and the document started loading.
                 # Better to not use any smarts from Playwright and just wait an arbitrary number of seconds