BrowserSteps - Wrong text taken from browser steps (#1911)

changedetectionio/content_fetcher.py

@@ -505,7 +505,7 @@ class base_html_playwright(Fetcher):
             extra_wait = int(os.getenv("WEBDRIVER_DELAY_BEFORE_CONTENT_READY", 5)) + self.render_extract_delay
             self.page.wait_for_timeout(extra_wait * 1000)
 
-            self.content = self.page.content()
+
             self.status_code = response.status
 
             if self.status_code != 200 and not ignore_status_codes:
@@ -533,6 +533,7 @@ class base_html_playwright(Fetcher):
                 "async () => {" + self.xpath_element_js.replace('%ELEMENTS%', visualselector_xpath_selectors) + "}")
             self.instock_data = self.page.evaluate("async () => {" + self.instock_data_js + "}")
 
+            self.content = self.page.content()
             # Bug 3 in Playwright screenshot handling
             # Some bug where it gives the wrong screenshot size, but making a request with the clip set first seems to solve it
             # JPEG is better here because the screenshots can be very very large

changedetectionio/tests/visualselector/test_fetch_data.py

@@ -91,14 +91,22 @@ def test_basic_browserstep(client, live_server):
               'browser_steps-0-operation': 'Goto site',
               'browser_steps-1-operation': 'Click element',
               'browser_steps-1-selector': 'button[name=test-button]',
-              'browser_steps-1-value': ''
+              'browser_steps-1-optional_value': ''
         },
         follow_redirects=True
     )
     assert b"unpaused" in res.data
     wait_for_all_checks(client)
+
     uuid = extract_UUID_from_client(client)
 
+    # Check HTML conversion detected and workd
+    res = client.get(
+        url_for("preview_page", uuid=uuid),
+        follow_redirects=True
+    )
+    assert b"This text should be removed" not in res.data
+    assert b"I smell JavaScript because the button was pressed" in res.data
 
     # now test for 404 errors
     res = client.post(
@@ -111,7 +119,7 @@ def test_basic_browserstep(client, live_server):
               'browser_steps-0-operation': 'Goto site',
               'browser_steps-1-operation': 'Click element',
               'browser_steps-1-selector': 'button[name=test-button]',
-              'browser_steps-1-value': ''
+              'browser_steps-1-optional_value': ''
         },
         follow_redirects=True
     )