From e4504fee49fc8016b4eac1a3e61f905567aa15b5 Mon Sep 17 00:00:00 2001 From: dgtlmoon Date: Wed, 15 May 2024 10:49:30 +0200 Subject: [PATCH] Browser Steps - Fixing "goto site" step #2330 #2337 (#2364) --- .../blueprint/browser_steps/__init__.py | 9 +++------ .../blueprint/browser_steps/browser_steps.py | 11 ++++++++++- changedetectionio/content_fetchers/base.py | 15 +++++++++------ changedetectionio/content_fetchers/playwright.py | 4 ++-- changedetectionio/static/js/browser-steps.js | 3 ++- .../tests/visualselector/test_fetch_data.py | 14 ++++++-------- 6 files changed, 32 insertions(+), 24 deletions(-) diff --git a/changedetectionio/blueprint/browser_steps/__init__.py b/changedetectionio/blueprint/browser_steps/__init__.py index 24440908..30797099 100644 --- a/changedetectionio/blueprint/browser_steps/__init__.py +++ b/changedetectionio/blueprint/browser_steps/__init__.py @@ -84,7 +84,9 @@ def construct_blueprint(datastore: ChangeDetectionStore): # Tell Playwright to connect to Chrome and setup a new session via our stepper interface browsersteps_start_session['browserstepper'] = browser_steps.browsersteps_live_ui( playwright_browser=browsersteps_start_session['browser'], - proxy=proxy) + proxy=proxy, + start_url=datastore.data['watching'][watch_uuid].get('url') + ) # For test #browsersteps_start_session['browserstepper'].action_goto_url(value="http://example.com?time="+str(time.time())) @@ -167,11 +169,6 @@ def construct_blueprint(datastore: ChangeDetectionStore): step_n = int(request.form.get('step_n')) is_last_step = strtobool(request.form.get('is_last_step')) - if step_operation == 'Goto site': - step_operation = 'goto_url' - step_optional_value = datastore.data['watching'][uuid].get('url') - step_selector = None - # @todo try.. accept.. nice errors not popups.. try: diff --git a/changedetectionio/blueprint/browser_steps/browser_steps.py b/changedetectionio/blueprint/browser_steps/browser_steps.py index 6aac2446..76f3d756 100644 --- a/changedetectionio/blueprint/browser_steps/browser_steps.py +++ b/changedetectionio/blueprint/browser_steps/browser_steps.py @@ -49,6 +49,10 @@ browser_step_ui_config = {'Choose one': '0 0', # ONLY Works in Playwright because we need the fullscreen screenshot class steppable_browser_interface(): page = None + start_url = None + + def __init__(self, start_url): + self.start_url = start_url # Convert and perform "Click Button" for example def call_action(self, action_name, selector=None, optional_value=None): @@ -87,6 +91,10 @@ class steppable_browser_interface(): logger.debug(f"Time to goto URL {time.time()-now:.2f}s") return response + # Incase they request to go back to the start + def action_goto_site(self, selector=None, value=None): + return self.action_goto_url(value=self.start_url) + def action_click_element_containing_text(self, selector=None, value=''): if not len(value.strip()): return @@ -194,10 +202,11 @@ class browsersteps_live_ui(steppable_browser_interface): browser_type = os.getenv("PLAYWRIGHT_BROWSER_TYPE", 'chromium').strip('"') - def __init__(self, playwright_browser, proxy=None, headers=None): + def __init__(self, playwright_browser, proxy=None, headers=None, start_url=None): self.headers = headers or {} self.age_start = time.time() self.playwright_browser = playwright_browser + self.start_url = start_url if self.context is None: self.connect(proxy=proxy) diff --git a/changedetectionio/content_fetchers/base.py b/changedetectionio/content_fetchers/base.py index ca2fd019..f817341d 100644 --- a/changedetectionio/content_fetchers/base.py +++ b/changedetectionio/content_fetchers/base.py @@ -112,23 +112,26 @@ class Fetcher(): def browser_steps_get_valid_steps(self): if self.browser_steps is not None and len(self.browser_steps): - valid_steps = filter( - lambda s: (s['operation'] and len(s['operation']) and s['operation'] != 'Choose one' and s['operation'] != 'Goto site'), - self.browser_steps) + valid_steps = list(filter( + lambda s: (s['operation'] and len(s['operation']) and s['operation'] != 'Choose one'), + self.browser_steps)) + + # Just incase they selected Goto site by accident with older JS + if valid_steps and valid_steps[0]['operation'] == 'Goto site': + del(valid_steps[0]) return valid_steps return None - def iterate_browser_steps(self): + def iterate_browser_steps(self, start_url=None): from changedetectionio.blueprint.browser_steps.browser_steps import steppable_browser_interface from playwright._impl._errors import TimeoutError, Error from changedetectionio.safe_jinja import render as jinja_render - step_n = 0 if self.browser_steps is not None and len(self.browser_steps): - interface = steppable_browser_interface() + interface = steppable_browser_interface(start_url=start_url) interface.page = self.page valid_steps = self.browser_steps_get_valid_steps() diff --git a/changedetectionio/content_fetchers/playwright.py b/changedetectionio/content_fetchers/playwright.py index 7950e033..04ab2759 100644 --- a/changedetectionio/content_fetchers/playwright.py +++ b/changedetectionio/content_fetchers/playwright.py @@ -119,7 +119,7 @@ class fetcher(Fetcher): # Re-use as much code from browser steps as possible so its the same from changedetectionio.blueprint.browser_steps.browser_steps import steppable_browser_interface - browsersteps_interface = steppable_browser_interface() + browsersteps_interface = steppable_browser_interface(start_url=url) browsersteps_interface.page = self.page response = browsersteps_interface.action_goto_url(value=url) @@ -172,7 +172,7 @@ class fetcher(Fetcher): # Run Browser Steps here if self.browser_steps_get_valid_steps(): - self.iterate_browser_steps() + self.iterate_browser_steps(start_url=url) self.page.wait_for_timeout(extra_wait * 1000) diff --git a/changedetectionio/static/js/browser-steps.js b/changedetectionio/static/js/browser-steps.js index 7c9c38d8..4e576bd4 100644 --- a/changedetectionio/static/js/browser-steps.js +++ b/changedetectionio/static/js/browser-steps.js @@ -26,7 +26,8 @@ $(document).ready(function () { set_scale(); }); // Should always be disabled - $('#browser_steps >li:first-child select').val('Goto site').attr('disabled', 'disabled'); + $('#browser_steps-0-operation option[value="Goto site"]').prop("selected", "selected"); + $('#browser_steps-0-operation').attr('disabled', 'disabled'); $('#browsersteps-click-start').click(function () { $("#browsersteps-click-start").fadeOut(); diff --git a/changedetectionio/tests/visualselector/test_fetch_data.py b/changedetectionio/tests/visualselector/test_fetch_data.py index 2f460d7c..15677f31 100644 --- a/changedetectionio/tests/visualselector/test_fetch_data.py +++ b/changedetectionio/tests/visualselector/test_fetch_data.py @@ -102,10 +102,9 @@ def test_basic_browserstep(client, live_server): "url": test_url, "tags": "", 'fetch_backend': "html_webdriver", - 'browser_steps-0-operation': 'Goto site', - 'browser_steps-1-operation': 'Click element', - 'browser_steps-1-selector': 'button[name=test-button]', - 'browser_steps-1-optional_value': '', + 'browser_steps-0-operation': 'Click element', + 'browser_steps-0-selector': 'button[name=test-button]', + 'browser_steps-0-optional_value': '', # For now, cookies doesnt work in headers because it must be a full cookiejar object 'headers': "testheader: yes\buser-agent: MyCustomAgent", }, @@ -141,10 +140,9 @@ def test_basic_browserstep(client, live_server): "url": four_o_four_url, "tags": "", 'fetch_backend': "html_webdriver", - 'browser_steps-0-operation': 'Goto site', - 'browser_steps-1-operation': 'Click element', - 'browser_steps-1-selector': 'button[name=test-button]', - 'browser_steps-1-optional_value': '' + 'browser_steps-0-operation': 'Click element', + 'browser_steps-0-selector': 'button[name=test-button]', + 'browser_steps-0-optional_value': '' }, follow_redirects=True )