diff --git a/changedetectionio/__init__.py b/changedetectionio/__init__.py index 2a97d7a9..cbd29567 100644 --- a/changedetectionio/__init__.py +++ b/changedetectionio/__init__.py @@ -605,12 +605,12 @@ def changedetection_app(config=None, datastore_o=None): if request.method == 'POST' and not form.validate(): flash("An error occurred, please see below.", "error") - output = render_template("edit.html", uuid=uuid, watch=datastore.data['watching'][uuid], form=form, has_empty_checktime=using_default_check_time, + using_global_webdriver_wait=default['webdriver_delay'] is None, current_base_url=datastore.data['settings']['application']['base_url'], emailprefix=os.getenv('NOTIFICATION_MAIL_BUTTON_PREFIX', False) ) diff --git a/changedetectionio/content_fetcher.py b/changedetectionio/content_fetcher.py index 27e24e58..3797d8c9 100644 --- a/changedetectionio/content_fetcher.py +++ b/changedetectionio/content_fetcher.py @@ -28,6 +28,9 @@ class Fetcher(): system_http_proxy = os.getenv('HTTP_PROXY') system_https_proxy = os.getenv('HTTPS_PROXY') + # Time ONTOP of the system defined env minimum time + render_extract_delay=0 + @abstractmethod def get_error(self): return self.error @@ -147,7 +150,7 @@ class base_html_playwright(Fetcher): # - `'commit'` - consider operation to be finished when network response is received and the document started loading. # Better to not use any smarts from Playwright and just wait an arbitrary number of seconds # This seemed to solve nearly all 'TimeoutErrors' - extra_wait = int(os.getenv("WEBDRIVER_DELAY_BEFORE_CONTENT_READY", 5)) + extra_wait = int(os.getenv("WEBDRIVER_DELAY_BEFORE_CONTENT_READY", 5)) + self.render_extract_delay page.wait_for_timeout(extra_wait * 1000) except playwright._impl._api_types.TimeoutError as e: raise EmptyReply(url=url, status_code=None) @@ -240,7 +243,7 @@ class base_html_webdriver(Fetcher): # raise EmptyReply(url=url, status_code=r.status_code) # @todo - dom wait loaded? - time.sleep(int(os.getenv("WEBDRIVER_DELAY_BEFORE_CONTENT_READY", 5))) + time.sleep(int(os.getenv("WEBDRIVER_DELAY_BEFORE_CONTENT_READY", 5)) + self.render_extract_delay) self.content = self.driver.page_source self.headers = {} self.screenshot = self.driver.get_screenshot_as_png() diff --git a/changedetectionio/fetch_site_status.py b/changedetectionio/fetch_site_status.py index 93e21663..0770a362 100644 --- a/changedetectionio/fetch_site_status.py +++ b/changedetectionio/fetch_site_status.py @@ -97,7 +97,13 @@ class perform_site_check(): proxy_args = self.set_proxy_from_list(watch) fetcher = klass(proxy_override=proxy_args) - # Proxy List support + # Configurable per-watch or global extra delay before extracting text (for webDriver types) + system_webdriver_delay = self.datastore.data['settings']['application'].get('webdriver_delay', None) + if watch['webdriver_delay'] is not None: + fetcher.render_extract_delay = watch['webdriver_delay'] + elif system_webdriver_delay is not None: + fetcher.render_extract_delay = system_webdriver_delay + fetcher.run(url, timeout, request_headers, request_body, request_method, ignore_status_code) # Fetching complete, now filters diff --git a/changedetectionio/forms.py b/changedetectionio/forms.py index 6d12267e..14d52cab 100644 --- a/changedetectionio/forms.py +++ b/changedetectionio/forms.py @@ -318,6 +318,7 @@ class commonSettingsForm(Form): notification_format = SelectField('Notification format', choices=valid_notification_formats.keys(), default=default_notification_format) fetch_backend = RadioField(u'Fetch method', choices=content_fetcher.available_fetchers(), validators=[ValidateContentFetcherIsReady()]) extract_title_as_title = BooleanField('Extract