diff --git a/changedetectionio/__init__.py b/changedetectionio/__init__.py index ff31bccd..3e0f384f 100644 --- a/changedetectionio/__init__.py +++ b/changedetectionio/__init__.py @@ -94,16 +94,6 @@ def init_app_secret(datastore_path): return secret -# Remember python is by reference -# populate_form in wtfors didnt work for me. (try using a setattr() obj type on datastore.watch?) -def populate_form_from_watch(form, watch): - for i in form.__dict__.keys(): - if i[0] != '_': - p = getattr(form, i) - if hasattr(p, 'data') and i in watch: - setattr(p, "data", watch[i]) - - # We use the whole watch object from the store/JSON so we can see if there's some related status in terms of a thread # running or something similar. @app.template_filter('format_last_checked_time') @@ -320,6 +310,7 @@ def changedetection_app(config=None, datastore_o=None): guid = "{}/{}".format(watch['uuid'], watch['last_changed']) fe = fg.add_entry() + # Include a link to the diff page, they will have to login here to see if password protection is enabled. # Description is the page you watch, link takes you to the diff JS UI page base_url = datastore.data['settings']['application']['base_url'] @@ -520,49 +511,46 @@ def changedetection_app(config=None, datastore_o=None): @app.route("/edit/", methods=['GET', 'POST']) @login_required + # https://stackoverflow.com/questions/42984453/wtforms-populate-form-with-data-if-data-exists + # https://wtforms.readthedocs.io/en/3.0.x/forms/#wtforms.form.Form.populate_obj ? + def edit_page(uuid): from changedetectionio import forms - form = forms.watchForm(request.form) + # More for testing, possible to return the first/only + if not datastore.data['watching'].keys(): + flash("No watches to edit", "error") + return redirect(url_for('index')) + if uuid == 'first': uuid = list(datastore.data['watching'].keys()).pop() + if not uuid in datastore.data['watching']: + flash("No watch with the UUID %s found." % (uuid), "error") + return redirect(url_for('index')) - if request.method == 'GET': - if not uuid in datastore.data['watching']: - flash("No watch with the UUID %s found." % (uuid), "error") - return redirect(url_for('index')) - - populate_form_from_watch(form, datastore.data['watching'][uuid]) + form = forms.watchForm(formdata=request.form if request.method == 'POST' else None, + data=datastore.data['watching'][uuid] + ) + if request.method == 'GET': + # Set some defaults that refer to the main config when None, we do the same in POST, + # probably there should be a nice little handler for this. if datastore.data['watching'][uuid]['fetch_backend'] is None: form.fetch_backend.data = datastore.data['settings']['application']['fetch_backend'] + if datastore.data['watching'][uuid]['minutes_between_check'] is None: + form.minutes_between_check.data = datastore.data['settings']['requests']['minutes_between_check'] if request.method == 'POST' and form.validate(): # Re #110, if they submit the same as the default value, set it to None, so we continue to follow the default if form.minutes_between_check.data == datastore.data['settings']['requests']['minutes_between_check']: form.minutes_between_check.data = None - if form.fetch_backend.data == datastore.data['settings']['application']['fetch_backend']: form.fetch_backend.data = None - update_obj = {'url': form.url.data.strip(), - 'minutes_between_check': form.minutes_between_check.data, - 'tag': form.tag.data.strip(), - 'title': form.title.data.strip(), - 'headers': form.headers.data, - 'body': form.body.data, - 'method': form.method.data, - 'ignore_status_codes': form.ignore_status_codes.data, - 'fetch_backend': form.fetch_backend.data, - 'trigger_text': form.trigger_text.data, - 'notification_title': form.notification_title.data, - 'notification_body': form.notification_body.data, - 'notification_format': form.notification_format.data, - 'extract_title_as_title': form.extract_title_as_title.data, - } + extra_update_obj = {} # Notification URLs datastore.data['watching'][uuid]['notification_urls'] = form.notification_urls.data @@ -574,18 +562,15 @@ def changedetection_app(config=None, datastore_o=None): # Reset the previous_md5 so we process a new snapshot including stripping ignore text. if form_ignore_text: if len(datastore.data['watching'][uuid]['history']): - update_obj['previous_md5'] = get_current_checksum_include_ignore_text(uuid=uuid) - - - datastore.data['watching'][uuid]['css_filter'] = form.css_filter.data.strip() - datastore.data['watching'][uuid]['subtractive_selectors'] = form.subtractive_selectors.data + extra_update_obj['previous_md5'] = get_current_checksum_include_ignore_text(uuid=uuid) # Reset the previous_md5 so we process a new snapshot including stripping ignore text. if form.css_filter.data.strip() != datastore.data['watching'][uuid]['css_filter']: if len(datastore.data['watching'][uuid]['history']): - update_obj['previous_md5'] = get_current_checksum_include_ignore_text(uuid=uuid) + extra_update_obj['previous_md5'] = get_current_checksum_include_ignore_text(uuid=uuid) - datastore.data['watching'][uuid].update(update_obj) + datastore.data['watching'][uuid].update(form.data) + datastore.data['watching'][uuid].update(extra_update_obj) flash("Updated watch.") @@ -610,17 +595,12 @@ def changedetection_app(config=None, datastore_o=None): if request.method == 'POST' and not form.validate(): flash("An error occurred, please see below.", "error") - # Re #110 offer the default minutes - using_default_minutes = False - if form.minutes_between_check.data == None: - form.minutes_between_check.data = datastore.data['settings']['requests']['minutes_between_check'] - using_default_minutes = True - + has_empty_checktime = datastore.data['watching'][uuid].has_empty_checktime output = render_template("edit.html", uuid=uuid, watch=datastore.data['watching'][uuid], form=form, - using_default_minutes=using_default_minutes, + has_empty_checktime=has_empty_checktime, current_base_url=datastore.data['settings']['application']['base_url'], emailprefix=os.getenv('NOTIFICATION_MAIL_BUTTON_PREFIX', False) ) @@ -630,61 +610,39 @@ def changedetection_app(config=None, datastore_o=None): @app.route("/settings", methods=['GET', "POST"]) @login_required def settings_page(): - from changedetectionio import content_fetcher, forms - form = forms.globalSettingsForm(request.form) + # Don't use form.data on POST so that it doesnt overrid the checkbox status from the POST status + form = forms.globalSettingsForm(formdata=request.form if request.method == 'POST' else None, + data=datastore.data['settings'] + ) - if request.method == 'GET': - form.minutes_between_check.data = int(datastore.data['settings']['requests']['minutes_between_check']) - form.notification_urls.data = datastore.data['settings']['application']['notification_urls'] - form.global_subtractive_selectors.data = datastore.data['settings']['application']['global_subtractive_selectors'] - form.global_ignore_text.data = datastore.data['settings']['application']['global_ignore_text'] - form.ignore_whitespace.data = datastore.data['settings']['application']['ignore_whitespace'] - form.render_anchor_tag_content.data = datastore.data['settings']['application']['render_anchor_tag_content'] - form.extract_title_as_title.data = datastore.data['settings']['application']['extract_title_as_title'] - form.fetch_backend.data = datastore.data['settings']['application']['fetch_backend'] - form.notification_title.data = datastore.data['settings']['application']['notification_title'] - form.notification_body.data = datastore.data['settings']['application']['notification_body'] - form.notification_format.data = datastore.data['settings']['application']['notification_format'] - form.base_url.data = datastore.data['settings']['application']['base_url'] - form.real_browser_save_screenshot.data = datastore.data['settings']['application']['real_browser_save_screenshot'] - - if request.method == 'POST' and form.data.get('removepassword_button') == True: + if request.method == 'POST': # Password unset is a GET, but we can lock the session to a salted env password to always need the password - if not os.getenv("SALTED_PASS", False): - datastore.data['settings']['application']['password'] = False - flash("Password protection removed.", 'notice') - flask_login.logout_user() - return redirect(url_for('settings_page')) + if form.application.form.data.get('removepassword_button', False): + # SALTED_PASS means the password is "locked" to what we set in the Env var + if not os.getenv("SALTED_PASS", False): + datastore.remove_password() + flash("Password protection removed.", 'notice') + flask_login.logout_user() + return redirect(url_for('settings_page')) + + if form.validate(): + datastore.data['settings']['application'].update(form.data['application']) + datastore.data['settings']['requests'].update(form.data['requests']) + datastore.needs_write = True - if request.method == 'POST' and form.validate(): - datastore.data['settings']['application']['notification_urls'] = form.notification_urls.data - datastore.data['settings']['requests']['minutes_between_check'] = form.minutes_between_check.data - datastore.data['settings']['application']['extract_title_as_title'] = form.extract_title_as_title.data - datastore.data['settings']['application']['fetch_backend'] = form.fetch_backend.data - datastore.data['settings']['application']['notification_title'] = form.notification_title.data - datastore.data['settings']['application']['notification_body'] = form.notification_body.data - datastore.data['settings']['application']['notification_format'] = form.notification_format.data - datastore.data['settings']['application']['notification_urls'] = form.notification_urls.data - datastore.data['settings']['application']['base_url'] = form.base_url.data - datastore.data['settings']['application']['global_subtractive_selectors'] = form.global_subtractive_selectors.data - datastore.data['settings']['application']['global_ignore_text'] = form.global_ignore_text.data - datastore.data['settings']['application']['ignore_whitespace'] = form.ignore_whitespace.data - datastore.data['settings']['application']['real_browser_save_screenshot'] = form.real_browser_save_screenshot.data - datastore.data['settings']['application']['render_anchor_tag_content'] = form.render_anchor_tag_content.data - - if not os.getenv("SALTED_PASS", False) and form.password.encrypted_password: - datastore.data['settings']['application']['password'] = form.password.encrypted_password - flash("Password protection enabled.", 'notice') - flask_login.logout_user() - return redirect(url_for('index')) + if not os.getenv("SALTED_PASS", False) and len(form.application.form.password.encrypted_password): + datastore.data['settings']['application']['password'] = form.application.form.password.encrypted_password + datastore.needs_write = True + flash("Password protection enabled.", 'notice') + flask_login.logout_user() + return redirect(url_for('index')) - datastore.needs_write = True - flash("Settings updated.") + flash("Settings updated.") - if request.method == 'POST' and not form.validate(): - flash("An error occurred, please see below.", "error") + else: + flash("An error occurred, please see below.", "error") output = render_template("settings.html", form=form, @@ -1172,8 +1130,6 @@ def notification_runner(): notification_debug_log = notification_debug_log[-100:] - - # Thread runner to check every minute, look for new watches to feed into the Queue. def ticker_thread_check_time_launch_checks(): from changedetectionio import update_worker @@ -1210,7 +1166,9 @@ def ticker_thread_check_time_launch_checks(): # Check for watches outside of the time threshold to put in the thread queue. now = time.time() - max_system_wide = int(copied_datastore.data['settings']['requests']['minutes_between_check']) * 60 + + recheck_time_minimum_seconds = int(os.getenv('MINIMUM_SECONDS_RECHECK_TIME', 60)) + recheck_time_system_seconds = int(copied_datastore.data['settings']['requests']['minutes_between_check']) * 60 for uuid, watch in copied_datastore.data['watching'].items(): @@ -1219,18 +1177,14 @@ def ticker_thread_check_time_launch_checks(): continue # If they supplied an individual entry minutes to threshold. - watch_minutes_between_check = watch.get('minutes_between_check', None) - if watch_minutes_between_check is not None: - # Cast to int just incase - max_time = int(watch_minutes_between_check) * 60 + threshold = now + if watch.threshold_seconds: + threshold -= watch.threshold_seconds else: - # Default system wide. - max_time = max_system_wide - - threshold = now - max_time + threshold -= recheck_time_system_seconds # Yeah, put it in the queue, it's more than time - if watch['last_checked'] <= threshold: + if watch['last_checked'] <= max(threshold, recheck_time_minimum_seconds): if not uuid in running_uuids and uuid not in update_q.queue: update_q.put(uuid) @@ -1238,4 +1192,4 @@ def ticker_thread_check_time_launch_checks(): time.sleep(3) # Should be low so we can break this out in testing - app.config.exit.wait(1) + app.config.exit.wait(1) \ No newline at end of file diff --git a/changedetectionio/forms.py b/changedetectionio/forms.py index 0d01665f..b0dd561c 100644 --- a/changedetectionio/forms.py +++ b/changedetectionio/forms.py @@ -25,6 +25,8 @@ from changedetectionio.notification import ( valid_notification_formats, ) +from wtforms.fields import FormField + valid_method = { 'GET', 'POST', @@ -121,7 +123,6 @@ class ValidateContentFetcherIsReady(object): def __call__(self, form, field): import urllib3.exceptions - from changedetectionio import content_fetcher # Better would be a radiohandler that keeps a reference to each class @@ -297,6 +298,7 @@ class quickWatchForm(Form): url = fields.URLField('URL', validators=[validateURL()]) tag = StringField('Group tag', [validators.Optional(), validators.Length(max=35)]) +# Common to a single watch and the global settings class commonSettingsForm(Form): notification_urls = StringListField('Notification URL list', validators=[validators.Optional(), ValidateNotificationBodyAndTitleWhenURLisSet(), ValidateAppRiseServers()]) @@ -342,19 +344,31 @@ class watchForm(commonSettingsForm): return result -class globalSettingsForm(commonSettingsForm): - password = SaltyPasswordField() + +# datastore.data['settings']['requests'].. +class globalSettingsRequestForm(Form): minutes_between_check = fields.IntegerField('Maximum time in minutes until recheck', [validators.NumberRange(min=1)]) - extract_title_as_title = BooleanField('Extract from document and use as watch title') +# datastore.data['settings']['application'].. +class globalSettingsApplicationForm(commonSettingsForm): + base_url = StringField('Base URL', validators=[validators.Optional()]) global_subtractive_selectors = StringListField('Remove elements', [ValidateCSSJSONXPATHInput(allow_xpath=False, allow_json=False)]) - global_ignore_text = StringListField('Ignore text', [ValidateListRegex()]) + global_ignore_text = StringListField('Ignore Text', [ValidateListRegex()]) ignore_whitespace = BooleanField('Ignore whitespace') + real_browser_save_screenshot = BooleanField('Save last screenshot when using Chrome?') + removepassword_button = SubmitField('Remove password', render_kw={"class": "pure-button pure-button-primary"}) + render_anchor_tag_content = BooleanField('Render anchor tag content', default=False) + fetch_backend = RadioField('Fetch Method', default="html_requests", choices=content_fetcher.available_fetchers(), validators=[ValidateContentFetcherIsReady()]) + password = SaltyPasswordField() + - render_anchor_tag_content = BooleanField('Render anchor tag content', - default=False) +class globalSettingsForm(Form): + # Define these as FormFields/"sub forms", this way it matches the JSON storage + # datastore.data['settings']['application'].. + # datastore.data['settings']['requests'].. + requests = FormField(globalSettingsRequestForm) + application = FormField(globalSettingsApplicationForm) save_button = SubmitField('Save', render_kw={"class": "pure-button pure-button-primary"}) - real_browser_save_screenshot = BooleanField('Save last screenshot when using Chrome') - removepassword_button = SubmitField('Remove password', render_kw={"class": "pure-button pure-button-primary"}) + diff --git a/changedetectionio/model/App.py b/changedetectionio/model/App.py new file mode 100644 index 00000000..0d9aa692 --- /dev/null +++ b/changedetectionio/model/App.py @@ -0,0 +1,49 @@ +import collections +import os + +import uuid as uuid_builder + +from changedetectionio.notification import ( + default_notification_body, + default_notification_format, + default_notification_title, +) + +class model(dict): + def __init__(self, *arg, **kw): + super(model, self).__init__(*arg, **kw) + self.update({ + 'note': "Hello! If you change this file manually, please be sure to restart your changedetection.io instance!", + 'watching': {}, + 'settings': { + 'headers': { + 'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.66 Safari/537.36', + 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9', + 'Accept-Encoding': 'gzip, deflate', # No support for brolti in python requests yet. + 'Accept-Language': 'en-GB,en-US;q=0.9,en;' + }, + 'requests': { + 'timeout': 15, # Default 15 seconds + # Default 3 hours + 'minutes_between_check': 3 * 60, # Default 3 hours + 'workers': 10 # Number of threads, lower is better for slow connections + }, + 'application': { + 'password': False, + 'base_url' : None, + 'extract_title_as_title': False, + 'fetch_backend': 'html_requests', + 'global_ignore_text': [], # List of text to ignore when calculating the comparison checksum + 'global_subtractive_selectors': [], + 'ignore_whitespace': False, + 'render_anchor_tag_content': False, + 'notification_urls': [], # Apprise URL list + # Custom notification content + 'notification_title': default_notification_title, + 'notification_body': default_notification_body, + 'notification_format': default_notification_format, + 'real_browser_save_screenshot': True, + 'schema_version' : 0 + } + } + }) diff --git a/changedetectionio/model/Watch.py b/changedetectionio/model/Watch.py new file mode 100644 index 00000000..e97ee91c --- /dev/null +++ b/changedetectionio/model/Watch.py @@ -0,0 +1,60 @@ +import os + +import uuid as uuid_builder + +minimum_seconds_recheck_time = int(os.getenv('MINIMUM_SECONDS_RECHECK_TIME', 5)) + +from changedetectionio.notification import ( + default_notification_body, + default_notification_format, + default_notification_title, +) + + +class model(dict): + def __init__(self, *arg, **kw): + super(model, self).__init__(*arg, **kw) + self.update({ + 'url': None, + 'tag': None, + 'last_checked': 0, + 'last_changed': 0, + 'paused': False, + 'last_viewed': 0, # history key value of the last viewed via the [diff] link + 'newest_history_key': "", + 'title': None, + 'previous_md5': "", + 'uuid': str(uuid_builder.uuid4()), + 'headers': {}, # Extra headers to send + 'body': None, + 'method': 'GET', + 'history': {}, # Dict of timestamp and output stripped filename + 'ignore_text': [], # List of text to ignore when calculating the comparison checksum + # Custom notification content + 'notification_urls': [], # List of URLs to add to the notification Queue (Usually AppRise) + 'notification_title': default_notification_title, + 'notification_body': default_notification_body, + 'notification_format': default_notification_format, + 'css_filter': "", + 'subtractive_selectors': [], + 'trigger_text': [], # List of text or regex to wait for until a change is detected + 'fetch_backend': None, + 'extract_title_as_title': False, + # Re #110, so then if this is set to None, we know to use the default value instead + # Requires setting to None on submit if it's the same as the default + # Should be all None by default, so we use the system default in this case. + 'minutes_between_check': None + }) + + @property + def has_empty_checktime(self): + if self.get('minutes_between_check', None): + return False + return True + + @property + def threshold_seconds(self): + sec = self.get('minutes_between_check', None) + if sec: + sec = sec * 60 + return sec diff --git a/changedetectionio/model/__init__.py b/changedetectionio/model/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/changedetectionio/store.py b/changedetectionio/store.py index 4aa5cc3e..6f9d69e4 100644 --- a/changedetectionio/store.py +++ b/changedetectionio/store.py @@ -8,11 +8,7 @@ from copy import deepcopy from os import mkdir, path, unlink from threading import Lock -from changedetectionio.notification import ( - default_notification_body, - default_notification_format, - default_notification_title, -) +from changedetectionio.model import Watch, App # Is there an existing library to ensure some data store (JSON etc) is in sync with CRUD methods? @@ -29,71 +25,10 @@ class ChangeDetectionStore: self.json_store_path = "{}/url-watches.json".format(self.datastore_path) self.stop_thread = False - self.__data = { - 'note': "Hello! If you change this file manually, please be sure to restart your changedetection.io instance!", - 'watching': {}, - 'settings': { - 'headers': { - 'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.66 Safari/537.36', - 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9', - 'Accept-Encoding': 'gzip, deflate', # No support for brolti in python requests yet. - 'Accept-Language': 'en-GB,en-US;q=0.9,en;' - }, - 'requests': { - 'timeout': 15, # Default 15 seconds - 'minutes_between_check': 3 * 60, # Default 3 hours - 'workers': 10 # Number of threads, lower is better for slow connections - }, - 'application': { - 'password': False, - 'base_url' : None, - 'extract_title_as_title': False, - 'fetch_backend': 'html_requests', - 'global_ignore_text': [], # List of text to ignore when calculating the comparison checksum - 'global_subtractive_selectors': [], - 'ignore_whitespace': False, - 'render_anchor_tag_content': False, - 'notification_urls': [], # Apprise URL list - # Custom notification content - 'notification_title': default_notification_title, - 'notification_body': default_notification_body, - 'notification_format': default_notification_format, - 'real_browser_save_screenshot': True, - } - } - } + self.__data = App.model() # Base definition for all watchers - self.generic_definition = { - 'url': None, - 'tag': None, - 'last_checked': 0, - 'last_changed': 0, - 'paused': False, - 'last_viewed': 0, # history key value of the last viewed via the [diff] link - 'newest_history_key': "", - 'title': None, - # Re #110, so then if this is set to None, we know to use the default value instead - # Requires setting to None on submit if it's the same as the default - 'minutes_between_check': None, - 'previous_md5': "", - 'uuid': str(uuid_builder.uuid4()), - 'headers': {}, # Extra headers to send - 'body': None, - 'method': 'GET', - 'history': {}, # Dict of timestamp and output stripped filename - 'ignore_text': [], # List of text to ignore when calculating the comparison checksum - # Custom notification content - 'notification_urls': [], # List of URLs to add to the notification Queue (Usually AppRise) - 'notification_title': default_notification_title, - 'notification_body': default_notification_body, - 'notification_format': default_notification_format, - 'css_filter': "", - 'subtractive_selectors': [], - 'trigger_text': [], # List of text or regex to wait for until a change is detected - 'fetch_backend': None, - 'extract_title_as_title': False - } + self.generic_definition = Watch.model() if path.isfile('changedetectionio/source.txt'): with open('changedetectionio/source.txt') as f: @@ -190,6 +125,10 @@ class ChangeDetectionStore: self.data['watching'][uuid].update({'last_viewed': int(timestamp)}) self.needs_write = True + def remove_password(self): + self.__data['settings']['application']['password'] = False + self.needs_write = True + def update_watch(self, uuid, update_obj): with self.lock: diff --git a/changedetectionio/templates/edit.html b/changedetectionio/templates/edit.html index 0517c028..f1cdc08f 100644 --- a/changedetectionio/templates/edit.html +++ b/changedetectionio/templates/edit.html @@ -42,7 +42,7 @@ </div> <div class="pure-control-group"> {{ render_field(form.minutes_between_check) }} - {% if using_default_minutes %} + {% if has_empty_checktime %} <span class="pure-form-message-inline">Currently using the <a href="{{ url_for('settings_page', uuid=uuid) }}">default global settings</a>, change to another value if you want to be specific.</span> {% else %} diff --git a/changedetectionio/templates/settings.html b/changedetectionio/templates/settings.html index 040eb507..385f64dc 100644 --- a/changedetectionio/templates/settings.html +++ b/changedetectionio/templates/settings.html @@ -28,15 +28,15 @@ <div class="tab-pane-inner" id="general"> <fieldset> <div class="pure-control-group"> - {{ render_field(form.minutes_between_check) }} + {{ render_field(form.requests.form.minutes_between_check) }} <span class="pure-form-message-inline">Default time for all watches, when the watch does not have a specific time setting.</span> </div> <div class="pure-control-group"> {% if not hide_remove_pass %} {% if current_user.is_authenticated %} - {{ render_button(form.removepassword_button) }} + {{ render_button(form.application.form.removepassword_button) }} {% else %} - {{ render_field(form.password) }} + {{ render_field(form.application.form.password) }} <span class="pure-form-message-inline">Password protection for your changedetection.io application.</span> {% endif %} {% else %} @@ -44,7 +44,7 @@ {% endif %} </div> <div class="pure-control-group"> - {{ render_field(form.base_url, placeholder="http://yoursite.com:5000/", + {{ render_field(form.application.form.base_url, placeholder="http://yoursite.com:5000/", class="m-d") }} <span class="pure-form-message-inline"> Base URL used for the {base_url} token in notifications and RSS links.<br/>Default value is the ENV var 'BASE_URL' (Currently "{{current_base_url}}"), @@ -53,12 +53,12 @@ </div> <div class="pure-control-group"> - {{ render_checkbox_field(form.extract_title_as_title) }} + {{ render_checkbox_field(form.application.form.extract_title_as_title) }} <span class="pure-form-message-inline">Note: This will automatically apply to all existing watches.</span> </div> <div class="pure-control-group"> - {{ render_checkbox_field(form.real_browser_save_screenshot) }} + {{ render_checkbox_field(form.application.form.real_browser_save_screenshot) }} <span class="pure-form-message-inline">When using a Chrome browser, a screenshot from the last check will be available on the Diff page</span> </div> @@ -68,14 +68,14 @@ <div class="tab-pane-inner" id="notifications"> <fieldset> <div class="field-group"> - {{ render_common_settings_form(form, current_base_url, emailprefix) }} + {{ render_common_settings_form(form.application.form, current_base_url, emailprefix) }} </div> </fieldset> </div> <div class="tab-pane-inner" id="fetching"> <div class="pure-control-group"> - {{ render_field(form.fetch_backend) }} + {{ render_field(form.application.form.fetch_backend) }} <span class="pure-form-message-inline"> <p>Use the <strong>Basic</strong> method (default) where your watched sites don't need Javascript to render.</p> <p>The <strong>Chrome/Javascript</strong> method requires a network connection to a running WebDriver+Chrome server, set by the ENV var 'WEBDRIVER_URL'. </p> @@ -87,20 +87,20 @@ <div class="tab-pane-inner" id="filters"> <fieldset class="pure-group"> - {{ render_checkbox_field(form.ignore_whitespace) }} + {{ render_checkbox_field(form.application.form.ignore_whitespace) }} <span class="pure-form-message-inline">Ignore whitespace, tabs and new-lines/line-feeds when considering if a change was detected.<br/> <i>Note:</i> Changing this will change the status of your existing watches, possibly trigger alerts etc. </span> </fieldset> <fieldset class="pure-group"> - {{ render_checkbox_field(form.render_anchor_tag_content) }} + {{ render_checkbox_field(form.application.form.render_anchor_tag_content) }} <span class="pure-form-message-inline">Render anchor tag content, default disabled, when enabled renders links as <code>(link text)[https://somesite.com]</code> <br/> <i>Note:</i> Changing this could affect the content of your existing watches, possibly trigger alerts etc. </span> </fieldset> <fieldset class="pure-group"> - {{ render_field(form.global_subtractive_selectors, rows=5, placeholder="header + {{ render_field(form.application.form.global_subtractive_selectors, rows=5, placeholder="header footer nav .stockticker") }} @@ -112,7 +112,7 @@ nav </span> </fieldset> <fieldset class="pure-group"> - {{ render_field(form.global_ignore_text, rows=5, placeholder="Some text to ignore in a line + {{ render_field(form.application.form.global_ignore_text, rows=5, placeholder="Some text to ignore in a line /some.regex\d{2}/ for case-INsensitive regex ") }} <span class="pure-form-message-inline">Note: This is applied globally in addition to the per-watch rules.</span><br/> diff --git a/changedetectionio/tests/conftest.py b/changedetectionio/tests/conftest.py index 98d4dd9e..258ce6a1 100644 --- a/changedetectionio/tests/conftest.py +++ b/changedetectionio/tests/conftest.py @@ -16,6 +16,7 @@ def cleanup(datastore_path): # Unlink test output files files = ['output.txt', 'url-watches.json', + 'secret.txt', 'notification.txt', 'count.txt', 'endpoint-content.txt' diff --git a/changedetectionio/tests/test_access_control.py b/changedetectionio/tests/test_access_control.py index 80eeb780..b1fa8492 100644 --- a/changedetectionio/tests/test_access_control.py +++ b/changedetectionio/tests/test_access_control.py @@ -1,5 +1,5 @@ from flask import url_for - +from . util import live_server_setup def test_check_access_control(app, client): # Still doesnt work, but this is closer. @@ -12,9 +12,9 @@ def test_check_access_control(app, client): # Enable password check. res = c.post( url_for("settings_page"), - data={"password": "foobar", - "minutes_between_check": 180, - 'fetch_backend': "html_requests"}, + data={"application-password": "foobar", + "requests-minutes_between_check": 180, + 'application-fetch_backend': "html_requests"}, follow_redirects=True ) @@ -49,74 +49,31 @@ def test_check_access_control(app, client): assert b"minutes_between_check" in res.data assert b"fetch_backend" in res.data + ################################################## + # Remove password button, and check that it worked + ################################################## res = c.post( url_for("settings_page"), data={ - "minutes_between_check": 180, - "tag": "", - "headers": "", - "fetch_backend": "html_webdriver", - "removepassword_button": "Remove password" + "requests-minutes_between_check": 180, + "application-fetch_backend": "html_webdriver", + "application-removepassword_button": "Remove password" }, follow_redirects=True, ) + assert b"Password protection removed." in res.data + assert b"LOG OUT" not in res.data -# There was a bug where saving the settings form would submit a blank password -def test_check_access_control_no_blank_password(app, client): - # Still doesnt work, but this is closer. - - with app.test_client() as c: - # Check we dont have any password protection enabled yet. - res = c.get(url_for("settings_page")) - assert b"Remove password" not in res.data - - # Enable password check. - res = c.post( - url_for("settings_page"), - data={"password": "", - "minutes_between_check": 180, - 'fetch_backend': "html_requests"}, - follow_redirects=True - ) - - assert b"Password protection enabled." not in res.data - assert b"Login" not in res.data - - -# There was a bug where saving the settings form would submit a blank password -def test_check_access_no_remote_access_to_remove_password(app, client): - # Still doesnt work, but this is closer. - - with app.test_client() as c: - # Check we dont have any password protection enabled yet. - res = c.get(url_for("settings_page")) - assert b"Remove password" not in res.data - - # Enable password check. + ############################################################ + # Be sure a blank password doesnt setup password protection + ############################################################ res = c.post( url_for("settings_page"), - data={"password": "password", - "minutes_between_check": 180, - 'fetch_backend': "html_requests"}, + data={"application-password": "", + "requests-minutes_between_check": 180, + 'application-fetch_backend': "html_requests"}, follow_redirects=True ) - assert b"Password protection enabled." in res.data - assert b"Login" in res.data - - res = c.post( - url_for("settings_page"), - data={ - "minutes_between_check": 180, - "tag": "", - "headers": "", - "fetch_backend": "html_webdriver", - "removepassword_button": "Remove password" - }, - follow_redirects=True, - ) - assert b"Password protection removed." not in res.data + assert b"Password protection enabled" not in res.data - res = c.get(url_for("index"), - follow_redirects=True) - assert b"watch-table-wrapper" not in res.data diff --git a/changedetectionio/tests/test_backend.py b/changedetectionio/tests/test_backend.py index 79143c4e..4d718d5e 100644 --- a/changedetectionio/tests/test_backend.py +++ b/changedetectionio/tests/test_backend.py @@ -109,7 +109,7 @@ def test_check_basic_change_detection_functionality(client, live_server): # Enable auto pickup of <title> in settings res = client.post( url_for("settings_page"), - data={"extract_title_as_title": "1", "minutes_between_check": 180, 'fetch_backend': "html_requests"}, + data={"application-extract_title_as_title": "1", "requests-minutes_between_check": 180, 'application-fetch_backend': "html_requests"}, follow_redirects=True ) diff --git a/changedetectionio/tests/test_ignore_text.py b/changedetectionio/tests/test_ignore_text.py index 022c4f56..da6ceb23 100644 --- a/changedetectionio/tests/test_ignore_text.py +++ b/changedetectionio/tests/test_ignore_text.py @@ -196,9 +196,9 @@ def test_check_global_ignore_text_functionality(client, live_server): res = client.post( url_for("settings_page"), data={ - "minutes_between_check": 180, - "global_ignore_text": ignore_text, - 'fetch_backend': "html_requests" + "requests-minutes_between_check": 180, + "application-global_ignore_text": ignore_text, + 'application-fetch_backend': "html_requests" }, follow_redirects=True ) diff --git a/changedetectionio/tests/test_ignorehyperlinks.py b/changedetectionio/tests/test_ignorehyperlinks.py index 9356305b..920d0864 100644 --- a/changedetectionio/tests/test_ignorehyperlinks.py +++ b/changedetectionio/tests/test_ignorehyperlinks.py @@ -55,9 +55,9 @@ def test_render_anchor_tag_content_true(client, live_server): res = client.post( url_for("settings_page"), data={ - "minutes_between_check": 180, - "render_anchor_tag_content": "true", - "fetch_backend": "html_requests", + "requests-minutes_between_check": 180, + "application-render_anchor_tag_content": "true", + "application-fetch_backend": "html_requests", }, follow_redirects=True, ) @@ -116,9 +116,9 @@ def test_render_anchor_tag_content_false(client, live_server): res = client.post( url_for("settings_page"), data={ - "minutes_between_check": 180, - "render_anchor_tag_content": "false", - "fetch_backend": "html_requests", + "requests-minutes_between_check": 180, + "application-render_anchor_tag_content": "false", + "application-fetch_backend": "html_requests", }, follow_redirects=True, ) @@ -175,8 +175,8 @@ def test_render_anchor_tag_content_default(client, live_server): res = client.post( url_for("settings_page"), data={ - "minutes_between_check": 180, - "fetch_backend": "html_requests", + "requests-minutes_between_check": 180, + "application-fetch_backend": "html_requests", }, follow_redirects=True, ) diff --git a/changedetectionio/tests/test_ignorestatuscode.py b/changedetectionio/tests/test_ignorestatuscode.py index 7f7ca280..03df6cf3 100644 --- a/changedetectionio/tests/test_ignorestatuscode.py +++ b/changedetectionio/tests/test_ignorestatuscode.py @@ -51,9 +51,9 @@ def test_normal_page_check_works_with_ignore_status_code(client, live_server): res = client.post( url_for("settings_page"), data={ - "minutes_between_check": 180, - "ignore_status_codes": "y", - 'fetch_backend': "html_requests" + "requests-minutes_between_check": 180, + "application-ignore_status_codes": "y", + 'application-fetch_backend': "html_requests" }, follow_redirects=True ) diff --git a/changedetectionio/tests/test_ignorewhitespace.py b/changedetectionio/tests/test_ignorewhitespace.py index 062efd70..fbc98495 100644 --- a/changedetectionio/tests/test_ignorewhitespace.py +++ b/changedetectionio/tests/test_ignorewhitespace.py @@ -61,9 +61,9 @@ def test_check_ignore_whitespace(client, live_server): res = client.post( url_for("settings_page"), data={ - "minutes_between_check": 180, - "ignore_whitespace": "y", - 'fetch_backend': "html_requests" + "requests-minutes_between_check": 180, + "application-ignore_whitespace": "y", + "application-fetch_backend": "html_requests" }, follow_redirects=True ) diff --git a/changedetectionio/tests/test_notification.py b/changedetectionio/tests/test_notification.py index c590f762..02f59cfa 100644 --- a/changedetectionio/tests/test_notification.py +++ b/changedetectionio/tests/test_notification.py @@ -195,11 +195,11 @@ def test_notification_validation(client, live_server): # Now adding a wrong token should give us an error res = client.post( url_for("settings_page"), - data={"notification_title": "New ChangeDetection.io Notification - {watch_url}", - "notification_body": "Rubbish: {rubbish}\n", - "notification_format": "Text", - "notification_urls": "json://localhost/foobar", - "time_between_check": {'seconds': 180}, + data={"application-notification_title": "New ChangeDetection.io Notification - {watch_url}", + "application-notification_body": "Rubbish: {rubbish}\n", + "application-notification_format": "Text", + "application-notification_urls": "json://localhost/foobar", + "requests-minutes_between_check": 180, "fetch_backend": "html_requests" }, follow_redirects=True diff --git a/changedetectionio/tests/test_request.py b/changedetectionio/tests/test_request.py index b745c7a2..4959ef89 100644 --- a/changedetectionio/tests/test_request.py +++ b/changedetectionio/tests/test_request.py @@ -84,9 +84,25 @@ def test_body_in_request(client, live_server): ) assert b"1 Imported" in res.data - body_value = 'Test Body Value' + time.sleep(3) + + # add the first 'version' + res = client.post( + url_for("edit_page", uuid="first"), + data={ + "url": test_url, + "tag": "", + "method": "POST", + "fetch_backend": "html_requests", + "body": "something something"}, + follow_redirects=True + ) + assert b"Updated watch." in res.data + + time.sleep(3) - # Add a properly formatted body with a proper method + # Now the change which should trigger a change + body_value = 'Test Body Value' res = client.post( url_for("edit_page", uuid="first"), data={ diff --git a/changedetectionio/tests/test_watch_fields_storage.py b/changedetectionio/tests/test_watch_fields_storage.py index a2929b79..b57273d7 100644 --- a/changedetectionio/tests/test_watch_fields_storage.py +++ b/changedetectionio/tests/test_watch_fields_storage.py @@ -56,8 +56,8 @@ def test_check_recheck_global_setting(client, live_server): res = client.post( url_for("settings_page"), data={ - "minutes_between_check": 1566, - 'fetch_backend': "html_requests" + "requests-minutes_between_check": 1566, + 'application-fetch_backend': "html_requests" }, follow_redirects=True ) @@ -88,8 +88,8 @@ def test_check_recheck_global_setting(client, live_server): res = client.post( url_for("settings_page"), data={ - "minutes_between_check": 222, - 'fetch_backend': "html_requests" + "requests-minutes_between_check": 222, + 'application-fetch_backend': "html_requests" }, follow_redirects=True ) @@ -124,8 +124,8 @@ def test_check_recheck_global_setting(client, live_server): res = client.post( url_for("settings_page"), data={ - "minutes_between_check": 666, - 'fetch_backend': "html_requests" + "requests-minutes_between_check": 666, + 'application-fetch_backend': "html_requests" }, follow_redirects=True ) diff --git a/changedetectionio/tests/util.py b/changedetectionio/tests/util.py index e2043747..d1457fab 100644 --- a/changedetectionio/tests/util.py +++ b/changedetectionio/tests/util.py @@ -85,6 +85,7 @@ def live_server_setup(live_server): # Just return the body in the request @live_server.app.route('/test-body', methods=['POST', 'GET']) def test_body(): + print ("TEST-BODY GOT", request.data, "returning") return request.data # Just return the verb in the request