diff --git a/changedetectionio/fetch_site_status.py b/changedetectionio/fetch_site_status.py index 4a162f3d..128a4e1d 100644 --- a/changedetectionio/fetch_site_status.py +++ b/changedetectionio/fetch_site_status.py @@ -4,8 +4,7 @@ import os import re import urllib3 import difflib -import requests -import json + from changedetectionio import content_fetcher, html_tools @@ -59,19 +58,6 @@ class perform_site_check(): # Tweak the base config with the per-watch ones request_headers = self.datastore.data['settings']['headers'].copy() - - if self.datastore.data['watching'][uuid].get('external_header_server') is not None and self.datastore.data['watching'][uuid].get('external_header_server') != "" and self.datastore.data['watching'][uuid].get('external_header_server') != "None": - try: - resp = requests.get(self.datastore.data['watching'][uuid].get('external_header_server')) - if resp.status_code != 200: - raise Exception("External header server returned non-200 response. Please check the URL for the server") - - data = json.loads(resp.text.strip()) - request_headers.update(resp.json()) - - except json.decoder.JSONDecodeError: - raise Exception("Failed to decode JSON response from external header server") - request_headers.update(extra_headers) # https://github.com/psf/requests/issues/4525 diff --git a/changedetectionio/forms.py b/changedetectionio/forms.py index a277a73f..84d164d6 100644 --- a/changedetectionio/forms.py +++ b/changedetectionio/forms.py @@ -370,7 +370,6 @@ class watchForm(commonSettingsForm): title = StringField('Title', default='') ignore_text = StringListField('Ignore text', [ValidateListRegex()]) - external_header_server = fields.URLField('External Header Server', validators=[validators.Optional(), validateURL()]) headers = StringDictKeyValue('Request headers') body = TextAreaField('Request body', [validators.Optional()]) method = SelectField('Request method', choices=valid_method, default=default_method) diff --git a/changedetectionio/model/Watch.py b/changedetectionio/model/Watch.py index 4988786b..61028485 100644 --- a/changedetectionio/model/Watch.py +++ b/changedetectionio/model/Watch.py @@ -26,7 +26,6 @@ class model(dict): 'previous_md5': False, 'uuid': str(uuid.uuid4()), 'headers': {}, # Extra headers to send - 'external_header_server': None, # URL to a server that will return headers 'body': None, 'method': 'GET', #'history': {}, # Dict of timestamp and output stripped filename @@ -203,13 +202,12 @@ class model(dict): def save_previous_text(self, contents): import logging - output_path = "{}/{}".format(self.__datastore_path, self['uuid']) + output_path = os.path.join(self.__datastore_path, self['uuid']) # Incase the operator deleted it, check and create. - if not os.path.isdir(output_path): - os.mkdir(output_path) + self.ensure_data_dir_exists() - snapshot_fname = "{}/previous.txt".format(output_path) + snapshot_fname = os.path.join(self.watch_data_dir, "previous.txt") logging.debug("Saving previous text {}".format(snapshot_fname)) with open(snapshot_fname, 'wb') as f: @@ -219,11 +217,8 @@ class model(dict): # Get previous text snapshot for diffing - used for calculating additions and deletions def get_previous_text(self): - from os import path - - output_path = "{}/{}".format(self.__datastore_path, self['uuid']) - snapshot_fname = "{}/previous.txt".format(output_path) + snapshot_fname = os.path.join(self.watch_data_dir, "previous.txt") if self.history_n < 1: return "" diff --git a/changedetectionio/store.py b/changedetectionio/store.py index 4857bcf2..95c3c2b5 100644 --- a/changedetectionio/store.py +++ b/changedetectionio/store.py @@ -598,9 +598,9 @@ class ChangeDetectionStore: # Check if the previous.txt exists - if not os.path.exists(os.path.join(self.datastore_path, uuid, "previous.txt")): + if not os.path.exists(os.path.join(watch.watch_data_dir, "previous.txt")): # Generate a previous.txt - with open(os.path.join(self.datastore_path, uuid, "previous.txt"), "wb") as f: + with open(os.path.join(watch.watch_data_dir, "previous.txt"), "wb") as f: # Fill it with the latest history latest_file_name = watch.history[watch.newest_history_key] with open(latest_file_name, "rb") as f2: diff --git a/changedetectionio/templates/edit.html b/changedetectionio/templates/edit.html index 1cca5ef0..90d1fa2f 100644 --- a/changedetectionio/templates/edit.html +++ b/changedetectionio/templates/edit.html @@ -120,12 +120,6 @@
{{ render_field(form.method) }}
-
- {{ render_field(form.external_header_server, placeholder="http://example.com/watch1") }} -
- The watch will perform a GET request before each check to this URL and will use the headers in addition to the ones listed below and in global settings. More help and examples here -
-
{{ render_field(form.headers, rows=5, placeholder="Example Cookie: foobar @@ -185,7 +179,7 @@ User-Agent: wonderbra 1.0") }} {{ render_checkbox_field(form.trigger_add, class="trigger-type") }} {{ render_checkbox_field(form.trigger_del, class="trigger-type") }} - Filters the change-detection of this watch to only this type of content change. Replacements (neither additions or deletions) are always included. + Filters the change-detection of this watch to only this type of content change. Replacements (neither additions nor deletions) are always included. The 'diff' will still include all changes.
diff --git a/changedetectionio/tests/test_auth.py b/changedetectionio/tests/test_auth.py index 38f89721..f8d1437e 100644 --- a/changedetectionio/tests/test_auth.py +++ b/changedetectionio/tests/test_auth.py @@ -23,7 +23,7 @@ def test_basic_auth(client, live_server): # Check form validation res = client.post( url_for("edit_page", uuid="first"), - data={"css_filter": "", "url": test_url, "tag": "", "headers": "", "external_header_server": "", 'fetch_backend': "html_requests"}, + data={"css_filter": "", "url": test_url, "tag": "", "headers": "", 'fetch_backend': "html_requests"}, follow_redirects=True ) assert b"Updated watch." in res.data diff --git a/changedetectionio/tests/test_backup.py b/changedetectionio/tests/test_backup.py index 89fd66a5..072cfa60 100644 --- a/changedetectionio/tests/test_backup.py +++ b/changedetectionio/tests/test_backup.py @@ -46,6 +46,6 @@ def test_backup(client, live_server): uuid4hex = re.compile('^[a-f0-9]{8}-?[a-f0-9]{4}-?4[a-f0-9]{3}-?[89ab][a-f0-9]{3}-?[a-f0-9]{12}.*txt', re.I) newlist = list(filter(uuid4hex.match, l)) # Read Note below - # Should be two txt files in the archive (history and the snapshot) - assert len(newlist) == 2 + # Should be three txt files in the archive (history and the snapshot) + assert len(newlist) == 3 diff --git a/changedetectionio/tests/test_css_selector.py b/changedetectionio/tests/test_css_selector.py index d7d0a1af..ab234ddb 100644 --- a/changedetectionio/tests/test_css_selector.py +++ b/changedetectionio/tests/test_css_selector.py @@ -98,7 +98,7 @@ def test_check_markup_css_filter_restriction(client, live_server): # Add our URL to the import page res = client.post( url_for("edit_page", uuid="first"), - data={"css_filter": css_filter, "url": test_url, "tag": "", "headers": "", "external_header_server": "",'fetch_backend': "html_requests"}, + data={"css_filter": css_filter, "url": test_url, "tag": "", "headers": "", 'fetch_backend': "html_requests"}, follow_redirects=True ) assert b"Updated watch." in res.data diff --git a/changedetectionio/tests/test_diff_filter_only_deletions.py b/changedetectionio/tests/test_diff_filter_only_deletions.py index 34119631..3bdcb005 100644 --- a/changedetectionio/tests/test_diff_filter_only_deletions.py +++ b/changedetectionio/tests/test_diff_filter_only_deletions.py @@ -66,8 +66,8 @@ def test_diff_filtering_no_add(client, live_server): # We should NOT see the change res = client.get(url_for("index")) # save res.data to a file - with open("./test-index.html", "w") as f: - f.write(res.data.decode("utf-8")) + + assert b'unviewed' not in res.data diff --git a/changedetectionio/tests/test_ignorestatuscode.py b/changedetectionio/tests/test_ignorestatuscode.py index 15aa1d78..aeafcdaa 100644 --- a/changedetectionio/tests/test_ignorestatuscode.py +++ b/changedetectionio/tests/test_ignorestatuscode.py @@ -114,7 +114,7 @@ def test_403_page_check_works_with_ignore_status_code(client, live_server): # Add our URL to the import page res = client.post( url_for("edit_page", uuid="first"), - data={"ignore_status_codes": "y", "url": test_url, "tag": "", "headers": "", "external_header_server": "",'fetch_backend': "html_requests"}, + data={"ignore_status_codes": "y", "url": test_url, "tag": "", "headers": "", 'fetch_backend': "html_requests"}, follow_redirects=True ) assert b"Updated watch." in res.data diff --git a/changedetectionio/tests/test_share_watch.py b/changedetectionio/tests/test_share_watch.py index 5f7eff36..620bda03 100644 --- a/changedetectionio/tests/test_share_watch.py +++ b/changedetectionio/tests/test_share_watch.py @@ -29,7 +29,7 @@ def test_share_watch(client, live_server): # Add our URL to the import page res = client.post( url_for("edit_page", uuid="first"), - data={"css_filter": css_filter, "url": test_url, "tag": "", "headers": "", "external_header_server": "",'fetch_backend': "html_requests"}, + data={"css_filter": css_filter, "url": test_url, "tag": "", "headers": "", 'fetch_backend': "html_requests"}, follow_redirects=True ) assert b"Updated watch." in res.data diff --git a/changedetectionio/tests/test_xpath_selector.py b/changedetectionio/tests/test_xpath_selector.py index 9be957d6..4e417a74 100644 --- a/changedetectionio/tests/test_xpath_selector.py +++ b/changedetectionio/tests/test_xpath_selector.py @@ -89,7 +89,7 @@ def test_check_xpath_filter_utf8(client, live_server): time.sleep(1) res = client.post( url_for("edit_page", uuid="first"), - data={"css_filter": filter, "url": test_url, "tag": "", "headers": "", "external_header_server": "",'fetch_backend': "html_requests"}, + data={"css_filter": filter, "url": test_url, "tag": "", "headers": "", 'fetch_backend': "html_requests"}, follow_redirects=True ) assert b"Updated watch." in res.data @@ -143,7 +143,7 @@ def test_check_xpath_text_function_utf8(client, live_server): time.sleep(1) res = client.post( url_for("edit_page", uuid="first"), - data={"css_filter": filter, "url": test_url, "tag": "", "headers": "", "external_header_server": "",'fetch_backend': "html_requests"}, + data={"css_filter": filter, "url": test_url, "tag": "", "headers": "", 'fetch_backend': "html_requests"}, follow_redirects=True ) assert b"Updated watch." in res.data @@ -192,7 +192,7 @@ def test_check_markup_xpath_filter_restriction(client, live_server): # Add our URL to the import page res = client.post( url_for("edit_page", uuid="first"), - data={"css_filter": xpath_filter, "url": test_url, "tag": "", "headers": "", "external_header_server": "",'fetch_backend': "html_requests"}, + data={"css_filter": xpath_filter, "url": test_url, "tag": "", "headers": "", 'fetch_backend': "html_requests"}, follow_redirects=True ) assert b"Updated watch." in res.data @@ -233,7 +233,7 @@ def test_xpath_validation(client, live_server): res = client.post( url_for("edit_page", uuid="first"), - data={"css_filter": "/something horrible", "url": test_url, "tag": "", "headers": "", "external_header_server": "",'fetch_backend': "html_requests"}, + data={"css_filter": "/something horrible", "url": test_url, "tag": "", "headers": "", 'fetch_backend': "html_requests"}, follow_redirects=True ) assert b"is not a valid XPath expression" in res.data @@ -263,7 +263,7 @@ def test_check_with_prefix_css_filter(client, live_server): res = client.post( url_for("edit_page", uuid="first"), - data={"css_filter": "xpath://*[contains(@class, 'sametext')]", "url": test_url, "tag": "", "headers": "", "external_header_server": "",'fetch_backend': "html_requests"}, + data={"css_filter": "xpath://*[contains(@class, 'sametext')]", "url": test_url, "tag": "", "headers": "", 'fetch_backend': "html_requests"}, follow_redirects=True )