From e867e893033ecf8883cff9d3bf7644c6a6d3a82e Mon Sep 17 00:00:00 2001 From: Brandon Wees Date: Thu, 27 Oct 2022 08:45:44 -0400 Subject: [PATCH 1/9] Update test_backup.py --- changedetectionio/tests/test_backup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/changedetectionio/tests/test_backup.py b/changedetectionio/tests/test_backup.py index 89fd66a5..072cfa60 100644 --- a/changedetectionio/tests/test_backup.py +++ b/changedetectionio/tests/test_backup.py @@ -46,6 +46,6 @@ def test_backup(client, live_server): uuid4hex = re.compile('^[a-f0-9]{8}-?[a-f0-9]{4}-?4[a-f0-9]{3}-?[89ab][a-f0-9]{3}-?[a-f0-9]{12}.*txt', re.I) newlist = list(filter(uuid4hex.match, l)) # Read Note below - # Should be two txt files in the archive (history and the snapshot) - assert len(newlist) == 2 + # Should be three txt files in the archive (history and the snapshot) + assert len(newlist) == 3 From 748328453e971417531afd8a59ea30e874bfd302 Mon Sep 17 00:00:00 2001 From: Brandon Wees Date: Thu, 27 Oct 2022 09:03:39 -0400 Subject: [PATCH 2/9] unmerge external header server. Sorry! --- changedetectionio/fetch_site_status.py | 16 +--------------- changedetectionio/forms.py | 1 - changedetectionio/model/Watch.py | 1 - changedetectionio/templates/edit.html | 6 ------ changedetectionio/tests/test_auth.py | 2 +- changedetectionio/tests/test_css_selector.py | 2 +- changedetectionio/tests/test_ignorestatuscode.py | 2 +- changedetectionio/tests/test_share_watch.py | 2 +- changedetectionio/tests/test_xpath_selector.py | 10 +++++----- 9 files changed, 10 insertions(+), 32 deletions(-) diff --git a/changedetectionio/fetch_site_status.py b/changedetectionio/fetch_site_status.py index 4a162f3d..128a4e1d 100644 --- a/changedetectionio/fetch_site_status.py +++ b/changedetectionio/fetch_site_status.py @@ -4,8 +4,7 @@ import os import re import urllib3 import difflib -import requests -import json + from changedetectionio import content_fetcher, html_tools @@ -59,19 +58,6 @@ class perform_site_check(): # Tweak the base config with the per-watch ones request_headers = self.datastore.data['settings']['headers'].copy() - - if self.datastore.data['watching'][uuid].get('external_header_server') is not None and self.datastore.data['watching'][uuid].get('external_header_server') != "" and self.datastore.data['watching'][uuid].get('external_header_server') != "None": - try: - resp = requests.get(self.datastore.data['watching'][uuid].get('external_header_server')) - if resp.status_code != 200: - raise Exception("External header server returned non-200 response. Please check the URL for the server") - - data = json.loads(resp.text.strip()) - request_headers.update(resp.json()) - - except json.decoder.JSONDecodeError: - raise Exception("Failed to decode JSON response from external header server") - request_headers.update(extra_headers) # https://github.com/psf/requests/issues/4525 diff --git a/changedetectionio/forms.py b/changedetectionio/forms.py index a277a73f..84d164d6 100644 --- a/changedetectionio/forms.py +++ b/changedetectionio/forms.py @@ -370,7 +370,6 @@ class watchForm(commonSettingsForm): title = StringField('Title', default='') ignore_text = StringListField('Ignore text', [ValidateListRegex()]) - external_header_server = fields.URLField('External Header Server', validators=[validators.Optional(), validateURL()]) headers = StringDictKeyValue('Request headers') body = TextAreaField('Request body', [validators.Optional()]) method = SelectField('Request method', choices=valid_method, default=default_method) diff --git a/changedetectionio/model/Watch.py b/changedetectionio/model/Watch.py index 4988786b..53cc9e73 100644 --- a/changedetectionio/model/Watch.py +++ b/changedetectionio/model/Watch.py @@ -26,7 +26,6 @@ class model(dict): 'previous_md5': False, 'uuid': str(uuid.uuid4()), 'headers': {}, # Extra headers to send - 'external_header_server': None, # URL to a server that will return headers 'body': None, 'method': 'GET', #'history': {}, # Dict of timestamp and output stripped filename diff --git a/changedetectionio/templates/edit.html b/changedetectionio/templates/edit.html index 1cca5ef0..fd3758d9 100644 --- a/changedetectionio/templates/edit.html +++ b/changedetectionio/templates/edit.html @@ -120,12 +120,6 @@
{{ render_field(form.method) }}
-
- {{ render_field(form.external_header_server, placeholder="http://example.com/watch1") }} -
- The watch will perform a GET request before each check to this URL and will use the headers in addition to the ones listed below and in global settings. More help and examples here -
-
{{ render_field(form.headers, rows=5, placeholder="Example Cookie: foobar diff --git a/changedetectionio/tests/test_auth.py b/changedetectionio/tests/test_auth.py index 38f89721..f8d1437e 100644 --- a/changedetectionio/tests/test_auth.py +++ b/changedetectionio/tests/test_auth.py @@ -23,7 +23,7 @@ def test_basic_auth(client, live_server): # Check form validation res = client.post( url_for("edit_page", uuid="first"), - data={"css_filter": "", "url": test_url, "tag": "", "headers": "", "external_header_server": "", 'fetch_backend': "html_requests"}, + data={"css_filter": "", "url": test_url, "tag": "", "headers": "", 'fetch_backend': "html_requests"}, follow_redirects=True ) assert b"Updated watch." in res.data diff --git a/changedetectionio/tests/test_css_selector.py b/changedetectionio/tests/test_css_selector.py index d7d0a1af..ab234ddb 100644 --- a/changedetectionio/tests/test_css_selector.py +++ b/changedetectionio/tests/test_css_selector.py @@ -98,7 +98,7 @@ def test_check_markup_css_filter_restriction(client, live_server): # Add our URL to the import page res = client.post( url_for("edit_page", uuid="first"), - data={"css_filter": css_filter, "url": test_url, "tag": "", "headers": "", "external_header_server": "",'fetch_backend': "html_requests"}, + data={"css_filter": css_filter, "url": test_url, "tag": "", "headers": "", 'fetch_backend': "html_requests"}, follow_redirects=True ) assert b"Updated watch." in res.data diff --git a/changedetectionio/tests/test_ignorestatuscode.py b/changedetectionio/tests/test_ignorestatuscode.py index 15aa1d78..aeafcdaa 100644 --- a/changedetectionio/tests/test_ignorestatuscode.py +++ b/changedetectionio/tests/test_ignorestatuscode.py @@ -114,7 +114,7 @@ def test_403_page_check_works_with_ignore_status_code(client, live_server): # Add our URL to the import page res = client.post( url_for("edit_page", uuid="first"), - data={"ignore_status_codes": "y", "url": test_url, "tag": "", "headers": "", "external_header_server": "",'fetch_backend': "html_requests"}, + data={"ignore_status_codes": "y", "url": test_url, "tag": "", "headers": "", 'fetch_backend': "html_requests"}, follow_redirects=True ) assert b"Updated watch." in res.data diff --git a/changedetectionio/tests/test_share_watch.py b/changedetectionio/tests/test_share_watch.py index 5f7eff36..620bda03 100644 --- a/changedetectionio/tests/test_share_watch.py +++ b/changedetectionio/tests/test_share_watch.py @@ -29,7 +29,7 @@ def test_share_watch(client, live_server): # Add our URL to the import page res = client.post( url_for("edit_page", uuid="first"), - data={"css_filter": css_filter, "url": test_url, "tag": "", "headers": "", "external_header_server": "",'fetch_backend': "html_requests"}, + data={"css_filter": css_filter, "url": test_url, "tag": "", "headers": "", 'fetch_backend': "html_requests"}, follow_redirects=True ) assert b"Updated watch." in res.data diff --git a/changedetectionio/tests/test_xpath_selector.py b/changedetectionio/tests/test_xpath_selector.py index 9be957d6..4e417a74 100644 --- a/changedetectionio/tests/test_xpath_selector.py +++ b/changedetectionio/tests/test_xpath_selector.py @@ -89,7 +89,7 @@ def test_check_xpath_filter_utf8(client, live_server): time.sleep(1) res = client.post( url_for("edit_page", uuid="first"), - data={"css_filter": filter, "url": test_url, "tag": "", "headers": "", "external_header_server": "",'fetch_backend': "html_requests"}, + data={"css_filter": filter, "url": test_url, "tag": "", "headers": "", 'fetch_backend': "html_requests"}, follow_redirects=True ) assert b"Updated watch." in res.data @@ -143,7 +143,7 @@ def test_check_xpath_text_function_utf8(client, live_server): time.sleep(1) res = client.post( url_for("edit_page", uuid="first"), - data={"css_filter": filter, "url": test_url, "tag": "", "headers": "", "external_header_server": "",'fetch_backend': "html_requests"}, + data={"css_filter": filter, "url": test_url, "tag": "", "headers": "", 'fetch_backend': "html_requests"}, follow_redirects=True ) assert b"Updated watch." in res.data @@ -192,7 +192,7 @@ def test_check_markup_xpath_filter_restriction(client, live_server): # Add our URL to the import page res = client.post( url_for("edit_page", uuid="first"), - data={"css_filter": xpath_filter, "url": test_url, "tag": "", "headers": "", "external_header_server": "",'fetch_backend': "html_requests"}, + data={"css_filter": xpath_filter, "url": test_url, "tag": "", "headers": "", 'fetch_backend': "html_requests"}, follow_redirects=True ) assert b"Updated watch." in res.data @@ -233,7 +233,7 @@ def test_xpath_validation(client, live_server): res = client.post( url_for("edit_page", uuid="first"), - data={"css_filter": "/something horrible", "url": test_url, "tag": "", "headers": "", "external_header_server": "",'fetch_backend': "html_requests"}, + data={"css_filter": "/something horrible", "url": test_url, "tag": "", "headers": "", 'fetch_backend': "html_requests"}, follow_redirects=True ) assert b"is not a valid XPath expression" in res.data @@ -263,7 +263,7 @@ def test_check_with_prefix_css_filter(client, live_server): res = client.post( url_for("edit_page", uuid="first"), - data={"css_filter": "xpath://*[contains(@class, 'sametext')]", "url": test_url, "tag": "", "headers": "", "external_header_server": "",'fetch_backend': "html_requests"}, + data={"css_filter": "xpath://*[contains(@class, 'sametext')]", "url": test_url, "tag": "", "headers": "", 'fetch_backend': "html_requests"}, follow_redirects=True ) From 4364521cfc0b2be50af5294eed75dae3f0bf97bf Mon Sep 17 00:00:00 2001 From: Brandon Wees Date: Thu, 27 Oct 2022 09:11:28 -0400 Subject: [PATCH 3/9] Update changedetectionio/templates/edit.html Co-authored-by: dgtlmoon --- changedetectionio/templates/edit.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/changedetectionio/templates/edit.html b/changedetectionio/templates/edit.html index fd3758d9..90d1fa2f 100644 --- a/changedetectionio/templates/edit.html +++ b/changedetectionio/templates/edit.html @@ -179,7 +179,7 @@ User-Agent: wonderbra 1.0") }} {{ render_checkbox_field(form.trigger_add, class="trigger-type") }} {{ render_checkbox_field(form.trigger_del, class="trigger-type") }} - Filters the change-detection of this watch to only this type of content change. Replacements (neither additions or deletions) are always included. + Filters the change-detection of this watch to only this type of content change. Replacements (neither additions nor deletions) are always included. The 'diff' will still include all changes.
From 7011a0439920f0984fba9975d28a0dbf45e83ae4 Mon Sep 17 00:00:00 2001 From: Brandon Wees Date: Thu, 27 Oct 2022 10:43:18 -0400 Subject: [PATCH 4/9] switching to os.path.join --- changedetectionio/model/Watch.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/changedetectionio/model/Watch.py b/changedetectionio/model/Watch.py index 53cc9e73..b0d89500 100644 --- a/changedetectionio/model/Watch.py +++ b/changedetectionio/model/Watch.py @@ -202,13 +202,13 @@ class model(dict): def save_previous_text(self, contents): import logging - output_path = "{}/{}".format(self.__datastore_path, self['uuid']) + output_path = os.path.join(self.__datastore_path, self['uuid']) # Incase the operator deleted it, check and create. if not os.path.isdir(output_path): os.mkdir(output_path) - snapshot_fname = "{}/previous.txt".format(output_path) + snapshot_fname = os.path.join(output_path, "previous.txt") logging.debug("Saving previous text {}".format(snapshot_fname)) with open(snapshot_fname, 'wb') as f: @@ -220,9 +220,9 @@ class model(dict): def get_previous_text(self): from os import path - output_path = "{}/{}".format(self.__datastore_path, self['uuid']) + output_path = os.path.join(self.__datastore_path, self['uuid']) - snapshot_fname = "{}/previous.txt".format(output_path) + snapshot_fname = os.path.join(output_path, "previous.txt") if self.history_n < 1: return "" From d24111f3a60af12e03f1972096a09abd0f726e29 Mon Sep 17 00:00:00 2001 From: Brandon Wees Date: Thu, 27 Oct 2022 10:52:20 -0400 Subject: [PATCH 5/9] Apply suggestions from code review Co-authored-by: dgtlmoon --- changedetectionio/model/Watch.py | 3 +-- changedetectionio/store.py | 4 ++-- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/changedetectionio/model/Watch.py b/changedetectionio/model/Watch.py index b0d89500..3191c113 100644 --- a/changedetectionio/model/Watch.py +++ b/changedetectionio/model/Watch.py @@ -208,7 +208,7 @@ class model(dict): if not os.path.isdir(output_path): os.mkdir(output_path) - snapshot_fname = os.path.join(output_path, "previous.txt") + snapshot_fname = os.path.join(self.watch_data_dir, "previous.txt") logging.debug("Saving previous text {}".format(snapshot_fname)) with open(snapshot_fname, 'wb') as f: @@ -220,7 +220,6 @@ class model(dict): def get_previous_text(self): from os import path - output_path = os.path.join(self.__datastore_path, self['uuid']) snapshot_fname = os.path.join(output_path, "previous.txt") if self.history_n < 1: diff --git a/changedetectionio/store.py b/changedetectionio/store.py index 4857bcf2..95c3c2b5 100644 --- a/changedetectionio/store.py +++ b/changedetectionio/store.py @@ -598,9 +598,9 @@ class ChangeDetectionStore: # Check if the previous.txt exists - if not os.path.exists(os.path.join(self.datastore_path, uuid, "previous.txt")): + if not os.path.exists(os.path.join(watch.watch_data_dir, "previous.txt")): # Generate a previous.txt - with open(os.path.join(self.datastore_path, uuid, "previous.txt"), "wb") as f: + with open(os.path.join(watch.watch_data_dir, "previous.txt"), "wb") as f: # Fill it with the latest history latest_file_name = watch.history[watch.newest_history_key] with open(latest_file_name, "rb") as f2: From 35d950fa743914d3ad5b2f62fd52e35ebd13c64d Mon Sep 17 00:00:00 2001 From: Brandon Wees Date: Thu, 27 Oct 2022 10:52:42 -0400 Subject: [PATCH 6/9] Update changedetectionio/model/Watch.py Co-authored-by: dgtlmoon --- changedetectionio/model/Watch.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/changedetectionio/model/Watch.py b/changedetectionio/model/Watch.py index 3191c113..fd9e7c38 100644 --- a/changedetectionio/model/Watch.py +++ b/changedetectionio/model/Watch.py @@ -221,7 +221,7 @@ class model(dict): from os import path - snapshot_fname = os.path.join(output_path, "previous.txt") + snapshot_fname = os.path.join(self.watch_data_dir, "previous.txt") if self.history_n < 1: return "" From c2c8bb4de86af5209f36fee9e5b0902b74556bc9 Mon Sep 17 00:00:00 2001 From: Brandon Wees Date: Thu, 27 Oct 2022 10:54:30 -0400 Subject: [PATCH 7/9] ensure_data_dir_exists call added --- changedetectionio/model/Watch.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/changedetectionio/model/Watch.py b/changedetectionio/model/Watch.py index fd9e7c38..fc610c88 100644 --- a/changedetectionio/model/Watch.py +++ b/changedetectionio/model/Watch.py @@ -205,8 +205,7 @@ class model(dict): output_path = os.path.join(self.__datastore_path, self['uuid']) # Incase the operator deleted it, check and create. - if not os.path.isdir(output_path): - os.mkdir(output_path) + self.ensure_data_dir_exists() snapshot_fname = os.path.join(self.watch_data_dir, "previous.txt") logging.debug("Saving previous text {}".format(snapshot_fname)) From 294bb7be15c2ef575fef54d443aac2617fb65c48 Mon Sep 17 00:00:00 2001 From: Brandon Wees Date: Thu, 27 Oct 2022 10:57:50 -0400 Subject: [PATCH 8/9] remvoe unneeded import --- changedetectionio/model/Watch.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/changedetectionio/model/Watch.py b/changedetectionio/model/Watch.py index fc610c88..61028485 100644 --- a/changedetectionio/model/Watch.py +++ b/changedetectionio/model/Watch.py @@ -217,8 +217,6 @@ class model(dict): # Get previous text snapshot for diffing - used for calculating additions and deletions def get_previous_text(self): - from os import path - snapshot_fname = os.path.join(self.watch_data_dir, "previous.txt") if self.history_n < 1: From e11c6aeb5f3e0b10513d63ff52651e9488faed16 Mon Sep 17 00:00:00 2001 From: Brandon Wees Date: Thu, 27 Oct 2022 10:59:14 -0400 Subject: [PATCH 9/9] Apply suggestions from code review Co-authored-by: dgtlmoon --- changedetectionio/tests/test_diff_filter_only_deletions.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/changedetectionio/tests/test_diff_filter_only_deletions.py b/changedetectionio/tests/test_diff_filter_only_deletions.py index 34119631..3bdcb005 100644 --- a/changedetectionio/tests/test_diff_filter_only_deletions.py +++ b/changedetectionio/tests/test_diff_filter_only_deletions.py @@ -66,8 +66,8 @@ def test_diff_filtering_no_add(client, live_server): # We should NOT see the change res = client.get(url_for("index")) # save res.data to a file - with open("./test-index.html", "w") as f: - f.write(res.data.decode("utf-8")) + + assert b'unviewed' not in res.data