Merge branch 'diff-filters' of https://github.com/bwees/changedetection.io into diff-filters

changedetectionio/fetch_site_status.py

@@ -4,8 +4,7 @@ import os
 import re
 import urllib3
 import difflib
-import requests
-import json
+
 
 from changedetectionio import content_fetcher, html_tools
 
@@ -59,19 +58,6 @@ class perform_site_check():
 
         # Tweak the base config with the per-watch ones
         request_headers = self.datastore.data['settings']['headers'].copy()
-
-        if self.datastore.data['watching'][uuid].get('external_header_server') is not None and self.datastore.data['watching'][uuid].get('external_header_server') != "" and self.datastore.data['watching'][uuid].get('external_header_server') != "None":
-            try:
-                resp = requests.get(self.datastore.data['watching'][uuid].get('external_header_server'))
-                if resp.status_code != 200:
-                    raise Exception("External header server returned non-200 response. Please check the URL for the server")
-            
-                data = json.loads(resp.text.strip())
-                request_headers.update(resp.json())
-
-            except json.decoder.JSONDecodeError:
-                raise Exception("Failed to decode JSON response from external header server")
-            
         request_headers.update(extra_headers)
 
         # https://github.com/psf/requests/issues/4525

changedetectionio/forms.py

@@ -370,7 +370,6 @@ class watchForm(commonSettingsForm):
     title = StringField('Title', default='')
 
     ignore_text = StringListField('Ignore text', [ValidateListRegex()])
-    external_header_server = fields.URLField('External Header Server', validators=[validators.Optional(), validateURL()])
     headers = StringDictKeyValue('Request headers')
     body = TextAreaField('Request body', [validators.Optional()])
     method = SelectField('Request method', choices=valid_method, default=default_method)

changedetectionio/model/Watch.py

@@ -26,7 +26,6 @@ class model(dict):
             'previous_md5': False,
             'uuid': str(uuid.uuid4()),
             'headers': {},  # Extra headers to send
-            'external_header_server': None,  # URL to a server that will return headers
             'body': None,
             'method': 'GET',
             #'history': {},  # Dict of timestamp and output stripped filename
@@ -203,13 +202,12 @@ class model(dict):
     def save_previous_text(self, contents):
         import logging
 
-        output_path = "{}/{}".format(self.__datastore_path, self['uuid'])
+        output_path = os.path.join(self.__datastore_path, self['uuid'])
 
         # Incase the operator deleted it, check and create.
-        if not os.path.isdir(output_path):
-            os.mkdir(output_path)
+        self.ensure_data_dir_exists()
 
-        snapshot_fname = "{}/previous.txt".format(output_path)
+        snapshot_fname = os.path.join(self.watch_data_dir, "previous.txt")
         logging.debug("Saving previous text {}".format(snapshot_fname))
 
         with open(snapshot_fname, 'wb') as f:
@@ -219,11 +217,8 @@ class model(dict):
 
     # Get previous text snapshot for diffing - used for calculating additions and deletions
     def get_previous_text(self):
-        from os import path
-
-        output_path = "{}/{}".format(self.__datastore_path, self['uuid'])
 
-        snapshot_fname = "{}/previous.txt".format(output_path)
+        snapshot_fname = os.path.join(self.watch_data_dir, "previous.txt")
         if self.history_n < 1:
             return ""
 

changedetectionio/store.py

@@ -598,9 +598,9 @@ class ChangeDetectionStore:
 
 
             # Check if the previous.txt exists
-            if not os.path.exists(os.path.join(self.datastore_path, uuid, "previous.txt")):
+            if not os.path.exists(os.path.join(watch.watch_data_dir, "previous.txt")):
                 # Generate a previous.txt
-                with open(os.path.join(self.datastore_path, uuid, "previous.txt"), "wb") as f:
+                with open(os.path.join(watch.watch_data_dir, "previous.txt"), "wb") as f:
                     # Fill it with the latest history
                     latest_file_name = watch.history[watch.newest_history_key]
                     with open(latest_file_name, "rb") as f2:

changedetectionio/templates/edit.html

@@ -120,12 +120,6 @@
                     <div class="pure-control-group" id="request-method">
                         {{ render_field(form.method) }}
                     </div>
-                    <div class="pure-control-group" id="external-header-server">
-                        {{ render_field(form.external_header_server, placeholder="http://example.com/watch1") }}
-                        <div class="pure-form-message-inline">
-                            The watch will perform a GET request before each check to this URL and will use the headers in addition to the ones listed below and in global settings. <a href="https://github.com/dgtlmoon/changedetection.io/wiki/Run-JavaScript-before-change-detection">More help and examples here</a>
-                        </div>
-                    </div>
                     <div class="pure-control-group" id="request-headers">
 {{ render_field(form.headers, rows=5, placeholder="Example
 Cookie: foobar
@@ -185,7 +179,7 @@ User-Agent: wonderbra 1.0") }}
                             {{ render_checkbox_field(form.trigger_add, class="trigger-type") }}
                             {{ render_checkbox_field(form.trigger_del, class="trigger-type") }}
                             <span class="pure-form-message-inline">
-                                Filters the change-detection of this watch to only this type of content change. <strong>Replacements</strong> (neither additions or deletions) are always included.
+                                Filters the change-detection of this watch to only this type of content change. <strong>Replacements</strong> (neither additions nor deletions) are always included. The 'diff' will still include all changes.
                             </span>
                         </div>
                     </fieldset>

changedetectionio/tests/test_auth.py

@@ -23,7 +23,7 @@ def test_basic_auth(client, live_server):
     # Check form validation
     res = client.post(
         url_for("edit_page", uuid="first"),
-        data={"css_filter": "", "url": test_url, "tag": "", "headers": "", "external_header_server": "", 'fetch_backend': "html_requests"},
+        data={"css_filter": "", "url": test_url, "tag": "", "headers": "", 'fetch_backend': "html_requests"},
         follow_redirects=True
     )
     assert b"Updated watch." in res.data

changedetectionio/tests/test_backup.py

@@ -46,6 +46,6 @@ def test_backup(client, live_server):
     uuid4hex = re.compile('^[a-f0-9]{8}-?[a-f0-9]{4}-?4[a-f0-9]{3}-?[89ab][a-f0-9]{3}-?[a-f0-9]{12}.*txt', re.I)
     newlist = list(filter(uuid4hex.match, l))  # Read Note below
 
-    # Should be two txt files in the archive (history and the snapshot)
-    assert len(newlist) == 2
+    # Should be three txt files in the archive (history and the snapshot)
+    assert len(newlist) == 3
 

changedetectionio/tests/test_css_selector.py

@@ -98,7 +98,7 @@ def test_check_markup_css_filter_restriction(client, live_server):
     # Add our URL to the import page
     res = client.post(
         url_for("edit_page", uuid="first"),
-        data={"css_filter": css_filter, "url": test_url, "tag": "", "headers": "", "external_header_server": "",'fetch_backend': "html_requests"},
+        data={"css_filter": css_filter, "url": test_url, "tag": "", "headers": "", 'fetch_backend': "html_requests"},
         follow_redirects=True
     )
     assert b"Updated watch." in res.data

changedetectionio/tests/test_diff_filter_only_deletions.py

@@ -66,8 +66,8 @@ def test_diff_filtering_no_add(client, live_server):
     # We should NOT see the change
     res = client.get(url_for("index"))
     # save res.data to a file
-    with open("./test-index.html", "w") as f:
-        f.write(res.data.decode("utf-8"))
+    
+        
         
     assert b'unviewed' not in res.data
 

changedetectionio/tests/test_ignorestatuscode.py

@@ -114,7 +114,7 @@ def test_403_page_check_works_with_ignore_status_code(client, live_server):
     # Add our URL to the import page
     res = client.post(
         url_for("edit_page", uuid="first"),
-        data={"ignore_status_codes": "y", "url": test_url, "tag": "", "headers": "", "external_header_server": "",'fetch_backend': "html_requests"},
+        data={"ignore_status_codes": "y", "url": test_url, "tag": "", "headers": "", 'fetch_backend': "html_requests"},
         follow_redirects=True
     )
     assert b"Updated watch." in res.data

changedetectionio/tests/test_share_watch.py

@@ -29,7 +29,7 @@ def test_share_watch(client, live_server):
     # Add our URL to the import page
     res = client.post(
         url_for("edit_page", uuid="first"),
-        data={"css_filter": css_filter, "url": test_url, "tag": "", "headers": "", "external_header_server": "",'fetch_backend': "html_requests"},
+        data={"css_filter": css_filter, "url": test_url, "tag": "", "headers": "", 'fetch_backend': "html_requests"},
         follow_redirects=True
     )
     assert b"Updated watch." in res.data

changedetectionio/tests/test_xpath_selector.py

@@ -89,7 +89,7 @@ def test_check_xpath_filter_utf8(client, live_server):
     time.sleep(1)
     res = client.post(
         url_for("edit_page", uuid="first"),
-        data={"css_filter": filter, "url": test_url, "tag": "", "headers": "", "external_header_server": "",'fetch_backend': "html_requests"},
+        data={"css_filter": filter, "url": test_url, "tag": "", "headers": "", 'fetch_backend': "html_requests"},
         follow_redirects=True
     )
     assert b"Updated watch." in res.data
@@ -143,7 +143,7 @@ def test_check_xpath_text_function_utf8(client, live_server):
     time.sleep(1)
     res = client.post(
         url_for("edit_page", uuid="first"),
-        data={"css_filter": filter, "url": test_url, "tag": "", "headers": "", "external_header_server": "",'fetch_backend': "html_requests"},
+        data={"css_filter": filter, "url": test_url, "tag": "", "headers": "", 'fetch_backend': "html_requests"},
         follow_redirects=True
     )
     assert b"Updated watch." in res.data
@@ -192,7 +192,7 @@ def test_check_markup_xpath_filter_restriction(client, live_server):
     # Add our URL to the import page
     res = client.post(
         url_for("edit_page", uuid="first"),
-        data={"css_filter": xpath_filter, "url": test_url, "tag": "", "headers": "", "external_header_server": "",'fetch_backend': "html_requests"},
+        data={"css_filter": xpath_filter, "url": test_url, "tag": "", "headers": "", 'fetch_backend': "html_requests"},
         follow_redirects=True
     )
     assert b"Updated watch." in res.data
@@ -233,7 +233,7 @@ def test_xpath_validation(client, live_server):
 
     res = client.post(
         url_for("edit_page", uuid="first"),
-        data={"css_filter": "/something horrible", "url": test_url, "tag": "", "headers": "", "external_header_server": "",'fetch_backend': "html_requests"},
+        data={"css_filter": "/something horrible", "url": test_url, "tag": "", "headers": "", 'fetch_backend': "html_requests"},
         follow_redirects=True
     )
     assert b"is not a valid XPath expression" in res.data
@@ -263,7 +263,7 @@ def test_check_with_prefix_css_filter(client, live_server):
 
     res = client.post(
         url_for("edit_page", uuid="first"),
-        data={"css_filter":  "xpath://*[contains(@class, 'sametext')]", "url": test_url, "tag": "", "headers": "", "external_header_server": "",'fetch_backend': "html_requests"},
+        data={"css_filter":  "xpath://*[contains(@class, 'sametext')]", "url": test_url, "tag": "", "headers": "", 'fetch_backend': "html_requests"},
         follow_redirects=True
     )