Merge branch 'master' into hours-day-schedule

README.md

@@ -1,6 +1,6 @@
 ## Web Site Change Detection, Monitoring and Notification.
 
-_Live your data-life pro-actively, track website and JSON content changes, trigger notifications via Discord, Email, Slack, Telegram, API calls and 70+ more._
+_Live your data-life pro-actively, Detect website changes and perform meaningful actions, trigger notifications via Discord, Email, Slack, Telegram, API calls and many more._
 
 
 [<img src="https://raw.githubusercontent.com/dgtlmoon/changedetection.io/master/docs/screenshot.png" style="max-width:100%;" alt="Self-hosted web page change monitoring"  title="Self-hosted web page change monitoring"  />](https://lemonade.changedetection.io/start?src=github)

changedetectionio/changedetection.py

@@ -102,8 +102,8 @@ def main():
                     has_password=datastore.data['settings']['application']['password'] != False
                     )
 
-    # Monitored websites will not receive a Referer header
+    # Monitored websites will not receive a Referer header when a user clicks on an outgoing link.
-    # when a user clicks on an outgoing link.
+    # @Note: Incompatible with password login (and maybe other features) for now, submit a PR!
     @app.after_request
     def hide_referrer(response):
         if os.getenv("HIDE_REFERER", False):

changedetectionio/model/Watch.py

@@ -16,43 +16,44 @@ class model(dict):
     __newest_history_key = None
     __history_n=0
     __base_config = {
-            'url': None,
+            #'history': {},  # Dict of timestamp and output stripped filename (removed)
-            'tag': None,
+            #'newest_history_key': 0, (removed, taken from history.txt index)
+            'body': None,
+            'check_unique_lines': False, # On change-detected, compare against all history if its something new
+            'check_count': 0,
+            'consecutive_filter_failures': 0, # Every time the CSS/xPath filter cannot be located, reset when all is fine.
+            'extract_text': [],  # Extract text by regex after filters
+            'extract_title_as_title': False,
+            'fetch_backend': None,
+            'filter_failure_notification_send': strtobool(os.getenv('FILTER_FAILURE_NOTIFICATION_SEND_DEFAULT', 'True')),
+            'headers': {},  # Extra headers to send
+            'ignore_text': [],  # List of text to ignore when calculating the comparison checksum
+            'include_filters': [],
             'last_checked': 0,
-            'paused': False,
+            'last_error': False,
             'last_viewed': 0,  # history key value of the last viewed via the [diff] link
-            #'newest_history_key': 0,
-            'title': None,
-            'previous_md5': False,
-            'uuid': str(uuid.uuid4()),
-            'headers': {},  # Extra headers to send
-            'body': None,
             'method': 'GET',
-            #'history': {},  # Dict of timestamp and output stripped filename
+             # Custom notification content
-            'ignore_text': [],  # List of text to ignore when calculating the comparison checksum
-            # Custom notification content
-            'notification_urls': [],  # List of URLs to add to the notification Queue (Usually AppRise)
-            'notification_title': None,
             'notification_body': None,
             'notification_format': default_notification_format_for_watch,
             'notification_muted': False,
-            'include_filters': [],
+            'notification_title': None,
-            'last_error': False,
+            'notification_urls': [],  # List of URLs to add to the notification Queue (Usually AppRise)
-            'extract_text': [],  # Extract text by regex after filters
+            'paused': False,
+            'previous_md5': False,
+            'proxy': None, # Preferred proxy connection
             'subtractive_selectors': [],
-            'trigger_text': [],  # List of text or regex to wait for until a change is detected
+            'tag': None,
             'text_should_not_be_present': [], # Text that should not present
-            'fetch_backend': None,
-            'filter_failure_notification_send': strtobool(os.getenv('FILTER_FAILURE_NOTIFICATION_SEND_DEFAULT', 'True')),
-            'consecutive_filter_failures': 0, # Every time the CSS/xPath filter cannot be located, reset when all is fine.
-            'extract_title_as_title': False,
-            'check_unique_lines': False, # On change-detected, compare against all history if its something new
-            'proxy': None, # Preferred proxy connection
             # Re #110, so then if this is set to None, we know to use the default value instead
             # Requires setting to None on submit if it's the same as the default
             # Should be all None by default, so we use the system default in this case.
             'time_between_check': {'weeks': None, 'days': None, 'hours': None, 'minutes': None, 'seconds': None},
             'time_schedule_check_limit': {'day_of_week': [0, 1, 2, 3, 4, 5, 6], 'time_from': '', 'time_until': ''},
+            'title': None,
+            'trigger_text': [],  # List of text or regex to wait for until a change is detected
+            'url': None,
+            'uuid': str(uuid.uuid4()),
             'webdriver_delay': None,
             'webdriver_js_execute_code': None, # Run before change-detection
         }

changedetectionio/update_worker.py

@@ -282,10 +282,12 @@ class update_worker(threading.Thread):
                             self.app.logger.error("Exception reached processing watch UUID: %s - %s", uuid, str(e))
                             self.datastore.update_watch(uuid=uuid, update_obj={'last_error': str(e)})
 
-
                     # Always record that we atleast tried
+                    count = self.datastore.data['watching'][uuid].get('check_count', 0) + 1
                     self.datastore.update_watch(uuid=uuid, update_obj={'fetch_time': round(time.time() - now, 3),
-                                                                       'last_checked': round(time.time())})
+                                                                       'last_checked': round(time.time()),
+                                                                       'check_count': count
+                                                                       })
 
                     # Always save the screenshot if it's available
                     if update_handler.screenshot: