Use a single thread for writing the sync json

backend/backend.py

@@ -208,8 +208,6 @@ def api_update():
                                              'tag': tag,
                                              'headers':extra_headers})
 
-    #@todo switch to prop/attr/observer
-    datastore.sync_to_json()
 
     messages.append({'class': 'ok', 'message': 'Updated watch.'})
 
@@ -262,27 +260,40 @@ def ticker_thread_check_time_launch_checks():
         launch_checks()
         time.sleep(60)
 
+# Thread runner, this helps with thread/write issues when there are many operations that want to update the JSON
+# by just running periodically in one thread.
+def save_datastore():
+    while True:
+        if datastore.needs_write:
+            datastore.sync_to_json()
+        time.sleep(5)
 
 def main(argv):
     ssl_mode = False
     port = 5000
 
-    # @todo handle ctrl break
-    ticker_thread = threading.Thread(target=ticker_thread_check_time_launch_checks).start()
-
     try:
-        opts, args = getopt.getopt(argv, "sp:")
+        opts, args = getopt.getopt(argv, "sp:", "purge")
     except getopt.GetoptError:
         print('backend.py -s SSL enable -p [port]')
         sys.exit(2)
 
     for opt, arg in opts:
+        if opt == '--purge':
+            # Remove history, the actual files you need to delete manually.
+            for uuid, watch in datastore.data['watching'].items():
+                watch.update({'history': {}, 'last_checked': 0, 'last_changed': 0, 'previous_md5': None})
+
         if opt == '-s':
             ssl_mode = True
 
         if opt == '-p':
             port = arg
 
+    # @todo handle ctrl break
+    ticker_thread = threading.Thread(target=ticker_thread_check_time_launch_checks).start()
+    save_data_thread = threading.Thread(target=save_datastore).start()
+
     # @todo finalise SSL config, but this should get you in the right direction if you need it.
     if ssl_mode:
         eventlet.wsgi.server(eventlet.wrap_ssl(eventlet.listen(('', port)),

backend/fetch_site_status.py

@@ -54,7 +54,7 @@ class perform_site_check(Thread):
         request_headers.update(extra_headers)
 
         print("Checking", self.url)
-        print(request_headers)
+        #print(request_headers)
 
         self.ensure_output_path()
 

backend/store.py

@@ -9,7 +9,9 @@ import validators
 class ChangeDetectionStore:
 
     def __init__(self):
-        self.data = {
+        self.needs_write = False
+
+        self.__data = {
             'watching': {},
             'settings': {
                 'headers': {
@@ -33,6 +35,7 @@ class ChangeDetectionStore:
             'last_checked': 0,
             'last_changed': 0,
             'title': None,
+            'previous_md5': None,
             'uuid': str(uuid_builder.uuid4()),
             'headers' : {}, # Extra headers to send
             'history' : {} # Dict of timestamp and output stripped filename
@@ -43,7 +46,7 @@ class ChangeDetectionStore:
             with open('/datastore/url-watches.json') as json_file:
                 from_disk = json.load(json_file)
 
-                self.data.update(from_disk)
+                self.__data.update(from_disk)
 
                 # Reinitialise each `watching` with our generic_definition in the case that we add a new var in the future.
                 # @todo pretty sure theres a python we todo this with an abstracted(?) object!
@@ -51,7 +54,7 @@ class ChangeDetectionStore:
                 for uuid, watch in self.data['watching'].items():
                     _blank = self.generic_definition.copy()
                     _blank.update(watch)
-                    self.data['watching'].update({uuid: _blank})
+                    self.__data['watching'].update({uuid: _blank})
                     print("Watching:", uuid, _blank['url'])
 
         # First time ran, doesnt exist.
@@ -61,10 +64,17 @@ class ChangeDetectionStore:
             self.add_watch(url='https://changedetection.io', tag='general')
             self.add_watch(url='http://www.quotationspage.com/random.php', tag='test')
 
+
+#        self.entryVariable.get()
     def update_watch(self, uuid, val, var):
 
-        self.data['watching'][uuid].update({val: var})
-        self.sync_to_json()
+        self.__data['watching'][uuid].update({val: var})
+        self.needs_write = True
+
+
+    @property
+    def data(self):
+        return self.__data
 
     def get_all_tags(self):
         tags=[]
@@ -81,8 +91,8 @@ class ChangeDetectionStore:
 
     def delete(self, uuid):
         # Probably their should be dict...
-        del(self.data['watching'][uuid])
-        self.sync_to_json()
+        del(self.__data['watching'][uuid])
+        self.needs_write = True
 
 
     def url_exists(self, url):
@@ -114,10 +124,11 @@ class ChangeDetectionStore:
 
         self.data['watching'].update({_blank['uuid']: _blank})
 
-        self.sync_to_json()
 
     def sync_to_json(self):
+        print ("Saving index....")
         with open('/datastore/url-watches.json', 'w') as json_file:
             json.dump(self.data, json_file, indent=4)
+        self.needs_write = False
 
 # body of the constructor