diff --git a/backend/backend.py b/backend/backend.py index 3b69c828..37a1b80f 100644 --- a/backend/backend.py +++ b/backend/backend.py @@ -208,8 +208,6 @@ def api_update(): 'tag': tag, 'headers':extra_headers}) - #@todo switch to prop/attr/observer - datastore.sync_to_json() messages.append({'class': 'ok', 'message': 'Updated watch.'}) @@ -262,27 +260,40 @@ def ticker_thread_check_time_launch_checks(): launch_checks() time.sleep(60) +# Thread runner, this helps with thread/write issues when there are many operations that want to update the JSON +# by just running periodically in one thread. +def save_datastore(): + while True: + if datastore.needs_write: + datastore.sync_to_json() + time.sleep(5) def main(argv): ssl_mode = False port = 5000 - # @todo handle ctrl break - ticker_thread = threading.Thread(target=ticker_thread_check_time_launch_checks).start() - try: - opts, args = getopt.getopt(argv, "sp:") + opts, args = getopt.getopt(argv, "sp:", "purge") except getopt.GetoptError: print('backend.py -s SSL enable -p [port]') sys.exit(2) for opt, arg in opts: + if opt == '--purge': + # Remove history, the actual files you need to delete manually. + for uuid, watch in datastore.data['watching'].items(): + watch.update({'history': {}, 'last_checked': 0, 'last_changed': 0, 'previous_md5': None}) + if opt == '-s': ssl_mode = True if opt == '-p': port = arg + # @todo handle ctrl break + ticker_thread = threading.Thread(target=ticker_thread_check_time_launch_checks).start() + save_data_thread = threading.Thread(target=save_datastore).start() + # @todo finalise SSL config, but this should get you in the right direction if you need it. if ssl_mode: eventlet.wsgi.server(eventlet.wrap_ssl(eventlet.listen(('', port)), diff --git a/backend/fetch_site_status.py b/backend/fetch_site_status.py index 84546d95..cac80f4f 100644 --- a/backend/fetch_site_status.py +++ b/backend/fetch_site_status.py @@ -54,7 +54,7 @@ class perform_site_check(Thread): request_headers.update(extra_headers) print("Checking", self.url) - print(request_headers) + #print(request_headers) self.ensure_output_path() diff --git a/backend/store.py b/backend/store.py index 27c2f1c4..5a0b787b 100644 --- a/backend/store.py +++ b/backend/store.py @@ -9,7 +9,9 @@ import validators class ChangeDetectionStore: def __init__(self): - self.data = { + self.needs_write = False + + self.__data = { 'watching': {}, 'settings': { 'headers': { @@ -33,6 +35,7 @@ class ChangeDetectionStore: 'last_checked': 0, 'last_changed': 0, 'title': None, + 'previous_md5': None, 'uuid': str(uuid_builder.uuid4()), 'headers' : {}, # Extra headers to send 'history' : {} # Dict of timestamp and output stripped filename @@ -43,7 +46,7 @@ class ChangeDetectionStore: with open('/datastore/url-watches.json') as json_file: from_disk = json.load(json_file) - self.data.update(from_disk) + self.__data.update(from_disk) # Reinitialise each `watching` with our generic_definition in the case that we add a new var in the future. # @todo pretty sure theres a python we todo this with an abstracted(?) object! @@ -51,7 +54,7 @@ class ChangeDetectionStore: for uuid, watch in self.data['watching'].items(): _blank = self.generic_definition.copy() _blank.update(watch) - self.data['watching'].update({uuid: _blank}) + self.__data['watching'].update({uuid: _blank}) print("Watching:", uuid, _blank['url']) # First time ran, doesnt exist. @@ -61,10 +64,17 @@ class ChangeDetectionStore: self.add_watch(url='https://changedetection.io', tag='general') self.add_watch(url='http://www.quotationspage.com/random.php', tag='test') + +# self.entryVariable.get() def update_watch(self, uuid, val, var): - self.data['watching'][uuid].update({val: var}) - self.sync_to_json() + self.__data['watching'][uuid].update({val: var}) + self.needs_write = True + + + @property + def data(self): + return self.__data def get_all_tags(self): tags=[] @@ -81,8 +91,8 @@ class ChangeDetectionStore: def delete(self, uuid): # Probably their should be dict... - del(self.data['watching'][uuid]) - self.sync_to_json() + del(self.__data['watching'][uuid]) + self.needs_write = True def url_exists(self, url): @@ -114,10 +124,11 @@ class ChangeDetectionStore: self.data['watching'].update({_blank['uuid']: _blank}) - self.sync_to_json() def sync_to_json(self): + print ("Saving index....") with open('/datastore/url-watches.json', 'w') as json_file: json.dump(self.data, json_file, indent=4) + self.needs_write = False # body of the constructor