From 934d8c6211fbefd995ba7a806b641811805f45b4 Mon Sep 17 00:00:00 2001 From: dgtlmoon Date: Thu, 1 Apr 2021 12:01:42 +0200 Subject: [PATCH] Re #30 - Delete history watch snapshots (#31) Re #30 - Delete history watch snapshots Scrub - Optionally delete history snapshots newer than timestamp --- backend/__init__.py | 13 +++----- backend/store.py | 59 ++++++++++++++++++++++++++++++++- backend/templates/scrub.html | 11 ++++-- backend/templates/settings.html | 2 +- 4 files changed, 72 insertions(+), 13 deletions(-) diff --git a/backend/__init__.py b/backend/__init__.py index 8293a2ba..72368e41 100644 --- a/backend/__init__.py +++ b/backend/__init__.py @@ -174,19 +174,16 @@ def changedetection_app(config=None, datastore_o=None): if request.method == 'POST': confirmtext = request.form.get('confirmtext') + limit_timestamp = int(request.form.get('limit_date')) if confirmtext == 'scrub': - for txt_file_path in Path(app.config['datastore_path']).rglob('*.txt'): - os.unlink(txt_file_path) - for uuid, watch in datastore.data['watching'].items(): - watch['last_checked'] = 0 - watch['last_changed'] = 0 - watch['previous_md5'] = None - watch['history'] = {} + if len(str(limit_timestamp)) == 10: + datastore.scrub_watch(uuid, limit_timestamp = limit_timestamp) + else: + datastore.scrub_watch(uuid) - datastore.needs_write = True messages.append({'class': 'ok', 'message': 'Cleaned all version history.'}) else: messages.append({'class': 'error', 'message': 'Wrong confirm text.'}) diff --git a/backend/store.py b/backend/store.py index 9f1d59a9..e03f9ef1 100644 --- a/backend/store.py +++ b/backend/store.py @@ -183,12 +183,28 @@ class ChangeDetectionStore: tags.sort() return tags + def unlink_history_file(self, path): + try: + os.unlink(path) + except (FileNotFoundError, IOError): + pass + + # Delete a single watch by UUID def delete(self, uuid): with self.lock: if uuid == 'all': self.__data['watching'] = {} + + # GitHub #30 also delete history records + for uuid in self.data['watching']: + for path in self.data['watching'][uuid]['history'].values(): + self.unlink_history_file(path) + else: - del (self.__data['watching'][uuid]) + for path in self.data['watching'][uuid]['history'].values(): + self.unlink_history_file(path) + + del self.data['watching'][uuid] self.needs_write = True @@ -205,6 +221,47 @@ class ChangeDetectionStore: # Probably their should be dict... return self.data['watching'][uuid].get(val) + # Remove a watchs data but keep the entry (URL etc) + def scrub_watch(self, uuid, limit_timestamp = False): + + import hashlib + del_timestamps = [] + + for timestamp, path in self.data['watching'][uuid]['history'].items(): + if not limit_timestamp or (limit_timestamp is not False and int(timestamp) > limit_timestamp): + self.unlink_history_file(path) + del_timestamps.append(timestamp) + + + + if not limit_timestamp: + self.data['watching'][uuid]['last_checked'] = 0 + self.data['watching'][uuid]['last_changed'] = 0 + self.data['watching'][uuid]['previous_md5'] = 0 + + for timestamp in del_timestamps: + del self.data['watching'][uuid]['history'][str(timestamp)] + + # If there was a limitstamp, we need to reset some meta data about the entry + # This has to happen after we remove the others from the list + if limit_timestamp: + newest_key = self.get_newest_history_key(uuid) + if newest_key: + self.data['watching'][uuid]['last_checked'] = int(newest_key) + # @todo should be the original value if it was less than newest key + self.data['watching'][uuid]['last_changed'] = int(newest_key) + try: + with open(self.data['watching'][uuid]['history'][str(newest_key)], "rb") as fp: + content = fp.read() + self.data['watching'][uuid]['previous_md5'] = hashlib.md5(content).hexdigest() + except (FileNotFoundError, IOError): + self.data['watching'][uuid]['previous_md5'] = False + pass + + + self.needs_write = True + + def add_watch(self, url, tag): with self.lock: # @todo use a common generic version of this diff --git a/backend/templates/scrub.html b/backend/templates/scrub.html index 12a140ba..3293ebfa 100644 --- a/backend/templates/scrub.html +++ b/backend/templates/scrub.html @@ -17,14 +17,19 @@

-
+
-
+
- +
+
+
+ +
+
diff --git a/backend/templates/settings.html b/backend/templates/settings.html index 8e1e6967..108edbd1 100644 --- a/backend/templates/settings.html +++ b/backend/templates/settings.html @@ -22,7 +22,7 @@
Back - Reset all version data + Delete history version data