Simplify scrub operation (simply cleans all) (#575)

pull/584/head
dgtlmoon 3 years ago committed by GitHub
parent 613308881c
commit 1d4474f5a3
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23

@ -434,42 +434,15 @@ def changedetection_app(config=None, datastore_o=None):
@login_required @login_required
def scrub_page(): def scrub_page():
import re
if request.method == 'POST': if request.method == 'POST':
confirmtext = request.form.get('confirmtext') confirmtext = request.form.get('confirmtext')
limit_date = request.form.get('limit_date')
limit_timestamp = 0
# Re #149 - allow empty/0 timestamp limit
if len(limit_date):
try:
limit_date = limit_date.replace('T', ' ')
# I noticed chrome will show '/' but actually submit '-'
limit_date = limit_date.replace('-', '/')
# In the case that :ss seconds are supplied
limit_date = re.sub(r'(\d\d:\d\d)(:\d\d)', '\\1', limit_date)
str_to_dt = datetime.datetime.strptime(limit_date, '%Y/%m/%d %H:%M')
limit_timestamp = int(str_to_dt.timestamp())
if limit_timestamp > time.time():
flash("Timestamp is in the future, cannot continue.", 'error')
return redirect(url_for('scrub_page'))
except ValueError:
flash('Incorrect date format, cannot continue.', 'error')
return redirect(url_for('scrub_page'))
if confirmtext == 'scrub': if confirmtext == 'scrub':
changes_removed = 0 changes_removed = 0
for uuid, watch in datastore.data['watching'].items(): for uuid in datastore.data['watching'].keys():
if limit_timestamp: datastore.scrub_watch(uuid)
changes_removed += datastore.scrub_watch(uuid, limit_timestamp=limit_timestamp)
else:
changes_removed += datastore.scrub_watch(uuid)
flash("Cleared snapshot history ({} snapshots removed)".format(changes_removed)) flash("Cleared all snapshot history")
else: else:
flash('Incorrect confirmation text.', 'error') flash('Incorrect confirmation text.', 'error')

@ -260,46 +260,14 @@ class ChangeDetectionStore:
return self.data['watching'][uuid].get(val) return self.data['watching'][uuid].get(val)
# Remove a watchs data but keep the entry (URL etc) # Remove a watchs data but keep the entry (URL etc)
def scrub_watch(self, uuid, limit_timestamp = False): def scrub_watch(self, uuid):
import pathlib
import hashlib
del_timestamps = []
changes_removed = 0
for timestamp, path in self.data['watching'][uuid]['history'].items(): self.__data['watching'][uuid].update({'history': {}, 'last_checked': 0, 'last_changed': 0, 'newest_history_key': 0, 'previous_md5': False})
if not limit_timestamp or (limit_timestamp is not False and int(timestamp) > limit_timestamp): self.needs_write_urgent = True
self.unlink_history_file(path)
del_timestamps.append(timestamp)
changes_removed += 1
if not limit_timestamp:
self.data['watching'][uuid]['last_checked'] = 0
self.data['watching'][uuid]['last_changed'] = 0
self.data['watching'][uuid]['previous_md5'] = ""
for timestamp in del_timestamps:
del self.data['watching'][uuid]['history'][str(timestamp)]
# If there was a limitstamp, we need to reset some meta data about the entry
# This has to happen after we remove the others from the list
if limit_timestamp:
newest_key = self.get_newest_history_key(uuid)
if newest_key:
self.data['watching'][uuid]['last_checked'] = int(newest_key)
# @todo should be the original value if it was less than newest key
self.data['watching'][uuid]['last_changed'] = int(newest_key)
try:
with open(self.data['watching'][uuid]['history'][str(newest_key)], "rb") as fp:
content = fp.read()
self.data['watching'][uuid]['previous_md5'] = hashlib.md5(content).hexdigest()
except (FileNotFoundError, IOError):
self.data['watching'][uuid]['previous_md5'] = ""
pass
self.needs_write = True for item in pathlib.Path(self.datastore_path).rglob(uuid+"/*.txt"):
return changes_removed unlink(item)
def add_watch(self, url, tag="", extras=None, write_to_disk_now=True): def add_watch(self, url, tag="", extras=None, write_to_disk_now=True):
if extras is None: if extras is None:
@ -453,7 +421,8 @@ class ChangeDetectionStore:
import pathlib import pathlib
# Only in the sub-directories # Only in the sub-directories
for item in pathlib.Path(self.datastore_path).rglob("*/*txt"): for uuid in self.data['watching']:
for item in pathlib.Path(self.datastore_path).rglob(uuid+"/*.txt"):
if not str(item) in index: if not str(item) in index:
print ("Removing",item) print ("Removing",item)
unlink(item) unlink(item)

@ -7,7 +7,7 @@
<input type="hidden" name="csrf_token" value="{{ csrf_token() }}"/> <input type="hidden" name="csrf_token" value="{{ csrf_token() }}"/>
<fieldset> <fieldset>
<div class="pure-control-group"> <div class="pure-control-group">
This will remove all version snapshots/data, but keep your list of URLs. <br/> This will remove ALL version snapshots/data, but keep your list of URLs. <br/>
You may like to use the <strong>BACKUP</strong> link first.<br/> You may like to use the <strong>BACKUP</strong> link first.<br/>
</div> </div>
<br/> <br/>
@ -17,12 +17,6 @@
<span class="pure-form-message-inline">Type in the word <strong>scrub</strong> to confirm that you understand!</span> <span class="pure-form-message-inline">Type in the word <strong>scrub</strong> to confirm that you understand!</span>
</div> </div>
<br/> <br/>
<div class="pure-control-group">
<label for="confirmtext">Optional: Limit deletion of snapshots to snapshots <i>newer</i> than date/time</label>
<input type="datetime-local" id="limit_date" name="limit_date" />
<span class="pure-form-message-inline">dd/mm/yyyy hh:mm (24 hour format)</span>
</div>
<br/>
<div class="pure-control-group"> <div class="pure-control-group">
<button type="submit" class="pure-button pure-button-primary">Scrub!</button> <button type="submit" class="pure-button pure-button-primary">Scrub!</button>
</div> </div>

Loading…
Cancel
Save