|
|
@ -10,6 +10,7 @@ import threading
|
|
|
|
import time
|
|
|
|
import time
|
|
|
|
import timeago
|
|
|
|
import timeago
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
from changedetectionio import queuedWatchMetaData
|
|
|
|
from copy import deepcopy
|
|
|
|
from copy import deepcopy
|
|
|
|
from distutils.util import strtobool
|
|
|
|
from distutils.util import strtobool
|
|
|
|
from feedgen.feed import FeedGenerator
|
|
|
|
from feedgen.feed import FeedGenerator
|
|
|
@ -404,7 +405,6 @@ def changedetection_app(config=None, datastore_o=None):
|
|
|
|
sorted_watches.append(watch)
|
|
|
|
sorted_watches.append(watch)
|
|
|
|
|
|
|
|
|
|
|
|
existing_tags = datastore.get_all_tags()
|
|
|
|
existing_tags = datastore.get_all_tags()
|
|
|
|
|
|
|
|
|
|
|
|
form = forms.quickWatchForm(request.form)
|
|
|
|
form = forms.quickWatchForm(request.form)
|
|
|
|
output = render_template("watch-overview.html",
|
|
|
|
output = render_template("watch-overview.html",
|
|
|
|
form=form,
|
|
|
|
form=form,
|
|
|
@ -416,7 +416,7 @@ def changedetection_app(config=None, datastore_o=None):
|
|
|
|
# Don't link to hosting when we're on the hosting environment
|
|
|
|
# Don't link to hosting when we're on the hosting environment
|
|
|
|
hosted_sticky=os.getenv("SALTED_PASS", False) == False,
|
|
|
|
hosted_sticky=os.getenv("SALTED_PASS", False) == False,
|
|
|
|
guid=datastore.data['app_guid'],
|
|
|
|
guid=datastore.data['app_guid'],
|
|
|
|
queued_uuids=[uuid for p,uuid in update_q.queue])
|
|
|
|
queued_uuids=[q_uuid.item['uuid'] for q_uuid in update_q.queue])
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
if session.get('share-link'):
|
|
|
|
if session.get('share-link'):
|
|
|
@ -596,25 +596,16 @@ def changedetection_app(config=None, datastore_o=None):
|
|
|
|
using_default_check_time = False
|
|
|
|
using_default_check_time = False
|
|
|
|
break
|
|
|
|
break
|
|
|
|
|
|
|
|
|
|
|
|
# Use the default if its the same as system wide
|
|
|
|
# Use the default if it's the same as system-wide.
|
|
|
|
if form.fetch_backend.data == datastore.data['settings']['application']['fetch_backend']:
|
|
|
|
if form.fetch_backend.data == datastore.data['settings']['application']['fetch_backend']:
|
|
|
|
extra_update_obj['fetch_backend'] = None
|
|
|
|
extra_update_obj['fetch_backend'] = None
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
# Ignore text
|
|
|
|
# Ignore text
|
|
|
|
form_ignore_text = form.ignore_text.data
|
|
|
|
form_ignore_text = form.ignore_text.data
|
|
|
|
datastore.data['watching'][uuid]['ignore_text'] = form_ignore_text
|
|
|
|
datastore.data['watching'][uuid]['ignore_text'] = form_ignore_text
|
|
|
|
|
|
|
|
|
|
|
|
# Reset the previous_md5 so we process a new snapshot including stripping ignore text.
|
|
|
|
|
|
|
|
if form_ignore_text:
|
|
|
|
|
|
|
|
if len(datastore.data['watching'][uuid].history):
|
|
|
|
|
|
|
|
extra_update_obj['previous_md5'] = get_current_checksum_include_ignore_text(uuid=uuid)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
# Reset the previous_md5 so we process a new snapshot including stripping ignore text.
|
|
|
|
|
|
|
|
if form.include_filters.data != datastore.data['watching'][uuid].get('include_filters', []):
|
|
|
|
|
|
|
|
if len(datastore.data['watching'][uuid].history):
|
|
|
|
|
|
|
|
extra_update_obj['previous_md5'] = get_current_checksum_include_ignore_text(uuid=uuid)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
# Be sure proxy value is None
|
|
|
|
# Be sure proxy value is None
|
|
|
|
if datastore.proxy_list is not None and form.data['proxy'] == '':
|
|
|
|
if datastore.proxy_list is not None and form.data['proxy'] == '':
|
|
|
|
extra_update_obj['proxy'] = None
|
|
|
|
extra_update_obj['proxy'] = None
|
|
|
@ -632,7 +623,7 @@ def changedetection_app(config=None, datastore_o=None):
|
|
|
|
datastore.needs_write_urgent = True
|
|
|
|
datastore.needs_write_urgent = True
|
|
|
|
|
|
|
|
|
|
|
|
# Queue the watch for immediate recheck, with a higher priority
|
|
|
|
# Queue the watch for immediate recheck, with a higher priority
|
|
|
|
update_q.put((1, uuid))
|
|
|
|
update_q.put(queuedWatchMetaData.PrioritizedItem(priority=1, item={'uuid': uuid, 'skip_when_checksum_same': False}))
|
|
|
|
|
|
|
|
|
|
|
|
# Diff page [edit] link should go back to diff page
|
|
|
|
# Diff page [edit] link should go back to diff page
|
|
|
|
if request.args.get("next") and request.args.get("next") == 'diff':
|
|
|
|
if request.args.get("next") and request.args.get("next") == 'diff':
|
|
|
@ -773,7 +764,7 @@ def changedetection_app(config=None, datastore_o=None):
|
|
|
|
importer = import_url_list()
|
|
|
|
importer = import_url_list()
|
|
|
|
importer.run(data=request.values.get('urls'), flash=flash, datastore=datastore)
|
|
|
|
importer.run(data=request.values.get('urls'), flash=flash, datastore=datastore)
|
|
|
|
for uuid in importer.new_uuids:
|
|
|
|
for uuid in importer.new_uuids:
|
|
|
|
update_q.put((1, uuid))
|
|
|
|
update_q.put(queuedWatchMetaData.PrioritizedItem(priority=1, item={'uuid': uuid, 'skip_when_checksum_same': True}))
|
|
|
|
|
|
|
|
|
|
|
|
if len(importer.remaining_data) == 0:
|
|
|
|
if len(importer.remaining_data) == 0:
|
|
|
|
return redirect(url_for('index'))
|
|
|
|
return redirect(url_for('index'))
|
|
|
@ -786,7 +777,7 @@ def changedetection_app(config=None, datastore_o=None):
|
|
|
|
d_importer = import_distill_io_json()
|
|
|
|
d_importer = import_distill_io_json()
|
|
|
|
d_importer.run(data=request.values.get('distill-io'), flash=flash, datastore=datastore)
|
|
|
|
d_importer.run(data=request.values.get('distill-io'), flash=flash, datastore=datastore)
|
|
|
|
for uuid in d_importer.new_uuids:
|
|
|
|
for uuid in d_importer.new_uuids:
|
|
|
|
update_q.put((1, uuid))
|
|
|
|
update_q.put(queuedWatchMetaData.PrioritizedItem(priority=1, item={'uuid': uuid, 'skip_when_checksum_same': True}))
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
@ -1151,7 +1142,7 @@ def changedetection_app(config=None, datastore_o=None):
|
|
|
|
|
|
|
|
|
|
|
|
if not add_paused and new_uuid:
|
|
|
|
if not add_paused and new_uuid:
|
|
|
|
# Straight into the queue.
|
|
|
|
# Straight into the queue.
|
|
|
|
update_q.put((1, new_uuid))
|
|
|
|
update_q.put(queuedWatchMetaData.PrioritizedItem(priority=1, item={'uuid': new_uuid}))
|
|
|
|
flash("Watch added.")
|
|
|
|
flash("Watch added.")
|
|
|
|
|
|
|
|
|
|
|
|
if add_paused:
|
|
|
|
if add_paused:
|
|
|
@ -1188,7 +1179,7 @@ def changedetection_app(config=None, datastore_o=None):
|
|
|
|
uuid = list(datastore.data['watching'].keys()).pop()
|
|
|
|
uuid = list(datastore.data['watching'].keys()).pop()
|
|
|
|
|
|
|
|
|
|
|
|
new_uuid = datastore.clone(uuid)
|
|
|
|
new_uuid = datastore.clone(uuid)
|
|
|
|
update_q.put((5, new_uuid))
|
|
|
|
update_q.put(queuedWatchMetaData.PrioritizedItem(priority=5, item={'uuid': new_uuid, 'skip_when_checksum_same': True}))
|
|
|
|
flash('Cloned.')
|
|
|
|
flash('Cloned.')
|
|
|
|
|
|
|
|
|
|
|
|
return redirect(url_for('index'))
|
|
|
|
return redirect(url_for('index'))
|
|
|
@ -1196,7 +1187,7 @@ def changedetection_app(config=None, datastore_o=None):
|
|
|
|
@app.route("/api/checknow", methods=['GET'])
|
|
|
|
@app.route("/api/checknow", methods=['GET'])
|
|
|
|
@login_required
|
|
|
|
@login_required
|
|
|
|
def form_watch_checknow():
|
|
|
|
def form_watch_checknow():
|
|
|
|
|
|
|
|
# Forced recheck will skip the 'skip if content is the same' rule (, 'reprocess_existing_data': True})))
|
|
|
|
tag = request.args.get('tag')
|
|
|
|
tag = request.args.get('tag')
|
|
|
|
uuid = request.args.get('uuid')
|
|
|
|
uuid = request.args.get('uuid')
|
|
|
|
i = 0
|
|
|
|
i = 0
|
|
|
@ -1205,11 +1196,9 @@ def changedetection_app(config=None, datastore_o=None):
|
|
|
|
for t in running_update_threads:
|
|
|
|
for t in running_update_threads:
|
|
|
|
running_uuids.append(t.current_uuid)
|
|
|
|
running_uuids.append(t.current_uuid)
|
|
|
|
|
|
|
|
|
|
|
|
# @todo check thread is running and skip
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
if uuid:
|
|
|
|
if uuid:
|
|
|
|
if uuid not in running_uuids:
|
|
|
|
if uuid not in running_uuids:
|
|
|
|
update_q.put((1, uuid))
|
|
|
|
update_q.put(queuedWatchMetaData.PrioritizedItem(priority=1, item={'uuid': uuid, 'skip_when_checksum_same': False}))
|
|
|
|
i = 1
|
|
|
|
i = 1
|
|
|
|
|
|
|
|
|
|
|
|
elif tag != None:
|
|
|
|
elif tag != None:
|
|
|
@ -1217,14 +1206,14 @@ def changedetection_app(config=None, datastore_o=None):
|
|
|
|
for watch_uuid, watch in datastore.data['watching'].items():
|
|
|
|
for watch_uuid, watch in datastore.data['watching'].items():
|
|
|
|
if (tag != None and tag in watch['tag']):
|
|
|
|
if (tag != None and tag in watch['tag']):
|
|
|
|
if watch_uuid not in running_uuids and not datastore.data['watching'][watch_uuid]['paused']:
|
|
|
|
if watch_uuid not in running_uuids and not datastore.data['watching'][watch_uuid]['paused']:
|
|
|
|
update_q.put((1, watch_uuid))
|
|
|
|
update_q.put(queuedWatchMetaData.PrioritizedItem(priority=1, item={'uuid': watch_uuid, 'skip_when_checksum_same': False}))
|
|
|
|
i += 1
|
|
|
|
i += 1
|
|
|
|
|
|
|
|
|
|
|
|
else:
|
|
|
|
else:
|
|
|
|
# No tag, no uuid, add everything.
|
|
|
|
# No tag, no uuid, add everything.
|
|
|
|
for watch_uuid, watch in datastore.data['watching'].items():
|
|
|
|
for watch_uuid, watch in datastore.data['watching'].items():
|
|
|
|
if watch_uuid not in running_uuids and not datastore.data['watching'][watch_uuid]['paused']:
|
|
|
|
if watch_uuid not in running_uuids and not datastore.data['watching'][watch_uuid]['paused']:
|
|
|
|
update_q.put((1, watch_uuid))
|
|
|
|
update_q.put(queuedWatchMetaData.PrioritizedItem(priority=1, item={'uuid': watch_uuid, 'skip_when_checksum_same': False}))
|
|
|
|
i += 1
|
|
|
|
i += 1
|
|
|
|
flash("{} watches are queued for rechecking.".format(i))
|
|
|
|
flash("{} watches are queued for rechecking.".format(i))
|
|
|
|
return redirect(url_for('index', tag=tag))
|
|
|
|
return redirect(url_for('index', tag=tag))
|
|
|
@ -1344,7 +1333,7 @@ def changedetection_app(config=None, datastore_o=None):
|
|
|
|
app.register_blueprint(browser_steps.construct_blueprint(datastore), url_prefix='/browser-steps')
|
|
|
|
app.register_blueprint(browser_steps.construct_blueprint(datastore), url_prefix='/browser-steps')
|
|
|
|
|
|
|
|
|
|
|
|
import changedetectionio.blueprint.price_data_follower as price_data_follower
|
|
|
|
import changedetectionio.blueprint.price_data_follower as price_data_follower
|
|
|
|
app.register_blueprint(price_data_follower.construct_blueprint(datastore), url_prefix='/price_data_follower')
|
|
|
|
app.register_blueprint(price_data_follower.construct_blueprint(datastore, update_q), url_prefix='/price_data_follower')
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
# @todo handle ctrl break
|
|
|
|
# @todo handle ctrl break
|
|
|
@ -1492,7 +1481,7 @@ def ticker_thread_check_time_launch_checks():
|
|
|
|
seconds_since_last_recheck = now - watch['last_checked']
|
|
|
|
seconds_since_last_recheck = now - watch['last_checked']
|
|
|
|
|
|
|
|
|
|
|
|
if seconds_since_last_recheck >= (threshold + watch.jitter_seconds) and seconds_since_last_recheck >= recheck_time_minimum_seconds:
|
|
|
|
if seconds_since_last_recheck >= (threshold + watch.jitter_seconds) and seconds_since_last_recheck >= recheck_time_minimum_seconds:
|
|
|
|
if not uuid in running_uuids and uuid not in [q_uuid for p,q_uuid in update_q.queue]:
|
|
|
|
if not uuid in running_uuids and uuid not in [q_uuid.item['uuid'] for q_uuid in update_q.queue]:
|
|
|
|
|
|
|
|
|
|
|
|
# Proxies can be set to have a limit on seconds between which they can be called
|
|
|
|
# Proxies can be set to have a limit on seconds between which they can be called
|
|
|
|
watch_proxy = datastore.get_preferred_proxy_for_watch(uuid=uuid)
|
|
|
|
watch_proxy = datastore.get_preferred_proxy_for_watch(uuid=uuid)
|
|
|
@ -1523,8 +1512,9 @@ def ticker_thread_check_time_launch_checks():
|
|
|
|
priority,
|
|
|
|
priority,
|
|
|
|
watch.jitter_seconds,
|
|
|
|
watch.jitter_seconds,
|
|
|
|
now - watch['last_checked']))
|
|
|
|
now - watch['last_checked']))
|
|
|
|
|
|
|
|
|
|
|
|
# Into the queue with you
|
|
|
|
# Into the queue with you
|
|
|
|
update_q.put((priority, uuid))
|
|
|
|
update_q.put(queuedWatchMetaData.PrioritizedItem(priority=priority, item={'uuid': uuid, 'skip_when_checksum_same': True}))
|
|
|
|
|
|
|
|
|
|
|
|
# Reset for next time
|
|
|
|
# Reset for next time
|
|
|
|
watch.jitter_seconds = 0
|
|
|
|
watch.jitter_seconds = 0
|
|
|
|