diff --git a/changedetectionio/flask_app.py b/changedetectionio/flask_app.py
index a1bb862e..bd6087af 100644
--- a/changedetectionio/flask_app.py
+++ b/changedetectionio/flask_app.py
@@ -13,7 +13,6 @@ import timeago
from .processors import find_processors, get_parent_module, get_custom_watch_obj_for_processor
from .safe_jinja import render as jinja_render
-from changedetectionio.strtobool import strtobool
from copy import deepcopy
from functools import wraps
from threading import Event
@@ -39,20 +38,10 @@ from flask_cors import CORS
from flask_wtf import CSRFProtect
from loguru import logger
-from changedetectionio import html_tools, __version__
+from changedetectionio import html_tools, __version__, strtobool
from changedetectionio import queuedWatchMetaData
from changedetectionio.api import api_v1
-datastore = None
-
-# Local
-running_update_threads = []
-ticker_thread = None
-
-extra_stylesheets = []
-
-update_q = queue.PriorityQueue()
-notification_q = queue.Queue()
app = Flask(__name__,
static_url_path="",
@@ -83,6 +72,18 @@ csrf = CSRFProtect()
csrf.init_app(app)
notification_debug_log=[]
+
+
+
+# Local
+app.datastore = None
+app.running_update_threads = []
+app.ticker_thread = None
+app.extra_stylesheets = []
+app.update_q = queue.PriorityQueue()
+app.notification_q = queue.Queue()
+
+
# get locale ready
default_locale = locale.getdefaultlocale()
logger.info(f"System locale default is {default_locale}")
@@ -110,7 +111,6 @@ def init_app_secret(datastore_path):
return secret
-
@app.template_global()
def get_darkmode_state():
css_dark_mode = request.cookies.get('css_dark_mode', 'false')
@@ -133,7 +133,7 @@ def _jinja2_filter_format_number_locale(value: float) -> str:
@app.template_filter('format_last_checked_time')
def _jinja2_filter_datetime(watch_obj, format="%Y-%m-%d %H:%M:%S"):
# Worker thread tells us which UUID it is currently processing.
- for t in running_update_threads:
+ for t in app.running_update_threads:
if t.current_uuid == watch_obj['uuid']:
return ' Checking now'
@@ -152,7 +152,7 @@ def _jinja2_filter_datetimestamp(timestamp, format="%Y-%m-%d %H:%M:%S"):
@app.template_filter('pagination_slice')
def _jinja2_filter_pagination_slice(arr, skip):
- per_page = datastore.data['settings']['application'].get('pager_size', 50)
+ per_page = app.datastore.data['settings']['application'].get('pager_size', 50)
if per_page:
return arr[skip:skip + per_page]
@@ -191,7 +191,7 @@ class User(flask_login.UserMixin):
raw_salt_pass = os.getenv("SALTED_PASS", False)
if not raw_salt_pass:
- raw_salt_pass = datastore.data['settings']['application'].get('password')
+ raw_salt_pass = app.datastore.data['settings']['application'].get('password')
raw_salt_pass = base64.b64decode(raw_salt_pass)
salt_from_storage = raw_salt_pass[:32] # 32 is the length of the salt
@@ -213,13 +213,13 @@ def login_optionally_required(func):
@wraps(func)
def decorated_view(*args, **kwargs):
- has_password_enabled = datastore.data['settings']['application'].get('password') or os.getenv("SALTED_PASS", False)
+ has_password_enabled = app.datastore.data['settings']['application'].get('password') or os.getenv("SALTED_PASS", False)
# Permitted
if request.endpoint == 'static_content' and request.view_args['group'] == 'styles':
return func(*args, **kwargs)
# Permitted
- elif request.endpoint == 'diff_history_page' and datastore.data['settings']['application'].get('shared_diff_access'):
+ elif request.endpoint == 'diff_history_page' and app.datastore.data['settings']['application'].get('shared_diff_access'):
return func(*args, **kwargs)
elif request.method in flask_login.config.EXEMPT_METHODS:
return func(*args, **kwargs)
@@ -234,13 +234,14 @@ def login_optionally_required(func):
def changedetection_app(config=None, datastore_o=None):
logger.trace("TRACE log is enabled")
+
+ global app
+ app.datastore = datastore_o
- global datastore
- datastore = datastore_o
# so far just for read-only via tests, but this will be moved eventually to be the main source
# (instead of the global var)
- app.config['DATASTORE'] = datastore_o
+ app.config['DATASTORE'] = app.datastore
login_manager = flask_login.LoginManager(app)
login_manager.login_view = 'login'
@@ -249,24 +250,24 @@ def changedetection_app(config=None, datastore_o=None):
watch_api.add_resource(api_v1.WatchSingleHistory,
'/api/v1/watch//history/',
- resource_class_kwargs={'datastore': datastore, 'update_q': update_q})
+ resource_class_kwargs={'datastore': app.datastore, 'update_q': app.update_q})
watch_api.add_resource(api_v1.WatchHistory,
'/api/v1/watch//history',
- resource_class_kwargs={'datastore': datastore})
+ resource_class_kwargs={'datastore': app.datastore})
watch_api.add_resource(api_v1.CreateWatch, '/api/v1/watch',
- resource_class_kwargs={'datastore': datastore, 'update_q': update_q})
+ resource_class_kwargs={'datastore': app.datastore, 'update_q': app.update_q})
watch_api.add_resource(api_v1.Watch, '/api/v1/watch/',
- resource_class_kwargs={'datastore': datastore, 'update_q': update_q})
+ resource_class_kwargs={'datastore': app.datastore, 'update_q': app.update_q})
watch_api.add_resource(api_v1.SystemInfo, '/api/v1/systeminfo',
- resource_class_kwargs={'datastore': datastore, 'update_q': update_q})
+ resource_class_kwargs={'datastore': app.datastore, 'update_q': app.update_q})
watch_api.add_resource(api_v1.Import,
'/api/v1/import',
- resource_class_kwargs={'datastore': datastore})
+ resource_class_kwargs={'datastore': app.datastore})
# Setup cors headers to allow all domains
# https://flask-cors.readthedocs.io/en/latest/
@@ -341,7 +342,7 @@ def changedetection_app(config=None, datastore_o=None):
def rss():
now = time.time()
# Always requires token set
- app_rss_token = datastore.data['settings']['application'].get('rss_access_token')
+ app_rss_token = app.datastore.data['settings']['application'].get('rss_access_token')
rss_url_token = request.args.get('token')
if rss_url_token != app_rss_token:
return "Access denied, bad token", 403
@@ -349,7 +350,7 @@ def changedetection_app(config=None, datastore_o=None):
from . import diff
limit_tag = request.args.get('tag', '').lower().strip()
# Be sure limit_tag is a uuid
- for uuid, tag in datastore.data['settings']['application'].get('tags', {}).items():
+ for uuid, tag in app.datastore.data['settings']['application'].get('tags', {}).items():
if limit_tag == tag.get('title', '').lower().strip():
limit_tag = uuid
@@ -357,9 +358,9 @@ def changedetection_app(config=None, datastore_o=None):
sorted_watches = []
# @todo needs a .itemsWithTag() or something - then we can use that in Jinaj2 and throw this away
- for uuid, watch in datastore.data['watching'].items():
+ for uuid, watch in app.datastore.data['watching'].items():
# @todo tag notification_muted skip also (improve Watch model)
- if datastore.data['settings']['application'].get('rss_hide_muted_watches') and watch.get('notification_muted'):
+ if app.datastore.data['settings']['application'].get('rss_hide_muted_watches') and watch.get('notification_muted'):
continue
if limit_tag and not limit_tag in watch['tags']:
continue
@@ -389,7 +390,7 @@ def changedetection_app(config=None, datastore_o=None):
# Include a link to the diff page, they will have to login here to see if password protection is enabled.
# Description is the page you watch, link takes you to the diff JS UI page
# Dict val base_url will get overriden with the env var if it is set.
- ext_base_url = datastore.data['settings']['application'].get('active_base_url')
+ ext_base_url = app.datastore.data['settings']['application'].get('active_base_url')
# Because we are called via whatever web server, flask should figure out the right path (
diff_link = {'href': url_for('diff_history_page', uuid=watch['uuid'], _external=True)}
@@ -426,7 +427,7 @@ def changedetection_app(config=None, datastore_o=None):
@app.route("/", methods=['GET'])
@login_optionally_required
def index():
- global datastore
+ global app
from changedetectionio import forms
active_tag_req = request.args.get('tag', '').lower().strip()
@@ -434,7 +435,7 @@ def changedetection_app(config=None, datastore_o=None):
# Be sure limit_tag is a uuid
if active_tag_req:
- for uuid, tag in datastore.data['settings']['application'].get('tags', {}).items():
+ for uuid, tag in app.datastore.data['settings']['application'].get('tags', {}).items():
if active_tag_req == tag.get('title', '').lower().strip() or active_tag_req == uuid:
active_tag = tag
active_tag_uuid = uuid
@@ -449,11 +450,11 @@ def changedetection_app(config=None, datastore_o=None):
if op:
uuid = request.args.get('uuid')
if op == 'pause':
- datastore.data['watching'][uuid].toggle_pause()
+ app.datastore.data['watching'][uuid].toggle_pause()
elif op == 'mute':
- datastore.data['watching'][uuid].toggle_mute()
+ app.datastore.data['watching'][uuid].toggle_mute()
- datastore.needs_write = True
+ app.datastore.needs_write = True
return redirect(url_for('index', tag = active_tag_uuid))
# Sort by last_changed and add the uuid which is usually the key..
@@ -461,7 +462,7 @@ def changedetection_app(config=None, datastore_o=None):
with_errors = request.args.get('with_errors') == "1"
errored_count = 0
search_q = request.args.get('q').strip().lower() if request.args.get('q') else False
- for uuid, watch in datastore.data['watching'].items():
+ for uuid, watch in app.datastore.data['watching'].items():
if with_errors and not watch.get('last_error'):
continue
@@ -484,28 +485,28 @@ def changedetection_app(config=None, datastore_o=None):
pagination = Pagination(page=page,
total=total_count,
- per_page=datastore.data['settings']['application'].get('pager_size', 50), css_framework="semantic")
+ per_page=app.datastore.data['settings']['application'].get('pager_size', 50), css_framework="semantic")
- sorted_tags = sorted(datastore.data['settings']['application'].get('tags').items(), key=lambda x: x[1]['title'])
+ sorted_tags = sorted(app.datastore.data['settings']['application'].get('tags').items(), key=lambda x: x[1]['title'])
output = render_template(
"watch-overview.html",
# Don't link to hosting when we're on the hosting environment
active_tag=active_tag,
active_tag_uuid=active_tag_uuid,
- app_rss_token=datastore.data['settings']['application'].get('rss_access_token'),
- datastore=datastore,
+ app_rss_token=app.datastore.data['settings']['application'].get('rss_access_token'),
+ datastore=app.datastore,
errored_count=errored_count,
form=form,
- guid=datastore.data['app_guid'],
- has_proxies=datastore.proxy_list,
- has_unviewed=datastore.has_unviewed,
+ guid=app.datastore.data['app_guid'],
+ has_proxies=app.datastore.proxy_list,
+ has_unviewed=app.datastore.has_unviewed,
hosted_sticky=os.getenv("SALTED_PASS", False) == False,
pagination=pagination,
- queued_uuids=[q_uuid.item['uuid'] for q_uuid in update_q.queue],
+ queued_uuids=[q_uuid.item['uuid'] for q_uuid in app.update_q.queue],
search_q=request.args.get('q','').strip(),
sort_attribute=request.args.get('sort') if request.args.get('sort') else request.cookies.get('sort'),
sort_order=request.args.get('order') if request.args.get('order') else request.cookies.get('order'),
- system_default_fetcher=datastore.data['settings']['application'].get('fetch_backend'),
+ system_default_fetcher=app.datastore.data['settings']['application'].get('fetch_backend'),
tags=sorted_tags,
watches=sorted_watches
)
@@ -543,11 +544,11 @@ def changedetection_app(config=None, datastore_o=None):
is_group_settings_form = request.args.get('mode', '') == 'group-settings'
# Use an existing random one on the global/main settings form
if not watch_uuid and (is_global_settings_form or is_group_settings_form) \
- and datastore.data.get('watching'):
+ and app.datastore.data.get('watching'):
logger.debug(f"Send test notification - Choosing random Watch {watch_uuid}")
- watch_uuid = random.choice(list(datastore.data['watching'].keys()))
- watch = datastore.data['watching'].get(watch_uuid)
+ watch_uuid = random.choice(list(app.datastore.data['watching'].keys()))
+ watch = app.datastore.data['watching'].get(watch_uuid)
else:
watch = None
@@ -558,14 +559,14 @@ def changedetection_app(config=None, datastore_o=None):
# On an edit page, we should also fire off to the tags if they have notifications
if request.form.get('tags') and request.form['tags'].strip():
for k in request.form['tags'].split(','):
- tag = datastore.tag_exists_by_name(k.strip())
+ tag = app.datastore.tag_exists_by_name(k.strip())
notification_urls = tag.get('notifications_urls') if tag and tag.get('notifications_urls') else None
if not notification_urls and not is_global_settings_form and not is_group_settings_form:
# In the global settings, use only what is typed currently in the text box
logger.debug("Test notification - Trying by global system settings notifications")
- if datastore.data['settings']['application'].get('notification_urls'):
- notification_urls = datastore.data['settings']['application']['notification_urls']
+ if app.datastore.data['settings']['application'].get('notification_urls'):
+ notification_urls = app.datastore.data['settings']['application']['notification_urls']
if not notification_urls:
@@ -594,8 +595,8 @@ def changedetection_app(config=None, datastore_o=None):
n_object['notification_body'] = request.form.get('notification_body', '').strip()
from . import update_worker
- new_worker = update_worker.update_worker(update_q, notification_q, app, datastore)
- new_worker.queue_notification_for_watch(notification_q=notification_q, n_object=n_object, watch=watch)
+ new_worker = update_worker.update_worker(app=app)
+ new_worker.queue_notification_for_watch(notification_q=app.notification_q, n_object=n_object, watch=watch)
except Exception as e:
return make_response({'error': str(e)}, 400)
@@ -606,7 +607,7 @@ def changedetection_app(config=None, datastore_o=None):
@login_optionally_required
def clear_watch_history(uuid):
try:
- datastore.clear_watch_history(uuid)
+ app.datastore.clear_watch_history(uuid)
except KeyError:
flash('Watch not found', 'error')
else:
@@ -623,8 +624,8 @@ def changedetection_app(config=None, datastore_o=None):
if confirmtext == 'clear':
changes_removed = 0
- for uuid in datastore.data['watching'].keys():
- datastore.clear_watch_history(uuid)
+ for uuid in app.datastore.data['watching'].keys():
+ app.datastore.clear_watch_history(uuid)
#TODO: KeyError not checked, as it is above
flash("Cleared snapshot history for all watches")
@@ -638,7 +639,7 @@ def changedetection_app(config=None, datastore_o=None):
def _watch_has_tag_options_set(watch):
"""This should be fixed better so that Tag is some proper Model, a tag is just a Watch also"""
- for tag_uuid, tag in datastore.data['settings']['application'].get('tags', {}).items():
+ for tag_uuid, tag in app.datastore.data['settings']['application'].get('tags', {}).items():
if tag_uuid in watch.get('tags', []) and (tag.get('include_filters') or tag.get('subtractive_selectors')):
return True
@@ -653,14 +654,14 @@ def changedetection_app(config=None, datastore_o=None):
import importlib
# More for testing, possible to return the first/only
- if not datastore.data['watching'].keys():
+ if not app.datastore.data['watching'].keys():
flash("No watches to edit", "error")
return redirect(url_for('index'))
if uuid == 'first':
- uuid = list(datastore.data['watching'].keys()).pop()
+ uuid = list(app.datastore.data['watching'].keys()).pop()
- if not uuid in datastore.data['watching']:
+ if not uuid in app.datastore.data['watching']:
flash("No watch with the UUID %s found." % (uuid), "error")
return redirect(url_for('index'))
@@ -668,24 +669,24 @@ def changedetection_app(config=None, datastore_o=None):
if switch_processor:
for p in processors.available_processors():
if p[0] == switch_processor:
- datastore.data['watching'][uuid]['processor'] = switch_processor
+ app.datastore.data['watching'][uuid]['processor'] = switch_processor
flash(f"Switched to mode - {p[1]}.")
- datastore.clear_watch_history(uuid)
+ app.datastore.clear_watch_history(uuid)
redirect(url_for('edit_page', uuid=uuid))
# be sure we update with a copy instead of accidently editing the live object by reference
- default = deepcopy(datastore.data['watching'][uuid])
+ default = deepcopy(app.datastore.data['watching'][uuid])
# Defaults for proxy choice
- if datastore.proxy_list is not None: # When enabled
+ if app.datastore.proxy_list is not None: # When enabled
# @todo
# Radio needs '' not None, or incase that the chosen one no longer exists
- if default['proxy'] is None or not any(default['proxy'] in tup for tup in datastore.proxy_list):
+ if default['proxy'] is None or not any(default['proxy'] in tup for tup in app.datastore.proxy_list):
default['proxy'] = ''
# proxy_override set to the json/text list of the items
# Does it use some custom form? does one exist?
- processor_name = datastore.data['watching'][uuid].get('processor', '')
+ processor_name = app.datastore.data['watching'][uuid].get('processor', '')
processor_classes = next((tpl for tpl in find_processors() if tpl[1] == processor_name), None)
if not processor_classes:
flash(f"Cannot load the edit form for processor/plugin '{processor_classes[1]}', plugin missing?", 'error')
@@ -711,33 +712,33 @@ def changedetection_app(config=None, datastore_o=None):
)
# For the form widget tag UUID back to "string name" for the field
- form.tags.datastore = datastore
+ form.tags.datastore = app.datastore
# Used by some forms that need to dig deeper
- form.datastore = datastore
+ form.datastore = app.datastore
form.watch = default
- for p in datastore.extra_browsers:
+ for p in app.datastore.extra_browsers:
form.fetch_backend.choices.append(p)
form.fetch_backend.choices.append(("system", 'System settings default'))
# form.browser_steps[0] can be assumed that we 'goto url' first
- if datastore.proxy_list is None:
+ if app.datastore.proxy_list is None:
# @todo - Couldn't get setattr() etc dynamic addition working, so remove it instead
del form.proxy
else:
form.proxy.choices = [('', 'Default')]
- for p in datastore.proxy_list:
- form.proxy.choices.append(tuple((p, datastore.proxy_list[p]['label'])))
+ for p in app.datastore.proxy_list:
+ form.proxy.choices.append(tuple((p, app.datastore.proxy_list[p]['label'])))
if request.method == 'POST' and form.validate():
# If they changed processor, it makes sense to reset it.
- if datastore.data['watching'][uuid].get('processor') != form.data.get('processor'):
- datastore.data['watching'][uuid].clear_watch()
+ if app.datastore.data['watching'][uuid].get('processor') != form.data.get('processor'):
+ app.datastore.data['watching'][uuid].clear_watch()
flash("Reset watch history due to change of processor")
extra_update_obj = {
@@ -752,10 +753,10 @@ def changedetection_app(config=None, datastore_o=None):
# Ignore text
form_ignore_text = form.ignore_text.data
- datastore.data['watching'][uuid]['ignore_text'] = form_ignore_text
+ app.datastore.data['watching'][uuid]['ignore_text'] = form_ignore_text
# Be sure proxy value is None
- if datastore.proxy_list is not None and form.data['proxy'] == '':
+ if app.datastore.proxy_list is not None and form.data['proxy'] == '':
extra_update_obj['proxy'] = None
# Unsetting all filter_text methods should make it go back to default
@@ -775,28 +776,29 @@ def changedetection_app(config=None, datastore_o=None):
extra_update_obj['tags'] = form.data.get('tags')
else:
for t in form.data.get('tags').split(','):
- tag_uuids.append(datastore.add_tag(name=t))
+ tag_uuids.append(app.datastore.add_tag(name=t))
extra_update_obj['tags'] = tag_uuids
- datastore.data['watching'][uuid].update(form.data)
- datastore.data['watching'][uuid].update(extra_update_obj)
+ app.datastore.data['watching'][uuid].update(form.data)
+ app.datastore.data['watching'][uuid].update(extra_update_obj)
- if not datastore.data['watching'][uuid].get('tags'):
+ if not app.datastore.data['watching'][uuid].get('tags'):
# Force it to be a list, because form.data['tags'] will be string if nothing found
# And del(form.data['tags'] ) wont work either for some reason
- datastore.data['watching'][uuid]['tags'] = []
+ app.datastore.data['watching'][uuid]['tags'] = []
# Recast it if need be to right data Watch handler
watch_class = get_custom_watch_obj_for_processor(form.data.get('processor'))
- datastore.data['watching'][uuid] = watch_class(datastore_path=datastore_o.datastore_path, default=datastore.data['watching'][uuid])
+ app.datastore.data['watching'][uuid] = watch_class(datastore_path=datastore_o.datastore_path, default=app.datastore.data['watching'][uuid])
+
flash("Updated watch - unpaused!" if request.args.get('unpause_on_save') else "Updated watch.")
# Re #286 - We wait for syncing new data to disk in another thread every 60 seconds
# But in the case something is added we should save straight away
- datastore.needs_write_urgent = True
+ app.datastore.needs_write_urgent = True
# Queue the watch for immediate recheck, with a higher priority
- update_q.put(queuedWatchMetaData.PrioritizedItem(priority=1, item={'uuid': uuid}))
+ app.update_q.put(queuedWatchMetaData.PrioritizedItem(priority=1, item={'uuid': uuid}))
# Diff page [edit] link should go back to diff page
if request.args.get("next") and request.args.get("next") == 'diff':
@@ -808,7 +810,7 @@ def changedetection_app(config=None, datastore_o=None):
if request.method == 'POST' and not form.validate():
flash("An error occurred, please see below.", "error")
- visualselector_data_is_ready = datastore.visualselector_data_is_ready(uuid)
+ visualselector_data_is_ready = app.datastore.visualselector_data_is_ready(uuid)
# JQ is difficult to install on windows and must be manually added (outside requirements.txt)
@@ -818,9 +820,9 @@ def changedetection_app(config=None, datastore_o=None):
except ModuleNotFoundError:
jq_support = False
- watch = datastore.data['watching'].get(uuid)
+ watch = app.datastore.data['watching'].get(uuid)
- system_uses_webdriver = datastore.data['settings']['application']['fetch_backend'] == 'html_webdriver'
+ system_uses_webdriver = app.datastore.data['settings']['application']['fetch_backend'] == 'html_webdriver'
is_html_webdriver = False
if (watch.get('fetch_backend') == 'system' and system_uses_webdriver) or watch.get('fetch_backend') == 'html_webdriver' or watch.get('fetch_backend', '').startswith('extra_browser_'):
@@ -834,15 +836,15 @@ def changedetection_app(config=None, datastore_o=None):
'emailprefix': os.getenv('NOTIFICATION_MAIL_BUTTON_PREFIX', False),
'extra_title': f" - Edit - {watch.label}",
'extra_processor_config': form.extra_tab_content(),
- 'extra_notification_token_placeholder_info': datastore.get_unique_notification_token_placeholders_available(),
+ 'extra_notification_token_placeholder_info': app.datastore.get_unique_notification_token_placeholders_available(),
'form': form,
- 'has_default_notification_urls': True if len(datastore.data['settings']['application']['notification_urls']) else False,
- 'has_extra_headers_file': len(datastore.get_all_headers_in_textfile_for_watch(uuid=uuid)) > 0,
+ 'has_default_notification_urls': True if len(app.datastore.data['settings']['application']['notification_urls']) else False,
+ 'has_extra_headers_file': len(app.datastore.get_all_headers_in_textfile_for_watch(uuid=uuid)) > 0,
'has_special_tag_options': _watch_has_tag_options_set(watch=watch),
'is_html_webdriver': is_html_webdriver,
'jq_support': jq_support,
'playwright_enabled': os.getenv('PLAYWRIGHT_DRIVER_URL', False),
- 'settings_application': datastore.data['settings']['application'],
+ 'settings_application': app.datastore.data['settings']['application'],
'using_global_webdriver_wait': not default['webdriver_delay'],
'uuid': uuid,
'visualselector_enabled': visualselector_enabled,
@@ -873,11 +875,11 @@ def changedetection_app(config=None, datastore_o=None):
def settings_page():
from changedetectionio import forms
- default = deepcopy(datastore.data['settings'])
- if datastore.proxy_list is not None:
- available_proxies = list(datastore.proxy_list.keys())
+ default = deepcopy(app.datastore.data['settings'])
+ if app.datastore.proxy_list is not None:
+ available_proxies = list(app.datastore.proxy_list.keys())
# When enabled
- system_proxy = datastore.data['settings']['requests']['proxy']
+ system_proxy = app.datastore.data['settings']['requests']['proxy']
# In the case it doesnt exist anymore
if not system_proxy in available_proxies:
system_proxy = None
@@ -890,19 +892,19 @@ def changedetection_app(config=None, datastore_o=None):
# Don't use form.data on POST so that it doesnt overrid the checkbox status from the POST status
form = forms.globalSettingsForm(formdata=request.form if request.method == 'POST' else None,
data=default,
- extra_notification_tokens=datastore.get_unique_notification_tokens_available()
+ extra_notification_tokens=app.datastore.get_unique_notification_tokens_available()
)
# Remove the last option 'System default'
form.application.form.notification_format.choices.pop()
- if datastore.proxy_list is None:
+ if app.datastore.proxy_list is None:
# @todo - Couldn't get setattr() etc dynamic addition working, so remove it instead
del form.requests.form.proxy
else:
form.requests.form.proxy.choices = []
- for p in datastore.proxy_list:
- form.requests.form.proxy.choices.append(tuple((p, datastore.proxy_list[p]['label'])))
+ for p in app.datastore.proxy_list:
+ form.requests.form.proxy.choices.append(tuple((p, app.datastore.proxy_list[p]['label'])))
if request.method == 'POST':
@@ -910,7 +912,7 @@ def changedetection_app(config=None, datastore_o=None):
if form.application.form.data.get('removepassword_button', False):
# SALTED_PASS means the password is "locked" to what we set in the Env var
if not os.getenv("SALTED_PASS", False):
- datastore.remove_password()
+ app.datastore.remove_password()
flash("Password protection removed.", 'notice')
flask_login.logout_user()
return redirect(url_for('settings_page'))
@@ -923,30 +925,30 @@ def changedetection_app(config=None, datastore_o=None):
if 'password' in app_update and not app_update['password']:
del (app_update['password'])
- datastore.data['settings']['application'].update(app_update)
- datastore.data['settings']['requests'].update(form.data['requests'])
+ app.datastore.data['settings']['application'].update(app_update)
+ app.datastore.data['settings']['requests'].update(form.data['requests'])
if not os.getenv("SALTED_PASS", False) and len(form.application.form.password.encrypted_password):
- datastore.data['settings']['application']['password'] = form.application.form.password.encrypted_password
- datastore.needs_write_urgent = True
+ app.datastore.data['settings']['application']['password'] = form.application.form.password.encrypted_password
+ app.datastore.needs_write_urgent = True
flash("Password protection enabled.", 'notice')
flask_login.logout_user()
return redirect(url_for('index'))
- datastore.needs_write_urgent = True
+ app.datastore.needs_write_urgent = True
flash("Settings updated.")
else:
flash("An error occurred, please see below.", "error")
output = render_template("settings.html",
- api_key=datastore.data['settings']['application'].get('api_access_token'),
+ api_key=app.datastore.data['settings']['application'].get('api_access_token'),
emailprefix=os.getenv('NOTIFICATION_MAIL_BUTTON_PREFIX', False),
- extra_notification_token_placeholder_info=datastore.get_unique_notification_token_placeholders_available(),
+ extra_notification_token_placeholder_info=app.datastore.get_unique_notification_token_placeholders_available(),
form=form,
hide_remove_pass=os.getenv("SALTED_PASS", False),
min_system_recheck_seconds=int(os.getenv('MINIMUM_SECONDS_RECHECK_TIME', 3)),
- settings_application=datastore.data['settings']['application']
+ settings_application=app.datastore.data['settings']['application']
)
return output
@@ -956,8 +958,8 @@ def changedetection_app(config=None, datastore_o=None):
def settings_reset_api_key():
import secrets
secret = secrets.token_hex(16)
- datastore.data['settings']['application']['api_access_token'] = secret
- datastore.needs_write_urgent = True
+ app.datastore.data['settings']['application']['api_access_token'] = secret
+ app.datastore.needs_write_urgent = True
flash("API Key was regenerated.")
return redirect(url_for('settings_page')+'#api')
@@ -975,9 +977,9 @@ def changedetection_app(config=None, datastore_o=None):
if request.values.get('urls') and len(request.values.get('urls').strip()):
# Import and push into the queue for immediate update check
importer = import_url_list()
- importer.run(data=request.values.get('urls'), flash=flash, datastore=datastore, processor=request.values.get('processor', 'text_json_diff'))
+ importer.run(data=request.values.get('urls'), flash=flash, datastore=app.datastore, processor=request.values.get('processor', 'text_json_diff'))
for uuid in importer.new_uuids:
- update_q.put(queuedWatchMetaData.PrioritizedItem(priority=1, item={'uuid': uuid}))
+ app.update_q.put(queuedWatchMetaData.PrioritizedItem(priority=1, item={'uuid': uuid}))
if len(importer.remaining_data) == 0:
return redirect(url_for('index'))
@@ -988,9 +990,9 @@ def changedetection_app(config=None, datastore_o=None):
if request.values.get('distill-io') and len(request.values.get('distill-io').strip()):
# Import and push into the queue for immediate update check
d_importer = import_distill_io_json()
- d_importer.run(data=request.values.get('distill-io'), flash=flash, datastore=datastore)
+ d_importer.run(data=request.values.get('distill-io'), flash=flash, datastore=app.datastore)
for uuid in d_importer.new_uuids:
- update_q.put(queuedWatchMetaData.PrioritizedItem(priority=1, item={'uuid': uuid}))
+ app.update_q.put(queuedWatchMetaData.PrioritizedItem(priority=1, item={'uuid': uuid}))
# XLSX importer
if request.files and request.files.get('xlsx_file'):
@@ -999,7 +1001,7 @@ def changedetection_app(config=None, datastore_o=None):
if request.values.get('file_mapping') == 'wachete':
w_importer = import_xlsx_wachete()
- w_importer.run(data=file, flash=flash, datastore=datastore)
+ w_importer.run(data=file, flash=flash, datastore=app.datastore)
else:
w_importer = import_xlsx_custom()
# Building mapping of col # to col # type
@@ -1011,10 +1013,10 @@ def changedetection_app(config=None, datastore_o=None):
map[int(c)] = v
w_importer.import_profile = map
- w_importer.run(data=file, flash=flash, datastore=datastore)
+ w_importer.run(data=file, flash=flash, datastore=app.datastore)
for uuid in w_importer.new_uuids:
- update_q.put(queuedWatchMetaData.PrioritizedItem(priority=1, item={'uuid': uuid}))
+ app.update_q.put(queuedWatchMetaData.PrioritizedItem(priority=1, item={'uuid': uuid}))
# Could be some remaining, or we could be on GET
form = forms.importForm(formdata=request.form if request.method == 'POST' else None)
@@ -1032,10 +1034,10 @@ def changedetection_app(config=None, datastore_o=None):
# Save the current newest history as the most recently viewed
with_errors = request.args.get('with_errors') == "1"
- for watch_uuid, watch in datastore.data['watching'].items():
+ for watch_uuid, watch in app.datastore.data['watching'].items():
if with_errors and not watch.get('last_error'):
continue
- datastore.set_last_viewed(watch_uuid, int(time.time()))
+ app.datastore.set_last_viewed(watch_uuid, int(time.time()))
return redirect(url_for('index'))
@@ -1047,11 +1049,11 @@ def changedetection_app(config=None, datastore_o=None):
# More for testing, possible to return the first/only
if uuid == 'first':
- uuid = list(datastore.data['watching'].keys()).pop()
+ uuid = list(app.datastore.data['watching'].keys()).pop()
extra_stylesheets = [url_for('static_content', group='styles', filename='diff.css')]
try:
- watch = datastore.data['watching'][uuid]
+ watch = app.datastore.data['watching'][uuid]
except KeyError:
flash("No history found for the specified link, bad link?", "error")
return redirect(url_for('index'))
@@ -1066,7 +1068,7 @@ def changedetection_app(config=None, datastore_o=None):
extract_regex = request.form.get('extract_regex').strip()
output = watch.extract_regex_from_all_history(extract_regex)
if output:
- watch_dir = os.path.join(datastore_o.datastore_path, uuid)
+ watch_dir = os.path.join(app.datastore_o.datastore_path, uuid)
response = make_response(send_from_directory(directory=watch_dir, path=output, as_attachment=True))
response.headers['Content-type'] = 'text/csv'
response.headers['Cache-Control'] = 'no-cache, no-store, must-revalidate'
@@ -1086,7 +1088,7 @@ def changedetection_app(config=None, datastore_o=None):
return redirect(url_for('index'))
# Save the current newest history as the most recently viewed
- datastore.set_last_viewed(uuid, time.time())
+ app.datastore.set_last_viewed(uuid, time.time())
# Read as binary and force decode as UTF-8
# Windows may fail decode in python if we just use 'r' mode (chardet decode exception)
@@ -1116,15 +1118,15 @@ def changedetection_app(config=None, datastore_o=None):
screenshot_url = watch.get_screenshot()
- system_uses_webdriver = datastore.data['settings']['application']['fetch_backend'] == 'html_webdriver'
+ system_uses_webdriver = app.datastore.data['settings']['application']['fetch_backend'] == 'html_webdriver'
is_html_webdriver = False
if (watch.get('fetch_backend') == 'system' and system_uses_webdriver) or watch.get('fetch_backend') == 'html_webdriver' or watch.get('fetch_backend', '').startswith('extra_browser_'):
is_html_webdriver = True
password_enabled_and_share_is_off = False
- if datastore.data['settings']['application'].get('password') or os.getenv("SALTED_PASS", False):
- password_enabled_and_share_is_off = not datastore.data['settings']['application'].get('shared_diff_access')
+ if app.datastore.data['settings']['application'].get('password') or os.getenv("SALTED_PASS", False):
+ password_enabled_and_share_is_off = not app.datastore.data['settings']['application'].get('shared_diff_access')
output = render_template("diff.html",
current_diff_url=watch['url'],
@@ -1160,22 +1162,22 @@ def changedetection_app(config=None, datastore_o=None):
# More for testing, possible to return the first/only
if uuid == 'first':
- uuid = list(datastore.data['watching'].keys()).pop()
+ uuid = list(app.datastore.data['watching'].keys()).pop()
try:
- watch = datastore.data['watching'][uuid]
+ watch = app.datastore.data['watching'][uuid]
except KeyError:
flash("No history found for the specified link, bad link?", "error")
return redirect(url_for('index'))
- system_uses_webdriver = datastore.data['settings']['application']['fetch_backend'] == 'html_webdriver'
+ system_uses_webdriver = app.datastore.data['settings']['application']['fetch_backend'] == 'html_webdriver'
extra_stylesheets = [url_for('static_content', group='styles', filename='diff.css')]
is_html_webdriver = False
if (watch.get('fetch_backend') == 'system' and system_uses_webdriver) or watch.get('fetch_backend') == 'html_webdriver' or watch.get('fetch_backend', '').startswith('extra_browser_'):
is_html_webdriver = True
triggered_line_numbers = []
- if datastore.data['watching'][uuid].history_n == 0 and (watch.get_error_text() or watch.get_error_snapshot()):
+ if app.datastore.data['watching'][uuid].history_n == 0 and (watch.get_error_text() or watch.get_error_snapshot()):
flash("Preview unavailable - No fetch/check completed or triggers not reached", "error")
else:
# So prepare the latest preview or not
@@ -1237,29 +1239,29 @@ def changedetection_app(config=None, datastore_o=None):
# Remove any existing backup file, for now we just keep one file
- for previous_backup_filename in Path(datastore_o.datastore_path).rglob('changedetection-backup-*.zip'):
+ for previous_backup_filename in Path(app.datastore_o.datastore_path).rglob('changedetection-backup-*.zip'):
os.unlink(previous_backup_filename)
# create a ZipFile object
timestamp = datetime.datetime.now().strftime("%Y%m%d%H%M%S")
backupname = "changedetection-backup-{}.zip".format(timestamp)
- backup_filepath = os.path.join(datastore_o.datastore_path, backupname)
+ backup_filepath = os.path.join(app.datastore_o.datastore_path, backupname)
with zipfile.ZipFile(backup_filepath, "w",
compression=zipfile.ZIP_DEFLATED,
compresslevel=8) as zipObj:
# Be sure we're written fresh
- datastore.sync_to_json()
+ app.datastore.sync_to_json()
# Add the index
- zipObj.write(os.path.join(datastore_o.datastore_path, "url-watches.json"), arcname="url-watches.json")
+ zipObj.write(os.path.join(app.datastore_o.datastore_path, "url-watches.json"), arcname="url-watches.json")
# Add the flask app secret
- zipObj.write(os.path.join(datastore_o.datastore_path, "secret.txt"), arcname="secret.txt")
+ zipObj.write(os.path.join(app.datastore_o.datastore_path, "secret.txt"), arcname="secret.txt")
# Add any data in the watch data directory.
- for uuid, w in datastore.data['watching'].items():
+ for uuid, w in app.datastore.data['watching'].items():
for f in Path(w.watch_data_dir).glob('*'):
zipObj.write(f,
# Use the full path to access the file, but make the file 'relative' in the Zip.
@@ -1269,35 +1271,35 @@ def changedetection_app(config=None, datastore_o=None):
# Create a list file with just the URLs, so it's easier to port somewhere else in the future
list_file = "url-list.txt"
- with open(os.path.join(datastore_o.datastore_path, list_file), "w") as f:
- for uuid in datastore.data["watching"]:
- url = datastore.data["watching"][uuid]["url"]
+ with open(os.path.join(app.datastore_o.datastore_path, list_file), "w") as f:
+ for uuid in app.datastore.data["watching"]:
+ url = app.datastore.data["watching"][uuid]["url"]
f.write("{}\r\n".format(url))
list_with_tags_file = "url-list-with-tags.txt"
with open(
- os.path.join(datastore_o.datastore_path, list_with_tags_file), "w"
+ os.path.join(app.datastore_o.datastore_path, list_with_tags_file), "w"
) as f:
- for uuid in datastore.data["watching"]:
- url = datastore.data["watching"][uuid].get('url')
- tag = datastore.data["watching"][uuid].get('tags', {})
+ for uuid in app.datastore.data["watching"]:
+ url = app.datastore.data["watching"][uuid].get('url')
+ tag = app.datastore.data["watching"][uuid].get('tags', {})
f.write("{} {}\r\n".format(url, tag))
# Add it to the Zip
zipObj.write(
- os.path.join(datastore_o.datastore_path, list_file),
+ os.path.join(app.datastore_o.datastore_path, list_file),
arcname=list_file,
compress_type=zipfile.ZIP_DEFLATED,
compresslevel=8,
)
zipObj.write(
- os.path.join(datastore_o.datastore_path, list_with_tags_file),
+ os.path.join(app.datastore_o.datastore_path, list_with_tags_file),
arcname=list_with_tags_file,
compress_type=zipfile.ZIP_DEFLATED,
compresslevel=8,
)
# Send_from_directory needs to be the full absolute path
- return send_from_directory(os.path.abspath(datastore_o.datastore_path), backupname, as_attachment=True)
+ return send_from_directory(os.path.abspath(app.datastore_o.datastore_path), backupname, as_attachment=True)
@app.route("/static//", methods=['GET'])
def static_content(group, filename):
@@ -1305,7 +1307,7 @@ def changedetection_app(config=None, datastore_o=None):
if group == 'screenshot':
# Could be sensitive, follow password requirements
- if datastore.data['settings']['application']['password'] and not flask_login.current_user.is_authenticated:
+ if app.datastore.data['settings']['application']['password'] and not flask_login.current_user.is_authenticated:
abort(403)
screenshot_filename = "last-screenshot.png" if not request.args.get('error_screenshot') else "last-error-screenshot.png"
@@ -1313,7 +1315,7 @@ def changedetection_app(config=None, datastore_o=None):
# These files should be in our subdirectory
try:
# set nocache, set content-type
- response = make_response(send_from_directory(os.path.join(datastore_o.datastore_path, filename), screenshot_filename))
+ response = make_response(send_from_directory(os.path.join(app.datastore_o.datastore_path, filename), screenshot_filename))
response.headers['Content-type'] = 'image/png'
response.headers['Cache-Control'] = 'no-cache, no-store, must-revalidate'
response.headers['Pragma'] = 'no-cache'
@@ -1326,13 +1328,13 @@ def changedetection_app(config=None, datastore_o=None):
if group == 'visual_selector_data':
# Could be sensitive, follow password requirements
- if datastore.data['settings']['application']['password'] and not flask_login.current_user.is_authenticated:
+ if app.datastore.data['settings']['application']['password'] and not flask_login.current_user.is_authenticated:
abort(403)
# These files should be in our subdirectory
try:
# set nocache, set content-type
- response = make_response(send_from_directory(os.path.join(datastore_o.datastore_path, filename), "elements.json"))
+ response = make_response(send_from_directory(os.path.join(app.datastore_o.datastore_path, filename), "elements.json"))
response.headers['Content-type'] = 'application/json'
response.headers['Cache-Control'] = 'no-cache, no-store, must-revalidate'
response.headers['Pragma'] = 'no-cache'
@@ -1355,7 +1357,7 @@ def changedetection_app(config=None, datastore_o=None):
from flask import send_file
import brotli
- watch = datastore.data['watching'].get(uuid)
+ watch = app.datastore.data['watching'].get(uuid)
if watch and watch.history.keys() and os.path.isdir(watch.watch_data_dir):
latest_filename = list(watch.history.keys())[-1]
html_fname = os.path.join(watch.watch_data_dir, f"{latest_filename}.html.br")
@@ -1380,7 +1382,7 @@ def changedetection_app(config=None, datastore_o=None):
def watch_get_preview_rendered(uuid):
'''For when viewing the "preview" of the rendered text from inside of Edit'''
from .processors.text_json_diff import prepare_filter_prevew
- return prepare_filter_prevew(watch_uuid=uuid, datastore=datastore)
+ return prepare_filter_prevew(watch_uuid=uuid, datastore=app.datastore)
@app.route("/form/add/quickwatch", methods=['POST'])
@@ -1395,12 +1397,12 @@ def changedetection_app(config=None, datastore_o=None):
return redirect(url_for('index'))
url = request.form.get('url').strip()
- if datastore.url_exists(url):
+ if app.datastore.url_exists(url):
flash(f'Warning, URL {url} already exists', "notice")
add_paused = request.form.get('edit_and_watch_submit_button') != None
processor = request.form.get('processor', 'text_json_diff')
- new_uuid = datastore.add_watch(url=url, tag=request.form.get('tags').strip(), extras={'paused': add_paused, 'processor': processor})
+ new_uuid = app.datastore.add_watch(url=url, tag=request.form.get('tags').strip(), extras={'paused': add_paused, 'processor': processor})
if new_uuid:
if add_paused:
@@ -1408,7 +1410,7 @@ def changedetection_app(config=None, datastore_o=None):
return redirect(url_for('edit_page', uuid=new_uuid, unpause_on_save=1))
else:
# Straight into the queue.
- update_q.put(queuedWatchMetaData.PrioritizedItem(priority=1, item={'uuid': new_uuid}))
+ app.update_q.put(queuedWatchMetaData.PrioritizedItem(priority=1, item={'uuid': new_uuid}))
flash("Watch added.")
return redirect(url_for('index'))
@@ -1420,14 +1422,14 @@ def changedetection_app(config=None, datastore_o=None):
def form_delete():
uuid = request.args.get('uuid')
- if uuid != 'all' and not uuid in datastore.data['watching'].keys():
+ if uuid != 'all' and not uuid in app.datastore.data['watching'].keys():
flash('The watch by UUID {} does not exist.'.format(uuid), 'error')
return redirect(url_for('index'))
# More for testing, possible to return the first/only
if uuid == 'first':
- uuid = list(datastore.data['watching'].keys()).pop()
- datastore.delete(uuid)
+ uuid = list(app.datastore.data['watching'].keys()).pop()
+ app.datastore.delete(uuid)
flash('Deleted.')
return redirect(url_for('index'))
@@ -1438,12 +1440,12 @@ def changedetection_app(config=None, datastore_o=None):
uuid = request.args.get('uuid')
# More for testing, possible to return the first/only
if uuid == 'first':
- uuid = list(datastore.data['watching'].keys()).pop()
+ uuid = list(app.datastore.data['watching'].keys()).pop()
- new_uuid = datastore.clone(uuid)
+ new_uuid = app.datastore.clone(uuid)
if new_uuid:
- if not datastore.data['watching'].get(uuid).get('paused'):
- update_q.put(queuedWatchMetaData.PrioritizedItem(priority=5, item={'uuid': new_uuid}))
+ if not app.datastore.data['watching'].get(uuid).get('paused'):
+ app.update_q.put(queuedWatchMetaData.PrioritizedItem(priority=5, item={'uuid': new_uuid}))
flash('Cloned.')
return redirect(url_for('index'))
@@ -1459,33 +1461,33 @@ def changedetection_app(config=None, datastore_o=None):
i = 0
running_uuids = []
- for t in running_update_threads:
+ for t in app.running_update_threads:
running_uuids.append(t.current_uuid)
if uuid:
if uuid not in running_uuids:
- update_q.put(queuedWatchMetaData.PrioritizedItem(priority=1, item={'uuid': uuid}))
+ app.update_q.put(queuedWatchMetaData.PrioritizedItem(priority=1, item={'uuid': uuid}))
i = 1
elif tag:
# Items that have this current tag
- for watch_uuid, watch in datastore.data['watching'].items():
+ for watch_uuid, watch in app.datastore.data['watching'].items():
if tag in watch.get('tags', {}):
if with_errors and not watch.get('last_error'):
continue
- if watch_uuid not in running_uuids and not datastore.data['watching'][watch_uuid]['paused']:
- update_q.put(
+ if watch_uuid not in running_uuids and not app.datastore.data['watching'][watch_uuid]['paused']:
+ app.update_q.put(
queuedWatchMetaData.PrioritizedItem(priority=1, item={'uuid': watch_uuid})
)
i += 1
else:
# No tag, no uuid, add everything.
- for watch_uuid, watch in datastore.data['watching'].items():
- if watch_uuid not in running_uuids and not datastore.data['watching'][watch_uuid]['paused']:
+ for watch_uuid, watch in app.datastore.data['watching'].items():
+ if watch_uuid not in running_uuids and not app.datastore.data['watching'][watch_uuid]['paused']:
if with_errors and not watch.get('last_error'):
continue
- update_q.put(queuedWatchMetaData.PrioritizedItem(priority=1, item={'uuid': watch_uuid}))
+ app.update_q.put(queuedWatchMetaData.PrioritizedItem(priority=1, item={'uuid': watch_uuid}))
i += 1
flash(f"{i} watches queued for rechecking.")
return redirect(url_for('index', tag=tag))
@@ -1499,65 +1501,65 @@ def changedetection_app(config=None, datastore_o=None):
if (op == 'delete'):
for uuid in uuids:
uuid = uuid.strip()
- if datastore.data['watching'].get(uuid):
- datastore.delete(uuid.strip())
+ if app.datastore.data['watching'].get(uuid):
+ app.datastore.delete(uuid.strip())
flash("{} watches deleted".format(len(uuids)))
elif (op == 'pause'):
for uuid in uuids:
uuid = uuid.strip()
- if datastore.data['watching'].get(uuid):
- datastore.data['watching'][uuid.strip()]['paused'] = True
+ if app.datastore.data['watching'].get(uuid):
+ app.datastore.data['watching'][uuid.strip()]['paused'] = True
flash("{} watches paused".format(len(uuids)))
elif (op == 'unpause'):
for uuid in uuids:
uuid = uuid.strip()
- if datastore.data['watching'].get(uuid):
- datastore.data['watching'][uuid.strip()]['paused'] = False
+ if app.datastore.data['watching'].get(uuid):
+ app.datastore.data['watching'][uuid.strip()]['paused'] = False
flash("{} watches unpaused".format(len(uuids)))
elif (op == 'mark-viewed'):
for uuid in uuids:
uuid = uuid.strip()
- if datastore.data['watching'].get(uuid):
- datastore.set_last_viewed(uuid, int(time.time()))
+ if app.datastore.data['watching'].get(uuid):
+ app.datastore.set_last_viewed(uuid, int(time.time()))
flash("{} watches updated".format(len(uuids)))
elif (op == 'mute'):
for uuid in uuids:
uuid = uuid.strip()
- if datastore.data['watching'].get(uuid):
- datastore.data['watching'][uuid.strip()]['notification_muted'] = True
+ if app.datastore.data['watching'].get(uuid):
+ app.datastore.data['watching'][uuid.strip()]['notification_muted'] = True
flash("{} watches muted".format(len(uuids)))
elif (op == 'unmute'):
for uuid in uuids:
uuid = uuid.strip()
- if datastore.data['watching'].get(uuid):
- datastore.data['watching'][uuid.strip()]['notification_muted'] = False
+ if app.datastore.data['watching'].get(uuid):
+ app.datastore.data['watching'][uuid.strip()]['notification_muted'] = False
flash("{} watches un-muted".format(len(uuids)))
elif (op == 'recheck'):
for uuid in uuids:
uuid = uuid.strip()
- if datastore.data['watching'].get(uuid):
+ if app.datastore.data['watching'].get(uuid):
# Recheck and require a full reprocessing
- update_q.put(queuedWatchMetaData.PrioritizedItem(priority=1, item={'uuid': uuid}))
+ app.update_q.put(queuedWatchMetaData.PrioritizedItem(priority=1, item={'uuid': uuid}))
flash("{} watches queued for rechecking".format(len(uuids)))
elif (op == 'clear-errors'):
for uuid in uuids:
uuid = uuid.strip()
- if datastore.data['watching'].get(uuid):
- datastore.data['watching'][uuid]["last_error"] = False
+ if app.datastore.data['watching'].get(uuid):
+ app.datastore.data['watching'][uuid]["last_error"] = False
flash(f"{len(uuids)} watches errors cleared")
elif (op == 'clear-history'):
for uuid in uuids:
uuid = uuid.strip()
- if datastore.data['watching'].get(uuid):
- datastore.clear_watch_history(uuid)
+ if app.datastore.data['watching'].get(uuid):
+ app.datastore.clear_watch_history(uuid)
flash("{} watches cleared/reset.".format(len(uuids)))
elif (op == 'notification-default'):
@@ -1566,26 +1568,26 @@ def changedetection_app(config=None, datastore_o=None):
)
for uuid in uuids:
uuid = uuid.strip()
- if datastore.data['watching'].get(uuid):
- datastore.data['watching'][uuid.strip()]['notification_title'] = None
- datastore.data['watching'][uuid.strip()]['notification_body'] = None
- datastore.data['watching'][uuid.strip()]['notification_urls'] = []
- datastore.data['watching'][uuid.strip()]['notification_format'] = default_notification_format_for_watch
+ if app.datastore.data['watching'].get(uuid):
+ app.datastore.data['watching'][uuid.strip()]['notification_title'] = None
+ app.datastore.data['watching'][uuid.strip()]['notification_body'] = None
+ app.datastore.data['watching'][uuid.strip()]['notification_urls'] = []
+ app.datastore.data['watching'][uuid.strip()]['notification_format'] = default_notification_format_for_watch
flash("{} watches set to use default notification settings".format(len(uuids)))
elif (op == 'assign-tag'):
op_extradata = request.form.get('op_extradata', '').strip()
if op_extradata:
- tag_uuid = datastore.add_tag(name=op_extradata)
+ tag_uuid = app.datastore.add_tag(name=op_extradata)
if op_extradata and tag_uuid:
for uuid in uuids:
uuid = uuid.strip()
- if datastore.data['watching'].get(uuid):
+ if app.datastore.data['watching'].get(uuid):
# Bug in old versions caused by bad edit page/tag handler
- if isinstance(datastore.data['watching'][uuid]['tags'], str):
- datastore.data['watching'][uuid]['tags'] = []
+ if isinstance(app.datastore.data['watching'][uuid]['tags'], str):
+ app.datastore.data['watching'][uuid]['tags'] = []
- datastore.data['watching'][uuid]['tags'].append(tag_uuid)
+ app.datastore.data['watching'][uuid]['tags'].append(tag_uuid)
flash(f"{len(uuids)} watches were tagged")
@@ -1602,10 +1604,10 @@ def changedetection_app(config=None, datastore_o=None):
# more for testing
if uuid == 'first':
- uuid = list(datastore.data['watching'].keys()).pop()
+ uuid = list(app.datastore.data['watching'].keys()).pop()
# copy it to memory as trim off what we dont need (history)
- watch = deepcopy(datastore.data['watching'][uuid])
+ watch = deepcopy(app.datastore.data['watching'][uuid])
# For older versions that are not a @property
if (watch.get('history')):
del (watch['history'])
@@ -1620,8 +1622,8 @@ def changedetection_app(config=None, datastore_o=None):
del (watch[r])
# Add the global stuff which may have an impact
- watch['ignore_text'] += datastore.data['settings']['application']['global_ignore_text']
- watch['subtractive_selectors'] += datastore.data['settings']['application']['global_subtractive_selectors']
+ watch['ignore_text'] += app.datastore.data['settings']['application']['global_ignore_text']
+ watch['subtractive_selectors'] += app.datastore.data['settings']['application']['global_subtractive_selectors']
watch_json = json.dumps(watch)
@@ -1629,7 +1631,7 @@ def changedetection_app(config=None, datastore_o=None):
r = requests.request(method="POST",
data={'watch': watch_json},
url="https://changedetection.io/share/share",
- headers={'App-Guid': datastore.data['app_guid']})
+ headers={'App-Guid': app.datastore.data['app_guid']})
res = r.json()
session['share-link'] = "https://changedetection.io/share/{}".format(res['share_key'])
@@ -1652,31 +1654,31 @@ def changedetection_app(config=None, datastore_o=None):
selection = request.form.get('selection')
uuid = request.args.get('uuid','')
- if datastore.data["watching"].get(uuid):
+ if app.datastore.data["watching"].get(uuid):
if mode == 'exact':
for l in selection.splitlines():
- datastore.data["watching"][uuid]['ignore_text'].append(l.strip())
+ app.datastore.data["watching"][uuid]['ignore_text'].append(l.strip())
elif mode == 'digit-regex':
for l in selection.splitlines():
# Replace any series of numbers with a regex
s = re.escape(l.strip())
s = re.sub(r'[0-9]+', r'\\d+', s)
- datastore.data["watching"][uuid]['ignore_text'].append('/' + s + '/')
+ app.datastore.data["watching"][uuid]['ignore_text'].append('/' + s + '/')
return f"Click to preview"
import changedetectionio.blueprint.browser_steps as browser_steps
- app.register_blueprint(browser_steps.construct_blueprint(datastore), url_prefix='/browser-steps')
+ app.register_blueprint(browser_steps.construct_blueprint(app.datastore), url_prefix='/browser-steps')
import changedetectionio.blueprint.price_data_follower as price_data_follower
- app.register_blueprint(price_data_follower.construct_blueprint(datastore, update_q), url_prefix='/price_data_follower')
+ app.register_blueprint(price_data_follower.construct_blueprint(app.datastore, app.update_q), url_prefix='/price_data_follower')
import changedetectionio.blueprint.tags as tags
- app.register_blueprint(tags.construct_blueprint(datastore), url_prefix='/tags')
+ app.register_blueprint(tags.construct_blueprint(app.datastore), url_prefix='/tags')
import changedetectionio.blueprint.check_proxies as check_proxies
- app.register_blueprint(check_proxies.construct_blueprint(datastore=datastore), url_prefix='/check_proxy')
+ app.register_blueprint(check_proxies.construct_blueprint(datastore=app.datastore), url_prefix='/check_proxy')
# @todo handle ctrl break
@@ -1700,8 +1702,8 @@ def check_for_new_version():
try:
r = requests.post("https://changedetection.io/check-ver.php",
data={'version': __version__,
- 'app_guid': datastore.data['app_guid'],
- 'watch_count': len(datastore.data['watching'])
+ 'app_guid': app.datastore.data['app_guid'],
+ 'watch_count': len(app.datastore.data['watching'])
},
verify=False)
@@ -1725,7 +1727,7 @@ def notification_runner():
while not app.config.exit.is_set():
try:
# At the moment only one thread runs (single runner)
- n_object = notification_q.get(block=False)
+ n_object = app.notification_q.get(block=False)
except queue.Empty:
time.sleep(1)
@@ -1737,23 +1739,23 @@ def notification_runner():
try:
from changedetectionio import notification
# Fallback to system config if not set
- if not n_object.get('notification_body') and datastore.data['settings']['application'].get('notification_body'):
- n_object['notification_body'] = datastore.data['settings']['application'].get('notification_body')
+ if not n_object.get('notification_body') and app.datastore.data['settings']['application'].get('notification_body'):
+ n_object['notification_body'] = app.datastore.data['settings']['application'].get('notification_body')
- if not n_object.get('notification_title') and datastore.data['settings']['application'].get('notification_title'):
- n_object['notification_title'] = datastore.data['settings']['application'].get('notification_title')
+ if not n_object.get('notification_title') and app.datastore.data['settings']['application'].get('notification_title'):
+ n_object['notification_title'] = app.datastore.data['settings']['application'].get('notification_title')
- if not n_object.get('notification_format') and datastore.data['settings']['application'].get('notification_format'):
- n_object['notification_format'] = datastore.data['settings']['application'].get('notification_format')
+ if not n_object.get('notification_format') and app.datastore.data['settings']['application'].get('notification_format'):
+ n_object['notification_format'] = app.datastore.data['settings']['application'].get('notification_format')
- sent_obj = notification.process_notification(n_object, datastore)
+ sent_obj = notification.process_notification(n_object, app.datastore)
except Exception as e:
logger.error(f"Watch URL: {n_object['watch_url']} Error {str(e)}")
# UUID wont be present when we submit a 'test' from the global settings
if 'uuid' in n_object:
- datastore.update_watch(uuid=n_object['uuid'],
+ app.datastore.update_watch(uuid=n_object['uuid'],
update_obj={'last_notification_error': "Notification error detected, goto notification log."})
log_lines = str(e).splitlines()
@@ -1776,17 +1778,17 @@ def ticker_thread_check_time_launch_checks():
# Spin up Workers that do the fetching
# Can be overriden by ENV or use the default settings
- n_workers = int(os.getenv("FETCH_WORKERS", datastore.data['settings']['requests']['workers']))
+ n_workers = int(os.getenv("FETCH_WORKERS", app.datastore.data['settings']['requests']['workers']))
for _ in range(n_workers):
- new_worker = update_worker.update_worker(update_q, notification_q, app, datastore)
- running_update_threads.append(new_worker)
+ new_worker = update_worker.update_worker(app=app)
+ app.running_update_threads.append(new_worker)
new_worker.start()
while not app.config.exit.is_set():
# Get a list of watches by UUID that are currently fetching data
running_uuids = []
- for t in running_update_threads:
+ for t in app.running_update_threads:
if t.current_uuid:
running_uuids.append(t.current_uuid)
@@ -1796,7 +1798,7 @@ def ticker_thread_check_time_launch_checks():
try:
# Get a list of watches sorted by last_checked, [1] because it gets passed a tuple
# This is so we examine the most over-due first
- for k in sorted(datastore.data['watching'].items(), key=lambda item: item[1].get('last_checked',0)):
+ for k in sorted(app.datastore.data['watching'].items(), key=lambda item: item[1].get('last_checked',0)):
watch_uuid_list.append(k[0])
except RuntimeError as e:
@@ -1806,16 +1808,16 @@ def ticker_thread_check_time_launch_checks():
break
# Re #438 - Don't place more watches in the queue to be checked if the queue is already large
- while update_q.qsize() >= 2000:
+ while app.update_q.qsize() >= 2000:
time.sleep(1)
- recheck_time_system_seconds = int(datastore.threshold_seconds)
+ recheck_time_system_seconds = int(app.datastore.threshold_seconds)
# Check for watches outside of the time threshold to put in the thread queue.
for uuid in watch_uuid_list:
now = time.time()
- watch = datastore.data['watching'].get(uuid)
+ watch = app.datastore.data['watching'].get(uuid)
if not watch:
logger.error(f"Watch: {uuid} no longer present.")
continue
@@ -1828,7 +1830,7 @@ def ticker_thread_check_time_launch_checks():
threshold = recheck_time_system_seconds if watch.get('time_between_check_use_default') else watch.threshold_seconds()
# #580 - Jitter plus/minus amount of time to make the check seem more random to the server
- jitter = datastore.data['settings']['requests'].get('jitter_seconds', 0)
+ jitter = app.datastore.data['settings']['requests'].get('jitter_seconds', 0)
if jitter > 0:
if watch.jitter_seconds == 0:
watch.jitter_seconds = random.uniform(-abs(jitter), jitter)
@@ -1836,13 +1838,13 @@ def ticker_thread_check_time_launch_checks():
seconds_since_last_recheck = now - watch['last_checked']
if seconds_since_last_recheck >= (threshold + watch.jitter_seconds) and seconds_since_last_recheck >= recheck_time_minimum_seconds:
- if not uuid in running_uuids and uuid not in [q_uuid.item['uuid'] for q_uuid in update_q.queue]:
+ if not uuid in running_uuids and uuid not in [q_uuid.item['uuid'] for q_uuid in app.update_q.queue]:
# Proxies can be set to have a limit on seconds between which they can be called
- watch_proxy = datastore.get_preferred_proxy_for_watch(uuid=uuid)
- if watch_proxy and watch_proxy in list(datastore.proxy_list.keys()):
+ watch_proxy = app.datastore.get_preferred_proxy_for_watch(uuid=uuid)
+ if watch_proxy and watch_proxy in list(app.datastore.proxy_list.keys()):
# Proxy may also have some threshold minimum
- proxy_list_reuse_time_minimum = int(datastore.proxy_list.get(watch_proxy, {}).get('reuse_time_minimum', 0))
+ proxy_list_reuse_time_minimum = int(app.datastore.proxy_list.get(watch_proxy, {}).get('reuse_time_minimum', 0))
if proxy_list_reuse_time_minimum:
proxy_last_used_time = proxy_last_called_time.get(watch_proxy, 0)
time_since_proxy_used = int(time.time() - proxy_last_used_time)
@@ -1867,7 +1869,7 @@ def ticker_thread_check_time_launch_checks():
f"{now - watch['last_checked']:0.2f}s since last checked")
# Into the queue with you
- update_q.put(queuedWatchMetaData.PrioritizedItem(priority=priority, item={'uuid': uuid}))
+ app.update_q.put(queuedWatchMetaData.PrioritizedItem(priority=priority, item={'uuid': uuid}))
# Reset for next time
watch.jitter_seconds = 0
diff --git a/changedetectionio/update_worker.py b/changedetectionio/update_worker.py
index af54eff4..800bf39c 100644
--- a/changedetectionio/update_worker.py
+++ b/changedetectionio/update_worker.py
@@ -19,11 +19,9 @@ from loguru import logger
class update_worker(threading.Thread):
current_uuid = None
- def __init__(self, q, notification_q, app, datastore, *args, **kwargs):
- self.q = q
+ def __init__(self, app, *args, **kwargs):
+
self.app = app
- self.notification_q = notification_q
- self.datastore = datastore
super().__init__(*args, **kwargs)
def queue_notification_for_watch(self, notification_q, n_object, watch):
@@ -102,19 +100,19 @@ class update_worker(threading.Thread):
v = watch.get(var_name)
if v and not watch.get('notification_muted'):
if var_name == 'notification_format' and v == default_notification_format_for_watch:
- return self.datastore.data['settings']['application'].get('notification_format')
+ return self.app.datastore.data['settings']['application'].get('notification_format')
return v
- tags = self.datastore.get_all_tags_for_watch(uuid=watch.get('uuid'))
+ tags = self.app.datastore.get_all_tags_for_watch(uuid=watch.get('uuid'))
if tags:
for tag_uuid, tag in tags.items():
v = tag.get(var_name)
if v and not tag.get('notification_muted'):
return v
- if self.datastore.data['settings']['application'].get(var_name):
- return self.datastore.data['settings']['application'].get(var_name)
+ if self.app.datastore.data['settings']['application'].get(var_name):
+ return self.app.datastore.data['settings']['application'].get(var_name)
# Otherwise could be defaults
if var_name == 'notification_format':
@@ -129,7 +127,7 @@ class update_worker(threading.Thread):
def send_content_changed_notification(self, watch_uuid):
n_object = {}
- watch = self.datastore.data['watching'].get(watch_uuid)
+ watch = self.app.datastore.data['watching'].get(watch_uuid)
if not watch:
return
@@ -156,17 +154,17 @@ class update_worker(threading.Thread):
queued = True
count = watch.get('notification_alert_count', 0) + 1
- self.datastore.update_watch(uuid=watch_uuid, update_obj={'notification_alert_count': count})
+ self.app.datastore.update_watch(uuid=watch_uuid, update_obj={'notification_alert_count': count})
- self.queue_notification_for_watch(notification_q=self.notification_q, n_object=n_object, watch=watch)
+ self.queue_notification_for_watch(notification_q=self.app.notification_q, n_object=n_object, watch=watch)
return queued
def send_filter_failure_notification(self, watch_uuid):
- threshold = self.datastore.data['settings']['application'].get('filter_failure_notification_threshold_attempts')
- watch = self.datastore.data['watching'].get(watch_uuid)
+ threshold = self.app.datastore.data['settings']['application'].get('filter_failure_notification_threshold_attempts')
+ watch = self.app.datastore.data['watching'].get(watch_uuid)
if not watch:
return
@@ -179,8 +177,8 @@ class update_worker(threading.Thread):
if len(watch['notification_urls']):
n_object['notification_urls'] = watch['notification_urls']
- elif len(self.datastore.data['settings']['application']['notification_urls']):
- n_object['notification_urls'] = self.datastore.data['settings']['application']['notification_urls']
+ elif len(self.app.datastore.data['settings']['application']['notification_urls']):
+ n_object['notification_urls'] = self.app.datastore.data['settings']['application']['notification_urls']
# Only prepare to notify if the rules above matched
if 'notification_urls' in n_object:
@@ -189,16 +187,16 @@ class update_worker(threading.Thread):
'uuid': watch_uuid,
'screenshot': None
})
- self.notification_q.put(n_object)
+ self.app.notification_q.put(n_object)
logger.debug(f"Sent filter not found notification for {watch_uuid}")
else:
logger.debug(f"NOT sending filter not found notification for {watch_uuid} - no notification URLs")
def send_step_failure_notification(self, watch_uuid, step_n):
- watch = self.datastore.data['watching'].get(watch_uuid, False)
+ watch = self.app.datastore.data['watching'].get(watch_uuid, False)
if not watch:
return
- threshold = self.datastore.data['settings']['application'].get('filter_failure_notification_threshold_attempts')
+ threshold = self.app.datastore.data['settings']['application'].get('filter_failure_notification_threshold_attempts')
n_object = {'notification_title': "Changedetection.io - Alert - Browser step at position {} could not be run".format(step_n+1),
'notification_body': "Your configured browser step at position {} for {{{{watch_url}}}} "
"did not appear on the page after {} attempts, did the page change layout? "
@@ -209,8 +207,8 @@ class update_worker(threading.Thread):
if len(watch['notification_urls']):
n_object['notification_urls'] = watch['notification_urls']
- elif len(self.datastore.data['settings']['application']['notification_urls']):
- n_object['notification_urls'] = self.datastore.data['settings']['application']['notification_urls']
+ elif len(self.app.datastore.data['settings']['application']['notification_urls']):
+ n_object['notification_urls'] = self.app.datastore.data['settings']['application']['notification_urls']
# Only prepare to notify if the rules above matched
if 'notification_urls' in n_object:
@@ -218,7 +216,7 @@ class update_worker(threading.Thread):
'watch_url': watch['url'],
'uuid': watch_uuid
})
- self.notification_q.put(n_object)
+ self.app.notification_q.put(n_object)
logger.error(f"Sent step not found notification for {watch_uuid}")
@@ -226,7 +224,7 @@ class update_worker(threading.Thread):
# All went fine, remove error artifacts
cleanup_files = ["last-error-screenshot.png", "last-error.txt"]
for f in cleanup_files:
- full_path = os.path.join(self.datastore.datastore_path, uuid, f)
+ full_path = os.path.join(self.app.datastore.datastore_path, uuid, f)
if os.path.isfile(full_path):
os.unlink(full_path)
@@ -237,23 +235,23 @@ class update_worker(threading.Thread):
update_handler = None
try:
- queued_item_data = self.q.get(block=False)
+ queued_item_data = self.app.update_q.get(block=False)
except queue.Empty:
pass
else:
uuid = queued_item_data.item.get('uuid')
self.current_uuid = uuid
- if uuid in list(self.datastore.data['watching'].keys()) and self.datastore.data['watching'][uuid].get('url'):
+ if uuid in list(self.app.datastore.data['watching'].keys()) and self.app.datastore.data['watching'][uuid].get('url'):
changed_detected = False
contents = b''
process_changedetection_results = True
update_obj = {}
# Clear last errors (move to preflight func?)
- self.datastore.data['watching'][uuid]['browser_steps_last_error_step'] = None
+ self.app.datastore.data['watching'][uuid]['browser_steps_last_error_step'] = None
- watch = self.datastore.data['watching'].get(uuid)
+ watch = self.app.datastore.data['watching'].get(uuid)
logger.info(f"Processing watch UUID {uuid} Priority {queued_item_data.priority} URL {watch['url']}")
now = time.time()
@@ -270,7 +268,7 @@ class update_worker(threading.Thread):
print(f"Processor module '{processor}' not found.")
raise e
- update_handler = processor_module.perform_site_check(datastore=self.datastore,
+ update_handler = processor_module.perform_site_check(datastore=self.app.datastore,
watch_uuid=uuid
)
@@ -294,7 +292,7 @@ class update_worker(threading.Thread):
watch.save_screenshot(screenshot=e.screenshot)
if e.xpath_data:
watch.save_xpath_data(data=e.xpath_data)
- self.datastore.update_watch(uuid=uuid, update_obj={'last_error': e.message})
+ self.app.datastore.update_watch(uuid=uuid, update_obj={'last_error': e.message})
process_changedetection_results = False
except content_fetchers_exceptions.ReplyWithContentButNoText as e:
@@ -311,7 +309,7 @@ class update_worker(threading.Thread):
else:
extra_help = ", it's possible that the filters were found, but contained no usable text."
- self.datastore.update_watch(uuid=uuid, update_obj={
+ self.app.datastore.update_watch(uuid=uuid, update_obj={
'last_error': f"Got HTML content but no text found (With {e.status_code} reply code){extra_help}"
})
@@ -343,15 +341,15 @@ class update_worker(threading.Thread):
if e.page_text:
watch.save_error_text(contents=e.page_text)
- self.datastore.update_watch(uuid=uuid, update_obj={'last_error': err_text})
+ self.app.datastore.update_watch(uuid=uuid, update_obj={'last_error': err_text})
process_changedetection_results = False
except FilterNotFoundInResponse as e:
- if not self.datastore.data['watching'].get(uuid):
+ if not self.app.datastore.data['watching'].get(uuid):
continue
err_text = "Warning, no filters were found, no change detection ran - Did the page change layout? update your Visual Filter if necessary."
- self.datastore.update_watch(uuid=uuid, update_obj={'last_error': err_text})
+ self.app.datastore.update_watch(uuid=uuid, update_obj={'last_error': err_text})
# Filter wasnt found, but we should still update the visual selector so that they can have a chance to set it up again
if e.screenshot:
@@ -365,7 +363,7 @@ class update_worker(threading.Thread):
c = watch.get('consecutive_filter_failures', 0)
c += 1
# Send notification if we reached the threshold?
- threshold = self.datastore.data['settings']['application'].get('filter_failure_notification_threshold_attempts', 0)
+ threshold = self.app.datastore.data['settings']['application'].get('filter_failure_notification_threshold_attempts', 0)
logger.debug(f"Filter for {uuid} not found, consecutive_filter_failures: {c} of threshold {threshold}")
if c >= threshold:
if not watch.get('notification_muted'):
@@ -374,7 +372,7 @@ class update_worker(threading.Thread):
c = 0
logger.debug(f"Reset filter failure count back to zero")
- self.datastore.update_watch(uuid=uuid, update_obj={'consecutive_filter_failures': c})
+ self.app.datastore.update_watch(uuid=uuid, update_obj={'consecutive_filter_failures': c})
else:
logger.trace(f"{uuid} - filter_failure_notification_send not enabled, skipping")
@@ -386,20 +384,20 @@ class update_worker(threading.Thread):
process_changedetection_results = False
changed_detected = False
except content_fetchers_exceptions.BrowserConnectError as e:
- self.datastore.update_watch(uuid=uuid,
+ self.app.datastore.update_watch(uuid=uuid,
update_obj={'last_error': e.msg
}
)
process_changedetection_results = False
except content_fetchers_exceptions.BrowserFetchTimedOut as e:
- self.datastore.update_watch(uuid=uuid,
+ self.app.datastore.update_watch(uuid=uuid,
update_obj={'last_error': e.msg
}
)
process_changedetection_results = False
except content_fetchers_exceptions.BrowserStepsStepException as e:
- if not self.datastore.data['watching'].get(uuid):
+ if not self.app.datastore.data['watching'].get(uuid):
continue
error_step = e.step_n + 1
@@ -417,7 +415,7 @@ class update_worker(threading.Thread):
logger.debug(f"BrowserSteps exception at step {error_step} {str(e.original_e)}")
- self.datastore.update_watch(uuid=uuid,
+ self.app.datastore.update_watch(uuid=uuid,
update_obj={'last_error': err_text,
'browser_steps_last_error_step': error_step
}
@@ -427,7 +425,7 @@ class update_worker(threading.Thread):
c = watch.get('consecutive_filter_failures', 0)
c += 1
# Send notification if we reached the threshold?
- threshold = self.datastore.data['settings']['application'].get('filter_failure_notification_threshold_attempts',
+ threshold = self.app.datastore.data['settings']['application'].get('filter_failure_notification_threshold_attempts',
0)
logger.error(f"Step for {uuid} not found, consecutive_filter_failures: {c}")
if threshold > 0 and c >= threshold:
@@ -435,26 +433,26 @@ class update_worker(threading.Thread):
self.send_step_failure_notification(watch_uuid=uuid, step_n=e.step_n)
c = 0
- self.datastore.update_watch(uuid=uuid, update_obj={'consecutive_filter_failures': c})
+ self.app.datastore.update_watch(uuid=uuid, update_obj={'consecutive_filter_failures': c})
process_changedetection_results = False
except content_fetchers_exceptions.EmptyReply as e:
# Some kind of custom to-str handler in the exception handler that does this?
err_text = "EmptyReply - try increasing 'Wait seconds before extracting text', Status Code {}".format(e.status_code)
- self.datastore.update_watch(uuid=uuid, update_obj={'last_error': err_text,
+ self.app.datastore.update_watch(uuid=uuid, update_obj={'last_error': err_text,
'last_check_status': e.status_code})
process_changedetection_results = False
except content_fetchers_exceptions.ScreenshotUnavailable as e:
err_text = "Screenshot unavailable, page did not render fully in the expected time or page was too long - try increasing 'Wait seconds before extracting text'"
- self.datastore.update_watch(uuid=uuid, update_obj={'last_error': err_text,
+ self.app.datastore.update_watch(uuid=uuid, update_obj={'last_error': err_text,
'last_check_status': e.status_code})
process_changedetection_results = False
except content_fetchers_exceptions.JSActionExceptions as e:
err_text = "Error running JS Actions - Page request - "+e.message
if e.screenshot:
watch.save_screenshot(screenshot=e.screenshot, as_error=True)
- self.datastore.update_watch(uuid=uuid, update_obj={'last_error': err_text,
+ self.app.datastore.update_watch(uuid=uuid, update_obj={'last_error': err_text,
'last_check_status': e.status_code})
process_changedetection_results = False
except content_fetchers_exceptions.PageUnloadable as e:
@@ -465,26 +463,26 @@ class update_worker(threading.Thread):
if e.screenshot:
watch.save_screenshot(screenshot=e.screenshot, as_error=True)
- self.datastore.update_watch(uuid=uuid, update_obj={'last_error': err_text,
+ self.app.datastore.update_watch(uuid=uuid, update_obj={'last_error': err_text,
'last_check_status': e.status_code,
'has_ldjson_price_data': None})
process_changedetection_results = False
except content_fetchers_exceptions.BrowserStepsInUnsupportedFetcher as e:
err_text = "This watch has Browser Steps configured and so it cannot run with the 'Basic fast Plaintext/HTTP Client', either remove the Browser Steps or select a Chrome fetcher."
- self.datastore.update_watch(uuid=uuid, update_obj={'last_error': err_text})
+ self.app.datastore.update_watch(uuid=uuid, update_obj={'last_error': err_text})
process_changedetection_results = False
logger.error(f"Exception (BrowserStepsInUnsupportedFetcher) reached processing watch UUID: {uuid}")
except Exception as e:
logger.error(f"Exception reached processing watch UUID: {uuid}")
logger.error(str(e))
- self.datastore.update_watch(uuid=uuid, update_obj={'last_error': "Exception: " + str(e)})
+ self.app.datastore.update_watch(uuid=uuid, update_obj={'last_error': "Exception: " + str(e)})
# Other serious error
process_changedetection_results = False
else:
# Crash protection, the watch entry could have been removed by this point (during a slow chrome fetch etc)
- if not self.datastore.data['watching'].get(uuid):
+ if not self.app.datastore.data['watching'].get(uuid):
continue
update_obj['content-type'] = update_handler.fetcher.get_all_headers().get('content-type', '').lower()
@@ -498,14 +496,14 @@ class update_worker(threading.Thread):
self.cleanup_error_artifacts(uuid)
- if not self.datastore.data['watching'].get(uuid):
+ if not self.app.datastore.data['watching'].get(uuid):
continue
#
# Different exceptions mean that we may or may not want to bump the snapshot, trigger notifications etc
if process_changedetection_results:
# Extract as title if possible/requested.
- if self.datastore.data['settings']['application'].get('extract_title_as_title') or watch['extract_title_as_title']:
+ if self.app.datastore.data['settings']['application'].get('extract_title_as_title') or watch['extract_title_as_title']:
if not watch['title'] or not len(watch['title']):
try:
update_obj['title'] = html_tools.extract_element(find='title', html_content=update_handler.fetcher.content)
@@ -516,7 +514,7 @@ class update_worker(threading.Thread):
# Now update after running everything
timestamp = round(time.time())
try:
- self.datastore.update_watch(uuid=uuid, update_obj=update_obj)
+ self.app.datastore.update_watch(uuid=uuid, update_obj=update_obj)
# Also save the snapshot on the first time checked, "last checked" will always be updated, so we just check history length.
@@ -554,7 +552,7 @@ class update_worker(threading.Thread):
# Catch everything possible here, so that if a worker crashes, we don't lose it until restart!
logger.critical("!!!! Exception in update_worker while processing process_changedetection_results !!!")
logger.critical(str(e))
- self.datastore.update_watch(uuid=uuid, update_obj={'last_error': str(e)})
+ self.app.datastore.update_watch(uuid=uuid, update_obj={'last_error': str(e)})
# Always record that we atleast tried
@@ -563,13 +561,13 @@ class update_worker(threading.Thread):
# Record the 'server' header reply, can be used for actions in the future like cloudflare/akamai workarounds
try:
server_header = update_handler.fetcher.headers.get('server', '').strip().lower()[:255]
- self.datastore.update_watch(uuid=uuid,
+ self.app.datastore.update_watch(uuid=uuid,
update_obj={'remote_server_reply': server_header}
)
except Exception as e:
pass
- self.datastore.update_watch(uuid=uuid, update_obj={'fetch_time': round(time.time() - now, 3),
+ self.app.datastore.update_watch(uuid=uuid, update_obj={'fetch_time': round(time.time() - now, 3),
'last_checked': round(time.time()),
'check_count': count
})