From 10068e1f243ff2f40f7986449f221fdf2bdf9f8c Mon Sep 17 00:00:00 2001 From: Kenny Root Date: Mon, 18 Nov 2024 20:43:12 -0800 Subject: [PATCH] Eliminate globals usage This will help running tests because the app isn't initialized automatically by touching the "changedetectionio" package. Moving things out of the __init__.py removes the side-effect of "import changedetection" which means tests can control the state without restarting. This is the first step in making the tests run with only calling "pytest". The fixture use and test setup need to be adjusted to not depend on test ordering. --- Dockerfile | 1 + changedetection.py | 4 +- changedetectionio/__init__.py | 190 ------------- changedetectionio/__main__.py | 204 ++++++++++++++ .../content_fetchers/requests.py | 2 +- changedetectionio/flask_app.py | 257 +++++++++--------- changedetectionio/model/__init__.py | 4 +- changedetectionio/tests/conftest.py | 8 +- changedetectionio/tests/test_security.py | 2 +- 9 files changed, 341 insertions(+), 331 deletions(-) create mode 100644 changedetectionio/__main__.py diff --git a/Dockerfile b/Dockerfile index c993ab24..857aa945 100644 --- a/Dockerfile +++ b/Dockerfile @@ -13,6 +13,7 @@ ARG CRYPTOGRAPHY_DONT_BUILD_RUST=1 RUN apt-get update && apt-get install -y --no-install-recommends \ g++ \ gcc \ + git \ libc-dev \ libffi-dev \ libjpeg-dev \ diff --git a/changedetection.py b/changedetection.py index ead2b8c5..b809123c 100755 --- a/changedetection.py +++ b/changedetection.py @@ -2,5 +2,5 @@ # Only exists for direct CLI usage -import changedetectionio -changedetectionio.main() +from changedetectionio.__main__ import main +main() diff --git a/changedetectionio/__init__.py b/changedetectionio/__init__.py index 70afc105..6bb8f32e 100644 --- a/changedetectionio/__init__.py +++ b/changedetectionio/__init__.py @@ -3,193 +3,3 @@ # Read more https://github.com/dgtlmoon/changedetection.io/wiki __version__ = '0.47.06' - -from changedetectionio.strtobool import strtobool -from json.decoder import JSONDecodeError -import os -os.environ['EVENTLET_NO_GREENDNS'] = 'yes' -import eventlet -import eventlet.wsgi -import getopt -import signal -import socket -import sys - -from changedetectionio import store -from changedetectionio.flask_app import changedetection_app -from loguru import logger - - -# Only global so we can access it in the signal handler -app = None -datastore = None - -# Parent wrapper or OS sends us a SIGTERM/SIGINT, do everything required for a clean shutdown -def sigshutdown_handler(_signo, _stack_frame): - global app - global datastore - name = signal.Signals(_signo).name - logger.critical(f'Shutdown: Got Signal - {name} ({_signo}), Saving DB to disk and calling shutdown') - datastore.sync_to_json() - logger.success('Sync JSON to disk complete.') - # This will throw a SystemExit exception, because eventlet.wsgi.server doesn't know how to deal with it. - # Solution: move to gevent or other server in the future (#2014) - datastore.stop_thread = True - app.config.exit.set() - sys.exit() - -def main(): - global datastore - global app - - datastore_path = None - do_cleanup = False - host = '' - ipv6_enabled = False - port = os.environ.get('PORT') or 5000 - ssl_mode = False - - # On Windows, create and use a default path. - if os.name == 'nt': - datastore_path = os.path.expandvars(r'%APPDATA%\changedetection.io') - os.makedirs(datastore_path, exist_ok=True) - else: - # Must be absolute so that send_from_directory doesnt try to make it relative to backend/ - datastore_path = os.path.join(os.getcwd(), "../datastore") - - try: - opts, args = getopt.getopt(sys.argv[1:], "6Ccsd:h:p:l:", "port") - except getopt.GetoptError: - print('backend.py -s SSL enable -h [host] -p [port] -d [datastore path] -l [debug level - TRACE, DEBUG(default), INFO, SUCCESS, WARNING, ERROR, CRITICAL]') - sys.exit(2) - - create_datastore_dir = False - - # Set a default logger level - logger_level = 'DEBUG' - # Set a logger level via shell env variable - # Used: Dockerfile for CICD - # To set logger level for pytest, see the app function in tests/conftest.py - if os.getenv("LOGGER_LEVEL"): - level = os.getenv("LOGGER_LEVEL") - logger_level = int(level) if level.isdigit() else level.upper() - - for opt, arg in opts: - if opt == '-s': - ssl_mode = True - - if opt == '-h': - host = arg - - if opt == '-p': - port = int(arg) - - if opt == '-d': - datastore_path = arg - - if opt == '-6': - logger.success("Enabling IPv6 listen support") - ipv6_enabled = True - - # Cleanup (remove text files that arent in the index) - if opt == '-c': - do_cleanup = True - - # Create the datadir if it doesnt exist - if opt == '-C': - create_datastore_dir = True - - if opt == '-l': - logger_level = int(arg) if arg.isdigit() else arg.upper() - - # Without this, a logger will be duplicated - logger.remove() - try: - log_level_for_stdout = { 'DEBUG', 'SUCCESS' } - logger.configure(handlers=[ - {"sink": sys.stdout, "level": logger_level, - "filter" : lambda record: record['level'].name in log_level_for_stdout}, - {"sink": sys.stderr, "level": logger_level, - "filter": lambda record: record['level'].name not in log_level_for_stdout}, - ]) - # Catch negative number or wrong log level name - except ValueError: - print("Available log level names: TRACE, DEBUG(default), INFO, SUCCESS," - " WARNING, ERROR, CRITICAL") - sys.exit(2) - - # isnt there some @thingy to attach to each route to tell it, that this route needs a datastore - app_config = {'datastore_path': datastore_path} - - if not os.path.isdir(app_config['datastore_path']): - if create_datastore_dir: - os.mkdir(app_config['datastore_path']) - else: - logger.critical( - f"ERROR: Directory path for the datastore '{app_config['datastore_path']}'" - f" does not exist, cannot start, please make sure the" - f" directory exists or specify a directory with the -d option.\n" - f"Or use the -C parameter to create the directory.") - sys.exit(2) - - try: - datastore = store.ChangeDetectionStore(datastore_path=app_config['datastore_path'], version_tag=__version__) - except JSONDecodeError as e: - # Dont' start if the JSON DB looks corrupt - logger.critical(f"ERROR: JSON DB or Proxy List JSON at '{app_config['datastore_path']}' appears to be corrupt, aborting.") - logger.critical(str(e)) - return - - app = changedetection_app(app_config, datastore) - - signal.signal(signal.SIGTERM, sigshutdown_handler) - signal.signal(signal.SIGINT, sigshutdown_handler) - - # Go into cleanup mode - if do_cleanup: - datastore.remove_unused_snapshots() - - app.config['datastore_path'] = datastore_path - - - @app.context_processor - def inject_version(): - return dict(right_sticky="v{}".format(datastore.data['version_tag']), - new_version_available=app.config['NEW_VERSION_AVAILABLE'], - has_password=datastore.data['settings']['application']['password'] != False - ) - - # Monitored websites will not receive a Referer header when a user clicks on an outgoing link. - # @Note: Incompatible with password login (and maybe other features) for now, submit a PR! - @app.after_request - def hide_referrer(response): - if strtobool(os.getenv("HIDE_REFERER", 'false')): - response.headers["Referrer-Policy"] = "no-referrer" - - return response - - # Proxy sub-directory support - # Set environment var USE_X_SETTINGS=1 on this script - # And then in your proxy_pass settings - # - # proxy_set_header Host "localhost"; - # proxy_set_header X-Forwarded-Prefix /app; - - - if os.getenv('USE_X_SETTINGS'): - logger.info("USE_X_SETTINGS is ENABLED") - from werkzeug.middleware.proxy_fix import ProxyFix - app.wsgi_app = ProxyFix(app.wsgi_app, x_prefix=1, x_host=1) - - s_type = socket.AF_INET6 if ipv6_enabled else socket.AF_INET - - if ssl_mode: - # @todo finalise SSL config, but this should get you in the right direction if you need it. - eventlet.wsgi.server(eventlet.wrap_ssl(eventlet.listen((host, port), s_type), - certfile='cert.pem', - keyfile='privkey.pem', - server_side=True), app) - - else: - eventlet.wsgi.server(eventlet.listen((host, int(port)), s_type), app) - diff --git a/changedetectionio/__main__.py b/changedetectionio/__main__.py new file mode 100644 index 00000000..f0fbe8eb --- /dev/null +++ b/changedetectionio/__main__.py @@ -0,0 +1,204 @@ +from flask import Flask +from changedetectionio.strtobool import strtobool +from json.decoder import JSONDecodeError +import os +os.environ['EVENTLET_NO_GREENDNS'] = 'yes' +import eventlet +import eventlet.wsgi +import getopt +import signal +import socket +import sys + +from changedetectionio import store +from changedetectionio.flask_app import changedetection_app +from loguru import logger + +from . import __version__ + + +# Parent wrapper or OS sends us a SIGTERM/SIGINT, do everything required for a clean shutdown +class SigShutdownHandler(object): + def __init__(self, app): + self.app = app + signal.signal(signal.SIGTERM, lambda _signum, _frame: self._signal_handler("SIGTERM")) + signal.signal(signal.SIGINT, lambda _signum, _frame: self._signal_handler("SIGINT")) + + def _signal_handler(self, signame): + logger.critical(f'Shutdown: Got Signal - {signame}, Saving DB to disk and calling shutdown') + datastore = self.app.config["DATASTORE"] + datastore.sync_to_json() + logger.success('Sync JSON to disk complete.') + # This will throw a SystemExit exception, because eventlet.wsgi.server doesn't know how to deal with it. + # Solution: move to gevent or other server in the future (#2014) + datastore.stop_thread = True + self.app.config.exit.set() + sys.exit(0) + +def create_application() -> Flask: + datastore_path = None + do_cleanup = False + host = '' + ipv6_enabled = False + port = os.environ.get('PORT') or 5000 + ssl_mode = False + + # On Windows, create and use a default path. + if os.name == 'nt': + datastore_path = os.path.expandvars(r'%APPDATA%\changedetection.io') + os.makedirs(datastore_path, exist_ok=True) + else: + # Must be absolute so that send_from_directory doesnt try to make it relative to backend/ + datastore_path = os.path.join(os.getcwd(), "../datastore") + + try: + opts, args = getopt.getopt(sys.argv[1:], "6Ccsd:h:p:l:", "port") + except getopt.GetoptError: + print('backend.py -s SSL enable -h [host] -p [port] -d [datastore path] -l [debug level - TRACE, DEBUG(default), INFO, SUCCESS, WARNING, ERROR, CRITICAL]') + sys.exit(2) + + create_datastore_dir = False + + # Set a default logger level + logger_level = 'DEBUG' + # Set a logger level via shell env variable + # Used: Dockerfile for CICD + # To set logger level for pytest, see the app function in tests/conftest.py + if os.getenv("LOGGER_LEVEL"): + level = os.getenv("LOGGER_LEVEL") + logger_level = int(level) if level.isdigit() else level.upper() + + for opt, arg in opts: + if opt == '-s': + ssl_mode = True + + if opt == '-h': + host = arg + + if opt == '-p': + port = int(arg) + + if opt == '-d': + datastore_path = arg + + if opt == '-6': + logger.success("Enabling IPv6 listen support") + ipv6_enabled = True + + # Cleanup (remove text files that arent in the index) + if opt == '-c': + do_cleanup = True + + # Create the datadir if it doesnt exist + if opt == '-C': + create_datastore_dir = True + + if opt == '-l': + logger_level = int(arg) if arg.isdigit() else arg.upper() + + # Without this, a logger will be duplicated + logger.remove() + try: + log_level_for_stdout = { 'DEBUG', 'SUCCESS' } + logger.configure(handlers=[ + {"sink": sys.stdout, "level": logger_level, + "filter" : lambda record: record['level'].name in log_level_for_stdout}, + {"sink": sys.stderr, "level": logger_level, + "filter": lambda record: record['level'].name not in log_level_for_stdout}, + ]) + # Catch negative number or wrong log level name + except ValueError: + print("Available log level names: TRACE, DEBUG(default), INFO, SUCCESS," + " WARNING, ERROR, CRITICAL") + sys.exit(2) + + # isnt there some @thingy to attach to each route to tell it, that this route needs a datastore + app_config = {'datastore_path': datastore_path} + + if not os.path.isdir(app_config['datastore_path']): + if create_datastore_dir: + os.mkdir(app_config['datastore_path']) + else: + logger.critical( + f"ERROR: Directory path for the datastore '{app_config['datastore_path']}'" + f" does not exist, cannot start, please make sure the" + f" directory exists or specify a directory with the -d option.\n" + f"Or use the -C parameter to create the directory.") + sys.exit(2) + + try: + datastore = store.ChangeDetectionStore(datastore_path=app_config['datastore_path'], version_tag=__version__) + except JSONDecodeError as e: + # Dont' start if the JSON DB looks corrupt + logger.critical(f"ERROR: JSON DB or Proxy List JSON at '{app_config['datastore_path']}' appears to be corrupt, aborting.") + logger.critical(str(e)) + raise(e) + + app = changedetection_app(app_config, datastore) + + sigshutdown_handler = SigShutdownHandler(app) + + # Go into cleanup mode + if do_cleanup: + datastore.remove_unused_snapshots() + + app.config['datastore_path'] = datastore_path + + + @app.context_processor + def inject_version(): + return dict(right_sticky="v{}".format(datastore.data['version_tag']), + new_version_available=app.config['NEW_VERSION_AVAILABLE'], + has_password=datastore.data['settings']['application']['password'] is not False + ) + + # Monitored websites will not receive a Referer header when a user clicks on an outgoing link. + # @Note: Incompatible with password login (and maybe other features) for now, submit a PR! + @app.after_request + def hide_referrer(response): + if strtobool(os.getenv("HIDE_REFERER", 'false')): + response.headers["Referrer-Policy"] = "no-referrer" + + return response + + # Proxy sub-directory support + # Set environment var USE_X_SETTINGS=1 on this script + # And then in your proxy_pass settings + # + # proxy_set_header Host "localhost"; + # proxy_set_header X-Forwarded-Prefix /app; + + + if os.getenv('USE_X_SETTINGS'): + logger.info("USE_X_SETTINGS is ENABLED") + from werkzeug.middleware.proxy_fix import ProxyFix + app.wsgi_app = ProxyFix(app.wsgi_app, x_prefix=1, x_host=1) + + app.config["USE_IPV6"] = ipv6_enabled + app.config["USE_SSL"] = ssl_mode + app.config["HOST"] = host + app.config["PORT"] = port + + return app + +app = create_application() + +def main(): + from .__main__ import app + s_type = socket.AF_INET6 if app.config["USE_IPV6"] else socket.AF_INET + + host = app.config["HOST"] + port = app.config["PORT"] + + if app.config["USE_SSL"]: + # @todo finalise SSL config, but this should get you in the right direction if you need it. + eventlet.wsgi.server(eventlet.wrap_ssl(eventlet.listen((host, port), s_type), + certfile='cert.pem', + keyfile='privkey.pem', + server_side=True), app) + + else: + eventlet.wsgi.server(eventlet.listen((host, int(port)), s_type), app) + +if __name__ == "__main__": + main() diff --git a/changedetectionio/content_fetchers/requests.py b/changedetectionio/content_fetchers/requests.py index 2519aa95..a04b106d 100644 --- a/changedetectionio/content_fetchers/requests.py +++ b/changedetectionio/content_fetchers/requests.py @@ -1,7 +1,7 @@ from loguru import logger import hashlib import os -from changedetectionio import strtobool +from changedetectionio.strtobool import strtobool from changedetectionio.content_fetchers.exceptions import BrowserStepsInUnsupportedFetcher, EmptyReply, Non200ErrorCodeReceived from changedetectionio.content_fetchers.base import Fetcher diff --git a/changedetectionio/flask_app.py b/changedetectionio/flask_app.py index 37da7699..72bbf73a 100644 --- a/changedetectionio/flask_app.py +++ b/changedetectionio/flask_app.py @@ -23,6 +23,7 @@ from flask import ( Flask, abort, flash, + g, make_response, redirect, render_template, @@ -45,8 +46,6 @@ from changedetectionio import html_tools, __version__ from changedetectionio import queuedWatchMetaData from changedetectionio.api import api_v1 -datastore = None - # Local running_update_threads = [] ticker_thread = None @@ -57,44 +56,14 @@ update_q = queue.PriorityQueue() notification_q = queue.Queue() MAX_QUEUE_SIZE = 2000 -app = Flask(__name__, - static_url_path="", - static_folder="static", - template_folder="templates") - -# Enable CORS, especially useful for the Chrome extension to operate from anywhere -CORS(app) - -# Super handy for compressing large BrowserSteps responses and others -FlaskCompress(app) - -# Stop browser caching of assets -app.config['SEND_FILE_MAX_AGE_DEFAULT'] = 0 -app.config.exit = Event() - -app.config['NEW_VERSION_AVAILABLE'] = False - -if os.getenv('FLASK_SERVER_NAME'): - app.config['SERVER_NAME'] = os.getenv('FLASK_SERVER_NAME') - -#app.config["EXPLAIN_TEMPLATE_LOADING"] = True - -# Disables caching of the templates -app.config['TEMPLATES_AUTO_RELOAD'] = True -app.jinja_env.add_extension('jinja2.ext.loopcontrols') -csrf = CSRFProtect() -csrf.init_app(app) -notification_debug_log=[] - -# Locale for correct presentation of prices etc -default_locale = locale.getdefaultlocale() -logger.info(f"System locale default is {default_locale}") -try: - locale.setlocale(locale.LC_ALL, default_locale) -except locale.Error: - logger.warning(f"Unable to set locale {default_locale}, locale is not installed maybe?") - -watch_api = Api(app, decorators=[csrf.exempt]) +def setup_locale(): + # get locale ready + default_locale = locale.getdefaultlocale() + logger.info(f"System locale default is {default_locale}") + try: + locale.setlocale(locale.LC_ALL, default_locale) + except locale.Error: + logger.warning(f"Unable to set locale {default_locale}, locale is not installed maybe?") def init_app_secret(datastore_path): secret = "" @@ -113,61 +82,6 @@ def init_app_secret(datastore_path): return secret - -@app.template_global() -def get_darkmode_state(): - css_dark_mode = request.cookies.get('css_dark_mode', 'false') - return 'true' if css_dark_mode and strtobool(css_dark_mode) else 'false' - -@app.template_global() -def get_css_version(): - return __version__ - -@app.template_filter('format_number_locale') -def _jinja2_filter_format_number_locale(value: float) -> str: - "Formats for example 4000.10 to the local locale default of 4,000.10" - # Format the number with two decimal places (locale format string will return 6 decimal) - formatted_value = locale.format_string("%.2f", value, grouping=True) - - return formatted_value - -# We use the whole watch object from the store/JSON so we can see if there's some related status in terms of a thread -# running or something similar. -@app.template_filter('format_last_checked_time') -def _jinja2_filter_datetime(watch_obj, format="%Y-%m-%d %H:%M:%S"): - # Worker thread tells us which UUID it is currently processing. - for t in running_update_threads: - if t.current_uuid == watch_obj['uuid']: - return ' Checking now' - - if watch_obj['last_checked'] == 0: - return 'Not yet' - - return timeago.format(int(watch_obj['last_checked']), time.time()) - -@app.template_filter('format_timestamp_timeago') -def _jinja2_filter_datetimestamp(timestamp, format="%Y-%m-%d %H:%M:%S"): - if not timestamp: - return 'Not yet' - - return timeago.format(int(timestamp), time.time()) - - -@app.template_filter('pagination_slice') -def _jinja2_filter_pagination_slice(arr, skip): - per_page = datastore.data['settings']['application'].get('pager_size', 50) - if per_page: - return arr[skip:skip + per_page] - - return arr - -@app.template_filter('format_seconds_ago') -def _jinja2_filter_seconds_precise(timestamp): - if timestamp == False: - return 'Not yet' - - return format(int(time.time()-timestamp), ',d') - # When nobody is logged in Flask-Login's current_user is set to an AnonymousUser object. class User(flask_login.UserMixin): id=None @@ -186,7 +100,7 @@ class User(flask_login.UserMixin): return str(self.id) # Compare given password against JSON store or Env var - def check_password(self, password): + def check_password(self, password, *, datastore): import base64 import hashlib @@ -216,39 +130,65 @@ def login_optionally_required(func): @wraps(func) def decorated_view(*args, **kwargs): - has_password_enabled = datastore.data['settings']['application'].get('password') or os.getenv("SALTED_PASS", False) + has_password_enabled = g.datastore.data['settings']['application'].get('password') or os.getenv("SALTED_PASS", False) # Permitted if request.endpoint == 'static_content' and request.view_args['group'] == 'styles': return func(*args, **kwargs) # Permitted - elif request.endpoint == 'diff_history_page' and datastore.data['settings']['application'].get('shared_diff_access'): + elif request.endpoint == 'diff_history_page' and g.datastore.data['settings']['application'].get('shared_diff_access'): return func(*args, **kwargs) elif request.method in flask_login.config.EXEMPT_METHODS: return func(*args, **kwargs) - elif app.config.get('LOGIN_DISABLED'): + elif g.app.config.get('LOGIN_DISABLED'): return func(*args, **kwargs) elif has_password_enabled and not current_user.is_authenticated: - return app.login_manager.unauthorized() + return g.app.login_manager.unauthorized() return func(*args, **kwargs) return decorated_view -def changedetection_app(config=None, datastore_o=None): +def changedetection_app(config, datastore): logger.trace("TRACE log is enabled") - global datastore - datastore = datastore_o + app = Flask(__name__, + static_url_path="", + static_folder="static", + template_folder="templates") - # so far just for read-only via tests, but this will be moved eventually to be the main source - # (instead of the global var) - app.config['DATASTORE'] = datastore_o + # Enable CORS, especially useful for the Chrome extension to operate from anywhere + CORS(app) + + # Super handy for compressing large BrowserSteps responses and others + FlaskCompress(app) + + # Stop browser caching of assets + app.config['SEND_FILE_MAX_AGE_DEFAULT'] = 0 + app.config.exit = Event() + + app.config['NEW_VERSION_AVAILABLE'] = False + + if os.getenv('FLASK_SERVER_NAME'): + app.config['SERVER_NAME'] = os.getenv('FLASK_SERVER_NAME') + + #app.config["EXPLAIN_TEMPLATE_LOADING"] = True + + # Disables caching of the templates + app.config['TEMPLATES_AUTO_RELOAD'] = True + app.jinja_env.add_extension('jinja2.ext.loopcontrols') + csrf = CSRFProtect() + csrf.init_app(app) + + app.config["notification_debug_log"] = [] + + app.config['DATASTORE'] = datastore login_manager = flask_login.LoginManager(app) login_manager.login_view = 'login' app.secret_key = init_app_secret(config['datastore_path']) + watch_api = Api(app, decorators=[csrf.exempt]) watch_api.add_resource(api_v1.WatchSingleHistory, '/api/v1/watch//history/', @@ -271,12 +211,61 @@ def changedetection_app(config=None, datastore_o=None): '/api/v1/import', resource_class_kwargs={'datastore': datastore}) - # Setup cors headers to allow all domains - # https://flask-cors.readthedocs.io/en/latest/ - # CORS(app) + # Flask Templates + @app.template_global() + def get_darkmode_state(): + css_dark_mode = request.cookies.get('css_dark_mode', 'false') + return 'true' if css_dark_mode and strtobool(css_dark_mode) else 'false' + + @app.template_global() + def get_css_version(): + return __version__ + + @app.template_filter('format_number_locale') + def _jinja2_filter_format_number_locale(value: float) -> str: + "Formats for example 4000.10 to the local locale default of 4,000.10" + # Format the number with two decimal places (locale format string will return 6 decimal) + formatted_value = locale.format_string("%.2f", value, grouping=True) + + return formatted_value + + # We use the whole watch object from the store/JSON so we can see if there's some related status in terms of a thread + # running or something similar. + @app.template_filter('format_last_checked_time') + def _jinja2_filter_datetime(watch_obj, format="%Y-%m-%d %H:%M:%S"): + # Worker thread tells us which UUID it is currently processing. + for t in running_update_threads: + if t.current_uuid == watch_obj['uuid']: + return ' Checking now' + + if watch_obj['last_checked'] == 0: + return 'Not yet' + + return timeago.format(int(watch_obj['last_checked']), time.time()) + + @app.template_filter('format_timestamp_timeago') + def _jinja2_filter_datetimestamp(timestamp, format="%Y-%m-%d %H:%M:%S"): + if not timestamp: + return 'Not yet' + + return timeago.format(int(timestamp), time.time()) + + @app.template_filter('pagination_slice') + def _jinja2_filter_pagination_slice(arr, skip): + per_page = datastore.data['settings']['application'].get('pager_size', 50) + if per_page: + return arr[skip:skip + per_page] + return arr + @app.template_filter('format_seconds_ago') + def _jinja2_filter_seconds_precise(timestamp): + if timestamp == False: + return 'Not yet' + return format(int(time.time()-timestamp), ',d') + + # Login Manager @login_manager.user_loader def user_loader(email): user = User() @@ -311,7 +300,7 @@ def changedetection_app(config=None, datastore_o=None): password = request.form.get('password') - if (user.check_password(password)): + if (user.check_password(password, datastore=datastore)): flask_login.login_user(user, remember=True) # For now there's nothing else interesting here other than the index/list page @@ -331,6 +320,11 @@ def changedetection_app(config=None, datastore_o=None): return redirect(url_for('login')) + @app.before_request + def remember_app_and_datastore(): + g.app = app + g.datastore = datastore + @app.before_request def before_request_handle_cookie_x_settings(): # Set the auth cookie path if we're running as X-settings/X-Forwarded-Prefix @@ -344,7 +338,7 @@ def changedetection_app(config=None, datastore_o=None): def rss(): now = time.time() # Always requires token set - app_rss_token = datastore.data['settings']['application'].get('rss_access_token') + app_rss_token = g.datastore.data['settings']['application'].get('rss_access_token') rss_url_token = request.args.get('token') if rss_url_token != app_rss_token: return "Access denied, bad token", 403 @@ -429,7 +423,6 @@ def changedetection_app(config=None, datastore_o=None): @app.route("/", methods=['GET']) @login_optionally_required def index(): - global datastore from changedetectionio import forms active_tag_req = request.args.get('tag', '').lower().strip() @@ -799,7 +792,7 @@ def changedetection_app(config=None, datastore_o=None): # Recast it if need be to right data Watch handler watch_class = get_custom_watch_obj_for_processor(form.data.get('processor')) - datastore.data['watching'][uuid] = watch_class(datastore_path=datastore_o.datastore_path, default=datastore.data['watching'][uuid]) + datastore.data['watching'][uuid] = watch_class(datastore_path=datastore.datastore_path, default=datastore.data['watching'][uuid]) flash("Updated watch - unpaused!" if request.args.get('unpause_on_save') else "Updated watch.") # Re #286 - We wait for syncing new data to disk in another thread every 60 seconds @@ -1083,7 +1076,7 @@ def changedetection_app(config=None, datastore_o=None): extract_regex = request.form.get('extract_regex').strip() output = watch.extract_regex_from_all_history(extract_regex) if output: - watch_dir = os.path.join(datastore_o.datastore_path, uuid) + watch_dir = os.path.join(datastore.datastore_path, uuid) response = make_response(send_from_directory(directory=watch_dir, path=output, as_attachment=True)) response.headers['Content-type'] = 'text/csv' response.headers['Cache-Control'] = 'no-cache, no-store, must-revalidate' @@ -1238,7 +1231,8 @@ def changedetection_app(config=None, datastore_o=None): @app.route("/settings/notification-logs", methods=['GET']) @login_optionally_required def notification_logs(): - global notification_debug_log + notification_debug_log = app.config["notification_debug_log"] + output = render_template("notification-log.html", logs=notification_debug_log if len(notification_debug_log) else ["Notification logs are empty - no notifications sent yet."]) @@ -1258,7 +1252,7 @@ def changedetection_app(config=None, datastore_o=None): # These files should be in our subdirectory try: # set nocache, set content-type - response = make_response(send_from_directory(os.path.join(datastore_o.datastore_path, filename), screenshot_filename)) + response = make_response(send_from_directory(os.path.join(datastore.datastore_path, filename), screenshot_filename)) response.headers['Content-type'] = 'image/png' response.headers['Cache-Control'] = 'no-cache, no-store, must-revalidate' response.headers['Pragma'] = 'no-cache' @@ -1278,7 +1272,7 @@ def changedetection_app(config=None, datastore_o=None): try: # set nocache, set content-type, # `filename` is actually directory UUID of the watch - watch_directory = str(os.path.join(datastore_o.datastore_path, filename)) + watch_directory = str(os.path.join(datastore.datastore_path, filename)) response = None if os.path.isfile(os.path.join(watch_directory, "elements.deflate")): response = make_response(send_from_directory(watch_directory, "elements.deflate")) @@ -1338,7 +1332,6 @@ def changedetection_app(config=None, datastore_o=None): from .processors.text_json_diff import prepare_filter_prevew return prepare_filter_prevew(watch_uuid=uuid, datastore=datastore) - @app.route("/form/add/quickwatch", methods=['POST']) @login_optionally_required def form_quick_watch_add(): @@ -1369,8 +1362,6 @@ def changedetection_app(config=None, datastore_o=None): return redirect(url_for('index')) - - @app.route("/api/delete", methods=['GET']) @login_optionally_required def form_delete(): @@ -1639,25 +1630,27 @@ def changedetection_app(config=None, datastore_o=None): # @todo handle ctrl break - ticker_thread = threading.Thread(target=ticker_thread_check_time_launch_checks).start() - threading.Thread(target=notification_runner).start() + threading.Thread(target=ticker_thread_check_time_launch_checks, args=(app,)).start() + threading.Thread(target=notification_runner, args=(app,)).start() # Check for new release version, but not when running in test/build or pytest if not os.getenv("GITHUB_REF", False) and not strtobool(os.getenv('DISABLE_VERSION_CHECK', 'no')): - threading.Thread(target=check_for_new_version).start() + threading.Thread(target=check_for_new_version, args=(app,)).start() return app # Check for new version and anonymous stats -def check_for_new_version(): +def check_for_new_version(app, url="https://changedetection.io/check-ver.php", delay_time=86400): import requests import urllib3 urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) + datastore = app.config["DATASTORE"] + while not app.config.exit.is_set(): try: - r = requests.post("https://changedetection.io/check-ver.php", + r = requests.post(url, data={'version': __version__, 'app_guid': datastore.data['app_guid'], 'watch_count': len(datastore.data['watching']) @@ -1674,11 +1667,10 @@ def check_for_new_version(): pass # Check daily - app.config.exit.wait(86400) + app.config.exit.wait(delay_time) -def notification_runner(): - global notification_debug_log +def notification_runner(app): from datetime import datetime import json while not app.config.exit.is_set(): @@ -1693,6 +1685,10 @@ def notification_runner(): now = datetime.now() sent_obj = None + notification_debug_log = app.config["notification_debug_log"] + + datastore = app.config["DATASTORE"] + try: from changedetectionio import notification # Fallback to system config if not set @@ -1724,9 +1720,12 @@ def notification_runner(): notification_debug_log = notification_debug_log[-100:] # Threaded runner, look for new watches to feed into the Queue. -def ticker_thread_check_time_launch_checks(): +def ticker_thread_check_time_launch_checks(app): import random from changedetectionio import update_worker + + datastore = app.config["DATASTORE"] + proxy_last_called_time = {} recheck_time_minimum_seconds = int(os.getenv('MINIMUM_SECONDS_RECHECK_TIME', 3)) diff --git a/changedetectionio/model/__init__.py b/changedetectionio/model/__init__.py index 36d12384..55d97b2a 100644 --- a/changedetectionio/model/__init__.py +++ b/changedetectionio/model/__init__.py @@ -1,7 +1,7 @@ import os import uuid -from changedetectionio import strtobool +from changedetectionio.strtobool import strtobool from changedetectionio.notification import default_notification_format_for_watch class watch_base(dict): @@ -73,4 +73,4 @@ class watch_base(dict): super(watch_base, self).__init__(*arg, **kw) if self.get('default'): - del self['default'] \ No newline at end of file + del self['default'] diff --git a/changedetectionio/tests/conftest.py b/changedetectionio/tests/conftest.py index 50f7104b..90c11088 100644 --- a/changedetectionio/tests/conftest.py +++ b/changedetectionio/tests/conftest.py @@ -4,21 +4,17 @@ import time from threading import Thread import pytest -from changedetectionio import changedetection_app +from changedetectionio.flask_app import changedetection_app from changedetectionio import store import os import sys from loguru import logger + # https://github.com/pallets/flask/blob/1.1.2/examples/tutorial/tests/test_auth.py # Much better boilerplate than the docs # https://www.python-boilerplate.com/py3+flask+pytest/ -global app - -# https://loguru.readthedocs.io/en/latest/resources/migration.html#replacing-caplog-fixture-from-pytest-library -# Show loguru logs only if CICD pytest fails. -from loguru import logger @pytest.fixture def reportlog(pytestconfig): logging_plugin = pytestconfig.pluginmanager.getplugin("logging-plugin") diff --git a/changedetectionio/tests/test_security.py b/changedetectionio/tests/test_security.py index 0dfbdcba..0772a726 100644 --- a/changedetectionio/tests/test_security.py +++ b/changedetectionio/tests/test_security.py @@ -4,7 +4,7 @@ from flask import url_for from .util import set_original_response, set_modified_response, live_server_setup, wait_for_all_checks import time -from .. import strtobool +from changedetectionio.strtobool import strtobool def test_setup(client, live_server, measure_memory_usage):