diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 67ef6cd3..413ce4b1 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -4,6 +4,10 @@ updates: directory: / schedule: interval: "weekly" + "caronc/apprise": + versioning-strategy: "increase" + schedule: + interval: "daily" groups: all: patterns: diff --git a/.github/workflows/test-only.yml b/.github/workflows/test-only.yml index 8703797d..3088432a 100644 --- a/.github/workflows/test-only.yml +++ b/.github/workflows/test-only.yml @@ -59,6 +59,7 @@ jobs: echo "run test with unittest" docker run test-changedetectionio bash -c 'python3 -m unittest changedetectionio.tests.unit.test_notification_diff' docker run test-changedetectionio bash -c 'python3 -m unittest changedetectionio.tests.unit.test_watch_model' + docker run test-changedetectionio bash -c 'python3 -m unittest changedetectionio.tests.unit.test_jinja2_security' # All tests echo "run test with pytest" diff --git a/README.md b/README.md index 52aabe9e..d941eccb 100644 --- a/README.md +++ b/README.md @@ -257,13 +257,7 @@ Supports managing the website watch list [via our API](https://changedetection.i Do you use changedetection.io to make money? does it save you time or money? Does it make your life easier? less stressful? Remember, we write this software when we should be doing actual paid work, we have to buy food and pay rent just like you. -Firstly, consider taking out an officially supported [website change detection subscription](https://changedetection.io?src=github) , even if you don't use it, you still get the warm fuzzy feeling of helping out the project. (And who knows, you might just use it!) - -Or directly donate an amount PayPal [![Donate](https://img.shields.io/badge/Donate-PayPal-green.svg)](https://www.paypal.com/donate/?hosted_button_id=7CP6HR9ZCNDYJ) - -Or BTC `1PLFN327GyUarpJd7nVe7Reqg9qHx5frNn` - -Support us! +Consider taking out an officially supported [website change detection subscription](https://changedetection.io?src=github) , even if you don't use it, you still get the warm fuzzy feeling of helping out the project. (And who knows, you might just use it!) ## Commercial Support diff --git a/changedetectionio/__init__.py b/changedetectionio/__init__.py index 72750107..8ec6bb8d 100644 --- a/changedetectionio/__init__.py +++ b/changedetectionio/__init__.py @@ -2,12 +2,12 @@ # Read more https://github.com/dgtlmoon/changedetection.io/wiki -__version__ = '0.45.17' +__version__ = '0.45.22' from changedetectionio.strtobool import strtobool from json.decoder import JSONDecodeError import os -#os.environ['EVENTLET_NO_GREENDNS'] = 'yes' +os.environ['EVENTLET_NO_GREENDNS'] = 'yes' import eventlet import eventlet.wsgi import getopt diff --git a/changedetectionio/blueprint/browser_steps/__init__.py b/changedetectionio/blueprint/browser_steps/__init__.py index 24440908..30797099 100644 --- a/changedetectionio/blueprint/browser_steps/__init__.py +++ b/changedetectionio/blueprint/browser_steps/__init__.py @@ -84,7 +84,9 @@ def construct_blueprint(datastore: ChangeDetectionStore): # Tell Playwright to connect to Chrome and setup a new session via our stepper interface browsersteps_start_session['browserstepper'] = browser_steps.browsersteps_live_ui( playwright_browser=browsersteps_start_session['browser'], - proxy=proxy) + proxy=proxy, + start_url=datastore.data['watching'][watch_uuid].get('url') + ) # For test #browsersteps_start_session['browserstepper'].action_goto_url(value="http://example.com?time="+str(time.time())) @@ -167,11 +169,6 @@ def construct_blueprint(datastore: ChangeDetectionStore): step_n = int(request.form.get('step_n')) is_last_step = strtobool(request.form.get('is_last_step')) - if step_operation == 'Goto site': - step_operation = 'goto_url' - step_optional_value = datastore.data['watching'][uuid].get('url') - step_selector = None - # @todo try.. accept.. nice errors not popups.. try: diff --git a/changedetectionio/blueprint/browser_steps/browser_steps.py b/changedetectionio/blueprint/browser_steps/browser_steps.py index 6bb58b38..76f3d756 100644 --- a/changedetectionio/blueprint/browser_steps/browser_steps.py +++ b/changedetectionio/blueprint/browser_steps/browser_steps.py @@ -7,6 +7,7 @@ from random import randint from loguru import logger from changedetectionio.content_fetchers.base import manage_user_agent +from changedetectionio.safe_jinja import render as jinja_render # Two flags, tell the JS which of the "Selector" or "Value" field should be enabled in the front end # 0- off, 1- on @@ -48,6 +49,10 @@ browser_step_ui_config = {'Choose one': '0 0', # ONLY Works in Playwright because we need the fullscreen screenshot class steppable_browser_interface(): page = None + start_url = None + + def __init__(self, start_url): + self.start_url = start_url # Convert and perform "Click Button" for example def call_action(self, action_name, selector=None, optional_value=None): @@ -64,14 +69,12 @@ class steppable_browser_interface(): action_handler = getattr(self, "action_" + call_action_name) # Support for Jinja2 variables in the value and selector - from jinja2 import Environment - jinja2_env = Environment(extensions=['jinja2_time.TimeExtension']) if selector and ('{%' in selector or '{{' in selector): - selector = str(jinja2_env.from_string(selector).render()) + selector = jinja_render(template_str=selector) if optional_value and ('{%' in optional_value or '{{' in optional_value): - optional_value = str(jinja2_env.from_string(optional_value).render()) + optional_value = jinja_render(template_str=optional_value) action_handler(selector, optional_value) self.page.wait_for_timeout(1.5 * 1000) @@ -88,6 +91,10 @@ class steppable_browser_interface(): logger.debug(f"Time to goto URL {time.time()-now:.2f}s") return response + # Incase they request to go back to the start + def action_goto_site(self, selector=None, value=None): + return self.action_goto_url(value=self.start_url) + def action_click_element_containing_text(self, selector=None, value=''): if not len(value.strip()): return @@ -195,10 +202,11 @@ class browsersteps_live_ui(steppable_browser_interface): browser_type = os.getenv("PLAYWRIGHT_BROWSER_TYPE", 'chromium').strip('"') - def __init__(self, playwright_browser, proxy=None, headers=None): + def __init__(self, playwright_browser, proxy=None, headers=None, start_url=None): self.headers = headers or {} self.age_start = time.time() self.playwright_browser = playwright_browser + self.start_url = start_url if self.context is None: self.connect(proxy=proxy) diff --git a/changedetectionio/blueprint/check_proxies/__init__.py b/changedetectionio/blueprint/check_proxies/__init__.py index db4bbe62..62a7dab3 100644 --- a/changedetectionio/blueprint/check_proxies/__init__.py +++ b/changedetectionio/blueprint/check_proxies/__init__.py @@ -31,9 +31,9 @@ def construct_blueprint(datastore: ChangeDetectionStore): import time from changedetectionio.content_fetchers import exceptions as content_fetcher_exceptions from changedetectionio.processors import text_json_diff + from changedetectionio.safe_jinja import render as jinja_render status = {'status': '', 'length': 0, 'text': ''} - from jinja2 import Environment, BaseLoader contents = '' now = time.time() @@ -64,7 +64,9 @@ def construct_blueprint(datastore: ChangeDetectionStore): status.update({'status': 'OK', 'length': len(contents), 'text': ''}) if status.get('text'): - status['text'] = Environment(loader=BaseLoader()).from_string('{{text|e}}').render({'text': status['text']}) + # parse 'text' as text for safety + v = {'text': status['text']} + status['text'] = jinja_render(template_str='{{text|e}}', **v) status['time'] = "{:.2f}s".format(time.time() - now) diff --git a/changedetectionio/blueprint/tags/__init__.py b/changedetectionio/blueprint/tags/__init__.py index ba20cb4a..7a49822b 100644 --- a/changedetectionio/blueprint/tags/__init__.py +++ b/changedetectionio/blueprint/tags/__init__.py @@ -12,9 +12,15 @@ def construct_blueprint(datastore: ChangeDetectionStore): from .form import SingleTag add_form = SingleTag(request.form) sorted_tags = sorted(datastore.data['settings']['application'].get('tags').items(), key=lambda x: x[1]['title']) + + from collections import Counter + + tag_count = Counter(tag for watch in datastore.data['watching'].values() if watch.get('tags') for tag in watch['tags']) + output = render_template("groups-overview.html", - form=add_form, available_tags=sorted_tags, + form=add_form, + tag_count=tag_count ) return output diff --git a/changedetectionio/blueprint/tags/templates/edit-tag.html b/changedetectionio/blueprint/tags/templates/edit-tag.html index 9834f566..1d297c81 100644 --- a/changedetectionio/blueprint/tags/templates/edit-tag.html +++ b/changedetectionio/blueprint/tags/templates/edit-tag.html @@ -1,9 +1,9 @@ {% extends 'base.html' %} {% block content %} -{% from '_helpers.jinja' import render_field, render_checkbox_field, render_button %} -{% from '_common_fields.jinja' import render_common_settings_form %} +{% from '_helpers.html' import render_field, render_checkbox_field, render_button %} +{% from '_common_fields.html' import render_common_settings_form %} diff --git a/changedetectionio/blueprint/tags/templates/groups-overview.html b/changedetectionio/blueprint/tags/templates/groups-overview.html index 7d942f43..30ddfe18 100644 --- a/changedetectionio/blueprint/tags/templates/groups-overview.html +++ b/changedetectionio/blueprint/tags/templates/groups-overview.html @@ -1,6 +1,6 @@ {% extends 'base.html' %} {% block content %} -{% from '_helpers.jinja' import render_simple_field, render_field %} +{% from '_helpers.html' import render_simple_field, render_field %}
@@ -27,6 +27,7 @@ + # Watches Tag / Label name @@ -45,7 +46,8 @@ Mute notifications - {{tag.title}} + {{ "{:,}".format(tag_count[uuid]) if uuid in tag_count else 0 }} + {{ tag.title }} Edit  Delete diff --git a/changedetectionio/content_fetchers/base.py b/changedetectionio/content_fetchers/base.py index 756a9bef..f817341d 100644 --- a/changedetectionio/content_fetchers/base.py +++ b/changedetectionio/content_fetchers/base.py @@ -112,24 +112,26 @@ class Fetcher(): def browser_steps_get_valid_steps(self): if self.browser_steps is not None and len(self.browser_steps): - valid_steps = filter( - lambda s: (s['operation'] and len(s['operation']) and s['operation'] != 'Choose one' and s['operation'] != 'Goto site'), - self.browser_steps) + valid_steps = list(filter( + lambda s: (s['operation'] and len(s['operation']) and s['operation'] != 'Choose one'), + self.browser_steps)) + + # Just incase they selected Goto site by accident with older JS + if valid_steps and valid_steps[0]['operation'] == 'Goto site': + del(valid_steps[0]) return valid_steps return None - def iterate_browser_steps(self): + def iterate_browser_steps(self, start_url=None): from changedetectionio.blueprint.browser_steps.browser_steps import steppable_browser_interface from playwright._impl._errors import TimeoutError, Error - from jinja2 import Environment - jinja2_env = Environment(extensions=['jinja2_time.TimeExtension']) - + from changedetectionio.safe_jinja import render as jinja_render step_n = 0 if self.browser_steps is not None and len(self.browser_steps): - interface = steppable_browser_interface() + interface = steppable_browser_interface(start_url=start_url) interface.page = self.page valid_steps = self.browser_steps_get_valid_steps() @@ -143,9 +145,9 @@ class Fetcher(): selector = step['selector'] # Support for jinja2 template in step values, with date module added if '{%' in step['optional_value'] or '{{' in step['optional_value']: - optional_value = str(jinja2_env.from_string(step['optional_value']).render()) + optional_value = jinja_render(template_str=step['optional_value']) if '{%' in step['selector'] or '{{' in step['selector']: - selector = str(jinja2_env.from_string(step['selector']).render()) + selector = jinja_render(template_str=step['selector']) getattr(interface, "call_action")(action_name=step['operation'], selector=selector, diff --git a/changedetectionio/content_fetchers/playwright.py b/changedetectionio/content_fetchers/playwright.py index 7950e033..04ab2759 100644 --- a/changedetectionio/content_fetchers/playwright.py +++ b/changedetectionio/content_fetchers/playwright.py @@ -119,7 +119,7 @@ class fetcher(Fetcher): # Re-use as much code from browser steps as possible so its the same from changedetectionio.blueprint.browser_steps.browser_steps import steppable_browser_interface - browsersteps_interface = steppable_browser_interface() + browsersteps_interface = steppable_browser_interface(start_url=url) browsersteps_interface.page = self.page response = browsersteps_interface.action_goto_url(value=url) @@ -172,7 +172,7 @@ class fetcher(Fetcher): # Run Browser Steps here if self.browser_steps_get_valid_steps(): - self.iterate_browser_steps() + self.iterate_browser_steps(start_url=url) self.page.wait_for_timeout(extra_wait * 1000) diff --git a/changedetectionio/flask_app.py b/changedetectionio/flask_app.py index 11c65d9b..49880b5d 100644 --- a/changedetectionio/flask_app.py +++ b/changedetectionio/flask_app.py @@ -5,11 +5,11 @@ import os import queue import threading import time -from copy import deepcopy +from .safe_jinja import render as jinja_render from changedetectionio.strtobool import strtobool +from copy import deepcopy from functools import wraps from threading import Event - import flask_login import pytz import timeago @@ -319,8 +319,6 @@ def changedetection_app(config=None, datastore_o=None): @app.route("/rss", methods=['GET']) def rss(): - from jinja2 import Environment, BaseLoader - jinja2_env = Environment(loader=BaseLoader) now = time.time() # Always requires token set app_rss_token = datastore.data['settings']['application'].get('rss_access_token') @@ -388,7 +386,7 @@ def changedetection_app(config=None, datastore_o=None): # @todo Make this configurable and also consider html-colored markup # @todo User could decide if goes to the diff page, or to the watch link rss_template = "\n

{{watch_title}}

\n

{{html_diff}}

\n\n" - content = jinja2_env.from_string(rss_template).render(watch_title=watch_title, html_diff=html_diff, watch_url=watch.link) + content = jinja_render(template_str=rss_template, watch_title=watch_title, html_diff=html_diff, watch_url=watch.link) fe.content(content=content, type='CDATA') @@ -452,6 +450,8 @@ def changedetection_app(config=None, datastore_o=None): if search_q: if (watch.get('title') and search_q in watch.get('title').lower()) or search_q in watch.get('url', '').lower(): sorted_watches.append(watch) + elif watch.get('last_error') and search_q in watch.get('last_error').lower(): + sorted_watches.append(watch) else: sorted_watches.append(watch) @@ -516,21 +516,38 @@ def changedetection_app(config=None, datastore_o=None): watch = datastore.data['watching'].get(watch_uuid) if watch_uuid else None - # validate URLS - if not len(request.form['notification_urls'].strip()): - return make_response({'error': 'No Notification URLs set'}, 400) + notification_urls = request.form['notification_urls'].strip().splitlines() + + if not notification_urls: + logger.debug("Test notification - Trying by group/tag in the edit form if available") + # On an edit page, we should also fire off to the tags if they have notifications + if request.form.get('tags') and request.form['tags'].strip(): + for k in request.form['tags'].split(','): + tag = datastore.tag_exists_by_name(k.strip()) + notification_urls = tag.get('notifications_urls') if tag and tag.get('notifications_urls') else None - for server_url in request.form['notification_urls'].splitlines(): - if len(server_url.strip()): - if not apobj.add(server_url): - message = '{} is not a valid AppRise URL.'.format(server_url) - return make_response({'error': message}, 400) + is_global_settings_form = request.args.get('mode', '') == 'global-settings' + is_group_settings_form = request.args.get('mode', '') == 'group-settings' + if not notification_urls and not is_global_settings_form and not is_group_settings_form: + # In the global settings, use only what is typed currently in the text box + logger.debug("Test notification - Trying by global system settings notifications") + if datastore.data['settings']['application'].get('notification_urls'): + notification_urls = datastore.data['settings']['application']['notification_urls'] + + + if not notification_urls: + return 'No Notification URLs set/found' + + for n_url in notification_urls: + if len(n_url.strip()): + if not apobj.add(n_url): + return f'Error - {n_url} is not a valid AppRise URL.' try: # use the same as when it is triggered, but then override it with the form test values n_object = { 'watch_url': request.form['window_url'], - 'notification_urls': request.form['notification_urls'].splitlines() + 'notification_urls': notification_urls } # Only use if present, if not set in n_object it should use the default system value @@ -549,7 +566,7 @@ def changedetection_app(config=None, datastore_o=None): except Exception as e: return make_response({'error': str(e)}, 400) - return 'OK' + return 'OK - Sent test notifications' @app.route("/clear_history/", methods=['GET']) @@ -586,6 +603,12 @@ def changedetection_app(config=None, datastore_o=None): output = render_template("clear_all_history.html") return output + def _watch_has_tag_options_set(watch): + """This should be fixed better so that Tag is some proper Model, a tag is just a Watch also""" + for tag_uuid, tag in datastore.data['settings']['application'].get('tags', {}).items(): + if tag_uuid in watch.get('tags', []) and (tag.get('include_filters') or tag.get('subtractive_selectors')): + return True + @app.route("/edit/", methods=['GET', 'POST']) @login_optionally_required # https://stackoverflow.com/questions/42984453/wtforms-populate-form-with-data-if-data-exists @@ -596,7 +619,6 @@ def changedetection_app(config=None, datastore_o=None): from .blueprint.browser_steps.browser_steps import browser_step_ui_config from . import processors - using_default_check_time = True # More for testing, possible to return the first/only if not datastore.data['watching'].keys(): flash("No watches to edit", "error") @@ -621,10 +643,6 @@ def changedetection_app(config=None, datastore_o=None): # be sure we update with a copy instead of accidently editing the live object by reference default = deepcopy(datastore.data['watching'][uuid]) - # Show system wide default if nothing configured - if all(value == 0 or value == None for value in datastore.data['watching'][uuid]['time_between_check'].values()): - default['time_between_check'] = deepcopy(datastore.data['settings']['requests']['time_between_check']) - # Defaults for proxy choice if datastore.proxy_list is not None: # When enabled # @todo @@ -662,18 +680,8 @@ def changedetection_app(config=None, datastore_o=None): if request.args.get('unpause_on_save'): extra_update_obj['paused'] = False - # Re #110, if they submit the same as the default value, set it to None, so we continue to follow the default - # Assume we use the default value, unless something relevant is different, then use the form value - # values could be None, 0 etc. - # Set to None unless the next for: says that something is different - extra_update_obj['time_between_check'] = dict.fromkeys(form.time_between_check.data) - for k, v in form.time_between_check.data.items(): - if v and v != datastore.data['settings']['requests']['time_between_check'][k]: - extra_update_obj['time_between_check'] = form.time_between_check.data - using_default_check_time = False - break - + extra_update_obj['time_between_check'] = form.time_between_check.data # Ignore text form_ignore_text = form.ignore_text.data @@ -754,8 +762,8 @@ def changedetection_app(config=None, datastore_o=None): extra_title=f" - Edit - {watch.label}", form=form, has_default_notification_urls=True if len(datastore.data['settings']['application']['notification_urls']) else False, - has_empty_checktime=using_default_check_time, has_extra_headers_file=len(datastore.get_all_headers_in_textfile_for_watch(uuid=uuid)) > 0, + has_special_tag_options=_watch_has_tag_options_set(watch=watch), is_html_webdriver=is_html_webdriver, jq_support=jq_support, playwright_enabled=os.getenv('PLAYWRIGHT_DRIVER_URL', False), @@ -839,11 +847,13 @@ def changedetection_app(config=None, datastore_o=None): flash("An error occurred, please see below.", "error") output = render_template("settings.html", - form=form, - hide_remove_pass=os.getenv("SALTED_PASS", False), api_key=datastore.data['settings']['application'].get('api_access_token'), emailprefix=os.getenv('NOTIFICATION_MAIL_BUTTON_PREFIX', False), - settings_application=datastore.data['settings']['application']) + form=form, + hide_remove_pass=os.getenv("SALTED_PASS", False), + min_system_recheck_seconds=int(os.getenv('MINIMUM_SECONDS_RECHECK_TIME', 3)), + settings_application=datastore.data['settings']['application'] + ) return output @@ -1279,9 +1289,8 @@ def changedetection_app(config=None, datastore_o=None): url = request.form.get('url').strip() if datastore.url_exists(url): - flash('The URL {} already exists'.format(url), "error") - return redirect(url_for('index')) - + flash(f'Warning, URL {url} already exists', "notice") + add_paused = request.form.get('edit_and_watch_submit_button') != None processor = request.form.get('processor', 'text_json_diff') new_uuid = datastore.add_watch(url=url, tag=request.form.get('tags').strip(), extras={'paused': add_paused, 'processor': processor}) @@ -1645,14 +1654,14 @@ def notification_runner(): # Trim the log length notification_debug_log = notification_debug_log[-100:] -# Thread runner to check every minute, look for new watches to feed into the Queue. +# Threaded runner, look for new watches to feed into the Queue. def ticker_thread_check_time_launch_checks(): import random from changedetectionio import update_worker proxy_last_called_time = {} - recheck_time_minimum_seconds = int(os.getenv('MINIMUM_SECONDS_RECHECK_TIME', 20)) + recheck_time_minimum_seconds = int(os.getenv('MINIMUM_SECONDS_RECHECK_TIME', 3)) logger.debug(f"System env MINIMUM_SECONDS_RECHECK_TIME {recheck_time_minimum_seconds}") # Spin up Workers that do the fetching @@ -1706,9 +1715,7 @@ def ticker_thread_check_time_launch_checks(): continue # If they supplied an individual entry minutes to threshold. - - watch_threshold_seconds = watch.threshold_seconds() - threshold = watch_threshold_seconds if watch_threshold_seconds > 0 else recheck_time_system_seconds + threshold = recheck_time_system_seconds if watch.get('time_between_check_use_default') else watch.threshold_seconds() # #580 - Jitter plus/minus amount of time to make the check seem more random to the server jitter = datastore.data['settings']['requests'].get('jitter_seconds', 0) diff --git a/changedetectionio/forms.py b/changedetectionio/forms.py index 4d408c61..2d64a227 100644 --- a/changedetectionio/forms.py +++ b/changedetectionio/forms.py @@ -236,21 +236,26 @@ class ValidateJinja2Template(object): def __call__(self, form, field): from changedetectionio import notification - from jinja2 import Environment, BaseLoader, TemplateSyntaxError, UndefinedError + from jinja2 import BaseLoader, TemplateSyntaxError, UndefinedError + from jinja2.sandbox import ImmutableSandboxedEnvironment from jinja2.meta import find_undeclared_variables + import jinja2.exceptions + # Might be a list of text, or might be just text (like from the apprise url list) + joined_data = ' '.join(map(str, field.data)) if isinstance(field.data, list) else f"{field.data}" try: - jinja2_env = Environment(loader=BaseLoader) + jinja2_env = ImmutableSandboxedEnvironment(loader=BaseLoader) jinja2_env.globals.update(notification.valid_tokens) - - rendered = jinja2_env.from_string(field.data).render() + jinja2_env.from_string(joined_data).render() except TemplateSyntaxError as e: raise ValidationError(f"This is not a valid Jinja2 template: {e}") from e except UndefinedError as e: raise ValidationError(f"A variable or function is not defined: {e}") from e + except jinja2.exceptions.SecurityError as e: + raise ValidationError(f"This is not a valid Jinja2 template: {e}") from e - ast = jinja2_env.parse(field.data) + ast = jinja2_env.parse(joined_data) undefined = ", ".join(find_undeclared_variables(ast)) if undefined: raise ValidationError( @@ -415,7 +420,7 @@ class quickWatchForm(Form): # Common to a single watch and the global settings class commonSettingsForm(Form): - notification_urls = StringListField('Notification URL List', validators=[validators.Optional(), ValidateAppRiseServers()]) + notification_urls = StringListField('Notification URL List', validators=[validators.Optional(), ValidateAppRiseServers(), ValidateJinja2Template()]) notification_title = StringField('Notification Title', default='ChangeDetection.io Notification - {{ watch_url }}', validators=[validators.Optional(), ValidateJinja2Template()]) notification_body = TextAreaField('Notification Body', default='{{ watch_url }} had a change.', validators=[validators.Optional(), ValidateJinja2Template()]) notification_format = SelectField('Notification format', choices=valid_notification_formats.keys()) @@ -448,6 +453,7 @@ class watchForm(commonSettingsForm): tags = StringTagUUID('Group tag', [validators.Optional()], default='') time_between_check = FormField(TimeBetweenCheckForm) + time_between_check_use_default = BooleanField('Use global settings for time between check', default=False) include_filters = StringListField('CSS/JSONPath/JQ/XPath Filters', [ValidateCSSJSONXPATHInput()], default='') @@ -499,11 +505,9 @@ class watchForm(commonSettingsForm): result = False # Attempt to validate jinja2 templates in the URL - from jinja2 import Environment - # Jinja2 available in URLs along with https://pypi.org/project/jinja2-time/ - jinja2_env = Environment(extensions=['jinja2_time.TimeExtension']) try: - ready_url = str(jinja2_env.from_string(self.url.data).render()) + from changedetectionio.safe_jinja import render as jinja_render + jinja_render(template_str=self.url.data) except Exception as e: self.url.errors.append('Invalid template syntax') result = False diff --git a/changedetectionio/html_tools.py b/changedetectionio/html_tools.py index 7c9844c8..a03653b9 100644 --- a/changedetectionio/html_tools.py +++ b/changedetectionio/html_tools.py @@ -169,14 +169,14 @@ def xpath1_filter(xpath_filter, html_content, append_pretty_line_formatting=Fals # And where the matched result doesn't include something that will cause Inscriptis to add a newline # (This way each 'match' reliably has a new-line in the diff) # Divs are converted to 4 whitespaces by inscriptis - if append_pretty_line_formatting and len(html_block) and (not hasattr( element, 'tag' ) or not element.tag in (['br', 'hr', 'div', 'p'])): + if append_pretty_line_formatting and len(html_block) and (not hasattr(element, 'tag') or not element.tag in (['br', 'hr', 'div', 'p'])): html_block += TEXT_FILTER_LIST_LINE_SUFFIX - if type(element) == etree._ElementStringResult: - html_block += str(element) - elif type(element) == etree._ElementUnicodeResult: - html_block += str(element) + # Some kind of text, UTF-8 or other + if isinstance(element, (str, bytes)): + html_block += element else: + # Return the HTML which will get parsed as text html_block += etree.tostring(element, pretty_print=True).decode('utf-8') return html_block diff --git a/changedetectionio/model/Watch.py b/changedetectionio/model/Watch.py index 044090b7..abaa9455 100644 --- a/changedetectionio/model/Watch.py +++ b/changedetectionio/model/Watch.py @@ -1,4 +1,5 @@ from changedetectionio.strtobool import strtobool +from changedetectionio.safe_jinja import render as jinja_render import os import re import time @@ -10,7 +11,7 @@ from loguru import logger # file:// is further checked by ALLOW_FILE_URI SAFE_PROTOCOL_REGEX='^(http|https|ftp|file):' -minimum_seconds_recheck_time = int(os.getenv('MINIMUM_SECONDS_RECHECK_TIME', 60)) +minimum_seconds_recheck_time = int(os.getenv('MINIMUM_SECONDS_RECHECK_TIME', 3)) mtable = {'seconds': 1, 'minutes': 60, 'hours': 3600, 'days': 86400, 'weeks': 86400 * 7} from changedetectionio.notification import ( @@ -67,6 +68,7 @@ base_config = { # Requires setting to None on submit if it's the same as the default # Should be all None by default, so we use the system default in this case. 'time_between_check': {'weeks': None, 'days': None, 'hours': None, 'minutes': None, 'seconds': None}, + 'time_between_check_use_default': True, 'title': None, 'trigger_text': [], # List of text or regex to wait for until a change is detected 'url': '', @@ -137,12 +139,11 @@ class model(dict): ready_url = url if '{%' in url or '{{' in url: - from jinja2 import Environment # Jinja2 available in URLs along with https://pypi.org/project/jinja2-time/ - jinja2_env = Environment(extensions=['jinja2_time.TimeExtension']) try: - ready_url = str(jinja2_env.from_string(url).render()) + ready_url = jinja_render(template_str=url) except Exception as e: + logger.critical(f"Invalid URL template for: '{url}' - {str(e)}") from flask import ( flash, Markup, url_for ) @@ -362,6 +363,7 @@ class model(dict): # @todo bump static cache of the last timestamp so we dont need to examine the file to set a proper ''viewed'' status return snapshot_fname + @property @property def has_empty_checktime(self): # using all() + dictionary comprehension diff --git a/changedetectionio/notification.py b/changedetectionio/notification.py index 21bda720..4fa35738 100644 --- a/changedetectionio/notification.py +++ b/changedetectionio/notification.py @@ -1,6 +1,5 @@ import apprise import time -from jinja2 import Environment, BaseLoader from apprise import NotifyFormat import json from loguru import logger @@ -49,7 +48,7 @@ from apprise.decorators import notify def apprise_custom_api_call_wrapper(body, title, notify_type, *args, **kwargs): import requests from apprise.utils import parse_url as apprise_parse_url - from apprise.URLBase import URLBase + from apprise import URLBase url = kwargs['meta'].get('url') @@ -116,6 +115,7 @@ def apprise_custom_api_call_wrapper(body, title, notify_type, *args, **kwargs): def process_notification(n_object, datastore): + from .safe_jinja import render as jinja_render now = time.time() if n_object.get('notification_timestamp'): logger.trace(f"Time since queued {now-n_object['notification_timestamp']:.3f}s") @@ -123,9 +123,9 @@ def process_notification(n_object, datastore): notification_parameters = create_notification_parameters(n_object, datastore) # Get the notification body from datastore - jinja2_env = Environment(loader=BaseLoader) - n_body = jinja2_env.from_string(n_object.get('notification_body', '')).render(**notification_parameters) - n_title = jinja2_env.from_string(n_object.get('notification_title', '')).render(**notification_parameters) + n_body = jinja_render(template_str=n_object.get('notification_body', ''), **notification_parameters) + n_title = jinja_render(template_str=n_object.get('notification_title', ''), **notification_parameters) + n_format = valid_notification_formats.get( n_object.get('notification_format', default_notification_format), valid_notification_formats[default_notification_format], @@ -157,7 +157,7 @@ def process_notification(n_object, datastore): continue logger.info(">> Process Notification: AppRise notifying {}".format(url)) - url = jinja2_env.from_string(url).render(**notification_parameters) + url = jinja_render(template_str=url, **notification_parameters) # Re 323 - Limit discord length to their 2000 char limit total or it wont send. # Because different notifications may require different pre-processing, run each sequentially :( diff --git a/changedetectionio/safe_jinja.py b/changedetectionio/safe_jinja.py new file mode 100644 index 00000000..8a6e1d38 --- /dev/null +++ b/changedetectionio/safe_jinja.py @@ -0,0 +1,18 @@ +""" +Safe Jinja2 render with max payload sizes + +See https://jinja.palletsprojects.com/en/3.1.x/sandbox/#security-considerations +""" + +import jinja2.sandbox +import typing as t +import os + +JINJA2_MAX_RETURN_PAYLOAD_SIZE = 1024 * int(os.getenv("JINJA2_MAX_RETURN_PAYLOAD_SIZE_KB", 1024 * 10)) + + +def render(template_str, **args: t.Any) -> str: + jinja2_env = jinja2.sandbox.ImmutableSandboxedEnvironment(extensions=['jinja2_time.TimeExtension']) + output = jinja2_env.from_string(template_str).render(args) + return output[:JINJA2_MAX_RETURN_PAYLOAD_SIZE] + diff --git a/changedetectionio/static/js/browser-steps.js b/changedetectionio/static/js/browser-steps.js index 7c9c38d8..4e576bd4 100644 --- a/changedetectionio/static/js/browser-steps.js +++ b/changedetectionio/static/js/browser-steps.js @@ -26,7 +26,8 @@ $(document).ready(function () { set_scale(); }); // Should always be disabled - $('#browser_steps >li:first-child select').val('Goto site').attr('disabled', 'disabled'); + $('#browser_steps-0-operation option[value="Goto site"]').prop("selected", "selected"); + $('#browser_steps-0-operation').attr('disabled', 'disabled'); $('#browsersteps-click-start').click(function () { $("#browsersteps-click-start").fadeOut(); diff --git a/changedetectionio/static/js/notifications.js b/changedetectionio/static/js/notifications.js index 046b645c..d3a0b81a 100644 --- a/changedetectionio/static/js/notifications.js +++ b/changedetectionio/static/js/notifications.js @@ -28,15 +28,11 @@ $(document).ready(function() { notification_format: $('#notification_format').val(), notification_title: $('#notification_title').val(), notification_urls: $('.notification-urls').val(), + tags: $('#tags').val(), window_url: window.location.href, } - if (!data['notification_urls'].length) { - alert("Notification URL list is empty, cannot send test.") - return; - } - $.ajax({ type: "POST", url: notification_base_url, @@ -49,7 +45,7 @@ $(document).ready(function() { } }).done(function(data){ console.log(data); - alert('Sent'); + alert(data); }).fail(function(data){ console.log(data); alert('There was an error communicating with the server.'); diff --git a/changedetectionio/static/js/watch-settings.js b/changedetectionio/static/js/watch-settings.js index 22bf48ed..73c66191 100644 --- a/changedetectionio/static/js/watch-settings.js +++ b/changedetectionio/static/js/watch-settings.js @@ -1,3 +1,17 @@ +function toggleOpacity(checkboxSelector, fieldSelector) { + const checkbox = document.querySelector(checkboxSelector); + const fields = document.querySelectorAll(fieldSelector); + function updateOpacity() { + const opacityValue = checkbox.checked ? 0.6 : 1; + fields.forEach(field => { + field.style.opacity = opacityValue; + }); + } + // Initial setup + updateOpacity(); + checkbox.addEventListener('change', updateOpacity); +} + $(document).ready(function () { $('#notification-setting-reset-to-default').click(function (e) { $('#notification_title').val(''); @@ -10,4 +24,7 @@ $(document).ready(function () { e.preventDefault(); $('#notification-tokens-info').toggle(); }); + + toggleOpacity('#time_between_check_use_default', '#time_between_check'); }); + diff --git a/changedetectionio/static/styles/diff.css b/changedetectionio/static/styles/diff.css index b1a07c20..4f264198 100644 --- a/changedetectionio/static/styles/diff.css +++ b/changedetectionio/static/styles/diff.css @@ -68,7 +68,7 @@ --color-last-checked: #bbb; --color-text-footer: #444; --color-border-watch-table-cell: #eee; - --color-text-watch-tag-list: #e70069; + --color-text-watch-tag-list: rgba(231, 0, 105, 0.4); --color-background-new-watch-form: rgba(0, 0, 0, 0.05); --color-background-new-watch-input: var(--color-white); --color-text-new-watch-input: var(--color-text); @@ -111,7 +111,7 @@ html[data-darkmode="true"] { --color-background-input: var(--color-grey-350); --color-text-input-description: var(--color-grey-600); --color-text-input-placeholder: var(--color-grey-600); - --color-text-watch-tag-list: #fa3e92; + --color-text-watch-tag-list: rgba(250, 62, 146, 0.4); --color-background-code: var(--color-grey-200); --color-background-tab: rgba(0, 0, 0, 0.2); --color-background-tab-hover: rgba(0, 0, 0, 0.5); diff --git a/changedetectionio/static/styles/scss/parts/_variables.scss b/changedetectionio/static/styles/scss/parts/_variables.scss index e9e9ccdf..aca4ed55 100644 --- a/changedetectionio/static/styles/scss/parts/_variables.scss +++ b/changedetectionio/static/styles/scss/parts/_variables.scss @@ -75,7 +75,7 @@ --color-text-footer: #444; --color-border-watch-table-cell: #eee; - --color-text-watch-tag-list: #e70069; + --color-text-watch-tag-list: rgba(231, 0, 105, 0.4); --color-background-new-watch-form: rgba(0, 0, 0, 0.05); --color-background-new-watch-input: var(--color-white); --color-text-new-watch-input: var(--color-text); @@ -127,7 +127,7 @@ html[data-darkmode="true"] { --color-background-input: var(--color-grey-350); --color-text-input-description: var(--color-grey-600); --color-text-input-placeholder: var(--color-grey-600); - --color-text-watch-tag-list: #fa3e92; + --color-text-watch-tag-list: rgba(250, 62, 146, 0.4); --color-background-code: var(--color-grey-200); --color-background-tab: rgba(0, 0, 0, 0.2); diff --git a/changedetectionio/static/styles/scss/styles.scss b/changedetectionio/static/styles/scss/styles.scss index aa5e7aa5..55bcc4cb 100644 --- a/changedetectionio/static/styles/scss/styles.scss +++ b/changedetectionio/static/styles/scss/styles.scss @@ -187,8 +187,11 @@ code { } .watch-tag-list { - color: var(--color-text-watch-tag-list); + color: var(--color-white); white-space: nowrap; + background: var(--color-text-watch-tag-list); + border-radius: 5px; + padding: 2px 5px; } .box { @@ -925,23 +928,26 @@ body.full-width { font-size: .875em; } } - .text-filtering { - h3 { - margin-top: 0; - } - border: 1px solid #ccc; - padding: 1rem; - border-radius: 5px; - margin-bottom: 1rem; - fieldset:last-of-type { +} + +.border-fieldset { + h3 { + margin-top: 0; + } + border: 1px solid #ccc; + padding: 1rem; + border-radius: 5px; + margin-bottom: 1rem; + fieldset:last-of-type { + padding-bottom: 0; + .pure-control-group { padding-bottom: 0; - .pure-control-group { - padding-bottom: 0; - } } } } + + ul { padding-left: 1em; padding-top: 0px; diff --git a/changedetectionio/static/styles/styles.css b/changedetectionio/static/styles/styles.css index ae37c680..c18d8d96 100644 --- a/changedetectionio/static/styles/styles.css +++ b/changedetectionio/static/styles/styles.css @@ -284,7 +284,7 @@ ul#requests-extra_browsers { --color-last-checked: #bbb; --color-text-footer: #444; --color-border-watch-table-cell: #eee; - --color-text-watch-tag-list: #e70069; + --color-text-watch-tag-list: rgba(231, 0, 105, 0.4); --color-background-new-watch-form: rgba(0, 0, 0, 0.05); --color-background-new-watch-input: var(--color-white); --color-text-new-watch-input: var(--color-text); @@ -327,7 +327,7 @@ html[data-darkmode="true"] { --color-background-input: var(--color-grey-350); --color-text-input-description: var(--color-grey-600); --color-text-input-placeholder: var(--color-grey-600); - --color-text-watch-tag-list: #fa3e92; + --color-text-watch-tag-list: rgba(250, 62, 146, 0.4); --color-background-code: var(--color-grey-200); --color-background-tab: rgba(0, 0, 0, 0.2); --color-background-tab-hover: rgba(0, 0, 0, 0.5); @@ -532,8 +532,11 @@ code { margin: 0 3px 0 5px; } .watch-tag-list { - color: var(--color-text-watch-tag-list); - white-space: nowrap; } + color: var(--color-white); + white-space: nowrap; + background: var(--color-text-watch-tag-list); + border-radius: 5px; + padding: 2px 5px; } .box { max-width: 80%; @@ -1038,17 +1041,18 @@ body.full-width .edit-form { color: var(--color-text-input-description); } .edit-form .pure-form-message-inline code { font-size: .875em; } - .edit-form .text-filtering { - border: 1px solid #ccc; - padding: 1rem; - border-radius: 5px; - margin-bottom: 1rem; } - .edit-form .text-filtering h3 { - margin-top: 0; } - .edit-form .text-filtering fieldset:last-of-type { + +.border-fieldset { + border: 1px solid #ccc; + padding: 1rem; + border-radius: 5px; + margin-bottom: 1rem; } + .border-fieldset h3 { + margin-top: 0; } + .border-fieldset fieldset:last-of-type { + padding-bottom: 0; } + .border-fieldset fieldset:last-of-type .pure-control-group { padding-bottom: 0; } - .edit-form .text-filtering fieldset:last-of-type .pure-control-group { - padding-bottom: 0; } ul { padding-left: 1em; diff --git a/changedetectionio/store.py b/changedetectionio/store.py index c9d28e79..884c617a 100644 --- a/changedetectionio/store.py +++ b/changedetectionio/store.py @@ -657,7 +657,10 @@ class ChangeDetectionStore: return res def tag_exists_by_name(self, tag_name): - return any(v.get('title', '').lower() == tag_name.lower() for k, v in self.__data['settings']['application']['tags'].items()) + # Check if any tag dictionary has a 'title' attribute matching the provided tag_name + tags = self.__data['settings']['application']['tags'].values() + return next((v for v in tags if v.get('title', '').lower() == tag_name.lower()), + None) def get_updates_available(self): import inspect @@ -869,3 +872,16 @@ class ChangeDetectionStore: self.__data["watching"][awatch]['include_filters'][num] = 'xpath1:' + selector if selector.startswith('xpath:'): self.__data["watching"][awatch]['include_filters'][num] = selector.replace('xpath:', 'xpath1:', 1) + + # Use more obvious default time setting + def update_15(self): + for uuid in self.__data["watching"]: + if self.__data["watching"][uuid]['time_between_check'] == self.__data['settings']['requests']['time_between_check']: + # What the old logic was, which was pretty confusing + self.__data["watching"][uuid]['time_between_check_use_default'] = True + elif all(value is None or value == 0 for value in self.__data["watching"][uuid]['time_between_check'].values()): + self.__data["watching"][uuid]['time_between_check_use_default'] = True + else: + # Something custom here + self.__data["watching"][uuid]['time_between_check_use_default'] = False + diff --git a/changedetectionio/templates/IMPORTANT.md b/changedetectionio/templates/IMPORTANT.md new file mode 100644 index 00000000..4b769264 --- /dev/null +++ b/changedetectionio/templates/IMPORTANT.md @@ -0,0 +1,6 @@ +# Important notes about templates + +Template names should always end in ".html", ".htm", ".xml", ".xhtml", ".svg", even the `import`'ed templates. + +Jinja2's `def select_jinja_autoescape(self, filename: str) -> bool:` will check the filename extension and enable autoescaping + diff --git a/changedetectionio/templates/_common_fields.jinja b/changedetectionio/templates/_common_fields.html similarity index 99% rename from changedetectionio/templates/_common_fields.jinja rename to changedetectionio/templates/_common_fields.html index 8b44ebbd..932f3fb7 100644 --- a/changedetectionio/templates/_common_fields.jinja +++ b/changedetectionio/templates/_common_fields.html @@ -1,5 +1,5 @@ -{% from '_helpers.jinja' import render_field %} +{% from '_helpers.html' import render_field %} {% macro render_common_settings_form(form, emailprefix, settings_application) %}
diff --git a/changedetectionio/templates/_helpers.jinja b/changedetectionio/templates/_helpers.html similarity index 100% rename from changedetectionio/templates/_helpers.jinja rename to changedetectionio/templates/_helpers.html diff --git a/changedetectionio/templates/diff.html b/changedetectionio/templates/diff.html index 0197bfc9..c82a559e 100644 --- a/changedetectionio/templates/diff.html +++ b/changedetectionio/templates/diff.html @@ -1,5 +1,5 @@ {% extends 'base.html' %} -{% from '_helpers.jinja' import render_field, render_checkbox_field, render_button %} +{% from '_helpers.html' import render_field, render_checkbox_field, render_button %} {% block content %} {% endif %} +{% set has_tag_filters_extra="WARNING: Watch has tag/groups set with special filters\n" if has_special_tag_options else '' %}
@@ -85,15 +87,9 @@ {{ render_field(form.tags) }} Organisational tag/group name used in the main listing page
-
+
{{ render_field(form.time_between_check, class="time-check-widget") }} - {% if has_empty_checktime %} - Currently using the default global settings, change to another value if you want to be specific. - {% else %} - Set to blank to use the default global settings. - {% endif %} + {{ render_checkbox_field(form.time_between_check_use_default, class="use-default-timecheck") }}
{{ render_checkbox_field(form.extract_title_as_title) }} @@ -280,7 +276,7 @@ User-Agent: wonderbra 1.0") }}
{% set field = render_field(form.include_filters, rows=5, - placeholder="#example + placeholder=has_tag_filters_extra+"#example xpath://body/div/span[contains(@class, 'example-class')]", class="m-d") %} @@ -316,7 +312,7 @@ xpath://body/div/span[contains(@class, 'example-class')]",
- {{ render_field(form.subtractive_selectors, rows=5, placeholder="header + {{ render_field(form.subtractive_selectors, rows=5, placeholder=has_tag_filters_extra+"header footer nav .stockticker") }} @@ -328,7 +324,7 @@ nav
-
+

Text filtering

Limit trigger/ignore/block/extract to;
@@ -485,13 +481,17 @@ Unavailable") }} {{ "{:,}".format(watch.history|length) }} - Last fetch time + Last fetch duration {{ watch.fetch_time }}s Notification alert count {{ watch.notification_alert_count }} + + Server type reply + {{ watch.get('remote_server_reply') }} +
diff --git a/changedetectionio/templates/import.html b/changedetectionio/templates/import.html index 1896080c..52d878de 100644 --- a/changedetectionio/templates/import.html +++ b/changedetectionio/templates/import.html @@ -1,6 +1,6 @@ {% extends 'base.html' %} {% block content %} -{% from '_helpers.jinja' import render_field %} +{% from '_helpers.html' import render_field %}
diff --git a/changedetectionio/templates/settings.html b/changedetectionio/templates/settings.html index d72a9265..e72c7818 100644 --- a/changedetectionio/templates/settings.html +++ b/changedetectionio/templates/settings.html @@ -1,10 +1,10 @@ {% extends 'base.html' %} {% block content %} -{% from '_helpers.jinja' import render_field, render_checkbox_field, render_button %} -{% from '_common_fields.jinja' import render_common_settings_form %} +{% from '_helpers.html' import render_field, render_checkbox_field, render_button %} +{% from '_common_fields.html' import render_common_settings_form %} @@ -169,7 +169,7 @@ {% if watch.uuid in queued_uuids %}Queued{% else %}Recheck{% endif %} - Edit + Edit {% if watch.history_n >= 2 %} {% if is_unviewed %} diff --git a/changedetectionio/tests/test_add_replace_remove_filter.py b/changedetectionio/tests/test_add_replace_remove_filter.py index 4ad9ecf8..f64d877b 100644 --- a/changedetectionio/tests/test_add_replace_remove_filter.py +++ b/changedetectionio/tests/test_add_replace_remove_filter.py @@ -1,5 +1,5 @@ #!/usr/bin/python3 - +import os.path import time from flask import url_for from .util import live_server_setup, wait_for_all_checks @@ -107,7 +107,6 @@ def test_check_add_line_contains_trigger(client, live_server): #live_server_setup(live_server) # Give the endpoint time to spin up - time.sleep(1) test_notification_url = url_for('test_notification_endpoint', _external=True).replace('http://', 'post://') + "?xxx={{ watch_url }}" res = client.post( @@ -166,6 +165,7 @@ def test_check_add_line_contains_trigger(client, live_server): # Takes a moment for apprise to fire time.sleep(3) + assert os.path.isfile("test-datastore/notification.txt"), "Notification fired because I can see the output file" with open("test-datastore/notification.txt", 'r') as f: response= f.read() assert '-Oh yes please-' in response diff --git a/changedetectionio/tests/test_group.py b/changedetectionio/tests/test_group.py index d9912a06..8904097d 100644 --- a/changedetectionio/tests/test_group.py +++ b/changedetectionio/tests/test_group.py @@ -100,6 +100,12 @@ def test_setup_group_tag(client, live_server): assert b'Should be only this' in res.data assert b'And never this' not in res.data + res = client.get( + url_for("edit_page", uuid="first"), + follow_redirects=True + ) + # 2307 the UI notice should appear in the placeholder + assert b'WARNING: Watch has tag/groups set with special filters' in res.data # RSS Group tag filter # An extra one that should be excluded diff --git a/changedetectionio/tests/test_jinja2.py b/changedetectionio/tests/test_jinja2.py index 771dc5ff..1e08691b 100644 --- a/changedetectionio/tests/test_jinja2.py +++ b/changedetectionio/tests/test_jinja2.py @@ -2,15 +2,15 @@ import time from flask import url_for -from .util import live_server_setup +from .util import live_server_setup, wait_for_all_checks -# If there was only a change in the whitespacing, then we shouldnt have a change detected -def test_jinja2_in_url_query(client, live_server): +def test_setup(client, live_server): live_server_setup(live_server) - # Give the endpoint time to spin up - time.sleep(1) +# If there was only a change in the whitespacing, then we shouldnt have a change detected +def test_jinja2_in_url_query(client, live_server): + #live_server_setup(live_server) # Add our URL to the import page test_url = url_for('test_return_query', _external=True) @@ -24,10 +24,35 @@ def test_jinja2_in_url_query(client, live_server): follow_redirects=True ) assert b"Watch added" in res.data - time.sleep(3) + wait_for_all_checks(client) + # It should report nothing found (no new 'unviewed' class) res = client.get( url_for("preview_page", uuid="first"), follow_redirects=True ) assert b'date=2' in res.data + +# https://techtonics.medium.com/secure-templating-with-jinja2-understanding-ssti-and-jinja2-sandbox-environment-b956edd60456 +def test_jinja2_security_url_query(client, live_server): + #live_server_setup(live_server) + + # Add our URL to the import page + test_url = url_for('test_return_query', _external=True) + + # because url_for() will URL-encode the var, but we dont here + full_url = "{}?{}".format(test_url, + "date={{ ''.__class__.__mro__[1].__subclasses__()}}", ) + res = client.post( + url_for("form_quick_watch_add"), + data={"url": full_url, "tags": "test"}, + follow_redirects=True + ) + assert b"Watch added" in res.data + wait_for_all_checks(client) + + # It should report nothing found (no new 'unviewed' class) + res = client.get(url_for("index")) + assert b'is invalid and cannot be used' in res.data + # Some of the spewed output from the subclasses + assert b'dict_values' not in res.data diff --git a/changedetectionio/tests/test_security.py b/changedetectionio/tests/test_security.py index 406a5401..c56c84492 100644 --- a/changedetectionio/tests/test_security.py +++ b/changedetectionio/tests/test_security.py @@ -2,9 +2,11 @@ from flask import url_for from .util import set_original_response, set_modified_response, live_server_setup, wait_for_all_checks import time +def test_setup(client, live_server): + live_server_setup(live_server) def test_bad_access(client, live_server): - live_server_setup(live_server) + #live_server_setup(live_server) res = client.post( url_for("import_page"), data={"urls": 'https://localhost'}, @@ -63,4 +65,25 @@ def test_bad_access(client, live_server): wait_for_all_checks(client) res = client.get(url_for("index")) - assert b'file:// type access is denied for security reasons.' in res.data \ No newline at end of file + assert b'file:// type access is denied for security reasons.' in res.data + +def test_xss(client, live_server): + #live_server_setup(live_server) + from changedetectionio.notification import ( + default_notification_format + ) + # the template helpers were named .jinja which meant they were not having jinja2 autoescape enabled. + res = client.post( + url_for("settings_page"), + data={"application-notification_urls": '">', + "application-notification_title": '">', + "application-notification_body": '">', + "application-notification_format": default_notification_format, + "requests-time_between_check-minutes": 180, + 'application-fetch_backend': "html_requests"}, + follow_redirects=True + ) + + assert b" + + + rpilocator.com + https://rpilocator.com + Find Raspberry Pi Computers in Stock + Thu, 19 May 2022 23:27:30 GMT + + https://rpilocator.com/favicon.png + rpilocator.com + https://rpilocator.com/ + 32 + 32 + + + Stock Alert (UK): RPi CM4 + something else unrelated + + + Stock Alert (UK): Big monitorěěěě + something else unrelated + + + '''.encode('utf-8') + + with open("test-datastore/endpoint-content.txt", "wb") as f: + f.write(d) + + + test_url = url_for('test_endpoint', _external=True) + res = client.post( + url_for("import_page"), + data={"urls": test_url}, + follow_redirects=True + ) + assert b"1 Imported" in res.data + wait_for_all_checks(client) + + res = client.post( + url_for("edit_page", uuid="first"), + data={"include_filters": "xpath1://title/text()", "url": test_url, "tags": "", "headers": "", + 'fetch_backend': "html_requests"}, + follow_redirects=True + ) + + ##### #2312 + wait_for_all_checks(client) + res = client.get(url_for("index")) + assert b'_ElementStringResult' not in res.data # tested with 5.1.1 when it was removed and 5.1.0 + assert b'Exception' not in res.data + res = client.get( + url_for("preview_page", uuid="first"), + follow_redirects=True + ) + + assert b"rpilocator.com" in res.data # in selector + assert "Stock Alert (UK): Big monitorěěěě".encode('utf-8') in res.data # not in selector + + ##### + def test_xpath1_validation(client, live_server): # Add our URL to the import page diff --git a/changedetectionio/tests/unit/test_jinja2_security.py b/changedetectionio/tests/unit/test_jinja2_security.py new file mode 100644 index 00000000..eb43db9d --- /dev/null +++ b/changedetectionio/tests/unit/test_jinja2_security.py @@ -0,0 +1,57 @@ +#!/usr/bin/python3 + +# run from dir above changedetectionio/ dir +# python3 -m unittest changedetectionio.tests.unit.test_jinja2_security + +import unittest +from changedetectionio import safe_jinja + + +# mostly +class TestJinja2SSTI(unittest.TestCase): + + def test_exception(self): + import jinja2 + + # Where sandbox should kick in + attempt_list = [ + "My name is {{ self.__init__.__globals__.__builtins__.__import__('os').system('id') }}", + "{{ self._TemplateReference__context.cycler.__init__.__globals__.os }}", + "{{ self.__init__.__globals__.__builtins__.__import__('os').popen('id').read() }}", + "{{cycler.__init__.__globals__.os.popen('id').read()}}", + "{{joiner.__init__.__globals__.os.popen('id').read()}}", + "{{namespace.__init__.__globals__.os.popen('id').read()}}", + "{{ ''.__class__.__mro__[2].__subclasses__()[40]('/tmp/hello.txt', 'w').write('Hello here !') }}", + "My name is {{ self.__init__.__globals__ }}", + "{{ dict.__base__.__subclasses__() }}" + ] + for attempt in attempt_list: + with self.assertRaises(jinja2.exceptions.SecurityError): + safe_jinja.render(attempt) + + def test_exception_debug_calls(self): + import jinja2 + # Where sandbox should kick in - configs and debug calls + attempt_list = [ + "{% debug %}", + ] + for attempt in attempt_list: + # Usually should be something like 'Encountered unknown tag 'debug'.' + with self.assertRaises(jinja2.exceptions.TemplateSyntaxError): + safe_jinja.render(attempt) + + # https://book.hacktricks.xyz/pentesting-web/ssti-server-side-template-injection/jinja2-ssti#accessing-global-objects + def test_exception_empty_calls(self): + import jinja2 + attempt_list = [ + "{{config}}", + "{{ debug }}" + "{{[].__class__}}", + ] + for attempt in attempt_list: + self.assertEqual(len(safe_jinja.render(attempt)), 0, f"string test '{attempt}' is correctly empty") + + + +if __name__ == '__main__': + unittest.main() diff --git a/changedetectionio/tests/util.py b/changedetectionio/tests/util.py index aab79163..186fc736 100644 --- a/changedetectionio/tests/util.py +++ b/changedetectionio/tests/util.py @@ -116,7 +116,7 @@ def extract_UUID_from_client(client): ) # {{api_key}} - m = re.search('edit/(.+?)"', str(res.data)) + m = re.search('edit/(.+?)[#"]', str(res.data)) uuid = m.group(1) return uuid.strip() diff --git a/changedetectionio/tests/visualselector/test_fetch_data.py b/changedetectionio/tests/visualselector/test_fetch_data.py index 2f460d7c..15677f31 100644 --- a/changedetectionio/tests/visualselector/test_fetch_data.py +++ b/changedetectionio/tests/visualselector/test_fetch_data.py @@ -102,10 +102,9 @@ def test_basic_browserstep(client, live_server): "url": test_url, "tags": "", 'fetch_backend': "html_webdriver", - 'browser_steps-0-operation': 'Goto site', - 'browser_steps-1-operation': 'Click element', - 'browser_steps-1-selector': 'button[name=test-button]', - 'browser_steps-1-optional_value': '', + 'browser_steps-0-operation': 'Click element', + 'browser_steps-0-selector': 'button[name=test-button]', + 'browser_steps-0-optional_value': '', # For now, cookies doesnt work in headers because it must be a full cookiejar object 'headers': "testheader: yes\buser-agent: MyCustomAgent", }, @@ -141,10 +140,9 @@ def test_basic_browserstep(client, live_server): "url": four_o_four_url, "tags": "", 'fetch_backend': "html_webdriver", - 'browser_steps-0-operation': 'Goto site', - 'browser_steps-1-operation': 'Click element', - 'browser_steps-1-selector': 'button[name=test-button]', - 'browser_steps-1-optional_value': '' + 'browser_steps-0-operation': 'Click element', + 'browser_steps-0-selector': 'button[name=test-button]', + 'browser_steps-0-optional_value': '' }, follow_redirects=True ) diff --git a/changedetectionio/update_worker.py b/changedetectionio/update_worker.py index 4f0bda8a..dcb9dbe7 100644 --- a/changedetectionio/update_worker.py +++ b/changedetectionio/update_worker.py @@ -462,7 +462,7 @@ class update_worker(threading.Thread): except Exception as e: logger.error(f"Exception reached processing watch UUID: {uuid}") logger.error(str(e)) - self.datastore.update_watch(uuid=uuid, update_obj={'last_error': str(e)}) + self.datastore.update_watch(uuid=uuid, update_obj={'last_error': "Exception: " + str(e)}) # Other serious error process_changedetection_results = False # import traceback diff --git a/docker-compose.yml b/docker-compose.yml index 1b5bd9af..4cf17605 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -54,7 +54,9 @@ services: # # Default number of parallel/concurrent fetchers # - FETCH_WORKERS=10 - + # + # Absolute minimum seconds to recheck, overrides any watch minimum, change to 0 to disable + # - MINIMUM_SECONDS_RECHECK_TIME=3 # Comment out ports: when using behind a reverse proxy , enable networks: etc. ports: - 5000:5000 diff --git a/requirements.txt b/requirements.txt index 54cf5e9d..049163ab 100644 --- a/requirements.txt +++ b/requirements.txt @@ -37,12 +37,12 @@ dnspython==2.3.0 # related to eventlet fixes # jq not available on Windows so must be installed manually # Notification library -apprise~=1.7.4 +apprise~=1.8.0 # apprise mqtt https://github.com/dgtlmoon/changedetection.io/issues/315 # and 2.0.0 https://github.com/dgtlmoon/changedetection.io/issues/2241 not yet compatible # use v1.x due to https://github.com/eclipse/paho.mqtt.python/issues/814 -paho-mqtt < 2.0.0 +paho-mqtt>=1.6.1,<2.0.0 # This mainly affects some ARM builds, which unlike the other builds ignores "ARG CRYPTOGRAPHY_DONT_BUILD_RUST=1" # so without this pinning, the newer versions on ARM will forcefully try to build rust, which results in "rust compiler not found" @@ -53,7 +53,10 @@ cryptography~=3.4 beautifulsoup4 # XPath filtering, lxml is required by bs4 anyway, but put it here to be safe. -lxml +# #2328 - 5.2.0 and 5.2.1 had extra CPU flag CFLAGS set which was not compatible on older hardware +# It could be advantageous to run its own pypi package here with those performance flags set +# https://bugs.launchpad.net/lxml/+bug/2059910/comments/16 +lxml >=4.8.0,<6,!=5.2.0,!=5.2.1 # XPath 2.0-3.1 support - 4.2.0 broke something? elementpath==4.1.5