diff --git a/README.md b/README.md index 97dcc408..9d1d7be8 100644 --- a/README.md +++ b/README.md @@ -164,6 +164,10 @@ BTC `1PLFN327GyUarpJd7nVe7Reqg9qHx5frNn` Support us! +### Commercial Support + +I offer commercial support, this software is depended on by network security, aerospace , data-science and data-journalist professionals just to name a few, please reach out at dgtlmoon@gmail.com for any enquiries, I am more than glad to work with your organisation to further the possibilities of what can be done with changedetection.io + [release-shield]: https://img.shields.io/github/v/release/dgtlmoon/changedetection.io?style=for-the-badge [docker-pulls]: https://img.shields.io/docker/pulls/dgtlmoon/changedetection.io?style=for-the-badge diff --git a/changedetectionio/__init__.py b/changedetectionio/__init__.py index 8a9d0a5b..e0393686 100644 --- a/changedetectionio/__init__.py +++ b/changedetectionio/__init__.py @@ -30,7 +30,7 @@ import datetime import pytz from copy import deepcopy -__version__ = '0.39.6' +__version__ = '0.39.7' datastore = None @@ -636,6 +636,7 @@ def changedetection_app(config=None, datastore_o=None): urls = request.values.get('urls').split("\n") for url in urls: url = url.strip() + # Flask wtform validators wont work with basic auth, use validators package if len(url) and validators.url(url): new_uuid = datastore.add_watch(url=url.strip(), tag="") # Straight into the queue. @@ -870,6 +871,26 @@ def changedetection_app(config=None, datastore_o=None): uuid=uuid) return output + @app.route("/api//snapshot/current", methods=['GET']) + @login_required + def api_snapshot(uuid): + + # More for testing, possible to return the first/only + if uuid == 'first': + uuid = list(datastore.data['watching'].keys()).pop() + + try: + watch = datastore.data['watching'][uuid] + except KeyError: + return abort(400, "No history found for the specified link, bad link?") + + newest = list(watch['history'].keys())[-1] + with open(watch['history'][newest], 'r') as f: + content = f.read() + + resp = make_response(content) + resp.headers['Content-Type'] = 'text/plain' + return resp @app.route("/favicon.ico", methods=['GET']) def favicon(): diff --git a/changedetectionio/forms.py b/changedetectionio/forms.py index bd40435a..9af85b31 100644 --- a/changedetectionio/forms.py +++ b/changedetectionio/forms.py @@ -130,6 +130,21 @@ class ValidateContentFetcherIsReady(object): raise ValidationError(message % (field.data, e)) +class ValidateNotificationBodyAndTitleWhenURLisSet(object): + """ + Validates that they entered something in both notification title+body when the URL is set + Due to https://github.com/dgtlmoon/changedetection.io/issues/360 + """ + + def __init__(self, message=None): + self.message = message + + def __call__(self, form, field): + if len(field.data): + if not len(form.notification_title.data) or not len(form.notification_body.data): + message = field.gettext('Notification Body and Title is required when a Notification URL is used') + raise ValidationError(message) + class ValidateAppRiseServers(object): """ Validates that each URL given is compatible with AppRise @@ -161,7 +176,24 @@ class ValidateTokensList(object): if not p.strip('{}') in notification.valid_tokens: message = field.gettext('Token \'%s\' is not a valid token.') raise ValidationError(message % (p)) + +class validateURL(object): + + """ + Flask wtform validators wont work with basic auth + """ + def __init__(self, message=None): + self.message = message + + def __call__(self, form, field): + import validators + try: + validators.url(field.data.strip()) + except validators.ValidationFailure: + message = field.gettext('\'%s\' is not a valid URL.' % (field.data.strip())) + raise ValidationError(message) + class ValidateListRegex(object): """ Validates that anything that looks like a regex passes as a regex @@ -229,12 +261,12 @@ class ValidateCSSJSONXPATHInput(object): class quickWatchForm(Form): # https://wtforms.readthedocs.io/en/2.3.x/fields/#module-wtforms.fields.html5 # `require_tld` = False is needed even for the test harness "http://localhost:5005.." to run - url = html5.URLField('URL', [validators.URL(require_tld=False)]) + url = html5.URLField('URL', validators=[validateURL()]) tag = StringField('Group tag', [validators.Optional(), validators.Length(max=35)]) class commonSettingsForm(Form): - notification_urls = StringListField('Notification URL List', validators=[validators.Optional(), ValidateAppRiseServers()]) + notification_urls = StringListField('Notification URL List', validators=[validators.Optional(), ValidateNotificationBodyAndTitleWhenURLisSet(), ValidateAppRiseServers()]) notification_title = StringField('Notification Title', default=default_notification_title, validators=[validators.Optional(), ValidateTokensList()]) notification_body = TextAreaField('Notification Body', default=default_notification_body, validators=[validators.Optional(), ValidateTokensList()]) notification_format = SelectField('Notification Format', choices=valid_notification_formats.keys(), default=default_notification_format) @@ -244,7 +276,7 @@ class commonSettingsForm(Form): class watchForm(commonSettingsForm): - url = html5.URLField('URL', [validators.URL(require_tld=False)]) + url = html5.URLField('URL', validators=[validateURL()]) tag = StringField('Group tag', [validators.Optional(), validators.Length(max=35)]) minutes_between_check = html5.IntegerField('Maximum time in minutes until recheck', diff --git a/changedetectionio/store.py b/changedetectionio/store.py index 7a669c24..7c1cceb3 100644 --- a/changedetectionio/store.py +++ b/changedetectionio/store.py @@ -49,9 +49,9 @@ class ChangeDetectionStore: 'ignore_whitespace': False, 'notification_urls': [], # Apprise URL list # Custom notification content - 'notification_title': None, - 'notification_body': None, - 'notification_format': None + 'notification_title': default_notification_title, + 'notification_body': default_notification_body, + 'notification_format': default_notification_format, } } } @@ -78,9 +78,9 @@ class ChangeDetectionStore: 'ignore_text': [], # List of text to ignore when calculating the comparison checksum # Custom notification content 'notification_urls': [], # List of URLs to add to the notification Queue (Usually AppRise) - 'notification_title': None, - 'notification_body': None, - 'notification_format': None, + 'notification_title': default_notification_title, + 'notification_body': default_notification_body, + 'notification_format': default_notification_format, 'css_filter': "", 'trigger_text': [], # List of text or regex to wait for until a change is detected 'fetch_backend': None, @@ -304,10 +304,10 @@ class ChangeDetectionStore: del_timestamps.append(timestamp) changes_removed += 1 - if not limit_timestamp: - self.data['watching'][uuid]['last_checked'] = 0 - self.data['watching'][uuid]['last_changed'] = 0 - self.data['watching'][uuid]['previous_md5'] = 0 + if not limit_timestamp: + self.data['watching'][uuid]['last_checked'] = 0 + self.data['watching'][uuid]['last_changed'] = 0 + self.data['watching'][uuid]['previous_md5'] = "" for timestamp in del_timestamps: @@ -326,7 +326,7 @@ class ChangeDetectionStore: content = fp.read() self.data['watching'][uuid]['previous_md5'] = hashlib.md5(content).hexdigest() except (FileNotFoundError, IOError): - self.data['watching'][uuid]['previous_md5'] = False + self.data['watching'][uuid]['previous_md5'] = "" pass self.needs_write = True diff --git a/changedetectionio/templates/edit.html b/changedetectionio/templates/edit.html index 8fb2e1b6..3ab30811 100644 --- a/changedetectionio/templates/edit.html +++ b/changedetectionio/templates/edit.html @@ -23,6 +23,7 @@
{{ render_field(form.url, placeholder="https://...", required=true, class="m-d") }} + Some sites use JavaScript to create the content, for this you should use the Chrome/WebDriver Fetcher
{{ render_field(form.title, class="m-d") }} @@ -77,7 +78,7 @@ User-Agent: wonderbra 1.0") }}
- Note: These settings override the global settings. + Note: These settings override the global settings for this watch.
{{ render_common_settings_form(form, current_base_url) }} @@ -121,11 +122,15 @@ User-Agent: wonderbra 1.0") }}
{{ render_field(form.trigger_text, rows=5, placeholder="Some text to wait for in a line /some.regex\d{2}/ for case-INsensitive regex - ") }}
- Text to wait for before triggering a change/notification, all text and regex are tested case-insensitive.
- Trigger text is processed from the result-text that comes out of any CSS/JSON Filters for this watch.
- Each line is process separately (think of each line as "OR")
- Note: Wrap in forward slash / to use regex example: /foo\d/ + ") }} + +
    +
  • Text to wait for before triggering a change/notification, all text and regex are tested case-insensitive.
  • +
  • Trigger text is processed from the result-text that comes out of any CSS/JSON Filters for this watch
  • +
  • Each line is process separately (think of each line as "OR")
  • +
  • Note: Wrap in forward slash / to use regex example: /foo\d/
  • +
+
diff --git a/changedetectionio/tests/test_api.py b/changedetectionio/tests/test_api.py new file mode 100644 index 00000000..b4f9e5f2 --- /dev/null +++ b/changedetectionio/tests/test_api.py @@ -0,0 +1,74 @@ +#!/usr/bin/python3 + +import time +from flask import url_for +from . util import live_server_setup + +def test_setup(live_server): + live_server_setup(live_server) + + +def set_response_data(test_return_data): + with open("test-datastore/endpoint-content.txt", "w") as f: + f.write(test_return_data) + + +def test_snapshot_api_detects_change(client, live_server): + + test_return_data = "Some initial text" + + test_return_data_modified = "Some NEW nice initial text" + + sleep_time_for_fetch_thread = 3 + + set_response_data(test_return_data) + + # Give the endpoint time to spin up + time.sleep(1) + + # Add our URL to the import page + test_url = url_for('test_endpoint', _external=True) + res = client.post( + url_for("import_page"), + data={"urls": test_url}, + follow_redirects=True + ) + assert b"1 Imported" in res.data + + # Trigger a check + client.get(url_for("api_watch_checknow"), follow_redirects=True) + + # Give the thread time to pick it up + time.sleep(sleep_time_for_fetch_thread) + + res = client.get( + url_for("api_snapshot", uuid="first"), + follow_redirects=True + ) + + assert test_return_data.encode() == res.data + + # Make a change + set_response_data(test_return_data_modified) + + # Trigger a check + client.get(url_for("api_watch_checknow"), follow_redirects=True) + # Give the thread time to pick it up + time.sleep(sleep_time_for_fetch_thread) + + res = client.get( + url_for("api_snapshot", uuid="first"), + follow_redirects=True + ) + + assert test_return_data_modified.encode() == res.data + +def test_snapshot_api_invalid_uuid(client, live_server): + + res = client.get( + url_for("api_snapshot", uuid="invalid"), + follow_redirects=True + ) + + assert res.status_code == 400 + diff --git a/changedetectionio/tests/test_auth.py b/changedetectionio/tests/test_auth.py new file mode 100644 index 00000000..45c8c909 --- /dev/null +++ b/changedetectionio/tests/test_auth.py @@ -0,0 +1,39 @@ +#!/usr/bin/python3 + +import time +from flask import url_for +from . util import live_server_setup + +def test_basic_auth(client, live_server): + + live_server_setup(live_server) + # Give the endpoint time to spin up + time.sleep(1) + + # Add our URL to the import page + test_url = url_for('test_basicauth_method', _external=True).replace("//","//myuser:mypass@") + + res = client.post( + url_for("import_page"), + data={"urls": test_url}, + follow_redirects=True + ) + assert b"1 Imported" in res.data + + # Check form validation + res = client.post( + url_for("edit_page", uuid="first"), + data={"css_filter": "", "url": test_url, "tag": "", "headers": "", 'fetch_backend': "html_requests"}, + follow_redirects=True + ) + assert b"Updated watch." in res.data + + # Trigger a check + client.get(url_for("api_watch_checknow"), follow_redirects=True) + time.sleep(1) + res = client.get( + url_for("preview_page", uuid="first"), + follow_redirects=True + ) + + assert b'myuser mypass basic' in res.data \ No newline at end of file diff --git a/changedetectionio/tests/test_errorhandling.py b/changedetectionio/tests/test_errorhandling.py index 423316d4..b9a01ac3 100644 --- a/changedetectionio/tests/test_errorhandling.py +++ b/changedetectionio/tests/test_errorhandling.py @@ -35,4 +35,28 @@ def test_error_handler(client, live_server): res = client.get(url_for("index")) assert b'unviewed' not in res.data assert b'Status Code 403' in res.data - assert bytes("just now".encode('utf-8')) in res.data \ No newline at end of file + assert bytes("just now".encode('utf-8')) in res.data + +# Just to be sure error text is properly handled +def test_error_text_handler(client, live_server): + # Give the endpoint time to spin up + time.sleep(1) + + # Add our URL to the import page + res = client.post( + url_for("import_page"), + data={"urls": "https://errorfuldomainthatnevereallyexists12356.com"}, + follow_redirects=True + ) + assert b"1 Imported" in res.data + + # Trigger a check + client.get(url_for("api_watch_checknow"), follow_redirects=True) + + # Give the thread time to pick it up + time.sleep(3) + + res = client.get(url_for("index")) + assert b'Name or service not known' in res.data + assert bytes("just now".encode('utf-8')) in res.data + diff --git a/changedetectionio/tests/test_notification.py b/changedetectionio/tests/test_notification.py index 21083066..79054d41 100644 --- a/changedetectionio/tests/test_notification.py +++ b/changedetectionio/tests/test_notification.py @@ -4,6 +4,7 @@ import re from flask import url_for from . util import set_original_response, set_modified_response, live_server_setup import logging +from changedetectionio.notification import default_notification_body, default_notification_title # Hard to just add more live server URLs when one test is already running (I think) # So we add our test here (was in a different file) @@ -15,6 +16,11 @@ def test_check_notification(client, live_server): # Give the endpoint time to spin up time.sleep(3) + # Re 360 - new install should have defaults set + res = client.get(url_for("settings_page")) + assert default_notification_body.encode() in res.data + assert default_notification_title.encode() in res.data + # When test mode is in BASE_URL env mode, we should see this already configured env_base_url = os.getenv('BASE_URL', '').strip() if len(env_base_url): @@ -117,7 +123,8 @@ def test_check_notification(client, live_server): assert test_url in notification_submission # Diff was correctly executed - assert "Diff Full: (changed) Which is across multiple lines" in notification_submission + assert "Diff Full: Some initial text" in notification_submission + assert "Diff: (changed) Which is across multiple lines" in notification_submission assert "(-> into) which has this one new line" in notification_submission @@ -201,3 +208,21 @@ def test_check_notification(client, live_server): ) assert bytes("is not a valid token".encode('utf-8')) in res.data + + # Re #360 some validation + res = client.post( + url_for("edit_page", uuid="first"), + data={"notification_urls": notification_url, + "notification_title": "", + "notification_body": "", + "notification_format": "Text", + "url": test_url, + "tag": "my tag", + "title": "my title", + "headers": "", + "fetch_backend": "html_requests", + "trigger_check": "y"}, + follow_redirects=True + ) + assert b"Notification Body and Title is required when a Notification URL is used" in res.data + diff --git a/changedetectionio/tests/util.py b/changedetectionio/tests/util.py index 54532680..86b78767 100644 --- a/changedetectionio/tests/util.py +++ b/changedetectionio/tests/util.py @@ -103,4 +103,14 @@ def live_server_setup(live_server): print("\n>> Test notification endpoint was hit.\n") return "Text was set" + + # Just return the verb in the request + @live_server.app.route('/test-basicauth', methods=['GET']) + def test_basicauth_method(): + + from flask import request + auth = request.authorization + ret = " ".join([auth.username, auth.password, auth.type]) + return ret + live_server.start() diff --git a/changedetectionio/update_worker.py b/changedetectionio/update_worker.py index 8f535829..441eb9ee 100644 --- a/changedetectionio/update_worker.py +++ b/changedetectionio/update_worker.py @@ -60,7 +60,7 @@ class update_worker(threading.Thread): self.datastore.update_watch(uuid=uuid, update_obj={'last_error': err_text, 'last_check_status': e.status_code}) except Exception as e: - self.app.logger.error("Exception reached processing watch UUID:%s - %s", uuid, str(e)) + self.app.logger.error("Exception reached processing watch UUID: %s - %s", uuid, str(e)) self.datastore.update_watch(uuid=uuid, update_obj={'last_error': str(e)}) else: @@ -127,8 +127,8 @@ class update_worker(threading.Thread): 'watch_url': watch['url'], 'uuid': uuid, 'current_snapshot': contents.decode('utf-8'), - 'diff_full': diff.render_diff(prev_fname, fname, line_feed_sep=line_feed_sep), - 'diff': diff.render_diff(prev_fname, fname, True, line_feed_sep=line_feed_sep) + 'diff': diff.render_diff(prev_fname, fname, line_feed_sep=line_feed_sep), + 'diff_full': diff.render_diff(prev_fname, fname, True, line_feed_sep=line_feed_sep) }) self.notification_q.put(n_object) diff --git a/requirements.txt b/requirements.txt index 688ad92b..6d74be41 100644 --- a/requirements.txt +++ b/requirements.txt @@ -17,7 +17,7 @@ wtforms ~= 2.3.3 jsonpath-ng ~= 1.5.3 # Notification library -apprise ~= 0.9 +apprise ~= 0.9.6 # apprise mqtt https://github.com/dgtlmoon/changedetection.io/issues/315 paho-mqtt