Self-Hosted
/
changedetection.io
mirror of https://github.com/dgtlmoon/changedetection.io.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Notifications - tokens/jinja2 templating (#1184)

Browse Source
pull/1196/head
dgtlmoon 1 year ago committed by GitHub
parent a048e4a02d
commit c12db2b725
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
12 changed files with 267 additions and 92 deletions
Show Stats Download Patch File Download Diff File

2
README.md
Unescape View File

@ -159,7 +159,7 @@ Just some examples
<img src="https://raw.githubusercontent.com/dgtlmoon/changedetection.io/master/docs/screenshot-notifications.png" style="max-width:100%;" alt="Self-hosted web page change monitoring notifications" title="Self-hosted web page change monitoring notifications" />
Now you can also customise your notification content!
Now you can also customise your notification content and use <a target="_new" href="https://jinja.palletsprojects.com/en/3.0.x/templates/">Jinja2 templating</a> for their title and body!
## JSON API Monitoring

34
changedetectionio/forms.py
Unescape View File

@ -193,7 +193,7 @@ class ValidateAppRiseServers(object):
message = field.gettext('\'%s\' is not a valid AppRise URL.' % (server_url))
raise ValidationError(message)
class ValidateTokensList(object):
class ValidateJinja2Template(object):
"""
Validates that a {token} is from a valid set
"""
@ -202,11 +202,24 @@ class ValidateTokensList(object):
def __call__(self, form, field):
from changedetectionio import notification
regex = re.compile('{.*?}')
for p in re.findall(regex, field.data):
if not p.strip('{}') in notification.valid_tokens:
message = field.gettext('Token \'%s\' is not a valid token.')
raise ValidationError(message % (p))
from jinja2 import Environment, BaseLoader, TemplateSyntaxError
from jinja2.meta import find_undeclared_variables
try:
jinja2_env = Environment(loader=BaseLoader)
jinja2_env.globals.update(notification.valid_tokens)
rendered = jinja2_env.from_string(field.data).render()
except TemplateSyntaxError as e:
raise ValidationError(f"This is not a valid Jinja2 template: {e}") from e
ast = jinja2_env.parse(field.data)
undefined = ", ".join(find_undeclared_variables(ast))
if undefined:
raise ValidationError(
f"The following tokens used in the notification are not valid: {undefined}"
)
class validateURL(object):
@ -225,6 +238,7 @@ class validateURL(object):
message = field.gettext('\'%s\' is not a valid URL.' % (field.data.strip()))
raise ValidationError(message)
class ValidateListRegex(object):
"""
Validates that anything that looks like a regex passes as a regex
@ -333,11 +347,11 @@ class quickWatchForm(Form):
# Common to a single watch and the global settings
class commonSettingsForm(Form):
notification_urls = StringListField('Notification URL list', validators=[validators.Optional(), ValidateAppRiseServers()])
notification_title = StringField('Notification title', validators=[validators.Optional(), ValidateTokensList()])
notification_body = TextAreaField('Notification body', validators=[validators.Optional(), ValidateTokensList()])
notification_urls = StringListField('Notification URL List', validators=[validators.Optional(), ValidateAppRiseServers()])
notification_title = StringField('Notification Title', default='ChangeDetection.io Notification - {{ watch_url }}', validators=[validators.Optional(), ValidateJinja2Template()])
notification_body = TextAreaField('Notification Body', default='{{ watch_url }} had a change.', validators=[validators.Optional(), ValidateJinja2Template()])
notification_format = SelectField('Notification format', choices=valid_notification_formats.keys())
fetch_backend = RadioField(u'Fetch method', choices=content_fetcher.available_fetchers(), validators=[ValidateContentFetcherIsReady()])
fetch_backend = RadioField(u'Fetch Method', choices=content_fetcher.available_fetchers(), validators=[ValidateContentFetcherIsReady()])
extract_title_as_title = BooleanField('Extract <title> from document and use as watch title', default=False)
webdriver_delay = IntegerField('Wait seconds before extracting text', validators=[validators.Optional(), validators.NumberRange(min=1,
message="Should contain one or more seconds")])

82
changedetectionio/notification.py
Unescape View File

@ -1,5 +1,7 @@
import apprise
from jinja2 import Environment, BaseLoader
from apprise import NotifyFormat
import json
valid_tokens = {
'base_url': '',
@ -16,8 +18,8 @@ valid_tokens = {
default_notification_format_for_watch = 'System default'
default_notification_format = 'Text'
default_notification_body = '{watch_url} had a change.\n---\n{diff}\n---\n'
default_notification_title = 'ChangeDetection.io Notification - {watch_url}'
default_notification_body = '{{watch_url}} had a change.\n---\n{{diff}}\n---\n'
default_notification_title = 'ChangeDetection.io Notification - {{watch_url}}'
valid_notification_formats = {
'Text': NotifyFormat.TEXT,
@ -27,25 +29,67 @@ valid_notification_formats = {
default_notification_format_for_watch: default_notification_format_for_watch
}
# include the decorator
from apprise.decorators import notify
@notify(on="delete")
@notify(on="deletes")
@notify(on="get")
@notify(on="gets")
@notify(on="post")
@notify(on="posts")
@notify(on="put")
@notify(on="puts")
def apprise_custom_api_call_wrapper(body, title, notify_type, *args, **kwargs):
import requests
url = kwargs['meta'].get('url')
if url.startswith('post'):
r = requests.post
elif url.startswith('get'):
r = requests.get
elif url.startswith('put'):
r = requests.put
elif url.startswith('delete'):
r = requests.delete
url = url.replace('post://', 'http://')
url = url.replace('posts://', 'https://')
url = url.replace('put://', 'http://')
url = url.replace('puts://', 'https://')
url = url.replace('get://', 'http://')
url = url.replace('gets://', 'https://')
url = url.replace('put://', 'http://')
url = url.replace('puts://', 'https://')
url = url.replace('delete://', 'http://')
url = url.replace('deletes://', 'https://')
# Try to auto-guess if it's JSON
headers = {}
try:
json.loads(body)
headers = {'Content-Type': 'application/json; charset=utf-8'}
except ValueError as e:
pass
r(url, headers=headers, data=body)
def process_notification(n_object, datastore):
# Insert variables into the notification content
notification_parameters = create_notification_parameters(n_object, datastore)
# Get the notification body from datastore
n_body = n_object.get('notification_body', default_notification_body)
n_title = n_object.get('notification_title', default_notification_title)
jinja2_env = Environment(loader=BaseLoader)
n_body = jinja2_env.from_string(n_object.get('notification_body', default_notification_body)).render(**notification_parameters)
n_title = jinja2_env.from_string(n_object.get('notification_title', default_notification_title)).render(**notification_parameters)
n_format = valid_notification_formats.get(
n_object['notification_format'],
valid_notification_formats[default_notification_format],
)
# Insert variables into the notification content
notification_parameters = create_notification_parameters(n_object, datastore)
for n_k in notification_parameters:
token = '{' + n_k + '}'
val = notification_parameters[n_k]
n_title = n_title.replace(token, val)
n_body = n_body.replace(token, val)
# https://github.com/caronc/apprise/wiki/Development_LogCapture
# Anything higher than or equal to WARNING (which covers things like Connection errors)
# raise it as an exception
@ -53,6 +97,7 @@ def process_notification(n_object, datastore):
sent_objs=[]
from .apprise_asset import asset
for url in n_object['notification_urls']:
url = jinja2_env.from_string(url).render(**notification_parameters)
apobj = apprise.Apprise(debug=True, asset=asset)
url = url.strip()
if len(url):
@ -66,7 +111,12 @@ def process_notification(n_object, datastore):
# So if no avatar_url is specified, add one so it can be correctly calculated into the total payload
k = '?' if not '?' in url else '&'
if not 'avatar_url' in url and not url.startswith('mail'):
if not 'avatar_url' in url \
and not url.startswith('mail') \
and not url.startswith('post') \
and not url.startswith('get') \
and not url.startswith('delete') \
and not url.startswith('put'):
url += k + 'avatar_url=https://raw.githubusercontent.com/dgtlmoon/changedetection.io/master/changedetectionio/static/images/avatar-256x256.png'
if url.startswith('tgram://'):
@ -144,7 +194,7 @@ def create_notification_parameters(n_object, datastore):
watch_url = n_object['watch_url']
# Re #148 - Some people have just {base_url} in the body or title, but this may break some notification services
# Re #148 - Some people have just {{ base_url }} in the body or title, but this may break some notification services
# like 'Join', so it's always best to atleast set something obvious so that they are not broken.
if base_url == '':
base_url = "<base-url-env-var-not-set>"

3
changedetectionio/static/styles/scss/styles.scss
Unescape View File

@ -877,6 +877,9 @@ body.full-width {
.pure-form-message-inline {
padding-left: 0;
color: var(--color-text-input-description);
code {
font-size: .875em;
}
}
}

2
changedetectionio/static/styles/styles.css
Unescape View File

@ -851,6 +851,8 @@ body.full-width .edit-form {
.edit-form .pure-form-message-inline {
padding-left: 0;
color: var(--color-text-input-description); }
.edit-form .pure-form-message-inline code {
font-size: .875em; }
ul {
padding-left: 1em;

42
changedetectionio/store.py
Unescape View File

@ -621,4 +621,44 @@ class ChangeDetectionStore:
watch['include_filters'] = [existing_filter]
except:
continue
return
return
# Convert old static notification tokens to jinja2 tokens
def update_9(self):
# Each watch
import re
# only { } not {{ or }}
r = r'(?<!{){(?!{)(\w+)(?<!})}(?!})'
for uuid, watch in self.data['watching'].items():
try:
n_body = watch.get('notification_body', '')
if n_body:
watch['notification_body'] = re.sub(r, r'{{\1}}', n_body)
n_title = watch.get('notification_title')
if n_title:
self.data['settings']['application']['notification_title'] = re.sub(r, r'{{\1}}', n_title)
n_urls = watch.get('notification_urls')
if n_urls:
for i, url in enumerate(n_urls):
watch['notification_urls'][i] = re.sub(r, r'{{\1}}', url)
except:
continue
# System wide
n_body = self.data['settings']['application'].get('notification_body')
if n_body:
self.data['settings']['application']['notification_body'] = re.sub(r, r'{{\1}}', n_body)
n_title = self.data['settings']['application'].get('notification_title')
if n_body:
self.data['settings']['application']['notification_title'] = re.sub(r, r'{{\1}}', n_title)
n_urls = self.data['settings']['application'].get('notification_urls')
if n_urls:
for i, url in enumerate(n_urls):
self.data['settings']['application']['notification_urls'][i] = re.sub(r, r'{{\1}}', url)
return

37
changedetectionio/templates/_common_fields.jinja
Unescape View File

@ -16,6 +16,7 @@
<li><code>discord://</code> only supports a maximum <strong>2,000 characters</strong> of notification text, including the title.</li>
<li><code>tgram://</code> bots cant send messages to other bots, so you should specify chat ID of non-bot user.</li>
<li><code>tgram://</code> only supports very limited HTML and can fail when extra tags are sent, <a href="https://core.telegram.org/bots/api#html-style">read more here</a> (or use plaintext/markdown format)</li>
<li><code>gets://</code>, <code>posts://</code>, <code>puts://</code>, <code>deletes://</code> for direct API calls (or omit the "<code>s</code>" for non-SSL ie <code>get://</code>)</li>
</ul>
</div>
<div class="notifications-wrapper">
@ -41,8 +42,9 @@
<span class="pure-form-message-inline">Format for all notifications</span>
</div>
<div class="pure-controls">
<span class="pure-form-message-inline">
These tokens can be used in the notification body and title to customise the notification text.
<p class="pure-form-message-inline">
You can use <a target="_new" href="https://jinja.palletsprojects.com/en/3.0.x/templates/">Jinja2</a> templating in the notification title, body and URL.
</p>
<table class="pure-table" id="token-table">
<thead>
@ -53,52 +55,49 @@
</thead>
<tbody>
<tr>
<td><code>{base_url}</code></td>
<td><code>{{ '{{ base_url }}' }}</code></td>
<td>The URL of the changedetection.io instance you are running.</td>
</tr>
<tr>
<td><code>{watch_url}</code></td>
<td><code>{{ '{{ watch_url }}' }}</code></td>
<td>The URL being watched.</td>
</tr>
<tr>
<td><code>{watch_uuid}</code></td>
<td><code>{{ '{{ watch_uuid }}' }}</code></td>
<td>The UUID of the watch.</td>
</tr>
<tr>
<td><code>{watch_title}</code></td>
<td><code>{{ '{{ watch_title }}' }}</code></td>
<td>The title of the watch.</td>
</tr>
<tr>
<td><code>{watch_tag}</code></td>
<td><code>{{ '{{ watch_tag }}' }}</code></td>
<td>The tag of the watch.</td>
</tr>
<tr>
<td><code>{preview_url}</code></td>
<td><code>{{ '{{ preview_url }}' }}</code></td>
<td>The URL of the preview page generated by changedetection.io.</td>
</tr>
<tr>
<td><code>{diff}</code></td>
<td><code>{{ '{{ diff_url }}' }}</code></td>
<td>The diff output - differences only</td>
</tr>
<tr>
<td><code>{diff_full}</code></td>
<td><code>{{ '{{ diff_full }}' }}</code></td>
<td>The diff output - full difference output</td>
</tr>
<tr>
<td><code>{diff_url}</code></td>
<td>The URL of the diff page generated by changedetection.io.</td>
</tr>
<tr>
<td><code>{current_snapshot}</code></td>
<td><code>{{ '{{ current_snapshot }}' }}</code></td>
<td>The current snapshot value, useful when combined with JSON or CSS filters
</td>
</tr>
</tbody>
</table>
<br/>
URLs generated by changedetection.io (such as <code>{diff_url}</code>) require the <code>BASE_URL</code> environment variable set.<br/>
Your <code>BASE_URL</code> var is currently "{{settings_application['current_base_url']}}"
</span>
<div class="pure-form-message-inline">
<br>
URLs generated by changedetection.io (such as <code>{{ '{{ diff_url }}' }}</code>) require the <code>BASE_URL</code> environment variable set.<br/>
Your <code>BASE_URL</code> var is currently "{{settings_application['current_base_url']}}"
</div>
</div>
</div>
{% endmacro %}

2
changedetectionio/templates/settings.html
Unescape View File

@ -60,7 +60,7 @@
{{ render_field(form.application.form.base_url, placeholder="http://yoursite.com:5000/",
class="m-d") }}
<span class="pure-form-message-inline">
Base URL used for the <code>{base_url}</code> token in notifications and RSS links.<br/>Default value is the ENV var 'BASE_URL' (Currently "{{settings_application['current_base_url']}}"),
Base URL used for the <code>{{ '{{ base_url }}' }}</code> token in notifications and RSS links.<br/>Default value is the ENV var 'BASE_URL' (Currently "{{settings_application['current_base_url']}}"),
<a href="https://github.com/dgtlmoon/changedetection.io/wiki/Configurable-BASE_URL-setting">read more here</a>.
</span>
</div>

22
changedetectionio/tests/test_filter_exist_changes.py
Unescape View File

@ -73,17 +73,17 @@ def test_filter_doesnt_exist_then_exists_should_get_notification(client, live_se
# Just a regular notification setting, this will be used by the special 'filter not found' notification
notification_form_data = {"notification_urls": notification_url,
"notification_title": "New ChangeDetection.io Notification - {watch_url}",
"notification_body": "BASE URL: {base_url}\n"
"Watch URL: {watch_url}\n"
"Watch UUID: {watch_uuid}\n"
"Watch title: {watch_title}\n"
"Watch tag: {watch_tag}\n"
"Preview: {preview_url}\n"
"Diff URL: {diff_url}\n"
"Snapshot: {current_snapshot}\n"
"Diff: {diff}\n"
"Diff Full: {diff_full}\n"
"notification_title": "New ChangeDetection.io Notification - {{watch_url}}",
"notification_body": "BASE URL: {{base_url}}\n"
"Watch URL: {{watch_url}}\n"
"Watch UUID: {{watch_uuid}}\n"
"Watch title: {{watch_title}}\n"
"Watch tag: {{watch_tag}}\n"
"Preview: {{preview_url}}\n"
"Diff URL: {{diff_url}}\n"
"Snapshot: {{current_snapshot}}\n"
"Diff: {{diff}}\n"
"Diff Full: {{diff_full}}\n"
":-)",
"notification_format": "Text"}

23
changedetectionio/tests/test_filter_failure_notification.py
Unescape View File

@ -56,17 +56,17 @@ def run_filter_test(client, content_filter):
# Just a regular notification setting, this will be used by the special 'filter not found' notification
notification_form_data = {"notification_urls": notification_url,
"notification_title": "New ChangeDetection.io Notification - {watch_url}",
"notification_body": "BASE URL: {base_url}\n"
"Watch URL: {watch_url}\n"
"Watch UUID: {watch_uuid}\n"
"Watch title: {watch_title}\n"
"Watch tag: {watch_tag}\n"
"Preview: {preview_url}\n"
"Diff URL: {diff_url}\n"
"Snapshot: {current_snapshot}\n"
"Diff: {diff}\n"
"Diff Full: {diff_full}\n"
"notification_title": "New ChangeDetection.io Notification - {{watch_url}}",
"notification_body": "BASE URL: {{base_url}}\n"
"Watch URL: {{watch_url}}\n"
"Watch UUID: {{watch_uuid}}\n"
"Watch title: {{watch_title}}\n"
"Watch tag: {{watch_tag}}\n"
"Preview: {{preview_url}}\n"
"Diff URL: {{diff_url}}\n"
"Snapshot: {{current_snapshot}}\n"
"Diff: {{diff}}\n"
"Diff Full: {{diff_full}}\n"
":-)",
"notification_format": "Text"}
@ -84,6 +84,7 @@ def run_filter_test(client, content_filter):
data=notification_form_data,
follow_redirects=True
)
assert b"Updated watch." in res.data
time.sleep(3)

107
changedetectionio/tests/test_notification.py
Unescape View File

@ -90,17 +90,17 @@ def test_check_notification(client, live_server):
print (">>>> Notification URL: "+notification_url)
notification_form_data = {"notification_urls": notification_url,
"notification_title": "New ChangeDetection.io Notification - {watch_url}",
"notification_body": "BASE URL: {base_url}\n"
"Watch URL: {watch_url}\n"
"Watch UUID: {watch_uuid}\n"
"Watch title: {watch_title}\n"
"Watch tag: {watch_tag}\n"
"Preview: {preview_url}\n"
"Diff URL: {diff_url}\n"
"Snapshot: {current_snapshot}\n"
"Diff: {diff}\n"
"Diff Full: {diff_full}\n"
"notification_title": "New ChangeDetection.io Notification - {{watch_url}}",
"notification_body": "BASE URL: {{base_url}}\n"
"Watch URL: {{watch_url}}\n"
"Watch UUID: {{watch_uuid}}\n"
"Watch title: {{watch_title}}\n"
"Watch tag: {{watch_tag}}\n"
"Preview: {{preview_url}}\n"
"Diff URL: {{diff_url}}\n"
"Snapshot: {{current_snapshot}}\n"
"Diff: {{diff}}\n"
"Diff Full: {{diff_full}}\n"
":-)",
"notification_screenshot": True,
"notification_format": "Text"}
@ -179,7 +179,6 @@ def test_check_notification(client, live_server):
logging.debug(">>> Skipping BASE_URL check")
# This should insert the {current_snapshot}
set_more_modified_response()
client.get(url_for("form_watch_checknow"), follow_redirects=True)
@ -237,10 +236,9 @@ def test_check_notification(client, live_server):
follow_redirects=True
)
def test_notification_validation(client, live_server):
#live_server_setup(live_server)
time.sleep(3)
time.sleep(1)
# re #242 - when you edited an existing new entry, it would not correctly show the notification settings
# Add our URL to the import page
test_url = url_for('test_endpoint', _external=True)
@ -269,19 +267,33 @@ def test_notification_validation(client, live_server):
# assert b"Notification Body and Title is required when a Notification URL is used" in res.data
# Now adding a wrong token should give us an error
# Disabled for now
# res = client.post(
# url_for("settings_page"),
# data={"application-notification_title": "New ChangeDetection.io Notification - {{watch_url}}",
# "application-notification_body": "Rubbish: {{rubbish}}\n",
# "application-notification_format": "Text",
# "application-notification_urls": "json://localhost/foobar",
# "requests-time_between_check-minutes": 180,
# "fetch_backend": "html_requests"
# },
# follow_redirects=True
# )
# assert bytes("Token 'rubbish' is not a valid token or is unknown".encode('utf-8')) in res.data
# And trying to define an invalid Jinja2 template should also throw an error
res = client.post(
url_for("settings_page"),
data={"application-notification_title": "New ChangeDetection.io Notification - {watch_url}",
"application-notification_body": "Rubbish: {rubbish}\n",
"application-notification_format": "Text",
"application-notification_urls": "json://localhost/foobar",
"requests-time_between_check-minutes": 180,
"fetch_backend": "html_requests"
data={"application-notification_title": "New ChangeDetection.io Notification - {{ watch_url }}",
"application-notification_body": "Rubbish: {{ rubbish }\n",
"application-notification_urls": "json://foobar.com",
"application-minutes_between_check": 180,
"application-fetch_backend": "html_requests"
},
follow_redirects=True
)
assert bytes("This is not a valid Jinja2 template".encode('utf-8')) in res.data
assert bytes("is not a valid token".encode('utf-8')) in res.data
# cleanup for the next
client.get(
@ -289,4 +301,55 @@ def test_notification_validation(client, live_server):
follow_redirects=True
)
def test_notification_jinja2(client, live_server):
#live_server_setup(live_server)
time.sleep(1)
# test_endpoint - that sends the contents of a file
# test_notification_endpoint - that takes a POST and writes it to file (test-datastore/notification.txt)
# CUSTOM JSON BODY CHECK for POST://
set_original_response()
test_notification_url = url_for('test_notification_endpoint', _external=True).replace('http://', 'post://')+"?xxx={{ watch_url }}"
res = client.post(
url_for("settings_page"),
data={"application-notification_title": "New ChangeDetection.io Notification - {{ watch_url }}",
"application-notification_body": '{ "url" : "{{ watch_url }}", "secret": 444 }',
# https://github.com/caronc/apprise/wiki/Notify_Custom_JSON#get-parameter-manipulation
"application-notification_urls": test_notification_url,
"application-minutes_between_check": 180,
"application-fetch_backend": "html_requests"
},
follow_redirects=True
)
assert b'Settings updated' in res.data
# Add a watch and trigger a HTTP POST
test_url = url_for('test_endpoint', _external=True)
res = client.post(
url_for("form_quick_watch_add"),
data={"url": test_url, "tag": 'nice one'},
follow_redirects=True
)
assert b"Watch added" in res.data
time.sleep(2)
set_modified_response()
client.get(url_for("form_watch_checknow"), follow_redirects=True)
time.sleep(2)
with open("test-datastore/notification.txt", 'r') as f:
x=f.read()
j = json.loads(x)
assert j['url'].startswith('http://localhost')
assert j['secret'] == 444
# URL check, this will always be converted to lowercase
assert os.path.isfile("test-datastore/notification-url.txt")
with open("test-datastore/notification-url.txt", 'r') as f:
notification_url = f.read()
assert 'xxx=http' in notification_url
os.unlink("test-datastore/notification-url.txt")

3
changedetectionio/tests/util.py
Unescape View File

@ -149,6 +149,9 @@ def live_server_setup(live_server):
if data != None:
f.write(data)
with open("test-datastore/notification-url.txt", "w") as f:
f.write(request.url)
print("\n>> Test notification endpoint was hit.\n", data)
return "Text was set"

Write Preview
Loading…
Cancel
Save