Merge branch 'master' of github.com:dgtlmoon/changedetection.io

pull/97/head
dgtlmoon 4 years ago
commit 4dbfd376f2

@ -6,7 +6,6 @@
# @todo enable https://urllib3.readthedocs.io/en/latest/user-guide.html#ssl as option?
# @todo option for interval day/6 hour/etc
# @todo on change detected, config for calling some API
# @todo make tables responsive!
# @todo fetch title into json
# https://distill.io/features
# proxy per check
@ -53,6 +52,8 @@ app.config['NEW_VERSION_AVAILABLE'] = False
app.config['LOGIN_DISABLED'] = False
#app.config["EXPLAIN_TEMPLATE_LOADING"] = True
# Disables caching of the templates
app.config['TEMPLATES_AUTO_RELOAD'] = True
@ -74,6 +75,17 @@ def init_app_secret(datastore_path):
return secret
# Remember python is by reference
# populate_form in wtfors didnt work for me. (try using a setattr() obj type on datastore.watch?)
def populate_form_from_watch(form, watch):
for i in form.__dict__.keys():
if i[0] != '_':
p = getattr(form, i)
if hasattr(p, 'data') and i in watch:
if not p.data:
setattr(p, "data", watch[i])
# We use the whole watch object from the store/JSON so we can see if there's some related status in terms of a thread
# running or something similar.
@app.template_filter('format_last_checked_time')
@ -345,82 +357,47 @@ def changedetection_app(config=None, datastore_o=None):
return datastore.data['watching'][uuid]['previous_md5']
@app.route("/edit/<string:uuid>", methods=['GET', 'POST'])
@login_required
def edit_page(uuid):
import validators
from backend import forms
form = forms.watchForm(request.form)
# More for testing, possible to return the first/only
if uuid == 'first':
uuid = list(datastore.data['watching'].keys()).pop()
if request.method == 'POST':
url = request.form.get('url').strip()
tag = request.form.get('tag').strip()
if request.method == 'GET':
populate_form_from_watch(form, datastore.data['watching'][uuid])
minutes_recheck = request.form.get('minutes')
if minutes_recheck:
minutes = int(minutes_recheck.strip())
if minutes >= 1:
datastore.data['watching'][uuid]['minutes_between_check'] = minutes
else:
flash("Must be atleast 1 minute between checks.", 'error')
return redirect(url_for('edit_page', uuid=uuid))
# Extra headers
form_headers = request.form.get('headers').strip().split("\n")
extra_headers = {}
if form_headers:
for header in form_headers:
if len(header):
parts = header.split(':', 1)
if len(parts) == 2:
extra_headers.update({parts[0].strip(): parts[1].strip()})
update_obj = {'url': url,
'tag': tag,
'headers': extra_headers
if request.method == 'POST' and form.validate():
update_obj = {'url': form.url.data.strip(),
'tag': form.tag.data.strip(),
'headers': form.headers.data
}
# Notification URLs
form_notification_text = request.form.get('notification_urls')
notification_urls = []
if form_notification_text:
for text in form_notification_text.strip().split("\n"):
text = text.strip()
if len(text):
notification_urls.append(text)
datastore.data['watching'][uuid]['notification_urls'] = notification_urls
datastore.data['watching'][uuid]['notification_urls'] = form.notification_urls.data
# Ignore text
form_ignore_text = request.form.get('ignore-text')
ignore_text = []
if form_ignore_text:
for text in form_ignore_text.strip().split("\n"):
text = text.strip()
if len(text):
ignore_text.append(text)
datastore.data['watching'][uuid]['ignore_text'] = ignore_text
form_ignore_text = form.ignore_text.data
datastore.data['watching'][uuid]['ignore_text'] = form_ignore_text
# Reset the previous_md5 so we process a new snapshot including stripping ignore text.
if form_ignore_text:
if len(datastore.data['watching'][uuid]['history']):
update_obj['previous_md5'] = get_current_checksum_include_ignore_text(uuid=uuid)
# CSS Filter
css_filter = request.form.get('css_filter')
if css_filter:
datastore.data['watching'][uuid]['css_filter'] = css_filter.strip()
datastore.data['watching'][uuid]['css_filter'] = form.css_filter.data.strip()
# Reset the previous_md5 so we process a new snapshot including stripping ignore text.
if form.css_filter.data.strip() != datastore.data['watching'][uuid]['css_filter']:
if len(datastore.data['watching'][uuid]['history']):
update_obj['previous_md5'] = get_current_checksum_include_ignore_text(uuid=uuid)
validators.url(url) # @todo switch to prop/attr/observer
datastore.data['watching'][uuid].update(update_obj)
datastore.needs_write = True
flash("Updated watch.")
@ -428,10 +405,9 @@ def changedetection_app(config=None, datastore_o=None):
# Queue the watch for immediate recheck
update_q.put(uuid)
trigger_n = request.form.get('trigger-test-notification')
if trigger_n:
n_object = {'watch_url': url,
'notification_urls': notification_urls}
if form.trigger_check.data:
n_object = {'watch_url': form.url.data.strip(),
'notification_urls': form.notification_urls.data}
notification_q.put(n_object)
flash('Notifications queued.')
@ -439,7 +415,7 @@ def changedetection_app(config=None, datastore_o=None):
return redirect(url_for('index'))
else:
output = render_template("edit.html", uuid=uuid, watch=datastore.data['watching'][uuid])
output = render_template("edit.html", uuid=uuid, watch=datastore.data['watching'][uuid], form=form)
return output
@ -447,11 +423,26 @@ def changedetection_app(config=None, datastore_o=None):
@login_required
def settings_page():
from backend import forms
form = forms.globalSettingsForm(request.form)
if request.method == 'GET':
if request.values.get('notification-test'):
url_count = len(datastore.data['settings']['application']['notification_urls'])
if url_count:
form.minutes_between_check.data = int(datastore.data['settings']['requests']['minutes_between_check'] / 60)
form.notification_urls.data = datastore.data['settings']['application']['notification_urls']
# Password unset is a GET
if request.values.get('removepassword') == 'true':
from pathlib import Path
datastore.data['settings']['application']['password'] = False
flash("Password protection removed.", 'notice')
flask_login.logout_user()
if request.method == 'POST' and form.validate():
datastore.data['settings']['application']['notification_urls'] = form.notification_urls.data
datastore.data['settings']['requests']['minutes_between_check'] = form.minutes_between_check.data * 60
if len(form.notification_urls.data):
import apprise
apobj = apprise.Apprise()
apobj.debug = True
@ -465,74 +456,29 @@ def changedetection_app(config=None, datastore_o=None):
)
if outcome:
flash("{} Notification URLs reached.".format(url_count), "notice")
flash("{} Notification URLs reached.".format(len(form.notification_urls.data)), "notice")
else:
flash("One or more Notification URLs failed", 'error')
return redirect(url_for('settings_page'))
if request.values.get('removepassword'):
from pathlib import Path
datastore.data['settings']['application']['password'] = False
flash("Password protection removed.", 'notice')
flask_login.logout_user()
return redirect(url_for('settings_page'))
if request.method == 'POST':
password = request.values.get('password')
if password:
import hashlib
import base64
import secrets
# Make a new salt on every new password and store it with the password
salt = secrets.token_bytes(32)
key = hashlib.pbkdf2_hmac('sha256', password.encode('utf-8'), salt, 100000)
store = base64.b64encode(salt + key).decode('ascii')
datastore.data['settings']['application']['password'] = store
flash("Password protection enabled.", 'notice')
flask_login.logout_user()
return redirect(url_for('index'))
try:
minutes = int(request.values.get('minutes').strip())
except ValueError:
flash("Invalid value given, use an integer.", "error")
else:
if minutes >= 1:
datastore.data['settings']['requests']['minutes_between_check'] = minutes
datastore.data['settings']['application']['notification_urls'] = form.notification_urls.data
datastore.needs_write = True
else:
flash("Must be atleast 1 minute.", 'error')
# 'validators' package doesnt work because its often a non-stanadard protocol. :(
datastore.data['settings']['application']['notification_urls'] = []
trigger_n = request.form.get('trigger-test-notification')
for n in request.values.get('notification_urls').strip().split("\n"):
url = n.strip()
datastore.data['settings']['application']['notification_urls'].append(url)
datastore.needs_write = True
if trigger_n:
if form.trigger_check.data:
n_object = {'watch_url': "Test from changedetection.io!",
'notification_urls': datastore.data['settings']['application']['notification_urls']}
'notification_urls': form.notification_urls.data}
notification_q.put(n_object)
flash('Notifications queued.')
flash("Settings updated.")
if form.password.encrypted_password:
datastore.data['settings']['application']['password'] = form.password.encrypted_password
flash("Password protection enabled.", 'notice')
flask_login.logout_user()
return redirect(url_for('index'))
flash("Settings updated.")
output = render_template("settings.html",
minutes=datastore.data['settings']['requests']['minutes_between_check'],
notification_urls="\r\n".join(
datastore.data['settings']['application']['notification_urls']))
output = render_template("settings.html", form=form)
return output
@app.route("/import", methods=['GET', "POST"])

@ -0,0 +1,109 @@
from wtforms import Form, BooleanField, StringField, PasswordField, validators, IntegerField, fields, TextAreaField, \
Field
from wtforms import widgets
from wtforms.fields import html5
class StringListField(StringField):
widget = widgets.TextArea()
def _value(self):
if self.data:
return "\r\n".join(self.data)
else:
return u''
# incoming
def process_formdata(self, valuelist):
if valuelist:
# Remove empty strings
cleaned = list(filter(None, valuelist[0].split("\n")))
self.data = [x.strip() for x in cleaned]
p = 1
else:
self.data = []
class SaltyPasswordField(StringField):
widget = widgets.PasswordInput()
encrypted_password = ""
def build_password(self, password):
import hashlib
import base64
import secrets
# Make a new salt on every new password and store it with the password
salt = secrets.token_bytes(32)
key = hashlib.pbkdf2_hmac('sha256', password.encode('utf-8'), salt, 100000)
store = base64.b64encode(salt + key).decode('ascii')
return store
# incoming
def process_formdata(self, valuelist):
if valuelist:
# Remove empty strings
self.encrypted_password = self.build_password(valuelist[0])
self.data=[]
else:
self.data = []
# Separated by key:value
class StringDictKeyValue(StringField):
widget = widgets.TextArea()
def _value(self):
if self.data:
output = u''
for k in self.data.keys():
output += "{}: {}\r\n".format(k, self.data[k])
return output
else:
return u''
# incoming
def process_formdata(self, valuelist):
if valuelist:
self.data = {}
# Remove empty strings
cleaned = list(filter(None, valuelist[0].split("\n")))
for s in cleaned:
parts = s.strip().split(':')
if len(parts) == 2:
self.data.update({parts[0].strip(): parts[1].strip()})
else:
self.data = {}
class watchForm(Form):
# https://wtforms.readthedocs.io/en/2.3.x/fields/#module-wtforms.fields.html5
# `require_tld` = False is needed even for the test harness "http://localhost:5005.." to run
url = html5.URLField('URL', [validators.URL(require_tld=False)])
tag = StringField('Tag', [validators.Optional(), validators.Length(max=35)])
minutes_between_check = html5.IntegerField('Maximum time in minutes until recheck',
[validators.Optional(), validators.NumberRange(min=1)])
css_filter = StringField('CSS Filter')
ignore_text = StringListField('Ignore Text')
notification_urls = StringListField('Notification URL List')
headers = StringDictKeyValue('Request Headers')
trigger_check = BooleanField('Send test notification on save')
class globalSettingsForm(Form):
password = SaltyPasswordField()
remove_password = BooleanField('Remove password')
minutes_between_check = html5.IntegerField('Maximum time in minutes until recheck',
[validators.NumberRange(min=1)])
notification_urls = StringListField('Notification URL List')
trigger_check = BooleanField('Send test notification on save')

@ -5,7 +5,7 @@
"main": "index.js",
"scripts": {
"test": "echo \"Error: no test specified\" && exit 1",
"scss": "node-sass --watch *.scss -o ."
"scss": "node-sass --watch styles.scss diff.scss -o ."
},
"author": "",
"license": "ISC",

@ -225,12 +225,36 @@ footer {
.paused-state.state-False:hover img {
opacity: 0.8; }
.monospaced-textarea textarea {
width: 100%;
font-family: monospace;
white-space: pre;
overflow-wrap: normal;
overflow-x: scroll; }
.pure-form {
/* The input fields with errors */
/* The list of errors */ }
.pure-form .pure-control-group, .pure-form .pure-group, .pure-form .pure-controls {
padding-bottom: 1em; }
.pure-form .pure-control-group dd, .pure-form .pure-group dd, .pure-form .pure-controls dd {
margin: 0px; }
.pure-form .error input {
background-color: #ffebeb; }
.pure-form ul.errors {
padding: .5em .6em;
border: 1px solid #dd0000;
border-radius: 4px;
vertical-align: middle;
-webkit-box-sizing: border-box;
box-sizing: border-box; }
.pure-form ul.errors li {
margin-left: 1em;
color: #dd0000; }
.pure-form label {
font-weight: bold; }
.pure-form input[type=url] {
width: 100%; }
.pure-form textarea {
width: 100%;
font-size: 14px; }

@ -297,8 +297,43 @@ footer {
}
}
.monospaced-textarea {
textarea {
width: 100%;
font-family: monospace;
white-space: pre;
overflow-wrap: normal;
overflow-x: scroll;
}
}
.pure-form {
.pure-control-group, .pure-group, .pure-controls {
padding-bottom: 1em;
dd {
margin: 0px;
}
}
/* The input fields with errors */
.error {
input {
background-color: #ffebeb;
}
}
/* The list of errors */
ul.errors {
padding: .5em .6em;
border: 1px solid #dd0000;
border-radius: 4px;
vertical-align: middle;
-webkit-box-sizing: border-box;
box-sizing: border-box;
li {
margin-left: 1em;
color: #dd0000;
}
}
label {
font-weight: bold;

@ -0,0 +1,12 @@
{% macro render_field(field) %}
<dt {% if field.errors %} class="error" {% endif %}>{{ field.label }}
<dd {% if field.errors %} class="error" {% endif %}>{{ field(**kwargs)|safe }}
{% if field.errors %}
<ul class=errors>
{% for error in field.errors %}
<li>{{ error }}</li>
{% endfor %}
</ul>
{% endif %}
</dd>
{% endmacro %}

@ -1,102 +1,58 @@
{% extends 'base.html' %}
{% block content %}
<div class="edit-form">
{% from '_helpers.jinja' import render_field %}
<div class="edit-form monospaced-textarea">
<form class="pure-form pure-form-stacked" action="/edit/{{uuid}}" method="POST">
<fieldset>
<div class="pure-control-group">
<label for="url">URL</label>
<input type="url" id="url" required="" placeholder="https://..." name="url" value="{{ watch.url}}"
size="50"/>
<span class="pure-form-message-inline">This is a required field.</span>
{{ render_field(form.url, placeholder="https://...", size=30, required=true) }}
</div>
<div class="pure-control-group">
<label for="tag">Tag</label>
<input type="text" placeholder="tag" size="10" id="tag" name="tag" value="{{ watch.tag}}"/>
<span class="pure-form-message-inline">Grouping tags, can be a comma separated list.</span>
{{ render_field(form.tag, size=10) }}
</div>
</br>
<div class="pure-control-group">
<label for="minutes">Maximum time in minutes until recheck.</label>
<input type="text" id="minutes" name="minutes" value="{{watch.minutes_between_check}}"
size="5"/>
<span class="pure-form-message-inline">Minimum 1 minute between recheck</span>
{{ render_field(form.minutes_between_check, size=5) }}
</div>
<br/>
<div class="pure-control-group">
<label for="minutes">CSS Filter</label>
<input type="text" id="css_filter" name="css_filter" value="{{watch.css_filter}}"
size="25" placeholder=".class-name or #some-id, or other CSS selector rule."/>
{{ render_field(form.css_filter, size=25, placeholder=".class-name or #some-id, or other CSS selector rule.") }}
<span class="pure-form-message-inline">Limit text to this CSS rule, only text matching this CSS rule is included.<br/>
Please be sure that you thoroughly understand how to write CSS selector rules before filing an issue on GitHub!<br/>
Go <a href="https://github.com/dgtlmoon/changedetection.io/wiki/CSS-Selector-help">here for more CSS selector help</a>
</span>
</div>
<br/>
<!-- @todo: move to tabs --->
<fieldset class="pure-group">
<label for="ignore-text">Ignore text</label>
<textarea id="ignore-text" name="ignore-text" class="pure-input-1-2" placeholder=""
style="width: 100%;
font-family:monospace;
white-space: pre;
overflow-wrap: normal;
overflow-x: scroll;" rows="5">{% for value in watch.ignore_text %}{{ value }}
{% endfor %}</textarea>
{{ render_field(form.ignore_text, rows=5) }}
<span class="pure-form-message-inline">Each line will be processed separately as an ignore rule.</span>
</fieldset>
<!-- @todo: move to tabs --->
<fieldset class="pure-group">
<label for="headers">Extra request headers</label>
<textarea id="headers" name="headers" class="pure-input-1-2" placeholder="Example
{{ render_field(form.headers, rows=5, placeholder="Example
Cookie: foobar
User-Agent: wonderbra 1.0"
style="width: 100%;
font-family:monospace;
white-space: pre;
overflow-wrap: normal;
overflow-x: scroll;" rows="5">{% for key, value in watch.headers.items() %}{{ key }}: {{ value }}
{% endfor %}</textarea>
<br/>
User-Agent: wonderbra 1.0") }}
</fieldset>
<div class="pure-control-group">
<label for="tag">Notification URLs</label>
<textarea id="notification_urls" name="notification_urls" class="pure-input-1-2" placeholder=""
style="width: 100%;
font-family:monospace;
white-space: pre;
overflow-wrap: normal;
overflow-x: scroll;" rows="5">{% for value in watch.notification_urls %}{{ value }}
{% endfor %}</textarea>
<span class="pure-form-message-inline">Use <a target=_new href="https://github.com/caronc/apprise">AppRise URLs</a> for notification to just about any service!</a> </span>
<br/>
<div class="pure-controls">
<span class="pure-form-message-inline"><label for="trigger-test-notification" class="pure-checkbox">
<input type="checkbox" id="trigger-test-notification" name="trigger-test-notification"> Send test notification on save.</label></span>
<div class="pure-control-group">
{{ render_field(form.notification_urls, rows=5, placeholder="Gitter - gitter://token/room
Office365 - o365://TenantID:AccountEmail/ClientID/ClientSecret/TargetEmail
AWS SNS - sns://AccessKeyID/AccessSecretKey/RegionName/+PhoneNo
SMTPS - mailtos://user:pass@mail.domain.com?to=receivingAddress@example.com
") }}
<span class="pure-form-message-inline">Use <a target=_new href="https://github.com/caronc/apprise">AppRise URLs</a> for notification to just about any service!</span>
</div>
<div class="pure-controls">
{{ render_field(form.trigger_check, rows=5) }}
</div>
<br/>
<div class="pure-control-group">
<button type="submit" class="pure-button pure-button-primary">Save</button>
</div>
<br/>
<div class="pure-control-group">
<a href="/" class="pure-button button-small button-cancel">Cancel</a>
<a href="/api/delete?uuid={{uuid}}"
class="pure-button button-small button-error ">Delete</a>
</div>
</fieldset>
</form>

@ -1,42 +1,28 @@
{% extends 'base.html' %}
{% block content %}
<div class="edit-form">
{% from '_helpers.jinja' import render_field %}
<div class="edit-form">
<form class="pure-form pure-form-stacked settings" action="/settings" method="POST">
<fieldset>
<div class="pure-control-group">
<label for="minutes">Maximum time in minutes until recheck.</label>
<input type="text" id="minutes" required="" name="minutes" value="{{minutes}}"
size="5"/>
<span class="pure-form-message-inline">This is a required field.</span><br/>
<span class="pure-form-message-inline">Minimum 1 minute between recheck</span>
{{ render_field(form.minutes_between_check, size=5) }}
</div>
<br/>
<hr>
<div class="pure-control-group">
<label for="minutes">Password protection</label>
<input type="password" id="password" name="password" size="15"/>
{% if current_user.is_authenticated %}
<a href="/settings?removepassword=true" class="pure-button pure-button-primary">Remove password</a>
{% else %}
{{ render_field(form.password, size=10) }}
{% endif %}
</div>
<br/>
<hr>
<div class="pure-control-group">
<label for="minutes">Global notification settings</label><br/>
Notification URLs <a href="https://github.com/caronc/apprise"> see Apprise examples</a>.
<textarea style="overflow-wrap: normal; overflow-x: scroll;" id="notification_urls" name="notification_urls" cols="80"
rows="6" wrap=off placeholder="Example:
Gitter - gitter://token/room
{{ render_field(form.notification_urls, rows=5, placeholder="Gitter - gitter://token/room
Office365 - o365://TenantID:AccountEmail/ClientID/ClientSecret/TargetEmail
AWS SNS - sns://AccessKeyID/AccessSecretKey/RegionName/+PhoneNo
SMTPS - mailtos://user:pass@mail.domain.com?to=receivingAddress@example.com
">{{notification_urls}}</textarea>
") }}
<span class="pure-form-message-inline">Use <a target=_new href="https://github.com/caronc/apprise">AppRise URLs</a> for notification to just about any service!</span>
</div>
<div class="pure-controls">
<span class="pure-form-message-inline"><label for="trigger-test-notification" class="pure-checkbox">

@ -82,7 +82,7 @@ def set_modified_ignore_response():
def test_check_ignore_text_functionality(client, live_server):
sleep_time_for_fetch_thread = 3
ignore_text = "XXXXX\nYYYYY\nZZZZZ"
ignore_text = "XXXXX\r\nYYYYY\r\nZZZZZ"
set_original_ignore_response()
# Give the endpoint time to spin up
@ -107,7 +107,7 @@ def test_check_ignore_text_functionality(client, live_server):
# Add our URL to the import page
res = client.post(
url_for("edit_page", uuid="first"),
data={"ignore-text": ignore_text, "url": test_url, "tag": "", "headers": ""},
data={"ignore_text": ignore_text, "url": test_url},
follow_redirects=True
)
assert b"Updated watch." in res.data

@ -11,6 +11,9 @@ feedgen ~= 0.9
flask-login ~= 0.5
pytz
urllib3
wtforms ~= 2.3.3
# Notification library
apprise ~= 0.9

Loading…
Cancel
Save