diff --git a/.github/workflows/image-javascript.yml b/.github/workflows/image-javascript.yml
new file mode 100644
index 00000000..a6b4d8ee
--- /dev/null
+++ b/.github/workflows/image-javascript.yml
@@ -0,0 +1,87 @@
+name: Javascript/Webdriver support - Test, build and push to Docker Hub :javascript tag
+
+on:
+ push:
+ branches: [ javascript-browser ]
+
+jobs:
+ build:
+ runs-on: ubuntu-latest
+ steps:
+
+ - uses: actions/checkout@v2
+ - name: Set up Python 3.9
+ uses: actions/setup-python@v2
+ with:
+ python-version: 3.9
+
+ - name: Install dependencies
+ run: |
+ python -m pip install --upgrade pip
+ pip install flake8 pytest
+ if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
+
+ - name: Lint with flake8
+ run: |
+ # stop the build if there are Python syntax errors or undefined names
+ flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
+ # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
+ flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
+
+ - name: Create release metadata
+ run: |
+ # COPY'ed by Dockerfile into backend/ of the image, then read by the server in store.py
+ echo ${{ github.sha }} > backend/source.txt
+ echo ${{ github.ref }} > backend/tag.txt
+
+ - name: Test with pytest
+ run: |
+ # Each test is totally isolated and performs its own cleanup/reset
+ cd backend; ./run_all_tests.sh
+
+ - name: Set up QEMU
+ uses: docker/setup-qemu-action@v1
+ with:
+ image: tonistiigi/binfmt:latest
+ platforms: all
+
+ - name: Login to Docker Hub
+ uses: docker/login-action@v1
+ with:
+ username: ${{ secrets.DOCKER_HUB_USERNAME }}
+ password: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }}
+
+ - name: Set up Docker Buildx
+ id: buildx
+ uses: docker/setup-buildx-action@v1
+ with:
+ install: true
+ version: latest
+ driver-opts: image=moby/buildkit:master
+
+ - name: Build and push
+ id: docker_build
+ uses: docker/build-push-action@v2
+ with:
+ context: ./
+ file: ./Dockerfile
+ push: true
+ tags: |
+ ${{ secrets.DOCKER_HUB_USERNAME }}/changedetection.io:javascript-dev
+ platforms: linux/amd64,linux/arm64,linux/arm/v6,linux/arm/v7
+ cache-from: type=local,src=/tmp/.buildx-cache
+ cache-to: type=local,dest=/tmp/.buildx-cache
+
+ - name: Image digest
+ run: echo step SHA ${{ steps.vars.outputs.sha_short }} tag ${{steps.vars.outputs.tag}} branch ${{steps.vars.outputs.branch}} digest ${{ steps.docker_build.outputs.digest }}
+
+# failed: Cache service responded with 503
+# - name: Cache Docker layers
+# uses: actions/cache@v2
+# with:
+# path: /tmp/.buildx-cache
+# key: ${{ runner.os }}-buildx-${{ github.sha }}
+# restore-keys: |
+# ${{ runner.os }}-buildx-
+
+
diff --git a/.github/workflows/image.yml b/.github/workflows/image.yml
index 3150c60e..7abf493c 100644
--- a/.github/workflows/image.yml
+++ b/.github/workflows/image.yml
@@ -44,6 +44,7 @@ jobs:
with:
image: tonistiigi/binfmt:latest
platforms: all
+
- name: Login to Docker Hub
uses: docker/login-action@v1
with:
@@ -66,10 +67,8 @@ jobs:
file: ./Dockerfile
push: true
tags: |
- ${{ secrets.DOCKER_HUB_USERNAME }}/changedetection.io:latest
- # ${{ secrets.DOCKER_HUB_USERNAME }}:/changedetection.io:${{ env.RELEASE_VERSION }}
+ ${{ secrets.DOCKER_HUB_USERNAME }}/changedetection.io
platforms: linux/amd64,linux/arm64,linux/arm/v6,linux/arm/v7
-# platforms: linux/amd64
cache-from: type=local,src=/tmp/.buildx-cache
cache-to: type=local,dest=/tmp/.buildx-cache
diff --git a/README.md b/README.md
index 601bb0c4..e9ca8dcc 100644
--- a/README.md
+++ b/README.md
@@ -32,6 +32,7 @@ Know when ...
_Need an actual Chrome runner with Javascript support? see the experimental Javascript/Chrome support changedetection.io branch!_
**Get monitoring now! super simple, one command!**
+
Run the python code on your own machine by cloning this repository, or with docker and/or docker-compose
With one docker-compose command
@@ -40,24 +41,18 @@ With one docker-compose command
docker-compose up -d
```
-or
-
+Then visit http://127.0.0.1:5000 , You should now be able to access the UI.
-```bash
-docker run -d --restart always -p "127.0.0.1:5000:5000" -v datastore-volume:/datastore --name changedetection.io dgtlmoon/changedetection.io
-```
+_Now with per-site configurable support for using a fast built in HTTP fetcher or use a Chrome based fetcher for monitoring of JavaScript websites!_
-Now visit http://127.0.0.1:5000 , You should now be able to access the UI.
-#### Updating to latest version
+#### Updating to the latest version
Highly recommended :)
```bash
docker pull dgtlmoon/changedetection.io
-docker kill $(docker ps -a|grep changedetection.io|awk '{print $1}')
-docker rm $(docker ps -a|grep changedetection.io|awk '{print $1}')
-docker run -d --restart always -p "127.0.0.1:5000:5000" -v datastore-volume:/datastore --name changedetection.io dgtlmoon/changedetection.io
+docker-compose up -d
```
### Screenshots
@@ -135,6 +130,7 @@ For more information see https://docs.python-requests.org/en/master/user/advance
This proxy support also extends to the notifications https://github.com/caronc/apprise/issues/387#issuecomment-841718867
+
### Notes
- ~~Does not yet support Javascript~~
@@ -143,6 +139,7 @@ This proxy support also extends to the notifications https://github.com/caronc/a
See the experimental Javascript/Chrome browser support!
+
### RaspberriPi support?
RaspberriPi and linux/arm/v6 linux/arm/v7 arm64 devices are supported!
diff --git a/backend/__init__.py b/backend/__init__.py
index bee1bc13..91608993 100644
--- a/backend/__init__.py
+++ b/backend/__init__.py
@@ -378,6 +378,7 @@ def changedetection_app(config=None, datastore_o=None):
if uuid == 'first':
uuid = list(datastore.data['watching'].keys()).pop()
+
if request.method == 'GET':
if not uuid in datastore.data['watching']:
flash("No watch with the UUID %s found." % (uuid), "error")
@@ -385,17 +386,25 @@ def changedetection_app(config=None, datastore_o=None):
populate_form_from_watch(form, datastore.data['watching'][uuid])
+ if datastore.data['watching'][uuid]['fetch_backend'] is None:
+ form.fetch_backend.data = datastore.data['settings']['application']['fetch_backend']
+
if request.method == 'POST' and form.validate():
# Re #110, if they submit the same as the default value, set it to None, so we continue to follow the default
if form.minutes_between_check.data == datastore.data['settings']['requests']['minutes_between_check']:
form.minutes_between_check.data = None
+ if form.fetch_backend.data == datastore.data['settings']['application']['fetch_backend']:
+ form.fetch_backend.data = None
+
+
update_obj = {'url': form.url.data.strip(),
'minutes_between_check': form.minutes_between_check.data,
'tag': form.tag.data.strip(),
'title': form.title.data.strip(),
- 'headers': form.headers.data
+ 'headers': form.headers.data,
+ 'fetch_backend': form.fetch_backend.data
}
# Notification URLs
@@ -428,8 +437,8 @@ def changedetection_app(config=None, datastore_o=None):
if form.trigger_check.data:
n_object = {'watch_url': form.url.data.strip(),
- 'notification_urls': form.notification_urls.data,
- 'uuid': uuid}
+ 'notification_urls': form.notification_urls.data
+ }
notification_q.put(n_object)
flash('Notifications queued.')
@@ -464,12 +473,15 @@ def changedetection_app(config=None, datastore_o=None):
def settings_page():
from backend import forms
+ from backend import content_fetcher
+
form = forms.globalSettingsForm(request.form)
if request.method == 'GET':
form.minutes_between_check.data = int(datastore.data['settings']['requests']['minutes_between_check'])
form.notification_urls.data = datastore.data['settings']['application']['notification_urls']
form.extract_title_as_title.data = datastore.data['settings']['application']['extract_title_as_title']
+ form.fetch_backend.data = datastore.data['settings']['application']['fetch_backend']
form.notification_title.data = datastore.data['settings']['application']['notification_title']
form.notification_body.data = datastore.data['settings']['application']['notification_body']
@@ -486,6 +498,7 @@ def changedetection_app(config=None, datastore_o=None):
datastore.data['settings']['application']['notification_urls'] = form.notification_urls.data
datastore.data['settings']['requests']['minutes_between_check'] = form.minutes_between_check.data
datastore.data['settings']['application']['extract_title_as_title'] = form.extract_title_as_title.data
+ datastore.data['settings']['application']['fetch_backend'] = form.fetch_backend.data
datastore.data['settings']['application']['notification_title'] = form.notification_title.data
datastore.data['settings']['application']['notification_body'] = form.notification_body.data
diff --git a/backend/content_fetcher.py b/backend/content_fetcher.py
new file mode 100644
index 00000000..40ff7327
--- /dev/null
+++ b/backend/content_fetcher.py
@@ -0,0 +1,137 @@
+import os
+import time
+from abc import ABC, abstractmethod
+from selenium import webdriver
+from selenium.webdriver.common.desired_capabilities import DesiredCapabilities
+from selenium.common.exceptions import WebDriverException
+import urllib3.exceptions
+
+
+class EmptyReply(Exception):
+ pass
+
+class Fetcher():
+ error = None
+ status_code = None
+ content = None # Should be bytes?
+
+ fetcher_description ="No description"
+
+ @abstractmethod
+ def get_error(self):
+ return self.error
+
+ @abstractmethod
+ def run(self, url, timeout, request_headers):
+ # Should set self.error, self.status_code and self.content
+ pass
+
+ @abstractmethod
+ def get_last_status_code(self):
+ return self.status_code
+
+ @abstractmethod
+ # Return true/false if this checker is ready to run, in the case it needs todo some special config check etc
+ def is_ready(self):
+ return True
+
+# Maybe for the future, each fetcher provides its own diff output, could be used for text, image
+# the current one would return javascript output (as we use JS to generate the diff)
+#
+# Returns tuple(mime_type, stream)
+# @abstractmethod
+# def return_diff(self, stream_a, stream_b):
+# return
+
+def available_fetchers():
+ import inspect
+ from backend import content_fetcher
+ p=[]
+ for name, obj in inspect.getmembers(content_fetcher):
+ if inspect.isclass(obj):
+ # @todo html_ is maybe better as fetcher_ or something
+ # In this case, make sure to edit the default one in store.py and fetch_site_status.py
+ if "html_" in name:
+ t=tuple([name,obj.fetcher_description])
+ p.append(t)
+
+ return p
+
+class html_webdriver(Fetcher):
+ fetcher_description = "WebDriver Chrome/Javascript"
+ command_executor = ''
+
+ def __init__(self):
+ self.command_executor = os.getenv("WEBDRIVER_URL",'http://browser-chrome:4444/wd/hub')
+
+ def run(self, url, timeout, request_headers):
+
+ # check env for WEBDRIVER_URL
+ driver = webdriver.Remote(
+ command_executor=self.command_executor,
+ desired_capabilities=DesiredCapabilities.CHROME)
+
+ try:
+ driver.get(url)
+ except WebDriverException as e:
+ # Be sure we close the session window
+ driver.quit()
+ raise
+
+ # @todo - how to check this? is it possible?
+ self.status_code = 200
+
+ # @todo - dom wait loaded?
+ time.sleep(5)
+ self.content = driver.page_source
+
+ driver.quit()
+
+
+ def is_ready(self):
+ from selenium import webdriver
+ from selenium.webdriver.common.desired_capabilities import DesiredCapabilities
+ from selenium.common.exceptions import WebDriverException
+
+ driver = webdriver.Remote(
+ command_executor='http://browser-chrome:4444/wd/hub',
+ desired_capabilities=DesiredCapabilities.CHROME)
+
+ # driver.quit() seems to cause better exceptions
+ driver.quit()
+
+
+ return True
+
+# "html_requests" is listed as the default fetcher in store.py!
+class html_requests(Fetcher):
+ fetcher_description = "Basic fast Plaintext/HTTP Client"
+
+ def run(self, url, timeout, request_headers):
+ import requests
+ try:
+ r = requests.get(url,
+ headers=request_headers,
+ timeout=timeout,
+ verify=False)
+
+ html = r.text
+
+ # Usually from networkIO/requests level
+ except (
+ requests.exceptions.ConnectionError, requests.exceptions.ReadTimeout,
+ requests.exceptions.MissingSchema) as e:
+ self.error = str(e)
+ return None
+
+ except Exception as e:
+ self.error = "Other exception" + str(e)
+ return None
+
+ # @todo test this
+ if not r or not html or not len(html):
+ raise EmptyReply(url)
+
+ self.status_code = r.status_code
+ self.content = html
+
diff --git a/backend/fetch_site_status.py b/backend/fetch_site_status.py
index 870a6515..242a46b1 100644
--- a/backend/fetch_site_status.py
+++ b/backend/fetch_site_status.py
@@ -1,11 +1,13 @@
import time
-import requests
+from backend import content_fetcher
import hashlib
from inscriptis import get_text
import urllib3
from . import html_tools
urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
+from selenium import webdriver
+from selenium.webdriver.common.desired_capabilities import DesiredCapabilities
# Some common stuff here that can be moved to a base class
@@ -52,8 +54,8 @@ class perform_site_check():
def run(self, uuid):
timestamp = int(time.time()) # used for storage etc too
- stripped_text_from_html = False
changed_detected = False
+ stripped_text_from_html = ""
update_obj = {'previous_md5': self.datastore.data['watching'][uuid]['previous_md5'],
'history': {},
@@ -72,71 +74,63 @@ class perform_site_check():
if 'Accept-Encoding' in request_headers and "br" in request_headers['Accept-Encoding']:
request_headers['Accept-Encoding'] = request_headers['Accept-Encoding'].replace(', br', '')
- try:
- timeout = self.datastore.data['settings']['requests']['timeout']
- except KeyError:
- # @todo yeah this should go back to the default value in store.py, but this whole object should abstract off it
- timeout = 15
+ # @todo check the failures are really handled how we expect
- try:
+ else:
+ timeout = self.datastore.data['settings']['requests']['timeout']
url = self.datastore.get_val(uuid, 'url')
- r = requests.get(url,
- headers=request_headers,
- timeout=timeout,
- verify=False)
+ # Pluggable content fetcher
+ prefer_backend = self.datastore.data['watching'][uuid]['fetch_backend']
+ if hasattr(content_fetcher, prefer_backend):
+ klass = getattr(content_fetcher, prefer_backend)
+ else:
+ # If the klass doesnt exist, just use a default
+ klass = getattr(content_fetcher, "html_requests")
+
- html = r.text
+ fetcher = klass()
+ fetcher.run(url, timeout, request_headers)
+ # Fetching complete, now filters
+ # @todo move to class / maybe inside of fetcher abstract base?
is_html = True
css_filter_rule = self.datastore.data['watching'][uuid]['css_filter']
if css_filter_rule and len(css_filter_rule.strip()):
if 'json:' in css_filter_rule:
- stripped_text_from_html = html_tools.extract_json_as_string(html, css_filter_rule)
+ stripped_text_from_html = html_tools.extract_json_as_string(content=fetcher.content, jsonpath_filter=css_filter_rule)
is_html = False
else:
# CSS Filter, extract the HTML that matches and feed that into the existing inscriptis::get_text
- html = html_tools.css_filter(css_filter=css_filter_rule, html_content=r.content)
+ stripped_text_from_html = html_tools.css_filter(css_filter=css_filter_rule, html_content=fetcher.content)
if is_html:
- stripped_text_from_html = get_text(html)
+ # CSS Filter, extract the HTML that matches and feed that into the existing inscriptis::get_text
+ html_content = fetcher.content
+ css_filter_rule = self.datastore.data['watching'][uuid]['css_filter']
+ if css_filter_rule and len(css_filter_rule.strip()):
+ html_content = html_tools.css_filter(css_filter=css_filter_rule, html_content=fetcher.content)
- # Usually from networkIO/requests level
- except (requests.exceptions.ConnectionError, requests.exceptions.ReadTimeout) as e:
- update_obj["last_error"] = str(e)
- print(str(e))
+ # get_text() via inscriptis
+ stripped_text_from_html = get_text(html_content)
- except requests.exceptions.MissingSchema:
- print("Skipping {} due to missing schema/bad url".format(uuid))
-
- # Usually from html2text level
- except Exception as e:
- # except UnicodeDecodeError as e:
- update_obj["last_error"] = str(e)
- print(str(e))
- # figure out how to deal with this cleaner..
- # 'utf-8' codec can't decode byte 0xe9 in position 480: invalid continuation byte
-
-
- else:
# We rely on the actual text in the html output.. many sites have random script vars etc,
# in the future we'll implement other mechanisms.
- update_obj["last_check_status"] = r.status_code
+ update_obj["last_check_status"] = fetcher.get_last_status_code()
update_obj["last_error"] = False
- if not len(r.text):
- update_obj["last_error"] = "Empty reply"
# If there's text to skip
# @todo we could abstract out the get_text() to handle this cleaner
if len(self.datastore.data['watching'][uuid]['ignore_text']):
- content = self.strip_ignore_text(stripped_text_from_html,
+ stripped_text_from_html = self.strip_ignore_text(stripped_text_from_html,
self.datastore.data['watching'][uuid]['ignore_text'])
else:
- content = stripped_text_from_html.encode('utf8')
+ stripped_text_from_html = stripped_text_from_html.encode('utf8')
+
- fetched_md5 = hashlib.md5(content).hexdigest()
+ fetched_md5 = hashlib.md5(stripped_text_from_html).hexdigest()
# could be None or False depending on JSON type
if self.datastore.data['watching'][uuid]['previous_md5'] != fetched_md5:
@@ -149,9 +143,9 @@ class perform_site_check():
update_obj["previous_md5"] = fetched_md5
# Extract title as title
- if self.datastore.data['settings']['application']['extract_title_as_title']:
+ if is_html and self.datastore.data['settings']['application']['extract_title_as_title']:
if not self.datastore.data['watching'][uuid]['title'] or not len(self.datastore.data['watching'][uuid]['title']):
- update_obj['title'] = html_tools.extract_element(find='title', html_content=html)
+ update_obj['title'] = html_tools.extract_element(find='title', html_content=fetcher.content)
return changed_detected, update_obj, stripped_text_from_html
diff --git a/backend/forms.py b/backend/forms.py
index 3abb2e55..28d74224 100644
--- a/backend/forms.py
+++ b/backend/forms.py
@@ -1,9 +1,9 @@
-from wtforms import Form, BooleanField, StringField, PasswordField, validators, IntegerField, fields, TextAreaField, \
+from wtforms import Form, SelectField, RadioField, BooleanField, StringField, PasswordField, validators, IntegerField, fields, TextAreaField, \
Field
from wtforms import widgets
from wtforms.validators import ValidationError
from wtforms.fields import html5
-
+from backend import content_fetcher
class StringListField(StringField):
widget = widgets.TextArea()
@@ -82,6 +82,40 @@ class StringDictKeyValue(StringField):
else:
self.data = {}
+class ValidateContentFetcherIsReady(object):
+ """
+ Validates that anything that looks like a regex passes as a regex
+ """
+ def __init__(self, message=None):
+ self.message = message
+
+ def __call__(self, form, field):
+ from backend import content_fetcher
+ import urllib3.exceptions
+
+ # Better would be a radiohandler that keeps a reference to each class
+ if field.data is not None:
+ klass = getattr(content_fetcher, field.data)
+ some_object = klass()
+ try:
+ ready = some_object.is_ready()
+
+ except urllib3.exceptions.MaxRetryError as e:
+ driver_url = some_object.command_executor
+ message = field.gettext('Content fetcher \'%s\' did not respond.' % (field.data))
+ message += '
'+field.gettext('Be sure that the selenium/webdriver runner is running and accessible via network from this container/host.')
+ message += '
' + field.gettext('Did you follow the instructions in the wiki?')
+ message += '
' + field.gettext('WebDriver Host: %s' % (driver_url))
+ message += '
Go here for more information'
+
+ raise ValidationError(message)
+
+ except Exception as e:
+ message = field.gettext('Content fetcher \'%s\' did not respond properly, unable to use it.\n %s')
+ raise ValidationError(message % (field.data, e))
+
+
+
class ValidateListRegex(object):
"""
Validates that anything that looks like a regex passes as a regex
@@ -138,6 +172,8 @@ class watchForm(quickWatchForm):
css_filter = StringField('CSS/JSON Filter', [ValidateCSSJSONInput()])
title = StringField('Title')
+ fetch_backend = RadioField(u'Fetch Method', choices=content_fetcher.available_fetchers(), validators=[ValidateContentFetcherIsReady()])
+
ignore_text = StringListField('Ignore Text', [ValidateListRegex()])
notification_urls = StringListField('Notification URL List')
headers = StringDictKeyValue('Request Headers')
@@ -152,6 +188,9 @@ class globalSettingsForm(Form):
[validators.NumberRange(min=1)])
notification_urls = StringListField('Notification URL List')
+
+ fetch_backend = RadioField(u'Fetch Method', choices=content_fetcher.available_fetchers(), validators=[ValidateContentFetcherIsReady()])
+
extract_title_as_title = BooleanField('Extract from document and use as watch title')
trigger_check = BooleanField('Send test notification on save')
diff --git a/backend/static/images/Google-Chrome-icon.png b/backend/static/images/Google-Chrome-icon.png
new file mode 100644
index 00000000..a4dd10f5
Binary files /dev/null and b/backend/static/images/Google-Chrome-icon.png differ
diff --git a/backend/static/js/settings.js b/backend/static/js/settings.js
index c6eba772..27b07dcc 100644
--- a/backend/static/js/settings.js
+++ b/backend/static/js/settings.js
@@ -14,3 +14,4 @@ window.addEventListener("load", (event) => {
toggleVisible("notification-customisation");
};
});
+
diff --git a/backend/static/js/tabs.js b/backend/static/js/tabs.js
new file mode 100644
index 00000000..36fecf70
--- /dev/null
+++ b/backend/static/js/tabs.js
@@ -0,0 +1,51 @@
+// Rewrite this is a plugin.. is all this JS really 'worth it?'
+
+
+window.addEventListener('hashchange', function() {
+ var tabs = document.getElementsByClassName('active');
+ while (tabs[0]) {
+ tabs[0].classList.remove('active')
+ }
+ set_active_tab();
+}, false);
+
+var has_errors=document.querySelectorAll(".messages .error");
+if (!has_errors.length) {
+ if (document.location.hash == "" ) {
+ document.location.hash = "#general";
+ document.getElementById("default-tab").className = "active";
+ } else {
+ set_active_tab();
+ }
+} else {
+ focus_error_tab();
+}
+
+
+function set_active_tab() {
+ var tab=document.querySelectorAll("a[href='"+location.hash+"']");
+ if (tab.length) {
+ tab[0].parentElement.className="active";
+ }
+ // hash could move the page down
+ window.scrollTo(0, 0);
+}
+
+function focus_error_tab() {
+ // time to use jquery or vuejs really,
+ // activate the tab with the error
+ var tabs = document.querySelectorAll('.tabs li a'),i;
+ for (i = 0; i < tabs.length; ++i) {
+ var tab_name=tabs[i].hash.replace('#','');
+ var pane_errors=document.querySelectorAll('#'+tab_name+' .error')
+ if (pane_errors.length) {
+ document.location.hash = '#'+tab_name;
+ return true;
+ }
+ }
+ return false;
+}
+
+
+
+
diff --git a/backend/static/styles/diff.scss b/backend/static/styles/diff.scss
index 7f08a5fa..98e5f0b5 100644
--- a/backend/static/styles/diff.scss
+++ b/backend/static/styles/diff.scss
@@ -65,4 +65,4 @@ ins {
body {
height: 99%; /* Hide scroll bar in Firefox */
}
-}
\ No newline at end of file
+}
diff --git a/backend/static/styles/styles.css b/backend/static/styles/styles.css
index 60285d35..f59ccd39 100644
--- a/backend/static/styles/styles.css
+++ b/backend/static/styles/styles.css
@@ -129,13 +129,6 @@ body:after, body:before {
max-width: 400px;
display: block; }
-.edit-form {
- background: #fff;
- padding: 2em;
- margin: 1em;
- border-radius: 5px;
- min-width: 70%; }
-
.button-secondary {
color: white;
border-radius: 4px;
@@ -221,15 +214,14 @@ body:after, body:before {
border-top-right-radius: 5px;
border-bottom-right-radius: 5px;
box-shadow: 5px 0 5px -2px #888; }
-
-#diff-jump a {
- color: #1b98f8;
- cursor: grabbing;
- -moz-user-select: none;
- -webkit-user-select: none;
- -ms-user-select: none;
- user-select: none;
- -o-user-select: none; }
+ #diff-jump a {
+ color: #1b98f8;
+ cursor: grabbing;
+ -moz-user-select: none;
+ -webkit-user-select: none;
+ -ms-user-select: none;
+ user-select: none;
+ -o-user-select: none; }
footer {
padding: 10px;
@@ -299,6 +291,11 @@ footer {
font-weight: bold; }
.pure-form textarea {
width: 100%; }
+ .pure-form ul#fetch_backend {
+ margin: 0px;
+ list-style: none; }
+ .pure-form ul#fetch_backend > li > * {
+ display: inline-block; }
@media only screen and (max-width: 760px), (min-device-width: 768px) and (max-device-width: 1024px) {
.box {
@@ -363,3 +360,44 @@ and also iPads specifically.
/* m-d is medium-desktop */
.m-d {
min-width: 80%; } }
+
+.tabs ul {
+ margin: 0px;
+ padding: 0px;
+ display: block; }
+ .tabs ul li {
+ margin-right: 3px;
+ display: inline-block;
+ color: #fff;
+ border-top-left-radius: 5px;
+ border-top-right-radius: 5px;
+ background-color: rgba(255, 255, 255, 0.2); }
+ .tabs ul li.active, .tabs ul li :target {
+ background-color: #fff; }
+ .tabs ul li.active a, .tabs ul li :target a {
+ color: #222;
+ font-weight: bold; }
+ .tabs ul li a {
+ display: block;
+ padding: 0.8em;
+ color: #fff; }
+
+.pure-form-stacked > div:first-child {
+ display: block; }
+
+.edit-form {
+ min-width: 70%; }
+ .edit-form .tab-pane-inner {
+ padding: 0px; }
+ .edit-form .tab-pane-inner:not(:target) {
+ display: none; }
+ .edit-form .tab-pane-inner:target {
+ display: block; }
+ .edit-form .box-wrap {
+ position: relative; }
+ .edit-form .inner {
+ background: #fff;
+ padding: 20px; }
+ .edit-form #actions {
+ display: block;
+ background: #fff; }
diff --git a/backend/static/styles/styles.scss b/backend/static/styles/styles.scss
index 559f1311..4a4b3b83 100644
--- a/backend/static/styles/styles.scss
+++ b/backend/static/styles/styles.scss
@@ -7,7 +7,6 @@ body {
color: #333;
background: #262626;
}
-
.pure-table-even {
background: #fff;
}
@@ -170,13 +169,6 @@ body:after, body:before {
display: block;
}
-.edit-form {
- background: #fff;
- padding: 2em;
- margin: 1em;
- border-radius: 5px;
- min-width: 70%;
-}
.button-secondary {
color: white;
@@ -294,16 +286,15 @@ body:after, body:before {
border-top-right-radius: 5px;
border-bottom-right-radius: 5px;
box-shadow: 5px 0 5px -2px #888;
-}
-
-#diff-jump a {
- color: #1b98f8;
- cursor: grabbing;
- -moz-user-select: none;
- -webkit-user-select: none;
- -ms-user-select: none;
- user-select: none;
- -o-user-select: none;
+ a {
+ color: #1b98f8;
+ cursor: grabbing;
+ -moz-user-select: none;
+ -webkit-user-select: none;
+ -ms-user-select: none;
+ user-select: none;
+ -o-user-select: none;
+ }
}
footer {
@@ -404,6 +395,15 @@ footer {
textarea {
width: 100%;
}
+ ul#fetch_backend {
+ margin: 0px;
+ list-style: none;
+ > li {
+ > * {
+ display: inline-block;
+ }
+ }
+ }
}
@media only screen and (max-width: 760px), (min-device-width: 768px) and (max-device-width: 1024px) {
@@ -417,7 +417,6 @@ footer {
#nav-menu {
overflow-x: scroll;
}
-
}
/*
@@ -425,6 +424,7 @@ Max width before this PARTICULAR table gets nasty
This query will take effect for any screen smaller than 760px
and also iPads specifically.
*/
+
@media only screen and (max-width: 760px), (min-device-width: 768px) and (max-device-width: 1024px) {
input[type='text'] {
@@ -507,3 +507,65 @@ and also iPads specifically.
}
}
+
+
+.tabs {
+ ul {
+ margin: 0px;
+ padding: 0px;
+ display:block;
+ li {
+ margin-right: 3px;
+ display: inline-block;
+ color: #fff;
+ border-top-left-radius: 5px;
+ border-top-right-radius: 5px;
+ background-color: rgba(255, 255, 255, 0.2);
+
+ &.active,:target {
+ background-color: #fff;
+ a {
+ color: #222;
+ font-weight: bold;
+ }
+ }
+ a {
+ display: block;
+ padding: 0.8em;
+ color: #fff;
+ }
+ }
+ }
+}
+
+$form-edge-padding: 20px;
+.pure-form-stacked {
+ >div:first-child {
+ display: block;
+ }
+}
+.edit-form {
+ min-width: 70%;
+ .tab-pane-inner {
+ &:not(:target) {
+ display: none;
+ }
+ &:target {
+ display: block;
+ }
+ // doesnt need padding because theres another row of buttons/activity
+ padding: 0px;
+ }
+ .box-wrap {
+ position: relative;
+ }
+ .inner {
+ background: #fff;;
+ padding: $form-edge-padding;
+ }
+ #actions {
+ display: block;
+ background: #fff;
+ }
+}
+
diff --git a/backend/store.py b/backend/store.py
index a345bfd4..2d3f7abf 100644
--- a/backend/store.py
+++ b/backend/store.py
@@ -39,6 +39,7 @@ class ChangeDetectionStore:
'application': {
'password': False,
'extract_title_as_title': False,
+ 'fetch_backend': 'html_requests',
'notification_urls': [], # Apprise URL list
# Custom notification content
'notification_title': 'ChangeDetection.io Notification - {watch_url}',
@@ -67,6 +68,7 @@ class ChangeDetectionStore:
'ignore_text': [], # List of text to ignore when calculating the comparison checksum
'notification_urls': [], # List of URLs to add to the notification Queue (Usually AppRise)
'css_filter': "",
+ 'fetch_backend': None,
}
if path.isfile('backend/source.txt'):
@@ -193,6 +195,10 @@ class ChangeDetectionStore:
if not self.__data['watching'][uuid]['title']:
self.__data['watching'][uuid]['title'] = None
+ # Default var for fetch_backend
+ if not self.__data['watching'][uuid]['fetch_backend']:
+ self.__data['watching'][uuid]['fetch_backend'] = self.__data['settings']['application']['fetch_backend']
+
self.__data['has_unviewed'] = has_unviewed
return self.__data
@@ -315,18 +321,15 @@ class ChangeDetectionStore:
# Save some text file to the appropriate path and bump the history
# result_obj from fetch_site_status.run()
- def save_history_text(self, uuid, result_obj, contents):
+ def save_history_text(self, watch_uuid, contents):
+ import uuid
- output_path = "{}/{}".format(self.datastore_path, uuid)
- fname = "{}/{}-{}.stripped.txt".format(output_path, result_obj['previous_md5'], str(time.time()))
- with open(fname, 'w') as f:
+ output_path = "{}/{}".format(self.datastore_path, watch_uuid)
+ fname = "{}/{}.stripped.txt".format(output_path, uuid.uuid4())
+ with open(fname, 'wb') as f:
f.write(contents)
f.close()
- # Update history with the stripped text for future reference, this will also mean we save the first
- # Should always be keyed by string(timestamp)
- self.update_watch(uuid, {"history": {str(result_obj["last_checked"]): fname}})
-
return fname
def sync_to_json(self):
diff --git a/backend/templates/diff.html b/backend/templates/diff.html
index c3969ad6..bc88a290 100644
--- a/backend/templates/diff.html
+++ b/backend/templates/diff.html
@@ -52,7 +52,9 @@
</div>
-<script src="/static/js/diff.js"></script>
+
+<script type="text/javascript" src="{{url_for('static_content', group='js', filename='diff.js')}}"></script>
+
<script defer="">
diff --git a/backend/templates/edit.html b/backend/templates/edit.html
index 77ee233f..54efce2e 100644
--- a/backend/templates/edit.html
+++ b/backend/templates/edit.html
@@ -1,80 +1,112 @@
{% extends 'base.html' %}
{% block content %}
{% from '_helpers.jinja' import render_field %}
+<script type="text/javascript" src="{{url_for('static_content', group='js', filename='tabs.js')}}" defer></script>
+
<div class="edit-form monospaced-textarea">
- <form class="pure-form pure-form-stacked" action="{{ url_for('edit_page', uuid=uuid, next = request.args.get('next') ) }}" method="POST">
- <fieldset>
- <div class="pure-control-group">
- {{ render_field(form.url, placeholder="https://...", required=true, class="m-d") }}
- </div>
- <div class="pure-control-group">
- {{ render_field(form.title, class="m-d") }}
- </div>
- <div class="pure-control-group">
- {{ render_field(form.tag) }}
+
+ <div class="tabs">
+ <ul>
+ <li class="tab" id="default-tab"><a href="#general">General</a></li>
+ <li class="tab"><a href="#notifications">Notifications</a></li>
+ <li class="tab"><a href="#filters">Filters</a></li>
+ </ul>
+ </div>
+
+ <div class="box-wrap inner">
+ <form class="pure-form pure-form-stacked"
+ action="{{ url_for('edit_page', uuid=uuid, next = request.args.get('next') ) }}" method="POST">
+
+ <div class="tab-pane-inner" id="general">
+ <fieldset>
+ <div class="pure-control-group">
+ {{ render_field(form.url, placeholder="https://...", required=true, class="m-d") }}
+ </div>
+ <div class="pure-control-group">
+ {{ render_field(form.title, class="m-d") }}
+ </div>
+ <div class="pure-control-group">
+ {{ render_field(form.tag) }}
+ </div>
+ <div class="pure-control-group">
+ {{ render_field(form.minutes_between_check) }}
+ {% if using_default_minutes %}
+ <span class="pure-form-message-inline">Currently using the <a
+ href="{{ url_for('settings_page', uuid=uuid) }}">default global settings</a>, change to another value if you want to be specific.</span>
+ {% else %}
+ <span class="pure-form-message-inline">Set to blank to use the <a
+ href="{{ url_for('settings_page', uuid=uuid) }}">default global settings</a>.</span>
+ {% endif %}
+ </div>
+ <fieldset class="pure-group">
+ {{ render_field(form.headers, rows=5, placeholder="Example
+ Cookie: foobar
+ User-Agent: wonderbra 1.0") }}
+ <span class="pure-form-message-inline">
+ Note: ONLY used by Basic fast Plaintext/HTTP Client
+ </span>
+ </fieldset>
+ </fieldset>
</div>
- <div class="pure-control-group">
- {{ render_field(form.minutes_between_check) }}
- {% if using_default_minutes %}
- <span class="pure-form-message-inline">Currently using the <a href="{{ url_for('settings_page', uuid=uuid) }}">default global settings</a>, change to another value if you want to be specific.</span>
- {% else %}
- <span class="pure-form-message-inline">Set to blank to use the <a href="{{ url_for('settings_page', uuid=uuid) }}">default global settings</a>.</span>
- {% endif %}
+ <div class="tab-pane-inner" id="notifications">
+ <fieldset>
+ <div class="pure-control-group">
+ {{ render_field(form.notification_urls, rows=5, placeholder="Examples:
+ Gitter - gitter://token/room
+ Office365 - o365://TenantID:AccountEmail/ClientID/ClientSecret/TargetEmail
+ AWS SNS - sns://AccessKeyID/AccessSecretKey/RegionName/+PhoneNo
+ SMTPS - mailtos://user:pass@mail.domain.com?to=receivingAddress@example.com
+ ") }}
+ <span class="pure-form-message-inline">Use <a target=_new
+ href="https://github.com/caronc/apprise">AppRise URLs</a> for notification to just about any service!</span>
+ <span class="pure-form-message-inline">Note: This overrides any global settings notification URLs</span>
+ </div>
+
+ <div class="pure-controls">
+ {{ render_field(form.trigger_check, rows=5) }}
+ </div>
+ </fieldset>
</div>
- <div class="pure-control-group">
- {{ render_field(form.css_filter, placeholder=".class-name or #some-id, or other CSS selector rule.", class="m-d") }}
- <span class="pure-form-message-inline">
+ <div class="tab-pane-inner" id="filters">
+ <fieldset>
+ <div class="pure-control-group">
+ {{ render_field(form.css_filter, placeholder=".class-name or #some-id, or other CSS selector rule.",
+ class="m-d") }}
+ <span class="pure-form-message-inline">
<ul>
<li>CSS - Limit text to this CSS rule, only text matching this CSS rule is included.</li>
- <li>JSON - Limit text to this JSON rule, using <a href="https://pypi.org/project/jsonpath-ng/">JSONPath</a>, prefix with <b>"json:"</b>, <a href="https://jsonpath.com/" target="new">test your JSONPath here</a></li>
+ <li>JSON - Limit text to this JSON rule, using <a href="https://pypi.org/project/jsonpath-ng/">JSONPath</a>, prefix with <b>"json:"</b>, <a
+ href="https://jsonpath.com/" target="new">test your JSONPath here</a></li>
</ul>
- Please be sure that you thoroughly understand how to write CSS or JSONPath selector rules before filing an issue on GitHub! <a href="https://github.com/dgtlmoon/changedetection.io/wiki/CSS-Selector-help">here for more CSS selector help</a>.<br/>
+ Please be sure that you thoroughly understand how to write CSS or JSONPath selector rules before filing an issue on GitHub! <a
+ href="https://github.com/dgtlmoon/changedetection.io/wiki/CSS-Selector-help">here for more CSS selector help</a>.<br/>
</span>
- </div>
- <!-- @todo: move to tabs --->
- <fieldset class="pure-group">
- {{ render_field(form.ignore_text, rows=5, placeholder="Some text to ignore in a line
-/some.regex\d{2}/ for case-INsensitive regex
-") }}
- <span class="pure-form-message-inline">
+ </div>
+
+ </fieldset>
+ <fieldset class="pure-group">
+ {{ render_field(form.ignore_text, rows=5, placeholder="Some text to ignore in a line
+/some.regex\d{2}/ for case-INsensitive regex
+ ") }}
+ <span class="pure-form-message-inline">
Each line processed separately, any line matching will be ignored.<br/>
Regular Expression support, wrap the line in forward slash <b>/regex/</b>.
</span>
</fieldset>
- <fieldset class="pure-group">
- {{ render_field(form.headers, rows=5, placeholder="Example
-Cookie: foobar
-User-Agent: wonderbra 1.0") }}
- </fieldset>
- <div class="pure-control-group">
- {{ render_field(form.notification_urls, rows=5, placeholder="Examples:
-Gitter - gitter://token/room
-Office365 - o365://TenantID:AccountEmail/ClientID/ClientSecret/TargetEmail
-AWS SNS - sns://AccessKeyID/AccessSecretKey/RegionName/+PhoneNo
-SMTPS - mailtos://user:pass@mail.domain.com?to=receivingAddress@example.com
-") }}
- <span class="pure-form-message-inline">Use <a target=_new href="https://github.com/caronc/apprise">AppRise URLs</a> for notification to just about any service!</span>
- <span class="pure-form-message-inline">Note: This overrides any global settings notification URLs</span>
</div>
+ <div id="actions">
+ <div class="pure-control-group">
- <div class="pure-controls">
- {{ render_field(form.trigger_check, rows=5) }}
+ <button type="submit" class="pure-button pure-button-primary">Save</button>
+ <a href="{{url_for('api_delete', uuid=uuid)}}"
+ class="pure-button button-small button-error ">Delete</a>
+ </div>
</div>
- <div class="pure-control-group">
- <button type="submit" class="pure-button pure-button-primary">Save</button>
- </div>
- <br/>
- <div class="pure-control-group">
- <a href="{{url_for('index')}}" class="pure-button button-small button-cancel">Cancel</a>
- <a href="{{url_for('api_delete', uuid=uuid)}}"
- class="pure-button button-small button-error ">Delete</a>
- </div>
- </fieldset>
- </form>
-
+ </form>
+ </div>
</div>
{% endblock %}
diff --git a/backend/templates/import.html b/backend/templates/import.html
index cd7012dc..77bd9b40 100644
--- a/backend/templates/import.html
+++ b/backend/templates/import.html
@@ -2,24 +2,21 @@
{% block content %}
<div class="edit-form">
-
-
- <form class="pure-form pure-form-aligned" action="{{url_for('import_page')}}" method="POST">
-
- <fieldset class="pure-group">
- <legend>One URL per line, URLs that do not pass validation will stay in the textarea.</legend>
-
- <textarea name="urls" class="pure-input-1-2" placeholder="https://"
- style="width: 100%;
- font-family:monospace;
- white-space: pre;
- overflow-wrap: normal;
- overflow-x: scroll;" rows="25">{{ remaining }}</textarea>
- </fieldset>
- <button type="submit" class="pure-button pure-input-1-2 pure-button-primary">Import</button>
-
- </form>
-
+ <div class="inner">
+ <form class="pure-form pure-form-aligned" action="{{url_for('import_page')}}" method="POST">
+ <fieldset class="pure-group">
+ <legend>One URL per line, URLs that do not pass validation will stay in the textarea.</legend>
+
+ <textarea name="urls" class="pure-input-1-2" placeholder="https://"
+ style="width: 100%;
+ font-family:monospace;
+ white-space: pre;
+ overflow-wrap: normal;
+ overflow-x: scroll;" rows="25">{{ remaining }}</textarea>
+ </fieldset>
+ <button type="submit" class="pure-button pure-input-1-2 pure-button-primary">Import</button>
+ </form>
+ </div>
</div>
{% endblock %}
diff --git a/backend/templates/login.html b/backend/templates/login.html
index 8cc798b1..6bcbdbd3 100644
--- a/backend/templates/login.html
+++ b/backend/templates/login.html
@@ -2,6 +2,8 @@
{% block content %}
<div class="edit-form">
+
+ <div class="inner">
<form class="pure-form pure-form-stacked" action="{{url_for('login')}}" method="POST">
<fieldset>
<div class="pure-control-group">
@@ -15,6 +17,7 @@
</div>
</fieldset>
</form>
-</div>
+ </div>
+ </div>
{% endblock %}
diff --git a/backend/templates/settings.html b/backend/templates/settings.html
index 1751df17..941760a4 100644
--- a/backend/templates/settings.html
+++ b/backend/templates/settings.html
@@ -2,107 +2,137 @@
{% block content %}
{% from '_helpers.jinja' import render_field %}
-<script type="text/javascript" src="static/js/settings.js"></script>
-<div class="edit-form">
- <form class="pure-form pure-form-stacked settings" action="{{url_for('settings_page')}}" method="POST">
- <fieldset>
- <div class="pure-control-group">
- {{ render_field(form.minutes_between_check) }}
- <span class="pure-form-message-inline">Default time for all watches, when the watch does not have a specific time setting.</span>
- </div>
- <div class="pure-control-group">
- {% if current_user.is_authenticated %}
- <a href="{{url_for('settings_page', removepassword='yes')}}" class="pure-button pure-button-primary">Remove password</a>
- {% else %}
- {{ render_field(form.password) }}
- <span class="pure-form-message-inline">Password protection for your changedetection.io application.</span>
- {% endif %}
- </div>
- <div class="pure-control-group">
- {{ render_field(form.extract_title_as_title) }}
- <span class="pure-form-message-inline">Note: This will automatically apply to all existing watches.</span>
- </div>
-
- <div class="field-group">
- <div class="pure-control-group">
- {{ render_field(form.notification_urls, rows=5, placeholder="Examples:
-Gitter - gitter://token/room
-Office365 - o365://TenantID:AccountEmail/ClientID/ClientSecret/TargetEmail
-AWS SNS - sns://AccessKeyID/AccessSecretKey/RegionName/+PhoneNo
-SMTPS - mailtos://user:pass@mail.domain.com?to=receivingAddress@example.com") }}
- <div class="pure-form-message-inline">Use <a target=_new href="https://github.com/caronc/apprise">AppRise URLs</a> for notification to just about any service!
- <a id="toggle-customise-notifications">Customise notification body: <i
- class="arrow down"></i></a>
- </div>
- </div>
- <div id="notification-customisation" style="display:none;">
+<script type="text/javascript" src="{{url_for('static_content', group='js', filename='settings.js')}}" defer></script>
+<script type="text/javascript" src="{{url_for('static_content', group='js', filename='tabs.js')}}" defer></script>
+<div class="edit-form">
+ <div class="tabs">
+ <ul>
+ <li class="tab" id="default-tab"><a href="#general">General</a></li>
+ <li class="tab"><a href="#notifications">Notifications</a></li>
+ <li class="tab"><a href="#fetching">Fetching</a></li>
+ </ul>
+ </div>
+ <div class="box-wrap inner">
+ <form class="pure-form pure-form-stacked settings" action="{{url_for('settings_page')}}" method="POST">
+ <div class="tab-pane-inner" id="general">
+ <fieldset>
<div class="pure-control-group">
- {{ render_field(form.notification_title, class="m-d") }}
- <span class="pure-form-message-inline">Title for all notifications</span>
+ {{ render_field(form.minutes_between_check) }}
+ <span class="pure-form-message-inline">Default time for all watches, when the watch does not have a specific time setting.</span>
</div>
<div class="pure-control-group">
- {{ render_field(form.notification_body , rows=5) }}
- <span class="pure-form-message-inline">Body for all notifications</span>
+ {% if current_user.is_authenticated %}
+ <a href="{{url_for('settings_page', removepassword='yes')}}"
+ class="pure-button pure-button-primary">Remove password</a>
+ {% else %}
+ {{ render_field(form.password) }}
+ <span class="pure-form-message-inline">Password protection for your changedetection.io application.</span>
+ {% endif %}
</div>
- <div class="pure-controls">
- <span class="pure-form-message-inline">
- These tokens can be used in the notification body and title to
- customise the notification text.
- </span>
- <table class="pure-table" id="token-table">
- <thead>
- <tr>
- <th>Token</th>
- <th>Description</th>
- </tr>
- </thead>
- <tbody>
- <tr>
- <td><code>{base_url}</code></td>
- <td>The URL of the changedetection.io instance you are running.</td>
- </tr>
- <tr>
- <td><code>{watch_url}</code></td>
- <td>The URL being watched.</td>
- </tr>
- <tr>
- <td><code>{preview_url}</code></td>
- <td>The URL of the preview page generated by changedetection.io.</td>
- </tr>
- <tr>
- <td><code>{diff_url}</code></td>
- <td>The URL of the diff page generated by changedetection.io.</td>
- </tr>
- <tr>
- <td><code>{current_snapshot}</code></td>
- <td>The current snapshot value, useful when combined with JSON or CSS filters</td>
- </tr>
- </tbody>
- </table>
- <span class="pure-form-message-inline">
- URLs generated by changedetection.io (such as <code>{diff_url}</code>) require the <code>BASE_URL</code> environment variable set.<br/>
- Your <code>BASE_URL</code> var is currently "{{base_url}}"
- </span>
+ <div class="pure-control-group">
+ {{ render_field(form.extract_title_as_title) }}
+ <span class="pure-form-message-inline">Note: This will automatically apply to all existing watches.</span>
</div>
- </div>
+ </fieldset>
+ </div>
+ <div class="tab-pane-inner" id="notifications">
+ <fieldset>
+ <div class="field-group">
+ <div class="pure-control-group">
+ {{ render_field(form.notification_urls, rows=5, placeholder="Examples:
+ Gitter - gitter://token/room
+ Office365 - o365://TenantID:AccountEmail/ClientID/ClientSecret/TargetEmail
+ AWS SNS - sns://AccessKeyID/AccessSecretKey/RegionName/+PhoneNo
+ SMTPS - mailtos://user:pass@mail.domain.com?to=receivingAddress@example.com")
+ }}
+ <div class="pure-form-message-inline">Use <a target=_new
+ href="https://github.com/caronc/apprise">AppRise
+ URLs</a> for notification to just about any service!
+ <a id="toggle-customise-notifications">Customise notification body: <i
+ class="arrow down"></i></a>
+ </div>
+ </div>
+ <div id="notification-customisation" style="display:none;">
+
+ <div class="pure-control-group">
+ {{ render_field(form.notification_title, class="m-d") }}
+ <span class="pure-form-message-inline">Title for all notifications</span>
+ </div>
+ <div class="pure-control-group">
+ {{ render_field(form.notification_body , rows=5) }}
+ <span class="pure-form-message-inline">Body for all notifications</span>
+ </div>
+ <div class="pure-controls">
+ <span class="pure-form-message-inline">
+ These tokens can be used in the notification body and title to
+ customise the notification text.
+ </span>
+ <table class="pure-table" id="token-table">
+ <thead>
+ <tr>
+ <th>Token</th>
+ <th>Description</th>
+ </tr>
+ </thead>
+ <tbody>
+ <tr>
+ <td><code>{base_url}</code></td>
+ <td>The URL of the changedetection.io instance you are running.</td>
+ </tr>
+ <tr>
+ <td><code>{watch_url}</code></td>
+ <td>The URL being watched.</td>
+ </tr>
+ <tr>
+ <td><code>{preview_url}</code></td>
+ <td>The URL of the preview page generated by changedetection.io.</td>
+ </tr>
+ <tr>
+ <td><code>{diff_url}</code></td>
+ <td>The URL of the diff page generated by changedetection.io.</td>
+ </tr>
+ <tr>
+ <td><code>{current_snapshot}</code></td>
+ <td>The current snapshot value, useful when combined with JSON or CSS filters
+ </td>
+ </tr>
+ </tbody>
+ </table>
+ <span class="pure-form-message-inline">
+ URLs generated by changedetection.io (such as <code>{diff_url}</code>) require the <code>BASE_URL</code> environment variable set.<br/>
+ Your <code>BASE_URL</code> var is currently "{{base_url}}"
+ </span>
+ </div>
+ </div>
+ <div class="pure-control-group">
+ {{ render_field(form.trigger_check) }}
+ </div>
+ </div>
+
+ </fieldset>
+ </div>
+ <div class="tab-pane-inner" id="fetching">
<div class="pure-control-group">
- {{ render_field(form.trigger_check) }}
+ {{ render_field(form.fetch_backend) }}
+ <span class="pure-form-message-inline">
+ <p>Use the <strong>Basic</strong> method (default) where your watched sites don't need Javascript to render.</p>
+ <p>The <strong>Chrome/Javascript</strong> method requires a network connection to a running WebDriver+Chrome server. </p>
+ </span>
</div>
</div>
- <div class="pure-control-group">
- <button type="submit" class="pure-button pure-button-primary">Save</button>
- </div>
- <br/>
- <div class="pure-control-group">
- <a href="{{url_for('index')}}" class="pure-button button-small button-cancel">Back</a>
- <a href="{{url_for('scrub_page')}}" class="pure-button button-small button-cancel">Delete History Snapshot Data</a>
+ <div id="actions">
+ <div class="pure-control-group">
+ <button type="submit" class="pure-button pure-button-primary">Save</button>
+ <a href="{{url_for('index')}}" class="pure-button button-small button-cancel">Back</a>
+ <a href="{{url_for('scrub_page')}}" class="pure-button button-small button-cancel">Delete
+ History
+ Snapshot Data</a>
+ </div>
</div>
- </fieldset>
- </form>
-
-
+ </form>
+ </div>
</div>
{% endblock %}
diff --git a/backend/templates/watch-overview.html b/backend/templates/watch-overview.html
index 9a024533..c992f6a1 100644
--- a/backend/templates/watch-overview.html
+++ b/backend/templates/watch-overview.html
@@ -49,6 +49,8 @@
<td class="title-col inline">{{watch.title if watch.title is not none and watch.title|length > 0 else watch.url}}
<a class="external" target="_blank" rel="noopener" href="{{ watch.url }}"></a>
+ {%if watch.fetch_backend == "html_webdriver" %}<img style="height: 1em; display:inline-block;" src="/static/images/Google-Chrome-icon.png" />{% endif %}
+
{% if watch.last_error is defined and watch.last_error != False %}
<div class="fetch-error">{{ watch.last_error }}</div>
{% endif %}
diff --git a/backend/tests/conftest.py b/backend/tests/conftest.py
index 3c620e6b..ef5cdc42 100644
--- a/backend/tests/conftest.py
+++ b/backend/tests/conftest.py
@@ -5,7 +5,6 @@ from backend import changedetection_app
from backend import store
import os
-
# https://github.com/pallets/flask/blob/1.1.2/examples/tutorial/tests/test_auth.py
# Much better boilerplate than the docs
# https://www.python-boilerplate.com/py3+flask+pytest/
@@ -39,10 +38,11 @@ def app(request):
# Enable a BASE_URL for notifications to work (so we can look for diff/ etc URLs)
os.environ["BASE_URL"] = "http://mysite.com/"
-
cleanup(datastore_path)
+
app_config = {'datastore_path': datastore_path}
+ cleanup(app_config['datastore_path'])
datastore = store.ChangeDetectionStore(datastore_path=app_config['datastore_path'], include_default_watches=False)
app = changedetection_app(app_config, datastore)
app.config['STOP_THREADS'] = True
@@ -50,8 +50,8 @@ def app(request):
def teardown():
datastore.stop_thread = True
app.config.exit.set()
- cleanup(datastore_path)
-
+ cleanup(app_config['datastore_path'])
+
+
request.addfinalizer(teardown)
yield app
-
diff --git a/backend/tests/test_access_control.py b/backend/tests/test_access_control.py
index 14e0880a..026929ba 100644
--- a/backend/tests/test_access_control.py
+++ b/backend/tests/test_access_control.py
@@ -12,7 +12,9 @@ def test_check_access_control(app, client):
# Enable password check.
res = c.post(
url_for("settings_page"),
- data={"password": "foobar", "minutes_between_check": 180},
+ data={"password": "foobar",
+ "minutes_between_check": 180,
+ 'fetch_backend': "html_requests"},
follow_redirects=True
)
@@ -66,8 +68,11 @@ def test_check_access_control_no_blank_password(app, client):
# Enable password check.
res = c.post(
url_for("settings_page"),
- data={"password": "", "minutes_between_check": 180},
- follow_redirects=True
+ data={"password": "",
+ "minutes_between_check": 180,
+ 'fetch_backend': "html_requests"},
+
+ follow_redirects=True
)
assert b"Password protection enabled." not in res.data
@@ -86,7 +91,8 @@ def test_check_access_no_remote_access_to_remove_password(app, client):
# Enable password check.
res = c.post(
url_for("settings_page"),
- data={"password": "password", "minutes_between_check": 180},
+ data={"password": "password", "minutes_between_check": 180,
+ 'fetch_backend': "html_requests"},
follow_redirects=True
)
diff --git a/backend/tests/test_backend.py b/backend/tests/test_backend.py
index 8f944605..3b013f61 100644
--- a/backend/tests/test_backend.py
+++ b/backend/tests/test_backend.py
@@ -88,7 +88,7 @@ def test_check_basic_change_detection_functionality(client, live_server):
# Enable auto pickup of <title> in settings
res = client.post(
url_for("settings_page"),
- data={"extract_title_as_title": "1", "minutes_between_check": 180},
+ data={"extract_title_as_title": "1", "minutes_between_check": 180, 'fetch_backend': "html_requests"},
follow_redirects=True
)
diff --git a/backend/tests/test_css_selector.py b/backend/tests/test_css_selector.py
index 6425600c..21183b06 100644
--- a/backend/tests/test_css_selector.py
+++ b/backend/tests/test_css_selector.py
@@ -22,7 +22,7 @@ def set_original_response():
</html>
"""
- with open("test-datastore/output.txt", "w") as f:
+ with open("test-datastore/endpoint-content.txt", "w") as f:
f.write(test_return_data)
return None
@@ -39,7 +39,7 @@ def set_modified_response():
</html>
"""
- with open("test-datastore/output.txt", "w") as f:
+ with open("test-datastore/endpoint-content.txt", "w") as f:
f.write(test_return_data)
return None
@@ -98,7 +98,7 @@ def test_check_markup_css_filter_restriction(client, live_server):
# Add our URL to the import page
res = client.post(
url_for("edit_page", uuid="first"),
- data={"css_filter": css_filter, "url": test_url, "tag": "", "headers": ""},
+ data={"css_filter": css_filter, "url": test_url, "tag": "", "headers": "", 'fetch_backend': "html_requests"},
follow_redirects=True
)
assert b"Updated watch." in res.data
diff --git a/backend/tests/test_headers.py b/backend/tests/test_headers.py
index 420fd39f..38943978 100644
--- a/backend/tests/test_headers.py
+++ b/backend/tests/test_headers.py
@@ -35,6 +35,7 @@ def test_headers_in_request(client, live_server):
data={
"url": test_url,
"tag": "",
+ "fetch_backend": "html_requests",
"headers": "xxx:ooo\ncool:yeah\r\ncookie:"+cookie_header},
follow_redirects=True
)
diff --git a/backend/tests/test_ignore_text.py b/backend/tests/test_ignore_text.py
index df6e349c..cdcb9bbb 100644
--- a/backend/tests/test_ignore_text.py
+++ b/backend/tests/test_ignore_text.py
@@ -41,7 +41,7 @@ def set_original_ignore_response():
"""
- with open("test-datastore/output.txt", "w") as f:
+ with open("test-datastore/endpoint-content.txt", "w") as f:
f.write(test_return_data)
@@ -57,7 +57,7 @@ def set_modified_original_ignore_response():
"""
- with open("test-datastore/output.txt", "w") as f:
+ with open("test-datastore/endpoint-content.txt", "w") as f:
f.write(test_return_data)
@@ -75,7 +75,7 @@ def set_modified_ignore_response():
"""
- with open("test-datastore/output.txt", "w") as f:
+ with open("test-datastore/endpoint-content.txt", "w") as f:
f.write(test_return_data)
@@ -107,7 +107,7 @@ def test_check_ignore_text_functionality(client, live_server):
# Add our URL to the import page
res = client.post(
url_for("edit_page", uuid="first"),
- data={"ignore_text": ignore_text, "url": test_url},
+ data={"ignore_text": ignore_text, "url": test_url, 'fetch_backend': "html_requests"},
follow_redirects=True
)
assert b"Updated watch." in res.data
diff --git a/backend/tests/test_jsonpath_selector.py b/backend/tests/test_jsonpath_selector.py
index 40f36556..91c46764 100644
--- a/backend/tests/test_jsonpath_selector.py
+++ b/backend/tests/test_jsonpath_selector.py
@@ -43,9 +43,6 @@ and it can also be repeated
html_tools.extract_json_as_string('COMPLETE GIBBERISH, NO JSON!', "$.id")
-def test_setup(live_server):
- live_server_setup(live_server)
-
def set_original_response():
test_return_data = """
{
@@ -66,7 +63,7 @@ def set_original_response():
}
}
"""
- with open("test-datastore/output.txt", "w") as f:
+ with open("test-datastore/endpoint-content.txt", "w") as f:
f.write(test_return_data)
return None
@@ -91,7 +88,7 @@ def set_modified_response():
}
"""
- with open("test-datastore/output.txt", "w") as f:
+ with open("test-datastore/endpoint-content.txt", "w") as f:
f.write(test_return_data)
return None
@@ -99,6 +96,7 @@ def set_modified_response():
def test_check_json_filter(client, live_server):
+ live_server_setup(live_server)
json_filter = 'json:boss.name'
@@ -126,7 +124,12 @@ def test_check_json_filter(client, live_server):
# Add our URL to the import page
res = client.post(
url_for("edit_page", uuid="first"),
- data={"css_filter": json_filter, "url": test_url, "tag": "", "headers": ""},
+ data={"css_filter": json_filter,
+ "url": test_url,
+ "tag": "",
+ "headers": "",
+ "fetch_backend": "html_requests"
+ },
follow_redirects=True
)
assert b"Updated watch." in res.data
@@ -148,7 +151,7 @@ def test_check_json_filter(client, live_server):
# Trigger a check
client.get(url_for("api_watch_checknow"), follow_redirects=True)
# Give the thread time to pick it up
- time.sleep(3)
+ time.sleep(4)
# It should have 'unviewed' still
res = client.get(url_for("index"))
diff --git a/backend/tests/test_notification.py b/backend/tests/test_notification.py
index e07d77ef..ec97be2d 100644
--- a/backend/tests/test_notification.py
+++ b/backend/tests/test_notification.py
@@ -37,6 +37,7 @@ def test_check_notification(client, live_server):
"url": test_url,
"tag": "",
"headers": "",
+ "fetch_backend": "html_requests",
"trigger_check": "y"},
follow_redirects=True
)
@@ -90,9 +91,8 @@ def test_check_notification(client, live_server):
#assert bytes("https://foobar.com".encode('utf-8')) in notification_submission
- ## Now configure something clever, we go into custom config (non-default) mode
-
- with open("test-datastore/output.txt", "w") as f:
+ ## Now configure something clever, we go into custom config (non-default) mode, this is returned by the endpoint
+ with open("test-datastore/endpoint-content.txt", "w") as f:
f.write(";jasdhflkjadshf kjhsdfkjl ahslkjf haslkjd hfaklsj hf\njl;asdhfkasj stuff we will detect\n")
res = client.post(
@@ -100,7 +100,9 @@ def test_check_notification(client, live_server):
data={"notification_title": "New ChangeDetection.io Notification - {watch_url}",
"notification_body": "{base_url}\n{watch_url}\n{preview_url}\n{diff_url}\n{current_snapshot}\n:-)",
"notification_urls": "json://foobar.com", #Re #143 should not see that it sent without [test checkbox]
- "minutes_between_check": 180},
+ "minutes_between_check": 180,
+ "fetch_backend": "html_requests",
+ },
follow_redirects=True
)
assert b"Settings updated." in res.data
@@ -122,6 +124,8 @@ def test_check_notification(client, live_server):
with open("test-datastore/notification.txt", "r") as f:
notification_submission = f.read()
+ # @todo regex that diff/uuid-31123-123-etc
+
assert "diff/" in notification_submission
assert "preview/" in notification_submission
assert ":-)" in notification_submission
diff --git a/backend/tests/test_watch_fields_storage.py b/backend/tests/test_watch_fields_storage.py
index f9ee7634..12f6474c 100644
--- a/backend/tests/test_watch_fields_storage.py
+++ b/backend/tests/test_watch_fields_storage.py
@@ -28,7 +28,7 @@ def test_check_watch_field_storage(client, live_server):
"url": test_url,
"tag": "woohoo",
"headers": "curl:foo",
-
+ 'fetch_backend': "html_requests"
},
follow_redirects=True
)
@@ -57,6 +57,7 @@ def test_check_recheck_global_setting(client, live_server):
url_for("settings_page"),
data={
"minutes_between_check": 1566,
+ 'fetch_backend': "html_requests"
},
follow_redirects=True
)
@@ -88,6 +89,7 @@ def test_check_recheck_global_setting(client, live_server):
url_for("settings_page"),
data={
"minutes_between_check": 222,
+ 'fetch_backend': "html_requests"
},
follow_redirects=True
)
@@ -107,6 +109,7 @@ def test_check_recheck_global_setting(client, live_server):
url_for("edit_page", uuid="first"),
data={"url": test_url,
"minutes_between_check": 55,
+ 'fetch_backend': "html_requests"
},
follow_redirects=True
)
@@ -122,6 +125,7 @@ def test_check_recheck_global_setting(client, live_server):
url_for("settings_page"),
data={
"minutes_between_check": 666,
+ 'fetch_backend': "html_requests"
},
follow_redirects=True
)
@@ -131,6 +135,7 @@ def test_check_recheck_global_setting(client, live_server):
url_for("edit_page", uuid="first"),
data={"url": test_url,
"minutes_between_check": "",
+ 'fetch_backend': "html_requests"
},
follow_redirects=True
)
diff --git a/backend/tests/util.py b/backend/tests/util.py
index 5d6f7c28..38e1c2b6 100644
--- a/backend/tests/util.py
+++ b/backend/tests/util.py
@@ -13,7 +13,7 @@ def set_original_response():
</html>
"""
- with open("test-datastore/output.txt", "w") as f:
+ with open("test-datastore/endpoint-content.txt", "w") as f:
f.write(test_return_data)
return None
@@ -29,7 +29,7 @@ def set_modified_response():
</html>
"""
- with open("test-datastore/output.txt", "w") as f:
+ with open("test-datastore/endpoint-content.txt", "w") as f:
f.write(test_return_data)
return None
@@ -41,7 +41,7 @@ def live_server_setup(live_server):
@live_server.app.route('/test-endpoint')
def test_endpoint():
# Tried using a global var here but didn't seem to work, so reading from a file instead.
- with open("test-datastore/output.txt", "r") as f:
+ with open("test-datastore/endpoint-content.txt", "r") as f:
return f.read()
# Just return the headers in the request
diff --git a/backend/update_worker.py b/backend/update_worker.py
index 9eb54ebd..e27bbfc9 100644
--- a/backend/update_worker.py
+++ b/backend/update_worker.py
@@ -1,5 +1,6 @@
import threading
import queue
+import time
# Requests for checking on the site use a pool of thread Workers managed by a Queue.
class update_worker(threading.Thread):
@@ -26,24 +27,45 @@ class update_worker(threading.Thread):
else:
self.current_uuid = uuid
+ from backend import content_fetcher
if uuid in list(self.datastore.data['watching'].keys()):
+
+ changed_detected = False
+ contents = ""
+ update_obj= {}
+
try:
- changed_detected, result, contents = update_handler.run(uuid)
+ now = time.time()
+ changed_detected, update_obj, contents = update_handler.run(uuid)
+
+ # Always record that we atleast tried
+ self.datastore.update_watch(uuid=uuid, update_obj={'fetch_time': round(time.time() - now, 3)})
except PermissionError as e:
self.app.logger.error("File permission error updating", uuid, str(e))
+ except content_fetcher.EmptyReply as e:
+ self.datastore.update_watch(uuid=uuid, update_obj={'last_error':str(e)})
+
+ #@todo how to handle when it's thrown from webdriver connecting?
except Exception as e:
self.app.logger.error("Exception reached", uuid, str(e))
+ self.datastore.update_watch(uuid=uuid, update_obj={'last_error': str(e)})
+
else:
- if result:
+ if update_obj:
try:
- self.datastore.update_watch(uuid=uuid, update_obj=result)
+ self.datastore.update_watch(uuid=uuid, update_obj=update_obj)
if changed_detected:
# A change was detected
newest_version_file_contents = ""
- self.datastore.save_history_text(uuid=uuid, contents=contents, result_obj=result)
+ fname = self.datastore.save_history_text(watch_uuid=uuid, contents=contents)
+
+ # Update history with the stripped text for future reference, this will also mean we save the first
+ # Should always be keyed by string(timestamp)
+ self.datastore.update_watch(uuid, {"history": {str(update_obj["last_checked"]): fname}})
+
watch = self.datastore.data['watching'][uuid]
print (">> Change detected in UUID {} - {}".format(uuid, watch['url']))
diff --git a/docker-compose.yml b/docker-compose.yml
index 8bb11bce..5acbb8d1 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -6,12 +6,14 @@ services:
hostname: changedetection.io
volumes:
- changedetection-data:/datastore
+
# environment:
# Default listening port, can also be changed with the -p option
# - PORT=5000
# - PUID=1000
# - PGID=1000
+ # - WEBDRIVER_URL="http://browser-chrome:4444/wd/hub"
# Proxy support example.
# - HTTP_PROXY="socks5h://10.10.1.10:1080"
# - HTTPS_PROXY="socks5h://10.10.1.10:1080"
@@ -27,8 +29,21 @@ services:
# Comment out ports: when using behind a reverse proxy , enable networks: etc.
ports:
- 5000:5000
+ restart: unless-stopped
+
+ # Used for fetching pages via WebDriver+Chrome where you need Javascript support.
+ # Does not work on rPi, https://github.com/dgtlmoon/changedetection.io/wiki/Fetching-pages-with-WebDriver
- restart: always
+# browser-chrome:
+# hostname: browser-chrome
+# image: selenium/standalone-chrome-debug:3.141.59
+# environment:
+# - VNC_NO_PASSWORD=1
+# volumes:
+# # Workaround to avoid the browser crashing inside a docker container
+# # See https://github.com/SeleniumHQ/docker-selenium#quick-start
+# - /dev/shm:/dev/shm
+# restart: unless-stopped
volumes:
changedetection-data:
diff --git a/requirements.txt b/requirements.txt
index 1d4c4b11..18bb875a 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -14,10 +14,10 @@ urllib3
wtforms ~= 2.3.3
jsonpath-ng ~= 1.5.3
-
# Notification library
apprise ~= 0.9
# Used for CSS filtering, replace with soupsieve and lxml for xpath
bs4
+selenium ~= 3.141
\ No newline at end of file