From a0e4f9b88ab7ea3f9fd03d46df6edddf343e2519 Mon Sep 17 00:00:00 2001 From: dgtlmoon Date: Thu, 17 Feb 2022 18:16:23 +0100 Subject: [PATCH 01/15] better checking of JSON type --- changedetectionio/fetch_site_status.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/changedetectionio/fetch_site_status.py b/changedetectionio/fetch_site_status.py index d75c0c6e..43cf7383 100644 --- a/changedetectionio/fetch_site_status.py +++ b/changedetectionio/fetch_site_status.py @@ -102,7 +102,7 @@ class perform_site_check(): # https://stackoverflow.com/questions/41817578/basic-method-chaining ? # return content().textfilter().jsonextract().checksumcompare() ? - is_json = fetcher.headers.get('Content-Type', '') == 'application/json' + is_json = 'application/json' in fetcher.headers.get('Content-Type', '') is_html = not is_json css_filter_rule = watch['css_filter'] From 85715120e2e3c432059035c755c190c3505ed1e4 Mon Sep 17 00:00:00 2001 From: Michael <75951847+michael-kerbel@users.noreply.github.com> Date: Sat, 19 Feb 2022 13:40:57 +0100 Subject: [PATCH 02/15] XPath RegularExpression support --- changedetectionio/html_tools.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/changedetectionio/html_tools.py b/changedetectionio/html_tools.py index 7a6b91c6..47b7f698 100644 --- a/changedetectionio/html_tools.py +++ b/changedetectionio/html_tools.py @@ -25,7 +25,7 @@ def xpath_filter(xpath_filter, html_content): tree = html.fromstring(html_content) html_block = "" - for item in tree.xpath(xpath_filter.strip()): + for item in tree.xpath(xpath_filter.strip(), namespaces={'re':'http://exslt.org/regular-expressions'}): html_block+= etree.tostring(item, pretty_print=True).decode('utf-8')+"
" return html_block From dd384619e01de975dfecbf84e525c0b2f97c1075 Mon Sep 17 00:00:00 2001 From: dgtlmoon Date: Sat, 19 Feb 2022 13:41:54 +0100 Subject: [PATCH 03/15] Update README.md --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index 297233a1..6e430023 100644 --- a/README.md +++ b/README.md @@ -99,6 +99,8 @@ See the wiki for more information https://github.com/dgtlmoon/changedetection.io ## Filters XPath, JSONPath and CSS support comes baked in! You can be as specific as you need, use XPath exported from various XPath element query creation tools. +(We support LXML re:test, re:math and re:replace.) + ## Notifications ChangeDetection.io supports a massive amount of notifications (including email, office365, custom APIs, etc) when a web-page has a change detected thanks to the apprise library. From 014fda905806e5199e5d337e1fbb14a71c7e9367 Mon Sep 17 00:00:00 2001 From: dgtlmoon Date: Wed, 23 Feb 2022 10:49:25 +0100 Subject: [PATCH 04/15] Ability to visualise trigger and filter rules against the current snapshot on the preview page --- changedetectionio/__init__.py | 57 ++- changedetectionio/fetch_site_status.py | 59 +-- changedetectionio/forms.py | 6 +- changedetectionio/html_tools.py | 49 ++- changedetectionio/static/styles/diff.css | 16 + changedetectionio/static/styles/diff.scss | 20 + changedetectionio/static/styles/package.json | 3 +- changedetectionio/static/styles/styles.css | 408 +++++++++++++++++- changedetectionio/static/styles/styles.scss | 5 + changedetectionio/templates/_helpers.jinja | 3 + changedetectionio/templates/diff.html | 1 + changedetectionio/templates/edit.html | 17 +- changedetectionio/templates/preview.html | 11 +- changedetectionio/templates/settings.html | 1 + .../tests/test_ignore_regex_text.py | 3 +- changedetectionio/tests/test_ignore_text.py | 19 +- changedetectionio/tests/test_trigger.py | 5 + .../tests/test_xpath_selector.py | 1 + 18 files changed, 612 insertions(+), 72 deletions(-) diff --git a/changedetectionio/__init__.py b/changedetectionio/__init__.py index bb1dd457..9698257e 100644 --- a/changedetectionio/__init__.py +++ b/changedetectionio/__init__.py @@ -35,6 +35,7 @@ from flask import ( url_for, ) from flask_login import login_required +from changedetectionio import html_tools __version__ = '0.39.8' @@ -441,7 +442,7 @@ def changedetection_app(config=None, datastore_o=None): raw_content = file.read() handler = fetch_site_status.perform_site_check(datastore=datastore) - stripped_content = handler.strip_ignore_text(raw_content, + stripped_content = html_tools.strip_ignore_text(raw_content, datastore.data['watching'][uuid]['ignore_text']) if datastore.data['settings']['application'].get('ignore_whitespace', False): @@ -546,10 +547,14 @@ def changedetection_app(config=None, datastore_o=None): flash('No notification URLs set, cannot send test.', 'error') # Diff page [edit] link should go back to diff page - if request.args.get("next") and request.args.get("next") == 'diff': + if request.args.get("next") and request.args.get("next") == 'diff' and not form.save_and_preview_button.data: return redirect(url_for('diff_history_page', uuid=uuid)) else: - return redirect(url_for('index')) + if form.save_and_preview_button.data: + flash('You may need to reload this page to see the new content.') + return redirect(url_for('preview_page', uuid=uuid)) + else: + return redirect(url_for('index')) else: if request.method == 'POST' and not form.validate(): @@ -721,8 +726,12 @@ def changedetection_app(config=None, datastore_o=None): # Save the current newest history as the most recently viewed datastore.set_last_viewed(uuid, dates[0]) newest_file = watch['history'][dates[0]] - with open(newest_file, 'r') as f: - newest_version_file_contents = f.read() + + try: + with open(newest_file, 'r') as f: + newest_version_file_contents = f.read() + except Exception as e: + newest_version_file_contents = "Unable to read {}.\n".format(newest_file) previous_version = request.args.get('previous_version') try: @@ -731,8 +740,11 @@ def changedetection_app(config=None, datastore_o=None): # Not present, use a default value, the second one in the sorted list. previous_file = watch['history'][dates[1]] - with open(previous_file, 'r') as f: - previous_version_file_contents = f.read() + try: + with open(previous_file, 'r') as f: + previous_version_file_contents = f.read() + except Exception as e: + previous_version_file_contents = "Unable to read {}.\n".format(previous_file) output = render_template("diff.html", watch_a=watch, newest=newest_version_file_contents, @@ -751,6 +763,7 @@ def changedetection_app(config=None, datastore_o=None): @app.route("/preview/", methods=['GET']) @login_required def preview_page(uuid): + content = [] # More for testing, possible to return the first/only if uuid == 'first': @@ -764,14 +777,38 @@ def changedetection_app(config=None, datastore_o=None): flash("No history found for the specified link, bad link?", "error") return redirect(url_for('index')) - newest = list(watch['history'].keys())[-1] - with open(watch['history'][newest], 'r') as f: - content = f.readlines() + if len(watch['history']): + timestamps = sorted(watch['history'].keys(), key=lambda x: int(x)) + filename = watch['history'][timestamps[-1]] + try: + with open(filename, 'r') as f: + content = f.readlines() + except: + content.append("File doesnt exist or unable to read file {}".format(filename)) + else: + content.append("No history found") + + # Get what needs to be highlighted + ignore_rules = watch.get('ignore_text', []) + datastore.data['settings']['application']['global_ignore_text'] + + # .readlines will keep the \n, but we will parse it here again, in the future tidy this up + ignored_line_numbers = html_tools.strip_ignore_text(content="".join(content), + wordlist=ignore_rules, + mode='line numbers' + ) + + trigger_line_numbers = html_tools.strip_ignore_text(content="".join(content), + wordlist=watch['trigger_text'], + mode='line numbers' + ) output = render_template("preview.html", content=content, extra_stylesheets=extra_stylesheets, + ignored_line_numbers=ignored_line_numbers, + triggered_line_numbers=trigger_line_numbers, current_diff_url=watch['url'], + watch=watch, uuid=uuid) return output diff --git a/changedetectionio/fetch_site_status.py b/changedetectionio/fetch_site_status.py index 43cf7383..f0f91961 100644 --- a/changedetectionio/fetch_site_status.py +++ b/changedetectionio/fetch_site_status.py @@ -1,5 +1,6 @@ import time from changedetectionio import content_fetcher +from changedetectionio import html_tools import hashlib from inscriptis import get_text import urllib3 @@ -16,40 +17,6 @@ class perform_site_check(): super().__init__(*args, **kwargs) self.datastore = datastore - def strip_ignore_text(self, content, list_ignore_text): - import re - ignore = [] - ignore_regex = [] - for k in list_ignore_text: - - # Is it a regex? - if k[0] == '/': - ignore_regex.append(k.strip(" /")) - else: - ignore.append(k) - - output = [] - for line in content.splitlines(): - - # Always ignore blank lines in this mode. (when this function gets called) - if len(line.strip()): - regex_matches = False - - # if any of these match, skip - for regex in ignore_regex: - try: - if re.search(regex, line, re.IGNORECASE): - regex_matches = True - except Exception as e: - continue - - if not regex_matches and not any(skip_text in line for skip_text in ignore): - output.append(line.encode('utf8')) - - return "\n".encode('utf8').join(output) - - - def run(self, uuid): timestamp = int(time.time()) # used for storage etc too @@ -147,7 +114,7 @@ class perform_site_check(): # @todo we could abstract out the get_text() to handle this cleaner text_to_ignore = watch.get('ignore_text', []) + self.datastore.data['settings']['application'].get('global_ignore_text', []) if len(text_to_ignore): - stripped_text_from_html = self.strip_ignore_text(stripped_text_from_html, text_to_ignore) + stripped_text_from_html = html_tools.strip_ignore_text(stripped_text_from_html, text_to_ignore) else: stripped_text_from_html = stripped_text_from_html.encode('utf8') @@ -165,22 +132,14 @@ class perform_site_check(): blocked_by_not_found_trigger_text = False if len(watch['trigger_text']): + # Yeah, lets block first until something matches blocked_by_not_found_trigger_text = True - for line in watch['trigger_text']: - # Because JSON wont serialize a re.compile object - if line[0] == '/' and line[-1] == '/': - regex = re.compile(line.strip('/'), re.IGNORECASE) - # Found it? so we don't wait for it anymore - r = re.search(regex, str(stripped_text_from_html)) - if r: - blocked_by_not_found_trigger_text = False - break - - elif line.lower() in str(stripped_text_from_html).lower(): - # We found it don't wait for it. - blocked_by_not_found_trigger_text = False - break - + # Filter and trigger works the same, so reuse it + result = html_tools.strip_ignore_text(content=str(stripped_text_from_html), + wordlist=watch['trigger_text'], + mode="line numbers") + if result: + blocked_by_not_found_trigger_text = False if not blocked_by_not_found_trigger_text and watch['previous_md5'] != fetched_md5: diff --git a/changedetectionio/forms.py b/changedetectionio/forms.py index 9af85b31..b2b53382 100644 --- a/changedetectionio/forms.py +++ b/changedetectionio/forms.py @@ -1,6 +1,7 @@ from wtforms import Form, SelectField, RadioField, BooleanField, StringField, PasswordField, validators, IntegerField, fields, TextAreaField, \ Field -from wtforms import widgets + +from wtforms import widgets, SubmitField from wtforms.validators import ValidationError from wtforms.fields import html5 from changedetectionio import content_fetcher @@ -290,6 +291,9 @@ class watchForm(commonSettingsForm): method = SelectField('Request Method', choices=valid_method, default=default_method) trigger_text = StringListField('Trigger/wait for text', [validators.Optional(), ValidateListRegex()]) + save_button = SubmitField('Save', render_kw={"class": "pure-button pure-button-primary"}) + save_and_preview_button = SubmitField('Save & Preview', render_kw={"class": "pure-button pure-button-primary"}) + def validate(self, **kwargs): if not super().validate(): return False diff --git a/changedetectionio/html_tools.py b/changedetectionio/html_tools.py index 47b7f698..4f500e74 100644 --- a/changedetectionio/html_tools.py +++ b/changedetectionio/html_tools.py @@ -1,7 +1,7 @@ import json from bs4 import BeautifulSoup from jsonpath_ng.ext import parse - +import re class JSONNotFound(ValueError): def __init__(self, msg): @@ -105,3 +105,50 @@ def extract_json_as_string(content, jsonpath_filter): return '' return stripped_text_from_html + +# Mode - "content" return the content without the matches (default) +# - "line numbers" return a list of line numbers that match (int list) +# +# wordlist - list of regex's (str) or words (str) +def strip_ignore_text(content, wordlist, mode="content"): + ignore = [] + ignore_regex = [] + + # @todo check this runs case insensitive + for k in wordlist: + + # Is it a regex? + if k[0] == '/': + ignore_regex.append(k.strip(" /")) + else: + ignore.append(k) + + i = 0 + output = [] + ignored_line_numbers = [] + for line in content.splitlines(): + i += 1 + # Always ignore blank lines in this mode. (when this function gets called) + if len(line.strip()): + regex_matches = False + + # if any of these match, skip + for regex in ignore_regex: + try: + if re.search(regex, line, re.IGNORECASE): + regex_matches = True + except Exception as e: + continue + + if not regex_matches and not any(skip_text in line for skip_text in ignore): + output.append(line.encode('utf8')) + else: + ignored_line_numbers.append(i) + + + + # Used for finding out what to highlight + if mode == "line numbers": + return ignored_line_numbers + + return "\n".encode('utf8').join(output) \ No newline at end of file diff --git a/changedetectionio/static/styles/diff.css b/changedetectionio/static/styles/diff.css index f2f8f545..6ab1009a 100644 --- a/changedetectionio/static/styles/diff.css +++ b/changedetectionio/static/styles/diff.css @@ -54,3 +54,19 @@ ins { body { height: 99%; /* Hide scroll bar in Firefox */ } } + +td#diff-col div { + text-align: justify; + white-space: pre-wrap; } + +.ignored { + background-color: #ccc; + /* border: #0d91fa 1px solid; */ + opacity: 0.7; } + +.triggered { + background-color: #1b98f8; } + +/* ignored and triggered? make it obvious error */ +.ignored.triggered { + background-color: #ff0000; } diff --git a/changedetectionio/static/styles/diff.scss b/changedetectionio/static/styles/diff.scss index c0b8315f..eaf06176 100644 --- a/changedetectionio/static/styles/diff.scss +++ b/changedetectionio/static/styles/diff.scss @@ -66,3 +66,23 @@ ins { height: 99%; /* Hide scroll bar in Firefox */ } } + +td#diff-col div { + text-align: justify; + white-space: pre-wrap; +} + +.ignored { + background-color: #ccc; + /* border: #0d91fa 1px solid; */ + opacity: 0.7; +} + +.triggered { + background-color: #1b98f8; +} + +/* ignored and triggered? make it obvious error */ +.ignored.triggered { + background-color: #ff0000; +} \ No newline at end of file diff --git a/changedetectionio/static/styles/package.json b/changedetectionio/static/styles/package.json index 89390c20..92862d71 100644 --- a/changedetectionio/static/styles/package.json +++ b/changedetectionio/static/styles/package.json @@ -4,8 +4,7 @@ "description": "", "main": "index.js", "scripts": { - "build": "node-sass styles.scss diff.scss -o .", - "watch": "node-sass --watch styles.scss diff.scss -o ." + "build": "node-sass styles.scss -o .;node-sass diff.scss -o ." }, "author": "", "license": "ISC", diff --git a/changedetectionio/static/styles/styles.css b/changedetectionio/static/styles/styles.css index 3461475c..794dc631 100644 --- a/changedetectionio/static/styles/styles.css +++ b/changedetectionio/static/styles/styles.css @@ -1 +1,407 @@ -body{color:#333;background:#262626}.pure-table-even{background:#fff}a{text-decoration:none;color:#1b98f8}a.github-link{color:#fff}.pure-menu-horizontal{background:#fff;padding:5px;display:flex;justify-content:space-between;border-bottom:2px solid #ed5900;align-items:center}section.content{padding-top:5em;padding-bottom:5em;flex-direction:column;display:flex;align-items:center;justify-content:center}.watch-table{width:100%}.watch-table tr.unviewed{font-weight:bold}.watch-table .error{color:#a00}.watch-table td{font-size:80%;white-space:nowrap}.watch-table td.title-col{word-break:break-all;white-space:normal}.watch-table th{white-space:nowrap}.watch-table .title-col a[target="_blank"]::after,.watch-table .current-diff-url::after{content:url(data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAoAAAAKCAYAAACNMs+9AAAAQElEQVR42qXKwQkAIAxDUUdxtO6/RBQkQZvSi8I/pL4BoGw/XPkh4XigPmsUgh0626AjRsgxHTkUThsG2T/sIlzdTsp52kSS1wAAAABJRU5ErkJggg==);margin:0 3px 0 5px}.watch-tag-list{color:#e70069;white-space:nowrap}.box{max-width:80%;flex-direction:column;display:flex;justify-content:center}#post-list-buttons{text-align:right;padding:0;margin:0}#post-list-buttons li{display:inline-block}#post-list-buttons a{border-top-left-radius:initial;border-top-right-radius:initial;border-bottom-left-radius:5px;border-bottom-right-radius:5px}body:after{content:"";background:linear-gradient(130deg, #ff7a18, #af002d 41.07%, #319197 76.05%)}body:after,body:before{display:block;height:600px;position:absolute;top:0;left:0;width:100%;z-index:-1}body::after{opacity:.91}body::before{content:"";background-image:url(../images/gradient-border.png)}body:before{background-size:cover}body:after,body:before{-webkit-clip-path:polygon(100% 0, 0 0, 0 77.5%, 1% 77.4%, 2% 77.1%, 3% 76.6%, 4% 75.9%, 5% 75.05%, 6% 74.05%, 7% 72.95%, 8% 71.75%, 9% 70.55%, 10% 69.3%, 11% 68.05%, 12% 66.9%, 13% 65.8%, 14% 64.8%, 15% 64%, 16% 63.35%, 17% 62.85%, 18% 62.6%, 19% 62.5%, 20% 62.65%, 21% 63%, 22% 63.5%, 23% 64.2%, 24% 65.1%, 25% 66.1%, 26% 67.2%, 27% 68.4%, 28% 69.65%, 29% 70.9%, 30% 72.15%, 31% 73.3%, 32% 74.35%, 33% 75.3%, 34% 76.1%, 35% 76.75%, 36% 77.2%, 37% 77.45%, 38% 77.5%, 39% 77.3%, 40% 76.95%, 41% 76.4%, 42% 75.65%, 43% 74.75%, 44% 73.75%, 45% 72.6%, 46% 71.4%, 47% 70.15%, 48% 68.9%, 49% 67.7%, 50% 66.55%, 51% 65.5%, 52% 64.55%, 53% 63.75%, 54% 63.15%, 55% 62.75%, 56% 62.55%, 57% 62.5%, 58% 62.7%, 59% 63.1%, 60% 63.7%, 61% 64.45%, 62% 65.4%, 63% 66.45%, 64% 67.6%, 65% 68.8%, 66% 70.05%, 67% 71.3%, 68% 72.5%, 69% 73.6%, 70% 74.65%, 71% 75.55%, 72% 76.35%, 73% 76.9%, 74% 77.3%, 75% 77.5%, 76% 77.45%, 77% 77.25%, 78% 76.8%, 79% 76.2%, 80% 75.4%, 81% 74.45%, 82% 73.4%, 83% 72.25%, 84% 71.05%, 85% 69.8%, 86% 68.55%, 87% 67.35%, 88% 66.2%, 89% 65.2%, 90% 64.3%, 91% 63.55%, 92% 63%, 93% 62.65%, 94% 62.5%, 95% 62.55%, 96% 62.8%, 97% 63.3%, 98% 63.9%, 99% 64.75%, 100% 65.7%);clip-path:polygon(100% 0, 0 0, 0 77.5%, 1% 77.4%, 2% 77.1%, 3% 76.6%, 4% 75.9%, 5% 75.05%, 6% 74.05%, 7% 72.95%, 8% 71.75%, 9% 70.55%, 10% 69.3%, 11% 68.05%, 12% 66.9%, 13% 65.8%, 14% 64.8%, 15% 64%, 16% 63.35%, 17% 62.85%, 18% 62.6%, 19% 62.5%, 20% 62.65%, 21% 63%, 22% 63.5%, 23% 64.2%, 24% 65.1%, 25% 66.1%, 26% 67.2%, 27% 68.4%, 28% 69.65%, 29% 70.9%, 30% 72.15%, 31% 73.3%, 32% 74.35%, 33% 75.3%, 34% 76.1%, 35% 76.75%, 36% 77.2%, 37% 77.45%, 38% 77.5%, 39% 77.3%, 40% 76.95%, 41% 76.4%, 42% 75.65%, 43% 74.75%, 44% 73.75%, 45% 72.6%, 46% 71.4%, 47% 70.15%, 48% 68.9%, 49% 67.7%, 50% 66.55%, 51% 65.5%, 52% 64.55%, 53% 63.75%, 54% 63.15%, 55% 62.75%, 56% 62.55%, 57% 62.5%, 58% 62.7%, 59% 63.1%, 60% 63.7%, 61% 64.45%, 62% 65.4%, 63% 66.45%, 64% 67.6%, 65% 68.8%, 66% 70.05%, 67% 71.3%, 68% 72.5%, 69% 73.6%, 70% 74.65%, 71% 75.55%, 72% 76.35%, 73% 76.9%, 74% 77.3%, 75% 77.5%, 76% 77.45%, 77% 77.25%, 78% 76.8%, 79% 76.2%, 80% 75.4%, 81% 74.45%, 82% 73.4%, 83% 72.25%, 84% 71.05%, 85% 69.8%, 86% 68.55%, 87% 67.35%, 88% 66.2%, 89% 65.2%, 90% 64.3%, 91% 63.55%, 92% 63%, 93% 62.65%, 94% 62.5%, 95% 62.55%, 96% 62.8%, 97% 63.3%, 98% 63.9%, 99% 64.75%, 100% 65.7%)}.arrow{border:solid #000;border-width:0 3px 3px 0;display:inline-block;padding:3px}.arrow.right{transform:rotate(-45deg);-webkit-transform:rotate(-45deg)}.arrow.left{transform:rotate(135deg);-webkit-transform:rotate(135deg)}.arrow.up{transform:rotate(-135deg);-webkit-transform:rotate(-135deg)}.arrow.down{transform:rotate(45deg);-webkit-transform:rotate(45deg)}.button-small{font-size:85%}.fetch-error{padding-top:1em;font-size:60%;max-width:400px;display:block}.button-secondary{color:#fff;border-radius:4px;text-shadow:0 1px 1px rgba(0,0,0,0.2)}.button-success{background:#1cb841}.button-tag{background:#636363;color:#fff;font-size:65%;border-bottom-left-radius:initial;border-bottom-right-radius:initial}.button-tag.active{background:#9c9c9c;font-weight:bold}.button-error{background:#ca3c3c}.button-warning{background:#df7514}.button-secondary{background:#42b8dd}.button-cancel{background:#c8c8c8}.messages li{list-style:none;padding:1em;border-radius:10px;color:#fff;font-weight:bold}.messages li.message{background:rgba(255,255,255,0.2)}.messages li.error{background:rgba(255,1,1,0.5)}.messages li.notice{background:rgba(255,255,255,0.5)}#notification-customisation{border:1px solid #ccc;padding:1rem;border-radius:5px}#token-table.pure-table td,#token-table.pure-table th{font-size:80%}#new-watch-form{background:rgba(0,0,0,0.05);padding:1em;border-radius:10px;margin-bottom:1em}#new-watch-form input{width:auto !important;display:inline-block}#new-watch-form .label{display:none}#new-watch-form legend{color:#fff}#diff-col{padding-left:40px}#diff-jump{position:fixed;left:0;top:120px;background:#fff;padding:10px;border-top-right-radius:5px;border-bottom-right-radius:5px;box-shadow:5px 0 5px -2px #888}#diff-jump a{color:#1b98f8;cursor:grabbing;-moz-user-select:none;-webkit-user-select:none;-ms-user-select:none;user-select:none;-o-user-select:none}footer{padding:10px;background:#fff;color:#444;text-align:center}#feed-icon{vertical-align:middle}.sticky-tab{position:absolute;top:80px;font-size:11px;background:#fff;padding:10px}.sticky-tab#left-sticky{left:0}.sticky-tab#right-sticky{right:0}#new-version-text a{color:#e07171}.paused-state.state-False img{opacity:.2}.paused-state.state-False:hover img{opacity:.8}.monospaced-textarea textarea{width:100%;font-family:monospace;white-space:pre;overflow-wrap:normal;overflow-x:scroll}.pure-form .pure-control-group,.pure-form .pure-group,.pure-form .pure-controls{padding-bottom:1em}.pure-form .pure-control-group div,.pure-form .pure-group div,.pure-form .pure-controls div{margin:0}.pure-form .error input{background-color:#ffebeb}.pure-form ul.errors{padding:.5em .6em;border:1px solid #d00;border-radius:4px;vertical-align:middle;-webkit-box-sizing:border-box;box-sizing:border-box}.pure-form ul.errors li{margin-left:1em;color:#d00}.pure-form label{font-weight:bold}.pure-form textarea{width:100%}.pure-form ul#fetch_backend{margin:0;list-style:none}.pure-form ul#fetch_backend > li > *{display:inline-block}@media only screen and (max-width: 760px), (min-device-width: 768px) and (max-device-width: 1024px){.box{max-width:95%}.edit-form{padding:.5em;margin:0}#nav-menu{overflow-x:scroll}}@media only screen and (max-width: 760px), (min-device-width: 768px) and (max-device-width: 1024px){input[type='text']{width:100%}.watch-table thead,.watch-table tbody,.watch-table th,.watch-table td,.watch-table tr{display:block}.watch-table .last-checked::before{color:#555;content:"Last Checked "}.watch-table .last-changed::before{color:#555;content:"Last Changed "}.watch-table td.inline{display:inline-block}.watch-table thead tr{position:absolute;top:-9999px;left:-9999px}.watch-table .pure-table td,.watch-table .pure-table th{border:none}.watch-table td{border:none;border-bottom:1px solid #eee}.watch-table td:before{top:6px;left:6px;width:45%;padding-right:10px;white-space:nowrap}.watch-table.pure-table-striped tr{background-color:#fff}.watch-table.pure-table-striped tr:nth-child(2n-1){background-color:#eee}.watch-table.pure-table-striped tr:nth-child(2n-1) td{background-color:inherit}}@media only screen and (min-width: 761px){.m-d{min-width:80%}}.tabs ul{margin:0;padding:0;display:block}.tabs ul li{margin-right:3px;display:inline-block;color:#fff;border-top-left-radius:5px;border-top-right-radius:5px;background-color:rgba(255,255,255,0.2)}.tabs ul li.active,.tabs ul li :target{background-color:#fff}.tabs ul li.active a,.tabs ul li :target a{color:#222;font-weight:bold}.tabs ul li a{display:block;padding:.8em;color:#fff}.pure-form-stacked > div:first-child{display:block}.edit-form{min-width:70%}.edit-form .tab-pane-inner{padding:0}.edit-form .tab-pane-inner:not(:target){display:none}.edit-form .tab-pane-inner:target{display:block}.edit-form .box-wrap{position:relative}.edit-form .inner{background:#fff;padding:20px}.edit-form #actions{display:block;background:#fff} \ No newline at end of file +/* + * -- BASE STYLES -- + * Most of these are inherited from Base, but I want to change a few. + * nvm use v14.18.1 + * npm install + * npm run build + * or npm run watch + */ +body { + color: #333; + background: #262626; } + +.pure-table-even { + background: #fff; } + +/* Some styles from https://css-tricks.com/ */ +a { + text-decoration: none; + color: #1b98f8; } + +a.github-link { + color: #fff; } + +.pure-menu-horizontal { + background: #fff; + padding: 5px; + display: flex; + justify-content: space-between; + border-bottom: 2px solid #ed5900; + align-items: center; } + +section.content { + padding-top: 5em; + padding-bottom: 5em; + flex-direction: column; + display: flex; + align-items: center; + justify-content: center; } + +/* table related */ +.watch-table { + width: 100%; } + .watch-table tr.unviewed { + font-weight: bold; } + .watch-table .error { + color: #a00; } + .watch-table td { + font-size: 80%; + white-space: nowrap; } + .watch-table td.title-col { + word-break: break-all; + white-space: normal; } + .watch-table th { + white-space: nowrap; } + .watch-table .title-col a[target="_blank"]::after, .watch-table .current-diff-url::after { + content: url(data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAoAAAAKCAYAAACNMs+9AAAAQElEQVR42qXKwQkAIAxDUUdxtO6/RBQkQZvSi8I/pL4BoGw/XPkh4XigPmsUgh0626AjRsgxHTkUThsG2T/sIlzdTsp52kSS1wAAAABJRU5ErkJggg==); + margin: 0 3px 0 5px; } + +.watch-tag-list { + color: #e70069; + white-space: nowrap; } + +.box { + max-width: 80%; + flex-direction: column; + display: flex; + justify-content: center; } + +#post-list-buttons { + text-align: right; + padding: 0px; + margin: 0px; } + #post-list-buttons li { + display: inline-block; } + #post-list-buttons a { + border-top-left-radius: initial; + border-top-right-radius: initial; + border-bottom-left-radius: 5px; + border-bottom-right-radius: 5px; } + +body:after { + content: ""; + background: linear-gradient(130deg, #ff7a18, #af002d 41.07%, #319197 76.05%); } + +body:after, body:before { + display: block; + height: 600px; + position: absolute; + top: 0; + left: 0; + width: 100%; + z-index: -1; } + +body::after { + opacity: 0.91; } + +body::before { + content: ""; + background-image: url(/static/images/gradient-border.png); } + +body:before { + background-size: cover; } + +body:after, body:before { + -webkit-clip-path: polygon(100% 0, 0 0, 0 77.5%, 1% 77.4%, 2% 77.1%, 3% 76.6%, 4% 75.9%, 5% 75.05%, 6% 74.05%, 7% 72.95%, 8% 71.75%, 9% 70.55%, 10% 69.3%, 11% 68.05%, 12% 66.9%, 13% 65.8%, 14% 64.8%, 15% 64%, 16% 63.35%, 17% 62.85%, 18% 62.6%, 19% 62.5%, 20% 62.65%, 21% 63%, 22% 63.5%, 23% 64.2%, 24% 65.1%, 25% 66.1%, 26% 67.2%, 27% 68.4%, 28% 69.65%, 29% 70.9%, 30% 72.15%, 31% 73.3%, 32% 74.35%, 33% 75.3%, 34% 76.1%, 35% 76.75%, 36% 77.2%, 37% 77.45%, 38% 77.5%, 39% 77.3%, 40% 76.95%, 41% 76.4%, 42% 75.65%, 43% 74.75%, 44% 73.75%, 45% 72.6%, 46% 71.4%, 47% 70.15%, 48% 68.9%, 49% 67.7%, 50% 66.55%, 51% 65.5%, 52% 64.55%, 53% 63.75%, 54% 63.15%, 55% 62.75%, 56% 62.55%, 57% 62.5%, 58% 62.7%, 59% 63.1%, 60% 63.7%, 61% 64.45%, 62% 65.4%, 63% 66.45%, 64% 67.6%, 65% 68.8%, 66% 70.05%, 67% 71.3%, 68% 72.5%, 69% 73.6%, 70% 74.65%, 71% 75.55%, 72% 76.35%, 73% 76.9%, 74% 77.3%, 75% 77.5%, 76% 77.45%, 77% 77.25%, 78% 76.8%, 79% 76.2%, 80% 75.4%, 81% 74.45%, 82% 73.4%, 83% 72.25%, 84% 71.05%, 85% 69.8%, 86% 68.55%, 87% 67.35%, 88% 66.2%, 89% 65.2%, 90% 64.3%, 91% 63.55%, 92% 63%, 93% 62.65%, 94% 62.5%, 95% 62.55%, 96% 62.8%, 97% 63.3%, 98% 63.9%, 99% 64.75%, 100% 65.7%); + clip-path: polygon(100% 0, 0 0, 0 77.5%, 1% 77.4%, 2% 77.1%, 3% 76.6%, 4% 75.9%, 5% 75.05%, 6% 74.05%, 7% 72.95%, 8% 71.75%, 9% 70.55%, 10% 69.3%, 11% 68.05%, 12% 66.9%, 13% 65.8%, 14% 64.8%, 15% 64%, 16% 63.35%, 17% 62.85%, 18% 62.6%, 19% 62.5%, 20% 62.65%, 21% 63%, 22% 63.5%, 23% 64.2%, 24% 65.1%, 25% 66.1%, 26% 67.2%, 27% 68.4%, 28% 69.65%, 29% 70.9%, 30% 72.15%, 31% 73.3%, 32% 74.35%, 33% 75.3%, 34% 76.1%, 35% 76.75%, 36% 77.2%, 37% 77.45%, 38% 77.5%, 39% 77.3%, 40% 76.95%, 41% 76.4%, 42% 75.65%, 43% 74.75%, 44% 73.75%, 45% 72.6%, 46% 71.4%, 47% 70.15%, 48% 68.9%, 49% 67.7%, 50% 66.55%, 51% 65.5%, 52% 64.55%, 53% 63.75%, 54% 63.15%, 55% 62.75%, 56% 62.55%, 57% 62.5%, 58% 62.7%, 59% 63.1%, 60% 63.7%, 61% 64.45%, 62% 65.4%, 63% 66.45%, 64% 67.6%, 65% 68.8%, 66% 70.05%, 67% 71.3%, 68% 72.5%, 69% 73.6%, 70% 74.65%, 71% 75.55%, 72% 76.35%, 73% 76.9%, 74% 77.3%, 75% 77.5%, 76% 77.45%, 77% 77.25%, 78% 76.8%, 79% 76.2%, 80% 75.4%, 81% 74.45%, 82% 73.4%, 83% 72.25%, 84% 71.05%, 85% 69.8%, 86% 68.55%, 87% 67.35%, 88% 66.2%, 89% 65.2%, 90% 64.3%, 91% 63.55%, 92% 63%, 93% 62.65%, 94% 62.5%, 95% 62.55%, 96% 62.8%, 97% 63.3%, 98% 63.9%, 99% 64.75%, 100% 65.7%); } + +.arrow { + border: solid black; + border-width: 0 3px 3px 0; + display: inline-block; + padding: 3px; } + .arrow.right { + transform: rotate(-45deg); + -webkit-transform: rotate(-45deg); } + .arrow.left { + transform: rotate(135deg); + -webkit-transform: rotate(135deg); } + .arrow.up { + transform: rotate(-135deg); + -webkit-transform: rotate(-135deg); } + .arrow.down { + transform: rotate(45deg); + -webkit-transform: rotate(45deg); } + +.button-small { + font-size: 85%; } + +.fetch-error { + padding-top: 1em; + font-size: 60%; + max-width: 400px; + display: block; } + +.button-secondary { + color: white; + border-radius: 4px; + text-shadow: 0 1px 1px rgba(0, 0, 0, 0.2); } + +.button-success { + background: #1cb841; + /* this is a green */ } + +.button-tag { + background: #636363; + color: #fff; + font-size: 65%; + border-bottom-left-radius: initial; + border-bottom-right-radius: initial; } + .button-tag.active { + background: #9c9c9c; + font-weight: bold; } + +.button-error { + background: #ca3c3c; + /* this is a maroon */ } + +.button-warning { + background: #df7514; + /* this is an orange */ } + +.button-secondary { + background: #42b8dd; + /* this is a light blue */ } + +.button-cancel { + background: #c8c8c8; + /* this is a green */ } + +.messages li { + list-style: none; + padding: 1em; + border-radius: 10px; + color: #fff; + font-weight: bold; } + .messages li.message { + background: rgba(255, 255, 255, 0.2); } + .messages li.error { + background: rgba(255, 1, 1, 0.5); } + .messages li.notice { + background: rgba(255, 255, 255, 0.5); } + +#notification-customisation { + border: 1px solid #ccc; + padding: 1rem; + border-radius: 5px; } + +#token-table.pure-table td, #token-table.pure-table th { + font-size: 80%; } + +#new-watch-form { + background: rgba(0, 0, 0, 0.05); + padding: 1em; + border-radius: 10px; + margin-bottom: 1em; } + #new-watch-form input { + width: auto !important; + display: inline-block; } + #new-watch-form .label { + display: none; } + #new-watch-form legend { + color: #fff; } + +#diff-col { + padding-left: 40px; } + +#diff-jump { + position: fixed; + left: 0px; + top: 120px; + background: #fff; + padding: 10px; + border-top-right-radius: 5px; + border-bottom-right-radius: 5px; + box-shadow: 5px 0 5px -2px #888; } + #diff-jump a { + color: #1b98f8; + cursor: grabbing; + -moz-user-select: none; + -webkit-user-select: none; + -ms-user-select: none; + user-select: none; + -o-user-select: none; } + +footer { + padding: 10px; + background: #fff; + color: #444; + text-align: center; } + +#feed-icon { + vertical-align: middle; } + +#top-right-menu { + /* + position: absolute; + right: 0px; + background: linear-gradient(to right, #fff0, #fff 10%); + padding-left: 20px; + padding-right: 10px; + */ } + +.sticky-tab { + position: absolute; + top: 80px; + font-size: 8px; + background: #fff; + padding: 10px; } + .sticky-tab#left-sticky { + left: 0px; } + .sticky-tab#right-sticky { + right: 0px; } + +#new-version-text a { + color: #e07171; } + +.paused-state.state-False img { + opacity: 0.2; } + +.paused-state.state-False:hover img { + opacity: 0.8; } + +.monospaced-textarea textarea { + width: 100%; + font-family: monospace; + white-space: pre; + overflow-wrap: normal; + overflow-x: scroll; } + +.pure-form { + /* The input fields with errors */ + /* The list of errors */ } + .pure-form .pure-control-group, .pure-form .pure-group, .pure-form .pure-controls { + padding-bottom: 1em; } + .pure-form .pure-control-group div, .pure-form .pure-group div, .pure-form .pure-controls div { + margin: 0px; } + .pure-form .error input { + background-color: #ffebeb; } + .pure-form ul.errors { + padding: .5em .6em; + border: 1px solid #dd0000; + border-radius: 4px; + vertical-align: middle; + -webkit-box-sizing: border-box; + box-sizing: border-box; } + .pure-form ul.errors li { + margin-left: 1em; + color: #dd0000; } + .pure-form label { + font-weight: bold; } + .pure-form textarea { + width: 100%; } + .pure-form ul#fetch_backend { + margin: 0px; + list-style: none; } + .pure-form ul#fetch_backend > li > * { + display: inline-block; } + +@media only screen and (max-width: 760px), (min-device-width: 768px) and (max-device-width: 1024px) { + .box { + max-width: 95%; } + .edit-form { + padding: 0.5em; + margin: 0; } + #nav-menu { + overflow-x: scroll; } } + +/* +Max width before this PARTICULAR table gets nasty +This query will take effect for any screen smaller than 760px +and also iPads specifically. +*/ +@media only screen and (max-width: 760px), (min-device-width: 768px) and (max-device-width: 1024px) { + input[type='text'] { + width: 100%; } + .watch-table { + /* Force table to not be like tables anymore */ + /* Force table to not be like tables anymore */ + /* Hide table headers (but not display: none;, for accessibility) */ } + .watch-table thead, .watch-table tbody, .watch-table th, .watch-table td, .watch-table tr { + display: block; } + .watch-table .last-checked::before { + color: #555; + content: "Last Checked "; } + .watch-table .last-changed::before { + color: #555; + content: "Last Changed "; } + .watch-table td.inline { + display: inline-block; } + .watch-table thead tr { + position: absolute; + top: -9999px; + left: -9999px; } + .watch-table .pure-table td, .watch-table .pure-table th { + border: none; } + .watch-table td { + /* Behave like a "row" */ + border: none; + border-bottom: 1px solid #eee; } + .watch-table td:before { + /* Top/left values mimic padding */ + top: 6px; + left: 6px; + width: 45%; + padding-right: 10px; + white-space: nowrap; } + .watch-table.pure-table-striped tr { + background-color: #fff; } + .watch-table.pure-table-striped tr:nth-child(2n-1) { + background-color: #eee; } + .watch-table.pure-table-striped tr:nth-child(2n-1) td { + background-color: inherit; } } + +/** Desktop vs mobile input field strategy +- We dont use 'size' with because `size` is too unreliable to override, and will often push-out +- Rely always on width in CSS +*/ +@media only screen and (min-width: 761px) { + /* m-d is medium-desktop */ + .m-d { + min-width: 80%; } } + +.tabs ul { + margin: 0px; + padding: 0px; + display: block; } + .tabs ul li { + margin-right: 3px; + display: inline-block; + color: #fff; + border-top-left-radius: 5px; + border-top-right-radius: 5px; + background-color: rgba(255, 255, 255, 0.2); } + .tabs ul li.active, .tabs ul li :target { + background-color: #fff; } + .tabs ul li.active a, .tabs ul li :target a { + color: #222; + font-weight: bold; } + .tabs ul li a { + display: block; + padding: 0.8em; + color: #fff; } + +.pure-form-stacked > div:first-child { + display: block; } + +.edit-form { + min-width: 70%; } + .edit-form .tab-pane-inner { + padding: 0px; } + .edit-form .tab-pane-inner:not(:target) { + display: none; } + .edit-form .tab-pane-inner:target { + display: block; } + .edit-form .box-wrap { + position: relative; } + .edit-form .inner { + background: #fff; + padding: 20px; } + .edit-form #actions { + display: block; + background: #fff; } + +ul { + padding-left: 1em; + padding-top: 0px; + margin-top: 4px; } diff --git a/changedetectionio/static/styles/styles.scss b/changedetectionio/static/styles/styles.scss index c07254b2..63b40a4e 100644 --- a/changedetectionio/static/styles/styles.scss +++ b/changedetectionio/static/styles/styles.scss @@ -567,3 +567,8 @@ $form-edge-padding: 20px; } } +ul { + padding-left: 1em; + padding-top: 0px; + margin-top: 4px; +} \ No newline at end of file diff --git a/changedetectionio/templates/_helpers.jinja b/changedetectionio/templates/_helpers.jinja index 734fc7e0..72187307 100644 --- a/changedetectionio/templates/_helpers.jinja +++ b/changedetectionio/templates/_helpers.jinja @@ -25,3 +25,6 @@ {% endmacro %} +{% macro render_button(field) %} + {{ field(**kwargs)|safe }} +{% endmacro %} \ No newline at end of file diff --git a/changedetectionio/templates/diff.html b/changedetectionio/templates/diff.html index bc88a290..9f697a5f 100644 --- a/changedetectionio/templates/diff.html +++ b/changedetectionio/templates/diff.html @@ -36,6 +36,7 @@ Jump
+
Pro-tip: Use show current snapshot tab to visualise what will be ignored.
diff --git a/changedetectionio/templates/edit.html b/changedetectionio/templates/edit.html index 3ab30811..29e8b6aa 100644 --- a/changedetectionio/templates/edit.html +++ b/changedetectionio/templates/edit.html @@ -1,6 +1,7 @@ {% extends 'base.html' %} {% block content %} {% from '_helpers.jinja' import render_field %} +{% from '_helpers.jinja' import render_button %} {% from '_common_fields.jinja' import render_common_settings_form %} @@ -88,6 +89,18 @@ User-Agent: wonderbra 1.0") }}
+
+ Pro-tips:
+
    +
  • + Use the preview page to see your filters and triggers highlighted. +
    • +
  • + Some sites use JavaScript to create the content, for this you should use the Chrome/WebDriver Fetcher +
    • +
+
+
{{ render_field(form.css_filter, placeholder=".class-name or #some-id, or other CSS selector rule.", class="m-d") }} @@ -114,6 +127,7 @@ User-Agent: wonderbra 1.0") }}
  • Each line processed separately, any line matching will be ignored (removed before creating the checksum)
  • Regular Expression support, wrap the line in forward slash /regex/
  • Changing this will affect the comparison checksum which may trigger an alert
    • +
  • Use the preview/show current tab to see ignores
    • @@ -138,7 +152,8 @@ User-Agent: wonderbra 1.0") }}
    - + {{ render_button(form.save_button) }} {{ render_button(form.save_and_preview_button) }} + Delete -

    Current

    +

    Current - {{watch.last_checked|format_timestamp_timeago}}

    + Grey lines are ignored Blue lines are triggers
    - {% for row in content %}{{row}}{% endfor %} + {% for row in content %} + {% set classes = [] %} + {% if (loop.index in ignored_line_numbers) %}{{ classes.append("ignored") }}{% endif %} + {% if (loop.index in triggered_line_numbers) %}{{ classes.append("triggered") }}{% endif %} +
    {{row}}
    + {% endfor %}
    - {% endblock %} \ No newline at end of file diff --git a/changedetectionio/templates/settings.html b/changedetectionio/templates/settings.html index fa76bc7d..3157bca6 100644 --- a/changedetectionio/templates/settings.html +++ b/changedetectionio/templates/settings.html @@ -95,6 +95,7 @@
  • Each line processed separately, any line matching will be ignored (removed before creating the checksum)
  • Regular Expression support, wrap the line in forward slash /regex/
  • Changing this will affect the comparison checksum which may trigger an alert
    • +
  • Use the preview/show current tab to see ignores
    • diff --git a/changedetectionio/tests/test_ignore_regex_text.py b/changedetectionio/tests/test_ignore_regex_text.py index e70d16a4..2d12e7a6 100644 --- a/changedetectionio/tests/test_ignore_regex_text.py +++ b/changedetectionio/tests/test_ignore_regex_text.py @@ -3,6 +3,7 @@ import time from flask import url_for from . util import live_server_setup +from changedetectionio import html_tools def test_setup(live_server): live_server_setup(live_server) @@ -23,7 +24,7 @@ def test_strip_regex_text_func(): ignore_lines = ["sometimes", "/\s\d{2,3}\s/", "/ignore-case text/"] fetcher = fetch_site_status.perform_site_check(datastore=False) - stripped_content = fetcher.strip_ignore_text(test_content, ignore_lines) + stripped_content = html_tools.strip_ignore_text(test_content, ignore_lines) assert b"but 1 lines" in stripped_content assert b"igNORe-cAse text" not in stripped_content diff --git a/changedetectionio/tests/test_ignore_text.py b/changedetectionio/tests/test_ignore_text.py index 726a6f9b..3acacb34 100644 --- a/changedetectionio/tests/test_ignore_text.py +++ b/changedetectionio/tests/test_ignore_text.py @@ -3,6 +3,7 @@ import time from flask import url_for from . util import live_server_setup +from changedetectionio import html_tools def test_setup(live_server): live_server_setup(live_server) @@ -23,7 +24,7 @@ def test_strip_text_func(): ignore_lines = ["sometimes"] fetcher = fetch_site_status.perform_site_check(datastore=False) - stripped_content = fetcher.strip_ignore_text(test_content, ignore_lines) + stripped_content = html_tools.strip_ignore_text(test_content, ignore_lines) assert b"sometimes" not in stripped_content assert b"Some content" in stripped_content @@ -52,6 +53,8 @@ def set_modified_original_ignore_response():

    Which is across multiple lines


    So let's see what happens.
    +

    new ignore stuff

    +

    blah

    @@ -82,7 +85,7 @@ def set_modified_ignore_response(): def test_check_ignore_text_functionality(client, live_server): sleep_time_for_fetch_thread = 3 - ignore_text = "XXXXX\r\nYYYYY\r\nZZZZZ" + ignore_text = "XXXXX\r\nYYYYY\r\nZZZZZ\r\nnew ignore stuff" set_original_ignore_response() # Give the endpoint time to spin up @@ -142,13 +145,25 @@ def test_check_ignore_text_functionality(client, live_server): assert b'unviewed' not in res.data assert b'/test-endpoint' in res.data + + + + # Just to be sure.. set a regular modified change.. set_modified_original_ignore_response() client.get(url_for("api_watch_checknow"), follow_redirects=True) time.sleep(sleep_time_for_fetch_thread) + res = client.get(url_for("index")) assert b'unviewed' in res.data + # Check the preview/highlighter, we should be able to see what we ignored, but it should be highlighted + # We only introduce the "modified" content that includes what we ignore so we can prove the newest version also displays + # at /preview + res = client.get(url_for("preview_page", uuid="first")) + # We should be able to see what we ignored + assert b'
    new ignore stuff' in res.data + res = client.get(url_for("api_delete", uuid="all"), follow_redirects=True) assert b'Deleted' in res.data diff --git a/changedetectionio/tests/test_trigger.py b/changedetectionio/tests/test_trigger.py index 284ec8d3..455c2bac 100644 --- a/changedetectionio/tests/test_trigger.py +++ b/changedetectionio/tests/test_trigger.py @@ -129,3 +129,8 @@ def test_trigger_functionality(client, live_server): time.sleep(sleep_time_for_fetch_thread) res = client.get(url_for("index")) assert b'unviewed' in res.data + + # Check the preview/highlighter, we should be able to see what we triggered on, but it should be highlighted + res = client.get(url_for("preview_page", uuid="first")) + # We should be able to see what we ignored + assert b'
    foobar' in res.data \ No newline at end of file diff --git a/changedetectionio/tests/test_xpath_selector.py b/changedetectionio/tests/test_xpath_selector.py index c5646c81..d1374834 100644 --- a/changedetectionio/tests/test_xpath_selector.py +++ b/changedetectionio/tests/test_xpath_selector.py @@ -96,6 +96,7 @@ def test_check_markup_xpath_filter_restriction(client, live_server): res = client.get(url_for("index")) assert b'unviewed' not in res.data + def test_xpath_validation(client, live_server): # Give the endpoint time to spin up From b401998030334e6177731ee8880c9d48d15fa8c0 Mon Sep 17 00:00:00 2001 From: dgtlmoon Date: Wed, 23 Feb 2022 12:01:11 +0100 Subject: [PATCH 05/15] Ensure string matching on the ignore filter is always case-INsensitive --- changedetectionio/html_tools.py | 2 +- changedetectionio/tests/test_ignore_text.py | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/changedetectionio/html_tools.py b/changedetectionio/html_tools.py index 4f500e74..d251aebc 100644 --- a/changedetectionio/html_tools.py +++ b/changedetectionio/html_tools.py @@ -140,7 +140,7 @@ def strip_ignore_text(content, wordlist, mode="content"): except Exception as e: continue - if not regex_matches and not any(skip_text in line for skip_text in ignore): + if not regex_matches and not any(skip_text.lower() in line.lower() for skip_text in ignore): output.append(line.encode('utf8')) else: ignored_line_numbers.append(i) diff --git a/changedetectionio/tests/test_ignore_text.py b/changedetectionio/tests/test_ignore_text.py index 3acacb34..022c4f56 100644 --- a/changedetectionio/tests/test_ignore_text.py +++ b/changedetectionio/tests/test_ignore_text.py @@ -70,7 +70,7 @@ def set_modified_ignore_response(): Some initial text

    Which is across multiple lines

    -

    ZZZZZ

    +

    ZZZZz


    So let's see what happens.
    @@ -85,7 +85,8 @@ def set_modified_ignore_response(): def test_check_ignore_text_functionality(client, live_server): sleep_time_for_fetch_thread = 3 - ignore_text = "XXXXX\r\nYYYYY\r\nZZZZZ\r\nnew ignore stuff" + # Use a mix of case in ZzZ to prove it works case-insensitive. + ignore_text = "XXXXX\r\nYYYYY\r\nzZzZZ\r\nnew ignore stuff" set_original_ignore_response() # Give the endpoint time to spin up From a51c55596421baa27bde5eaf766d3e976cce3307 Mon Sep 17 00:00:00 2001 From: dgtlmoon Date: Wed, 23 Feb 2022 12:30:36 +0100 Subject: [PATCH 06/15] Fix small issue in highlight trigger/ignore preview page with setting the background colours, add test --- changedetectionio/__init__.py | 47 ++++++++++++++++-------- changedetectionio/templates/preview.html | 5 +-- 2 files changed, 32 insertions(+), 20 deletions(-) diff --git a/changedetectionio/__init__.py b/changedetectionio/__init__.py index 9698257e..1232b43a 100644 --- a/changedetectionio/__init__.py +++ b/changedetectionio/__init__.py @@ -764,6 +764,8 @@ def changedetection_app(config=None, datastore_o=None): @login_required def preview_page(uuid): content = [] + ignored_line_numbers = [] + trigger_line_numbers = [] # More for testing, possible to return the first/only if uuid == 'first': @@ -782,25 +784,38 @@ def changedetection_app(config=None, datastore_o=None): filename = watch['history'][timestamps[-1]] try: with open(filename, 'r') as f: - content = f.readlines() - except: - content.append("File doesnt exist or unable to read file {}".format(filename)) - else: - content.append("No history found") + tmp = f.readlines() + + # Get what needs to be highlighted + ignore_rules = watch.get('ignore_text', []) + datastore.data['settings']['application']['global_ignore_text'] + + # .readlines will keep the \n, but we will parse it here again, in the future tidy this up + ignored_line_numbers = html_tools.strip_ignore_text(content="".join(tmp), + wordlist=ignore_rules, + mode='line numbers' + ) + + trigger_line_numbers = html_tools.strip_ignore_text(content="".join(tmp), + wordlist=watch['trigger_text'], + mode='line numbers' + ) + # Prepare the classes and lines used in the template + i=0 + for l in tmp: + classes=[] + i+=1 + if i in ignored_line_numbers: + classes.append('ignored') + if i in trigger_line_numbers: + classes.append('triggered') + content.append({'line': l, 'classes': ' '.join(classes)}) - # Get what needs to be highlighted - ignore_rules = watch.get('ignore_text', []) + datastore.data['settings']['application']['global_ignore_text'] - # .readlines will keep the \n, but we will parse it here again, in the future tidy this up - ignored_line_numbers = html_tools.strip_ignore_text(content="".join(content), - wordlist=ignore_rules, - mode='line numbers' - ) + except Exception as e: + content.append({'line': "File doesnt exist or unable to read file {}".format(filename), 'classes': ''}) + else: + content.append({'line': "No history found", 'classes': ''}) - trigger_line_numbers = html_tools.strip_ignore_text(content="".join(content), - wordlist=watch['trigger_text'], - mode='line numbers' - ) output = render_template("preview.html", content=content, diff --git a/changedetectionio/templates/preview.html b/changedetectionio/templates/preview.html index 17b1e0af..2e66cbc2 100644 --- a/changedetectionio/templates/preview.html +++ b/changedetectionio/templates/preview.html @@ -13,10 +13,7 @@ {% for row in content %} - {% set classes = [] %} - {% if (loop.index in ignored_line_numbers) %}{{ classes.append("ignored") }}{% endif %} - {% if (loop.index in triggered_line_numbers) %}{{ classes.append("triggered") }}{% endif %} -
    {{row}}
    +
    {{row.line}}
    {% endfor %} From fda93c3798aa786328529588a40718df090d3dcc Mon Sep 17 00:00:00 2001 From: dgtlmoon Date: Thu, 24 Feb 2022 16:36:24 +0100 Subject: [PATCH 07/15] Better file exception handling on saving index JSON --- changedetectionio/store.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/changedetectionio/store.py b/changedetectionio/store.py index c23bf204..5b69a955 100644 --- a/changedetectionio/store.py +++ b/changedetectionio/store.py @@ -398,13 +398,10 @@ class ChangeDetectionStore: # system was out of memory, out of RAM etc with open(self.json_store_path+".tmp", 'w') as json_file: json.dump(data, json_file, indent=4) - + os.rename(self.json_store_path+".tmp", self.json_store_path) except Exception as e: logging.error("Error writing JSON!! (Main JSON file save was skipped) : %s", str(e)) - else: - os.rename(self.json_store_path+".tmp", self.json_store_path) - self.needs_write = False # Thread runner, this helps with thread/write issues when there are many operations that want to update the JSON From a89ffffc7659f09959182f99c89645bb8fe39fea Mon Sep 17 00:00:00 2001 From: dgtlmoon Date: Thu, 24 Feb 2022 16:49:48 +0100 Subject: [PATCH 08/15] "Recheck" button should work when entry is in paused state --- changedetectionio/store.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/changedetectionio/store.py b/changedetectionio/store.py index 5b69a955..86081735 100644 --- a/changedetectionio/store.py +++ b/changedetectionio/store.py @@ -184,10 +184,6 @@ class ChangeDetectionStore: def update_watch(self, uuid, update_obj): - # Skip if 'paused' state - if self.__data['watching'][uuid]['paused']: - return - with self.lock: # In python 3.9 we have the |= dict operator, but that still will lose data on nested structures... From 3240ed2339d2ddfd907aff3ef8ec27f238556079 Mon Sep 17 00:00:00 2001 From: dgtlmoon Date: Thu, 24 Feb 2022 16:58:51 +0100 Subject: [PATCH 09/15] Minor reliability upgrade for large datasets - retry deepcopy (#436) --- changedetectionio/__init__.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/changedetectionio/__init__.py b/changedetectionio/__init__.py index 1232b43a..73c60bbf 100644 --- a/changedetectionio/__init__.py +++ b/changedetectionio/__init__.py @@ -1118,7 +1118,14 @@ def ticker_thread_check_time_launch_checks(): running_uuids.append(t.current_uuid) # Re #232 - Deepcopy the data incase it changes while we're iterating through it all - copied_datastore = deepcopy(datastore) + while True: + try: + copied_datastore = deepcopy(datastore) + except RuntimeError as e: + # RuntimeError: dictionary changed size during iteration + time.sleep(0.1) + else: + break # Check for watches outside of the time threshold to put in the thread queue. for uuid, watch in copied_datastore.data['watching'].items(): From 883aa968fd3f3c3fb0df8979384adc43b43f0b70 Mon Sep 17 00:00:00 2001 From: dgtlmoon Date: Thu, 24 Feb 2022 17:02:50 +0100 Subject: [PATCH 10/15] 0.39.9 --- changedetectionio/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/changedetectionio/__init__.py b/changedetectionio/__init__.py index 73c60bbf..87a583ab 100644 --- a/changedetectionio/__init__.py +++ b/changedetectionio/__init__.py @@ -37,7 +37,7 @@ from flask import ( from flask_login import login_required from changedetectionio import html_tools -__version__ = '0.39.8' +__version__ = '0.39.9' datastore = None From 8b1e9f6591aa1405d8e8c4b0921bac3ddde07f40 Mon Sep 17 00:00:00 2001 From: dgtlmoon Date: Sat, 26 Feb 2022 18:42:54 +0100 Subject: [PATCH 11/15] Update README.md with hosting options --- README.md | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 6e430023..8eae318c 100644 --- a/README.md +++ b/README.md @@ -15,13 +15,19 @@ Open source web page monitoring, notification and change detection. Self-hosted web page change monitoring -**Get your own instance now on Lemonade!** +**Get your own private instance now! Let us host it for you!** [![Deploy to Lemonade](https://lemonade.changedetection.io/static/images/lemonade.svg)](https://lemonade.changedetection.io/start) + +[_Let us host your own private instance - We accept PayPal and Bitcoin, Support the further development of changedetection.io!_](https://lemonade.changedetection.io/start) + + + - Automatic Updates, Automatic Backups, No Heroku "paused application", don't miss a change! - Javascript browser included -- Pay with Bitcoin +- Unlimited checks and watches! + #### Example use cases From 75ca7ec504359f1eac4e81d56f6a83e6537a18c9 Mon Sep 17 00:00:00 2001 From: dgtlmoon Date: Mon, 28 Feb 2022 15:08:51 +0100 Subject: [PATCH 12/15] Improved CPU usage around the loop responsible for what sites needs to be checked --- changedetectionio/__init__.py | 27 ++++++++++++++++++++------- changedetectionio/update_worker.py | 3 +++ 2 files changed, 23 insertions(+), 7 deletions(-) diff --git a/changedetectionio/__init__.py b/changedetectionio/__init__.py index 87a583ab..1708d5ec 100644 --- a/changedetectionio/__init__.py +++ b/changedetectionio/__init__.py @@ -756,7 +756,7 @@ def changedetection_app(config=None, datastore_o=None): current_previous_version=str(previous_version), current_diff_url=watch['url'], extra_title=" - Diff - {}".format(watch['title'] if watch['title'] else watch['url']), - left_sticky= True ) + left_sticky=True) return output @@ -1127,20 +1127,33 @@ def ticker_thread_check_time_launch_checks(): else: break + # Re #438 - Don't place more watches in the queue to be checked if the queue is already large + while update_q.qsize() >= 2000: + time.sleep(1) + # Check for watches outside of the time threshold to put in the thread queue. + now = time.time() + max_system_wide = int(copied_datastore.data['settings']['requests']['minutes_between_check']) * 60 + for uuid, watch in copied_datastore.data['watching'].items(): + + # No need todo further processing if it's paused + if watch['paused']: + continue + # If they supplied an individual entry minutes to threshold. - if 'minutes_between_check' in watch and watch['minutes_between_check'] is not None: + watch_minutes_between_check = watch.get('minutes_between_check', None) + if watch_minutes_between_check is not None: # Cast to int just incase - max_time = int(watch['minutes_between_check']) * 60 + max_time = int(watch_minutes_between_check) * 60 else: # Default system wide. - max_time = int(copied_datastore.data['settings']['requests']['minutes_between_check']) * 60 + max_time = max_system_wide - threshold = time.time() - max_time + threshold = now - max_time - # Yeah, put it in the queue, it's more than time. - if not watch['paused'] and watch['last_checked'] <= threshold: + # Yeah, put it in the queue, it's more than time + if watch['last_checked'] <= threshold: if not uuid in running_uuids and uuid not in update_q.queue: update_q.put(uuid) diff --git a/changedetectionio/update_worker.py b/changedetectionio/update_worker.py index 84993d80..9411f333 100644 --- a/changedetectionio/update_worker.py +++ b/changedetectionio/update_worker.py @@ -147,4 +147,7 @@ class update_worker(threading.Thread): self.current_uuid = None # Done self.q.task_done() + # Give the CPU time to interrupt + time.sleep(0.1) + self.app.config.exit.wait(1) From fd45fcce2f52a9ef0f3c0abb29e270d4537ba95b Mon Sep 17 00:00:00 2001 From: dgtlmoon Date: Mon, 28 Feb 2022 15:47:59 +0100 Subject: [PATCH 13/15] Include link to changedetection.io hosted option (#439) --- changedetectionio/__init__.py | 4 +++- changedetectionio/static/styles/styles.css | 6 +++++- changedetectionio/static/styles/styles.scss | 7 ++++++- changedetectionio/templates/base.html | 2 +- 4 files changed, 15 insertions(+), 4 deletions(-) diff --git a/changedetectionio/__init__.py b/changedetectionio/__init__.py index 1708d5ec..ac8d5bc3 100644 --- a/changedetectionio/__init__.py +++ b/changedetectionio/__init__.py @@ -368,7 +368,9 @@ def changedetection_app(config=None, datastore_o=None): tags=existing_tags, active_tag=limit_tag, app_rss_token=datastore.data['settings']['application']['rss_access_token'], - has_unviewed=datastore.data['has_unviewed']) + has_unviewed=datastore.data['has_unviewed'], + # Don't link to hosting when we're on the hosting environment + hosted_sticky=os.getenv("SALTED_PASS", False) == False) return output diff --git a/changedetectionio/static/styles/styles.css b/changedetectionio/static/styles/styles.css index 794dc631..cf850004 100644 --- a/changedetectionio/static/styles/styles.css +++ b/changedetectionio/static/styles/styles.css @@ -242,7 +242,7 @@ footer { .sticky-tab { position: absolute; - top: 80px; + top: 100px; font-size: 8px; background: #fff; padding: 10px; } @@ -250,6 +250,10 @@ footer { left: 0px; } .sticky-tab#right-sticky { right: 0px; } + .sticky-tab#hosted-sticky { + right: 0px; + top: 60px; + font-weight: bold; } #new-version-text a { color: #e07171; } diff --git a/changedetectionio/static/styles/styles.scss b/changedetectionio/static/styles/styles.scss index 63b40a4e..39451c46 100644 --- a/changedetectionio/static/styles/styles.scss +++ b/changedetectionio/static/styles/styles.scss @@ -321,7 +321,7 @@ footer { .sticky-tab { position: absolute; - top: 80px; + top: 100px; font-size: 8px; background: #fff; padding: 10px; @@ -331,6 +331,11 @@ footer { &#right-sticky { right: 0px; } + &#hosted-sticky { + right: 0px; + top: 60px; + font-weight: bold; + } } #new-version-text a { diff --git a/changedetectionio/templates/base.html b/changedetectionio/templates/base.html index d4533fde..99be4c6d 100644 --- a/changedetectionio/templates/base.html +++ b/changedetectionio/templates/base.html @@ -68,7 +68,7 @@
    - +{% if hosted_sticky %}    {% endif %} {% if left_sticky %} {% endif %} {% if right_sticky %}
    {{ right_sticky }}
    {% endif %}
    From 615fa2c5b2695b624c000b6f0a6cc916056745a9 Mon Sep 17 00:00:00 2001 From: dgtlmoon Date: Mon, 28 Feb 2022 22:39:32 +0100 Subject: [PATCH 14/15] Tweak support tabs and text (#440) --- changedetectionio/__init__.py | 3 ++- changedetectionio/static/styles/styles.css | 4 ++-- changedetectionio/static/styles/styles.scss | 6 ++---- changedetectionio/templates/base.html | 2 +- 4 files changed, 7 insertions(+), 8 deletions(-) diff --git a/changedetectionio/__init__.py b/changedetectionio/__init__.py index ac8d5bc3..9788db3c 100644 --- a/changedetectionio/__init__.py +++ b/changedetectionio/__init__.py @@ -370,7 +370,8 @@ def changedetection_app(config=None, datastore_o=None): app_rss_token=datastore.data['settings']['application']['rss_access_token'], has_unviewed=datastore.data['has_unviewed'], # Don't link to hosting when we're on the hosting environment - hosted_sticky=os.getenv("SALTED_PASS", False) == False) + hosted_sticky=os.getenv("SALTED_PASS", False) == False, + guid=datastore.data['app_guid']) return output diff --git a/changedetectionio/static/styles/styles.css b/changedetectionio/static/styles/styles.css index cf850004..c456afb2 100644 --- a/changedetectionio/static/styles/styles.css +++ b/changedetectionio/static/styles/styles.css @@ -242,7 +242,7 @@ footer { .sticky-tab { position: absolute; - top: 100px; + top: 60px; font-size: 8px; background: #fff; padding: 10px; } @@ -252,7 +252,7 @@ footer { right: 0px; } .sticky-tab#hosted-sticky { right: 0px; - top: 60px; + top: 100px; font-weight: bold; } #new-version-text a { diff --git a/changedetectionio/static/styles/styles.scss b/changedetectionio/static/styles/styles.scss index 39451c46..a1c1fb7c 100644 --- a/changedetectionio/static/styles/styles.scss +++ b/changedetectionio/static/styles/styles.scss @@ -317,11 +317,9 @@ footer { */ } - - .sticky-tab { position: absolute; - top: 100px; + top: 60px; font-size: 8px; background: #fff; padding: 10px; @@ -333,7 +331,7 @@ footer { } &#hosted-sticky { right: 0px; - top: 60px; + top: 100px; font-weight: bold; } } diff --git a/changedetectionio/templates/base.html b/changedetectionio/templates/base.html index 99be4c6d..d36dfca8 100644 --- a/changedetectionio/templates/base.html +++ b/changedetectionio/templates/base.html @@ -68,7 +68,7 @@ -{% if hosted_sticky %}{% endif %} +{% if hosted_sticky %}{% endif %} {% if left_sticky %} {% endif %} {% if right_sticky %}
    {{ right_sticky }}
    {% endif %}
    From 96664ffb106760ead0bfc5d04ba79f7016bc6223 Mon Sep 17 00:00:00 2001 From: dgtlmoon Date: Tue, 1 Mar 2022 17:50:15 +0100 Subject: [PATCH 15/15] Better text/plain detection and refactor tests (#443) --- changedetectionio/__init__.py | 1 + changedetectionio/fetch_site_status.py | 11 +++++--- changedetectionio/tests/test_api.py | 3 +-- changedetectionio/tests/test_backend.py | 7 +++++ .../tests/test_jsonpath_selector.py | 8 +++--- changedetectionio/tests/util.py | 26 +++---------------- 6 files changed, 24 insertions(+), 32 deletions(-) diff --git a/changedetectionio/__init__.py b/changedetectionio/__init__.py index 9788db3c..6b202a5f 100644 --- a/changedetectionio/__init__.py +++ b/changedetectionio/__init__.py @@ -838,6 +838,7 @@ def changedetection_app(config=None, datastore_o=None): logs=notification_debug_log if len(notification_debug_log) else ["No errors or warnings detected"]) return output + @app.route("/api//snapshot/current", methods=['GET']) @login_required def api_snapshot(uuid): diff --git a/changedetectionio/fetch_site_status.py b/changedetectionio/fetch_site_status.py index f0f91961..35ef2a01 100644 --- a/changedetectionio/fetch_site_status.py +++ b/changedetectionio/fetch_site_status.py @@ -86,8 +86,13 @@ class perform_site_check(): if is_html: # CSS Filter, extract the HTML that matches and feed that into the existing inscriptis::get_text html_content = fetcher.content - if not fetcher.headers.get('Content-Type', '') == 'text/plain': + # If not JSON, and if it's not text/plain.. + if 'text/plain' in fetcher.headers.get('Content-Type', '').lower(): + # Don't run get_text or xpath/css filters on plaintext + stripped_text_from_html = html_content + else: + # Then we assume HTML if has_filter_rule: # For HTML/XML we offer xpath as an option, just start a regular xPath "/.." if css_filter_rule[0] == '/': @@ -98,9 +103,7 @@ class perform_site_check(): # get_text() via inscriptis stripped_text_from_html = get_text(html_content) - else: - # Don't run get_text or xpath/css filters on plaintext - stripped_text_from_html = html_content + # Re #340 - return the content before the 'ignore text' was applied text_content_before_ignored_filter = stripped_text_from_html.encode('utf-8') diff --git a/changedetectionio/tests/test_api.py b/changedetectionio/tests/test_api.py index b4f9e5f2..370fd10a 100644 --- a/changedetectionio/tests/test_api.py +++ b/changedetectionio/tests/test_api.py @@ -14,7 +14,6 @@ def set_response_data(test_return_data): def test_snapshot_api_detects_change(client, live_server): - test_return_data = "Some initial text" test_return_data_modified = "Some NEW nice initial text" @@ -27,7 +26,7 @@ def test_snapshot_api_detects_change(client, live_server): time.sleep(1) # Add our URL to the import page - test_url = url_for('test_endpoint', _external=True) + test_url = url_for('test_endpoint', content_type="text/plain", _external=True) res = client.post( url_for("import_page"), data={"urls": test_url}, diff --git a/changedetectionio/tests/test_backend.py b/changedetectionio/tests/test_backend.py index 92b81891..b279251f 100644 --- a/changedetectionio/tests/test_backend.py +++ b/changedetectionio/tests/test_backend.py @@ -7,6 +7,13 @@ from . util import set_original_response, set_modified_response, live_server_set sleep_time_for_fetch_thread = 3 +# Basic test to check inscriptus is not adding return line chars, basically works etc +def test_inscriptus(): + from inscriptis import get_text + html_content="test!
    ok man" + stripped_text_from_html = get_text(html_content) + assert stripped_text_from_html == 'test!\nok man' + def test_check_basic_change_detection_functionality(client, live_server): set_original_response() diff --git a/changedetectionio/tests/test_jsonpath_selector.py b/changedetectionio/tests/test_jsonpath_selector.py index 5a4b7959..a5329b67 100644 --- a/changedetectionio/tests/test_jsonpath_selector.py +++ b/changedetectionio/tests/test_jsonpath_selector.py @@ -162,7 +162,7 @@ def test_check_json_without_filter(client, live_server): time.sleep(1) # Add our URL to the import page - test_url = url_for('test_endpoint_json', _external=True) + test_url = url_for('test_endpoint', content_type="application/json", _external=True) client.post( url_for("import_page"), data={"urls": test_url}, @@ -193,7 +193,7 @@ def test_check_json_filter(client, live_server): time.sleep(1) # Add our URL to the import page - test_url = url_for('test_endpoint', _external=True) + test_url = url_for('test_endpoint', content_type="application/json", _external=True) res = client.post( url_for("import_page"), data={"urls": test_url}, @@ -258,7 +258,7 @@ def test_check_json_filter_bool_val(client, live_server): # Give the endpoint time to spin up time.sleep(1) - test_url = url_for('test_endpoint', _external=True) + test_url = url_for('test_endpoint', content_type="application/json", _external=True) res = client.post( url_for("import_page"), @@ -313,7 +313,7 @@ def test_check_json_ext_filter(client, live_server): time.sleep(1) # Add our URL to the import page - test_url = url_for('test_endpoint', _external=True) + test_url = url_for('test_endpoint', content_type="application/json", _external=True) res = client.post( url_for("import_page"), data={"urls": test_url}, diff --git a/changedetectionio/tests/util.py b/changedetectionio/tests/util.py index 86b78767..12aefb6e 100644 --- a/changedetectionio/tests/util.py +++ b/changedetectionio/tests/util.py @@ -1,5 +1,6 @@ #!/usr/bin/python3 +from flask import make_response, request def set_original_response(): test_return_data = """ @@ -40,24 +41,16 @@ def live_server_setup(live_server): @live_server.app.route('/test-endpoint') def test_endpoint(): - # Tried using a global var here but didn't seem to work, so reading from a file instead. - with open("test-datastore/endpoint-content.txt", "r") as f: - return f.read() - - @live_server.app.route('/test-endpoint-json') - def test_endpoint_json(): - - from flask import make_response + ctype = request.args.get('content_type') + # Tried using a global var here but didn't seem to work, so reading from a file instead. with open("test-datastore/endpoint-content.txt", "r") as f: resp = make_response(f.read()) - resp.headers['Content-Type'] = 'application/json' + resp.headers['Content-Type'] = ctype if ctype else 'text/html' return resp @live_server.app.route('/test-403') def test_endpoint_403_error(): - - from flask import make_response resp = make_response('', 403) return resp @@ -65,7 +58,6 @@ def live_server_setup(live_server): @live_server.app.route('/test-headers') def test_headers(): - from flask import request output= [] for header in request.headers: @@ -76,24 +68,16 @@ def live_server_setup(live_server): # Just return the body in the request @live_server.app.route('/test-body', methods=['POST', 'GET']) def test_body(): - - from flask import request - return request.data # Just return the verb in the request @live_server.app.route('/test-method', methods=['POST', 'GET', 'PATCH']) def test_method(): - - from flask import request - return request.method # Where we POST to as a notification @live_server.app.route('/test_notification_endpoint', methods=['POST', 'GET']) def test_notification_endpoint(): - from flask import request - with open("test-datastore/notification.txt", "wb") as f: # Debug method, dump all POST to file also, used to prove #65 data = request.stream.read() @@ -107,8 +91,6 @@ def live_server_setup(live_server): # Just return the verb in the request @live_server.app.route('/test-basicauth', methods=['GET']) def test_basicauth_method(): - - from flask import request auth = request.authorization ret = " ".join([auth.username, auth.password, auth.type]) return ret