From 06517bfd22a1eb1a9628e3a1e72ae230b1c76d56 Mon Sep 17 00:00:00 2001 From: dgtlmoon Date: Tue, 26 Apr 2022 10:52:08 +0200 Subject: [PATCH 1/5] Ability to 'Share' a watch by a generated link, this will include all filters and triggers - see Wiki (#563) --- changedetectionio/__init__.py | 96 +++++++++++++++---- changedetectionio/static/images/copy.svg | 40 ++++++++ changedetectionio/static/images/spread.svg | 46 +++++++++ changedetectionio/static/js/watch-overview.js | 18 ++++ changedetectionio/static/styles/styles.css | 3 + changedetectionio/static/styles/styles.scss | 5 + changedetectionio/store.py | 34 ++++++- changedetectionio/templates/base.html | 7 ++ .../templates/watch-overview.html | 5 +- changedetectionio/tests/test_request.py | 1 + changedetectionio/tests/test_share_watch.py | 76 +++++++++++++++ 11 files changed, 308 insertions(+), 23 deletions(-) create mode 100644 changedetectionio/static/images/copy.svg create mode 100644 changedetectionio/static/images/spread.svg create mode 100644 changedetectionio/tests/test_share_watch.py diff --git a/changedetectionio/__init__.py b/changedetectionio/__init__.py index 4baa923e..f8794e80 100644 --- a/changedetectionio/__init__.py +++ b/changedetectionio/__init__.py @@ -32,6 +32,7 @@ from flask import ( render_template, request, send_from_directory, + session, url_for, ) from flask_login import login_required @@ -393,7 +394,8 @@ def changedetection_app(config=None, datastore_o=None): hosted_sticky=os.getenv("SALTED_PASS", False) == False, guid=datastore.data['app_guid'], queued_uuids=update_q.queue) - + if session.get('share-link'): + del(session['share-link']) return output @@ -688,12 +690,14 @@ def changedetection_app(config=None, datastore_o=None): # Up to 5000 per batch so we dont flood the server if len(url) and validators.url(url.replace('source:', '')) and good < 5000: new_uuid = datastore.add_watch(url=url.strip(), tag=" ".join(tags), write_to_disk_now=False) - # Straight into the queue. - update_q.put(new_uuid) - good += 1 - else: - if len(url): - remaining_urls.append(url) + if new_uuid: + # Straight into the queue. + update_q.put(new_uuid) + good += 1 + continue + + if len(url.strip()): + remaining_urls.append(url) flash("{} Imported in {:.2f}s, {} Skipped.".format(good, time.time()-now,len(remaining_urls))) datastore.needs_write = True @@ -1000,23 +1004,24 @@ def changedetection_app(config=None, datastore_o=None): from changedetectionio import forms form = forms.quickWatchForm(request.form) - if form.validate(): + if not form.validate(): + flash("Error") + return redirect(url_for('index')) - url = request.form.get('url').strip() - if datastore.url_exists(url): - flash('The URL {} already exists'.format(url), "error") - return redirect(url_for('index')) + url = request.form.get('url').strip() + if datastore.url_exists(url): + flash('The URL {} already exists'.format(url), "error") + return redirect(url_for('index')) - # @todo add_watch should throw a custom Exception for validation etc - new_uuid = datastore.add_watch(url=url, tag=request.form.get('tag').strip()) + # @todo add_watch should throw a custom Exception for validation etc + new_uuid = datastore.add_watch(url=url, tag=request.form.get('tag').strip()) + if new_uuid: # Straight into the queue. update_q.put(new_uuid) - flash("Watch added.") - return redirect(url_for('index')) - else: - flash("Error") - return redirect(url_for('index')) + + return redirect(url_for('index')) + @app.route("/api/delete", methods=['GET']) @@ -1082,6 +1087,59 @@ def changedetection_app(config=None, datastore_o=None): flash("{} watches are queued for rechecking.".format(i)) return redirect(url_for('index', tag=tag)) + @app.route("/api/share-url", methods=['GET']) + @login_required + def api_share_put_watch(): + """Given a watch UUID, upload the info and return a share-link + the share-link can be imported/added""" + import requests + import json + tag = request.args.get('tag') + uuid = request.args.get('uuid') + + # more for testing + if uuid == 'first': + uuid = list(datastore.data['watching'].keys()).pop() + + # copy it to memory as trim off what we dont need (history) + watch = deepcopy(datastore.data['watching'][uuid]) + if (watch.get('history')): + del (watch['history']) + + # for safety/privacy + for k in list(watch.keys()): + if k.startswith('notification_'): + del watch[k] + + for r in['uuid', 'last_checked', 'last_changed']: + if watch.get(r): + del (watch[r]) + + # Add the global stuff which may have an impact + watch['ignore_text'] += datastore.data['settings']['application']['global_ignore_text'] + watch['subtractive_selectors'] += datastore.data['settings']['application']['global_subtractive_selectors'] + + watch_json = json.dumps(watch) + + try: + r = requests.request(method="POST", + data={'watch': watch_json}, + url="https://changedetection.io/share/share", + headers={'App-Guid': datastore.data['app_guid']}) + res = r.json() + + session['share-link'] = "https://changedetection.io/share/{}".format(res['share_key']) + + + except Exception as e: + flash("Could not share, something went wrong while communicating with the share server.", 'error') + + # https://changedetection.io/share/VrMv05wpXyQa + # in the browser - should give you a nice info page - wtf + # paste in etc + return redirect(url_for('index')) + + # @todo handle ctrl break ticker_thread = threading.Thread(target=ticker_thread_check_time_launch_checks).start() diff --git a/changedetectionio/static/images/copy.svg b/changedetectionio/static/images/copy.svg new file mode 100644 index 00000000..b14994ab --- /dev/null +++ b/changedetectionio/static/images/copy.svg @@ -0,0 +1,40 @@ + + diff --git a/changedetectionio/static/images/spread.svg b/changedetectionio/static/images/spread.svg new file mode 100644 index 00000000..757cb631 --- /dev/null +++ b/changedetectionio/static/images/spread.svg @@ -0,0 +1,46 @@ + + + + + + + diff --git a/changedetectionio/static/js/watch-overview.js b/changedetectionio/static/js/watch-overview.js index acee1e35..1431b1b9 100644 --- a/changedetectionio/static/js/watch-overview.js +++ b/changedetectionio/static/js/watch-overview.js @@ -3,4 +3,22 @@ $(function () { $('.diff-link').click(function () { $(this).closest('.unviewed').removeClass('unviewed'); }); + + $('.with-share-link > *').click(function () { + $("#copied-clipboard").remove(); + + var range = document.createRange(); + var n=$("#share-link")[0]; + range.selectNode(n); + window.getSelection().removeAllRanges(); + window.getSelection().addRange(range); + document.execCommand("copy"); + window.getSelection().removeAllRanges(); + + $('.with-share-link').append('Copied to clipboard'); + $("#copied-clipboard").fadeOut(2500, function() { + $(this).remove(); + }); + }); }); + diff --git a/changedetectionio/static/styles/styles.css b/changedetectionio/static/styles/styles.css index 174c9aea..71ff2f5e 100644 --- a/changedetectionio/static/styles/styles.css +++ b/changedetectionio/static/styles/styles.css @@ -180,6 +180,9 @@ body:after, body:before { .messages li.notice { background: rgba(255, 255, 255, 0.5); } +.messages.with-share-link > *:hover { + cursor: pointer; } + #notification-customisation { border: 1px solid #ccc; padding: 0.5rem; diff --git a/changedetectionio/static/styles/styles.scss b/changedetectionio/static/styles/styles.scss index 3b305b45..a79c051b 100644 --- a/changedetectionio/static/styles/styles.scss +++ b/changedetectionio/static/styles/styles.scss @@ -237,6 +237,11 @@ body:after, body:before { background: rgba(255, 255, 255, .5); } } + &.with-share-link { + > *:hover { + cursor:pointer; + } + } } #notification-customisation { diff --git a/changedetectionio/store.py b/changedetectionio/store.py index 351f2b4c..bb4bab11 100644 --- a/changedetectionio/store.py +++ b/changedetectionio/store.py @@ -1,3 +1,6 @@ +from flask import ( + flash +) import json import logging import os @@ -8,6 +11,7 @@ from copy import deepcopy from os import mkdir, path, unlink from threading import Lock import re +import requests from changedetectionio.model import Watch, App @@ -295,6 +299,33 @@ class ChangeDetectionStore: def add_watch(self, url, tag="", extras=None, write_to_disk_now=True): if extras is None: extras = {} + # Incase these are copied across, assume it's a reference and deepcopy() + apply_extras = deepcopy(extras) + + # Was it a share link? try to fetch the data + if (url.startswith("https://changedetection.io/share/")): + try: + r = requests.request(method="GET", + url=url, + # So we know to return the JSON instead of the human-friendly "help" page + headers={'App-Guid': self.__data['app_guid']}) + res = r.json() + + # List of permisable stuff we accept from the wild internet + for k in ['url', 'tag', + 'paused', 'title', + 'previous_md5', 'headers', + 'body', 'method', + 'ignore_text', 'css_filter', + 'subtractive_selectors', 'trigger_text', + 'extract_title_as_title']: + if res.get(k): + apply_extras[k] = res[k] + + except Exception as e: + logging.error("Error fetching metadata for shared watch link", url, str(e)) + flash("Error fetching metadata for {}".format(url), 'error') + return False with self.lock: # @todo use a common generic version of this @@ -304,8 +335,7 @@ class ChangeDetectionStore: 'tag': tag }) - # Incase these are copied across, assume it's a reference and deepcopy() - apply_extras = deepcopy(extras) + for k in ['uuid', 'history', 'last_checked', 'last_changed', 'newest_history_key', 'previous_md5', 'viewed']: if k in apply_extras: del apply_extras[k] diff --git a/changedetectionio/templates/base.html b/changedetectionio/templates/base.html index 41105f16..9a9c6541 100644 --- a/changedetectionio/templates/base.html +++ b/changedetectionio/templates/base.html @@ -94,6 +94,13 @@ {% endif %} {% endwith %} + + {% if session['share-link'] %} + + {% endif %} + {% block content %} {% endblock %} diff --git a/changedetectionio/templates/watch-overview.html b/changedetectionio/templates/watch-overview.html index 02bb20fb..fc5ee1d0 100644 --- a/changedetectionio/templates/watch-overview.html +++ b/changedetectionio/templates/watch-overview.html @@ -13,8 +13,7 @@ {{ render_simple_field(form.tag, value=active_tag if active_tag else '', placeholder="watch group") }} - - + Tip: You can also add 'shared' watches. More info
All @@ -53,6 +52,8 @@ {{watch.title if watch.title is not none and watch.title|length > 0 else watch.url}} + + {%if watch.fetch_backend == "html_webdriver" %}{% endif %} {% if watch.last_error is defined and watch.last_error != False %} diff --git a/changedetectionio/tests/test_request.py b/changedetectionio/tests/test_request.py index 4959ef89..6d9b28d6 100644 --- a/changedetectionio/tests/test_request.py +++ b/changedetectionio/tests/test_request.py @@ -27,6 +27,7 @@ def test_headers_in_request(client, live_server): ) assert b"1 Imported" in res.data + time.sleep(3) cookie_header = '_ga=GA1.2.1022228332; cookie-preferences=analytics:accepted;' diff --git a/changedetectionio/tests/test_share_watch.py b/changedetectionio/tests/test_share_watch.py new file mode 100644 index 00000000..4ffa0d2d --- /dev/null +++ b/changedetectionio/tests/test_share_watch.py @@ -0,0 +1,76 @@ +#!/usr/bin/python3 + +import time +from flask import url_for +from urllib.request import urlopen +from .util import set_original_response, set_modified_response, live_server_setup +import re + +sleep_time_for_fetch_thread = 3 + + +def test_share_watch(client, live_server): + set_original_response() + live_server_setup(live_server) + + test_url = url_for('test_endpoint', _external=True) + css_filter = ".nice-filter" + + # Add our URL to the import page + res = client.post( + url_for("import_page"), + data={"urls": test_url}, + follow_redirects=True + ) + + assert b"1 Imported" in res.data + + # Goto the edit page, add our ignore text + # Add our URL to the import page + res = client.post( + url_for("edit_page", uuid="first"), + data={"css_filter": css_filter, "url": test_url, "tag": "", "headers": "", 'fetch_backend': "html_requests"}, + follow_redirects=True + ) + assert b"Updated watch." in res.data + # Check it saved + res = client.get( + url_for("edit_page", uuid="first"), + ) + assert bytes(css_filter.encode('utf-8')) in res.data + + # click share the link + res = client.get( + url_for("api_share_put_watch", uuid="first"), + follow_redirects=True + ) + + assert b"Share this link:" in res.data + assert b"https://changedetection.io/share/" in res.data + + html = res.data.decode() + share_link_search = re.search('(.*)', html, re.IGNORECASE) + assert share_link_search + + # Now delete what we have, we will try to re-import it + # Cleanup everything + res = client.get(url_for("api_delete", uuid="all"), follow_redirects=True) + assert b'Deleted' in res.data + + # Add our URL to the import page + res = client.post( + url_for("import_page"), + data={"urls": share_link_search.group(1)}, + follow_redirects=True + ) + + assert b"1 Imported" in res.data + + # Now hit edit, we should see what we expect + # that the import fetched the meta-data + + # Check it saved + res = client.get( + url_for("edit_page", uuid="first"), + ) + assert bytes(css_filter.encode('utf-8')) in res.data From 014dc61222af31f0b3fe92ffd08ffb712d6b1e5b Mon Sep 17 00:00:00 2001 From: dgtlmoon Date: Fri, 29 Apr 2022 09:39:40 +0200 Subject: [PATCH 2/5] Upgrade notifications library - fixing marketup in email subject --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 621a8a08..8042181c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -17,7 +17,7 @@ wtforms ~= 3.0 jsonpath-ng ~= 1.5.3 # Notification library -apprise ~= 0.9.8.2 +apprise ~= 0.9.8.3 # apprise mqtt https://github.com/dgtlmoon/changedetection.io/issues/315 paho-mqtt From c0d0424e7ee17676135404a5dff436a3f5c12abf Mon Sep 17 00:00:00 2001 From: dgtlmoon Date: Fri, 29 Apr 2022 18:26:15 +0200 Subject: [PATCH 3/5] Data storage bug fix #569 --- changedetectionio/__init__.py | 5 +++++ changedetectionio/model/Watch.py | 3 ++- changedetectionio/store.py | 12 ++++++++---- changedetectionio/tests/test_notification.py | 7 +++---- 4 files changed, 18 insertions(+), 9 deletions(-) diff --git a/changedetectionio/__init__.py b/changedetectionio/__init__.py index f8794e80..50c0d436 100644 --- a/changedetectionio/__init__.py +++ b/changedetectionio/__init__.py @@ -1028,6 +1028,11 @@ def changedetection_app(config=None, datastore_o=None): @login_required def api_delete(): uuid = request.args.get('uuid') + + if uuid != 'all' and not uuid in datastore.data['watching'].keys(): + flash('The watch by UUID {} does not exist.'.format(uuid), 'error') + return redirect(url_for('index')) + # More for testing, possible to return the first/only if uuid == 'first': uuid = list(datastore.data['watching'].keys()).pop() diff --git a/changedetectionio/model/Watch.py b/changedetectionio/model/Watch.py index b148f584..c0313868 100644 --- a/changedetectionio/model/Watch.py +++ b/changedetectionio/model/Watch.py @@ -22,7 +22,8 @@ class model(dict): 'newest_history_key': 0, 'title': None, 'previous_md5': False, - 'uuid': str(uuid_builder.uuid4()), +# UUID not needed, should be generated only as a key +# 'uuid': 'headers': {}, # Extra headers to send 'body': None, 'method': 'GET', diff --git a/changedetectionio/store.py b/changedetectionio/store.py index bb4bab11..c95c172c 100644 --- a/changedetectionio/store.py +++ b/changedetectionio/store.py @@ -38,7 +38,8 @@ class ChangeDetectionStore: self.__data = App.model() # Base definition for all watchers - self.generic_definition = Watch.model() + # deepcopy part of #569 - not sure why its needed exactly + self.generic_definition = deepcopy(Watch.model()) if path.isfile('changedetectionio/source.txt'): with open('changedetectionio/source.txt') as f: @@ -231,7 +232,7 @@ class ChangeDetectionStore: del self.data['watching'][uuid] - self.needs_write = True + self.needs_write_urgent = True # Clone a watch by UUID def clone(self, uuid): @@ -330,10 +331,13 @@ class ChangeDetectionStore: with self.lock: # @todo use a common generic version of this new_uuid = str(uuid_builder.uuid4()) - new_watch = Watch.model({ + # #Re 569 + # Not sure why deepcopy was needed here, sometimes new watches would appear to already have 'history' set + # I assumed this would instantiate a new object but somehow an existing dict was getting used + new_watch = deepcopy(Watch.model({ 'url': url, 'tag': tag - }) + })) for k in ['uuid', 'history', 'last_checked', 'last_changed', 'newest_history_key', 'previous_md5', 'viewed']: diff --git a/changedetectionio/tests/test_notification.py b/changedetectionio/tests/test_notification.py index 5cd2d0c7..178a152b 100644 --- a/changedetectionio/tests/test_notification.py +++ b/changedetectionio/tests/test_notification.py @@ -156,7 +156,7 @@ def test_check_notification(client, live_server): # cleanup for the next client.get( - url_for("api_delete", uuid="first"), + url_for("api_delete", uuid="all"), follow_redirects=True ) @@ -172,8 +172,7 @@ def test_notification_validation(client, live_server): data={"url": test_url, "tag": 'nice one'}, follow_redirects=True ) - with open("xxx.bin", "wb") as f: - f.write(res.data) + assert b"Watch added" in res.data # Re #360 some validation @@ -209,6 +208,6 @@ def test_notification_validation(client, live_server): # cleanup for the next client.get( - url_for("api_delete", uuid="first"), + url_for("api_delete", uuid="all"), follow_redirects=True ) From f69585b276623bfabcbaf98ac1b27cfa1f2c6d96 Mon Sep 17 00:00:00 2001 From: dgtlmoon Date: Fri, 29 Apr 2022 20:26:02 +0200 Subject: [PATCH 5/5] Improving support info in README.md --- README.md | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index cc3c566d..defa0a5f 100644 --- a/README.md +++ b/README.md @@ -170,9 +170,12 @@ Raspberry Pi and linux/arm/v6 linux/arm/v7 arm64 devices are supported! See the Do you use changedetection.io to make money? does it save you time or money? Does it make your life easier? less stressful? Remember, we write this software when we should be doing actual paid work, we have to buy food and pay rent just like you. -Please support us, even small amounts help a LOT. -BTC `1PLFN327GyUarpJd7nVe7Reqg9qHx5frNn` +Firstly, consider taking out a [change detection monthly subscription - unlimited checks and watches](https://lemonade.changedetection.io/start) , even if you don't use it, you still get the warm fuzzy feeling of helping out the project. (And who knows, you might just use it!) + +Or directly donate an amount PayPal [![Donate](https://img.shields.io/badge/Donate-PayPal-green.svg)](https://www.paypal.com/donate/?hosted_button_id=7CP6HR9ZCNDYJ) + +Or BTC `1PLFN327GyUarpJd7nVe7Reqg9qHx5frNn` Support us!