Merge branch 'master' into 550-visual-selector

README.md

@@ -170,9 +170,12 @@ Raspberry Pi and linux/arm/v6 linux/arm/v7 arm64 devices are supported! See the
 
 Do you use changedetection.io to make money? does it save you time or money? Does it make your life easier? less stressful? Remember, we write this software when we should be doing actual paid work, we have to buy food and pay rent just like you.
 
-Please support us, even small amounts help a LOT.
 
-BTC `1PLFN327GyUarpJd7nVe7Reqg9qHx5frNn`
+Firstly, consider taking out a [change detection monthly subscription - unlimited checks and watches](https://lemonade.changedetection.io/start) , even if you don't use it, you still get the warm fuzzy feeling of helping out the project. (And who knows, you might just use it!)
+
+Or directly donate an amount PayPal [![Donate](https://img.shields.io/badge/Donate-PayPal-green.svg)](https://www.paypal.com/donate/?hosted_button_id=7CP6HR9ZCNDYJ)
+
+Or BTC `1PLFN327GyUarpJd7nVe7Reqg9qHx5frNn`
 
 <img src="https://raw.githubusercontent.com/dgtlmoon/changedetection.io/master/btc-support.png" style="max-width:50%;" alt="Support us!"  />
 

changedetectionio/__init__.py

@@ -32,6 +32,7 @@ from flask import (
     render_template,
     request,
     send_from_directory,
+    session,
     url_for,
 )
 from flask_login import login_required
@@ -393,7 +394,8 @@ def changedetection_app(config=None, datastore_o=None):
                                  hosted_sticky=os.getenv("SALTED_PASS", False) == False,
                                  guid=datastore.data['app_guid'],
                                  queued_uuids=update_q.queue)
-
+        if session.get('share-link'):
+            del(session['share-link'])
         return output
 
 
@@ -688,12 +690,14 @@ def changedetection_app(config=None, datastore_o=None):
                 # Up to 5000 per batch so we dont flood the server
                 if len(url) and validators.url(url.replace('source:', '')) and good < 5000:
                     new_uuid = datastore.add_watch(url=url.strip(), tag=" ".join(tags), write_to_disk_now=False)
-                    # Straight into the queue.
-                    update_q.put(new_uuid)
-                    good += 1
-                else:
-                    if len(url):
-                        remaining_urls.append(url)
+                    if new_uuid:
+                        # Straight into the queue.
+                        update_q.put(new_uuid)
+                        good += 1
+                        continue
+
+                if len(url.strip()):
+                    remaining_urls.append(url)
 
             flash("{} Imported in {:.2f}s, {} Skipped.".format(good, time.time()-now,len(remaining_urls)))
             datastore.needs_write = True
@@ -1019,29 +1023,35 @@ def changedetection_app(config=None, datastore_o=None):
         from changedetectionio import forms
         form = forms.quickWatchForm(request.form)
 
-        if form.validate():
+        if not form.validate():
+            flash("Error")
+            return redirect(url_for('index'))
 
-            url = request.form.get('url').strip()
-            if datastore.url_exists(url):
-                flash('The URL {} already exists'.format(url), "error")
-                return redirect(url_for('index'))
+        url = request.form.get('url').strip()
+        if datastore.url_exists(url):
+            flash('The URL {} already exists'.format(url), "error")
+            return redirect(url_for('index'))
 
-            # @todo add_watch should throw a custom Exception for validation etc
-            new_uuid = datastore.add_watch(url=url, tag=request.form.get('tag').strip())
+        # @todo add_watch should throw a custom Exception for validation etc
+        new_uuid = datastore.add_watch(url=url, tag=request.form.get('tag').strip())
+        if new_uuid:
             # Straight into the queue.
             update_q.put(new_uuid)
-
             flash("Watch added.")
-            return redirect(url_for('index'))
-        else:
-            flash("Error")
-            return redirect(url_for('index'))
+
+        return redirect(url_for('index'))
+
 
 
     @app.route("/api/delete", methods=['GET'])
     @login_required
     def api_delete():
         uuid = request.args.get('uuid')
+
+        if uuid != 'all' and not uuid in datastore.data['watching'].keys():
+            flash('The watch by UUID {} does not exist.'.format(uuid), 'error')
+            return redirect(url_for('index'))
+
         # More for testing, possible to return the first/only
         if uuid == 'first':
             uuid = list(datastore.data['watching'].keys()).pop()
@@ -1101,6 +1111,57 @@ def changedetection_app(config=None, datastore_o=None):
         flash("{} watches are queued for rechecking.".format(i))
         return redirect(url_for('index', tag=tag))
 
+    @app.route("/api/share-url", methods=['GET'])
+    @login_required
+    def api_share_put_watch():
+        """Given a watch UUID, upload the info and return a share-link
+           the share-link can be imported/added"""
+        import requests
+        import json
+        tag = request.args.get('tag')
+        uuid = request.args.get('uuid')
+
+        # more for testing
+        if uuid == 'first':
+            uuid = list(datastore.data['watching'].keys()).pop()
+
+        # copy it to memory as trim off what we dont need (history)
+        watch = deepcopy(datastore.data['watching'][uuid])
+        if (watch.get('history')):
+            del (watch['history'])
+
+        # for safety/privacy
+        for k in list(watch.keys()):
+            if k.startswith('notification_'):
+                del watch[k]
+
+        for r in['uuid', 'last_checked', 'last_changed']:
+            if watch.get(r):
+                del (watch[r])
+
+        # Add the global stuff which may have an impact
+        watch['ignore_text'] += datastore.data['settings']['application']['global_ignore_text']
+        watch['subtractive_selectors'] += datastore.data['settings']['application']['global_subtractive_selectors']
+
+        watch_json = json.dumps(watch)
+
+        try:
+            r = requests.request(method="POST",
+                                 data={'watch': watch_json},
+                                 url="https://changedetection.io/share/share",
+                                 headers={'App-Guid': datastore.data['app_guid']})
+            res = r.json()
+
+            session['share-link'] = "https://changedetection.io/share/{}".format(res['share_key'])
+
+
+        except Exception as e:
+            flash("Could not share, something went wrong while communicating with the share server.", 'error')
+
+        # https://changedetection.io/share/VrMv05wpXyQa
+        # in the browser - should give you a nice info page - wtf
+        # paste in etc
+        return redirect(url_for('index'))
 
     # @todo handle ctrl break
     ticker_thread = threading.Thread(target=ticker_thread_check_time_launch_checks).start()

changedetectionio/model/Watch.py

@@ -22,7 +22,8 @@ class model(dict):
             'newest_history_key': 0,
             'title': None,
             'previous_md5': False,
-            'uuid': str(uuid_builder.uuid4()),
+#           UUID not needed, should be generated only as a key
+#            'uuid':
             'headers': {},  # Extra headers to send
             'body': None,
             'method': 'GET',

changedetectionio/static/images/copy.svg

@@ -0,0 +1,40 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<svg
+   version="1.1"
+   id="Layer_1"
+   x="0px"
+   y="0px"
+   viewBox="0 0 115.77 122.88"
+   style="enable-background:new 0 0 115.77 122.88"
+   xml:space="preserve"
+   sodipodi:docname="copy.svg"
+   inkscape:version="1.1.1 (1:1.1+202109281949+c3084ef5ed)"
+   xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape"
+   xmlns:sodipodi="http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd"
+   xmlns="http://www.w3.org/2000/svg"
+   xmlns:svg="http://www.w3.org/2000/svg"><defs
+     id="defs11" /><sodipodi:namedview
+     id="namedview9"
+     pagecolor="#ffffff"
+     bordercolor="#666666"
+     borderopacity="1.0"
+     inkscape:pageshadow="2"
+     inkscape:pageopacity="0.0"
+     inkscape:pagecheckerboard="0"
+     showgrid="false"
+     inkscape:zoom="5.5501303"
+     inkscape:cx="57.83648"
+     inkscape:cy="61.439999"
+     inkscape:window-width="1920"
+     inkscape:window-height="1056"
+     inkscape:window-x="1920"
+     inkscape:window-y="0"
+     inkscape:window-maximized="1"
+     inkscape:current-layer="g6" /><style
+     type="text/css"
+     id="style2">.st0{fill-rule:evenodd;clip-rule:evenodd;}</style><g
+     id="g6"><path
+       class="st0"
+       d="M89.62,13.96v7.73h12.19h0.01v0.02c3.85,0.01,7.34,1.57,9.86,4.1c2.5,2.51,4.06,5.98,4.07,9.82h0.02v0.02 v73.27v0.01h-0.02c-0.01,3.84-1.57,7.33-4.1,9.86c-2.51,2.5-5.98,4.06-9.82,4.07v0.02h-0.02h-61.7H40.1v-0.02 c-3.84-0.01-7.34-1.57-9.86-4.1c-2.5-2.51-4.06-5.98-4.07-9.82h-0.02v-0.02V92.51H13.96h-0.01v-0.02c-3.84-0.01-7.34-1.57-9.86-4.1 c-2.5-2.51-4.06-5.98-4.07-9.82H0v-0.02V13.96v-0.01h0.02c0.01-3.85,1.58-7.34,4.1-9.86c2.51-2.5,5.98-4.06,9.82-4.07V0h0.02h61.7 h0.01v0.02c3.85,0.01,7.34,1.57,9.86,4.1c2.5,2.51,4.06,5.98,4.07,9.82h0.02V13.96L89.62,13.96z M79.04,21.69v-7.73v-0.02h0.02 c0-0.91-0.39-1.75-1.01-2.37c-0.61-0.61-1.46-1-2.37-1v0.02h-0.01h-61.7h-0.02v-0.02c-0.91,0-1.75,0.39-2.37,1.01 c-0.61,0.61-1,1.46-1,2.37h0.02v0.01v64.59v0.02h-0.02c0,0.91,0.39,1.75,1.01,2.37c0.61,0.61,1.46,1,2.37,1v-0.02h0.01h12.19V35.65 v-0.01h0.02c0.01-3.85,1.58-7.34,4.1-9.86c2.51-2.5,5.98-4.06,9.82-4.07v-0.02h0.02H79.04L79.04,21.69z M105.18,108.92V35.65v-0.02 h0.02c0-0.91-0.39-1.75-1.01-2.37c-0.61-0.61-1.46-1-2.37-1v0.02h-0.01h-61.7h-0.02v-0.02c-0.91,0-1.75,0.39-2.37,1.01 c-0.61,0.61-1,1.46-1,2.37h0.02v0.01v73.27v0.02h-0.02c0,0.91,0.39,1.75,1.01,2.37c0.61,0.61,1.46,1,2.37,1v-0.02h0.01h61.7h0.02 v0.02c0.91,0,1.75-0.39,2.37-1.01c0.61-0.61,1-1.46,1-2.37h-0.02V108.92L105.18,108.92z"
+       id="path4"
+       style="fill:#ffffff;fill-opacity:1" /></g></svg>

changedetectionio/static/images/spread.svg

@@ -0,0 +1,46 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<svg
+   width="18"
+   height="19.92"
+   viewBox="0 0 18 19.92"
+   version="1.1"
+   id="svg6"
+   sodipodi:docname="spread.svg"
+   inkscape:version="1.1.1 (1:1.1+202109281949+c3084ef5ed)"
+   xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape"
+   xmlns:sodipodi="http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd"
+   xmlns="http://www.w3.org/2000/svg"
+   xmlns:svg="http://www.w3.org/2000/svg">
+  <defs
+     id="defs10" />
+  <sodipodi:namedview
+     id="namedview8"
+     pagecolor="#ffffff"
+     bordercolor="#666666"
+     borderopacity="1.0"
+     inkscape:pageshadow="2"
+     inkscape:pageopacity="0.0"
+     inkscape:pagecheckerboard="0"
+     showgrid="false"
+     fit-margin-top="0"
+     fit-margin-left="0"
+     fit-margin-right="0"
+     fit-margin-bottom="0"
+     inkscape:zoom="28.416667"
+     inkscape:cx="9.0087975"
+     inkscape:cy="9.9941348"
+     inkscape:window-width="1920"
+     inkscape:window-height="1056"
+     inkscape:window-x="1920"
+     inkscape:window-y="0"
+     inkscape:window-maximized="1"
+     inkscape:current-layer="svg6" />
+  <path
+     d="M -3,-2 H 21 V 22 H -3 Z"
+     fill="none"
+     id="path2" />
+  <path
+     d="m 15,14.08 c -0.76,0 -1.44,0.3 -1.96,0.77 L 5.91,10.7 C 5.96,10.47 6,10.24 6,10 6,9.76 5.96,9.53 5.91,9.3 L 12.96,5.19 C 13.5,5.69 14.21,6 15,6 16.66,6 18,4.66 18,3 18,1.34 16.66,0 15,0 c -1.66,0 -3,1.34 -3,3 0,0.24 0.04,0.47 0.09,0.7 L 5.04,7.81 C 4.5,7.31 3.79,7 3,7 1.34,7 0,8.34 0,10 c 0,1.66 1.34,3 3,3 0.79,0 1.5,-0.31 2.04,-0.81 l 7.12,4.16 c -0.05,0.21 -0.08,0.43 -0.08,0.65 0,1.61 1.31,2.92 2.92,2.92 1.61,0 2.92,-1.31 2.92,-2.92 0,-1.61 -1.31,-2.92 -2.92,-2.92 z"
+     id="path4"
+     style="fill:#0078e7;fill-opacity:1" />
+</svg>

changedetectionio/static/js/watch-overview.js

@@ -4,4 +4,23 @@ $(function () {
     $(this).closest('.unviewed').removeClass('unviewed');
   });
 
+
+  $('.with-share-link > *').click(function () {
+      $("#copied-clipboard").remove();
+
+      var range = document.createRange();
+      var n=$("#share-link")[0];
+      range.selectNode(n);
+      window.getSelection().removeAllRanges();
+      window.getSelection().addRange(range);
+      document.execCommand("copy");
+      window.getSelection().removeAllRanges();
+
+      $('.with-share-link').append('<span style="font-size: 80%; color: #fff;" id="copied-clipboard">Copied to clipboard</span>');
+      $("#copied-clipboard").fadeOut(2500, function() {
+       $(this).remove();
+      });
+  });
+
 });
+

changedetectionio/static/styles/styles.css

@@ -180,6 +180,9 @@ body:after, body:before {
   .messages li.notice {
     background: rgba(255, 255, 255, 0.5); }
 
+.messages.with-share-link > *:hover {
+  cursor: pointer; }
+
 #notification-customisation {
   border: 1px solid #ccc;
   padding: 0.5rem;

changedetectionio/static/styles/styles.scss

@@ -237,6 +237,11 @@ body:after, body:before {
             background: rgba(255, 255, 255, .5);
         }
     }
+    &.with-share-link {
+     > *:hover {
+       cursor:pointer;
+     }
+    }
 }
 
 #notification-customisation {

changedetectionio/store.py

@@ -1,3 +1,6 @@
+from flask import (
+    flash
+)
 import json
 import logging
 import os
@@ -8,6 +11,7 @@ from copy import deepcopy
 from os import mkdir, path, unlink
 from threading import Lock
 import re
+import requests
 
 from changedetectionio.model import Watch, App
 
@@ -34,7 +38,8 @@ class ChangeDetectionStore:
         self.__data = App.model()
 
         # Base definition for all watchers
-        self.generic_definition = Watch.model()
+        # deepcopy part of #569 - not sure why its needed exactly
+        self.generic_definition = deepcopy(Watch.model())
 
         if path.isfile('changedetectionio/source.txt'):
             with open('changedetectionio/source.txt') as f:
@@ -227,7 +232,7 @@ class ChangeDetectionStore:
 
                 del self.data['watching'][uuid]
 
-            self.needs_write = True
+            self.needs_write_urgent = True
 
     # Clone a watch by UUID
     def clone(self, uuid):
@@ -295,17 +300,46 @@ class ChangeDetectionStore:
     def add_watch(self, url, tag="", extras=None, write_to_disk_now=True):
         if extras is None:
             extras = {}
+        # Incase these are copied across, assume it's a reference and deepcopy()
+        apply_extras = deepcopy(extras)
+
+        # Was it a share link? try to fetch the data
+        if (url.startswith("https://changedetection.io/share/")):
+            try:
+                r = requests.request(method="GET",
+                                     url=url,
+                                     # So we know to return the JSON instead of the human-friendly "help" page
+                                     headers={'App-Guid': self.__data['app_guid']})
+                res = r.json()
+
+                # List of permisable stuff we accept from the wild internet
+                for k in ['url', 'tag',
+                                   'paused', 'title',
+                                   'previous_md5', 'headers',
+                                   'body', 'method',
+                                   'ignore_text', 'css_filter',
+                                   'subtractive_selectors', 'trigger_text',
+                                   'extract_title_as_title']:
+                    if res.get(k):
+                        apply_extras[k] = res[k]
+
+            except Exception as e:
+                logging.error("Error fetching metadata for shared watch link", url, str(e))
+                flash("Error fetching metadata for {}".format(url), 'error')
+                return False
 
         with self.lock:
             # @todo use a common generic version of this
             new_uuid = str(uuid_builder.uuid4())
-            new_watch = Watch.model({
+            # #Re 569
+            # Not sure why deepcopy was needed here, sometimes new watches would appear to already have 'history' set
+            # I assumed this would instantiate a new object but somehow an existing dict was getting used
+            new_watch = deepcopy(Watch.model({
                 'url': url,
                 'tag': tag
-            })
+            }))
+
 
-            # Incase these are copied across, assume it's a reference and deepcopy()
-            apply_extras = deepcopy(extras)
             for k in ['uuid', 'history', 'last_checked', 'last_changed', 'newest_history_key', 'previous_md5', 'viewed']:
                 if k in apply_extras:
                     del apply_extras[k]

changedetectionio/templates/base.html

@@ -94,6 +94,13 @@
         </ul>
       {% endif %}
     {% endwith %}
+
+    {% if session['share-link'] %}
+        <ul class="messages with-share-link">
+          <li class="message">Share this link: <span id="share-link">{{ session['share-link'] }}</span> <img style="height: 1em;display:inline-block;" src="{{url_for('static_content', group='images', filename='copy.svg')}}" /></li>
+        </ul>
+    {% endif %}
+
     {% block content %}
 
     {% endblock %}

changedetectionio/templates/watch-overview.html

@@ -14,8 +14,7 @@
                 {{ render_simple_field(form.tag, value=active_tag if active_tag else '', placeholder="watch group") }}
             <button type="submit" class="pure-button pure-button-primary">Watch</button>
         </fieldset>
-        <!-- add extra stuff, like do a http POST and send headers -->
-        <!-- user/pass r = requests.get('https://api.github.com/user', auth=('user', 'pass')) -->
+        <span style="color:#eee; font-size: 80%;"><img style="height: 1em;display:inline-block;" src="{{url_for('static_content', group='images', filename='spread.svg')}}" /> Tip: You can also add 'shared' watches. <a href="#">More info</a></a></span>
     </form>
     <div>
         <a href="{{url_for('index')}}" class="pure-button button-tag {{'active' if not active_tag }}">All</a>
@@ -54,6 +53,8 @@
 
                 <td class="title-col inline">{{watch.title if watch.title is not none and watch.title|length > 0 else watch.url}}
                     <a class="external" target="_blank" rel="noopener" href="{{ watch.url.replace('source:','') }}"></a>
+                    <a href="{{url_for('api_share_put_watch', uuid=watch.uuid)}}"><img style="height: 1em;display:inline-block;" src="{{url_for('static_content', group='images', filename='spread.svg')}}" /></a>
+
                     {%if watch.fetch_backend == "html_webdriver" %}<img style="height: 1em; display:inline-block;" src="{{url_for('static_content', group='images', filename='Google-Chrome-icon.png')}}" />{% endif %}
 
                     {% if watch.last_error is defined and watch.last_error != False %}

changedetectionio/tests/test_notification.py

@@ -156,7 +156,7 @@ def test_check_notification(client, live_server):
 
     # cleanup for the next
     client.get(
-        url_for("api_delete", uuid="first"),
+        url_for("api_delete", uuid="all"),
         follow_redirects=True
     )
 
@@ -172,8 +172,7 @@ def test_notification_validation(client, live_server):
         data={"url": test_url, "tag": 'nice one'},
         follow_redirects=True
     )
-    with open("xxx.bin", "wb") as f:
-        f.write(res.data)
+
     assert b"Watch added" in res.data
 
     # Re #360 some validation
@@ -209,6 +208,6 @@ def test_notification_validation(client, live_server):
 
     # cleanup for the next
     client.get(
-        url_for("api_delete", uuid="first"),
+        url_for("api_delete", uuid="all"),
         follow_redirects=True
     )

changedetectionio/tests/test_request.py

@@ -27,6 +27,7 @@ def test_headers_in_request(client, live_server):
     )
     assert b"1 Imported" in res.data
 
+    time.sleep(3)
     cookie_header = '_ga=GA1.2.1022228332; cookie-preferences=analytics:accepted;'
 
 

changedetectionio/tests/test_share_watch.py

@@ -0,0 +1,76 @@
+#!/usr/bin/python3
+
+import time
+from flask import url_for
+from urllib.request import urlopen
+from .util import set_original_response, set_modified_response, live_server_setup
+import re
+
+sleep_time_for_fetch_thread = 3
+
+
+def test_share_watch(client, live_server):
+    set_original_response()
+    live_server_setup(live_server)
+
+    test_url = url_for('test_endpoint', _external=True)
+    css_filter = ".nice-filter"
+
+    # Add our URL to the import page
+    res = client.post(
+        url_for("import_page"),
+        data={"urls": test_url},
+        follow_redirects=True
+    )
+
+    assert b"1 Imported" in res.data
+
+    # Goto the edit page, add our ignore text
+    # Add our URL to the import page
+    res = client.post(
+        url_for("edit_page", uuid="first"),
+        data={"css_filter": css_filter, "url": test_url, "tag": "", "headers": "", 'fetch_backend': "html_requests"},
+        follow_redirects=True
+    )
+    assert b"Updated watch." in res.data
+    # Check it saved
+    res = client.get(
+        url_for("edit_page", uuid="first"),
+    )
+    assert bytes(css_filter.encode('utf-8')) in res.data
+
+    # click share the link
+    res = client.get(
+        url_for("api_share_put_watch", uuid="first"),
+        follow_redirects=True
+    )
+
+    assert b"Share this link:" in res.data
+    assert b"https://changedetection.io/share/" in res.data
+
+    html = res.data.decode()
+    share_link_search = re.search('<span id="share-link">(.*)</span>', html, re.IGNORECASE)
+    assert share_link_search
+
+    # Now delete what we have, we will try to re-import it
+    # Cleanup everything
+    res = client.get(url_for("api_delete", uuid="all"), follow_redirects=True)
+    assert b'Deleted' in res.data
+
+    # Add our URL to the import page
+    res = client.post(
+        url_for("import_page"),
+        data={"urls": share_link_search.group(1)},
+        follow_redirects=True
+    )
+
+    assert b"1 Imported" in res.data
+
+    # Now hit edit, we should see what we expect
+    # that the import fetched the meta-data
+
+    # Check it saved
+    res = client.get(
+        url_for("edit_page", uuid="first"),
+    )
+    assert bytes(css_filter.encode('utf-8')) in res.data

requirements.txt

@@ -17,7 +17,7 @@ wtforms ~= 3.0
 jsonpath-ng ~= 1.5.3
 
 # Notification library
-apprise ~= 0.9.8.2
+apprise ~= 0.9.8.3
 
 # apprise mqtt https://github.com/dgtlmoon/changedetection.io/issues/315
 paho-mqtt