From 0855017dcaa632a452e468d3631083fe07f724ea Mon Sep 17 00:00:00 2001
From: Leigh Morresi <275001+dgtlmoon@users.noreply.github.com>
Date: Fri, 26 Feb 2021 16:52:14 +0100
Subject: [PATCH 01/28] Validation of added headers, should contain key/val (2
 parts)

---
 backend/__init__.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/backend/__init__.py b/backend/__init__.py
index 101e0094..530e5570 100644
--- a/backend/__init__.py
+++ b/backend/__init__.py
@@ -162,13 +162,15 @@ def changedetection_app(config=None, datastore_o=None):
             url = request.form.get('url').strip()
             tag = request.form.get('tag').strip()
 
+            # Extra headers
             form_headers = request.form.get('headers').strip().split("\n")
             extra_headers = {}
             if form_headers:
                 for header in form_headers:
                     if len(header):
                         parts = header.split(':', 1)
-                        extra_headers.update({parts[0].strip(): parts[1].strip()})
+                        if len(parts) == 2:
+                            extra_headers.update({parts[0].strip(): parts[1].strip()})
 
             validators.url(url)  # @todo switch to prop/attr/observer
             datastore.data['watching'][uuid].update({'url': url,

From 468184bc3acbd201db75e03af48752c03f13b6f5 Mon Sep 17 00:00:00 2001
From: Leigh Morresi <275001+dgtlmoon@users.noreply.github.com>
Date: Fri, 26 Feb 2021 20:07:26 +0100
Subject: [PATCH 02/28] Issue #14 - Tweaks to edit, create ignore text, tests
 for ignore text, integrate ignore text

---
 backend/__init__.py                   |  34 +++++--
 backend/fetch_site_status.py          |  20 +++-
 backend/pytest.ini                    |   4 +-
 backend/store.py                      |   3 +-
 backend/templates/edit.html           |  20 +++-
 backend/templates/watch-overview.html |   5 +-
 backend/tests/conftest.py             |   8 +-
 backend/tests/test_ignore_text.py     | 135 ++++++++++++++++++++++++++
 8 files changed, 213 insertions(+), 16 deletions(-)
 create mode 100644 backend/tests/test_ignore_text.py

diff --git a/backend/__init__.py b/backend/__init__.py
index 530e5570..d46a06fb 100644
--- a/backend/__init__.py
+++ b/backend/__init__.py
@@ -151,13 +151,16 @@ def changedetection_app(config=None, datastore_o=None):
 
         return render_template("scrub.html")
 
-    @app.route("/edit", methods=['GET', 'POST'])
-    def edit_page():
+    @app.route("/edit/<string:uuid>", methods=['GET', 'POST'])
+    def edit_page(uuid):
         global messages
         import validators
 
+        # More for testing, possible to return the first/only
+        if uuid == 'first':
+            uuid = list(datastore.data['watching'].keys()).pop()
+
         if request.method == 'POST':
-            uuid = request.args.get('uuid')
 
             url = request.form.get('url').strip()
             tag = request.form.get('tag').strip()
@@ -172,10 +175,27 @@ def changedetection_app(config=None, datastore_o=None):
                         if len(parts) == 2:
                             extra_headers.update({parts[0].strip(): parts[1].strip()})
 
+            update_obj = {'url': url,
+                          'tag': tag,
+                          'headers': extra_headers
+                          }
+
+            # Ignore text
+            form_ignore_text = request.form.get('ignore-text').strip()
+            ignore_text = []
+            if form_ignore_text:
+                for text in form_ignore_text.split("\n"):
+                    text = text.strip()
+                    if len(text):
+                        ignore_text.append(text)
+
+                # Reset the previous_md5 so we process a new snapshot including stripping ignore text.
+                update_obj['previous_md5'] = ""
+
+            update_obj['ignore_text'] = ignore_text
+
             validators.url(url)  # @todo switch to prop/attr/observer
-            datastore.data['watching'][uuid].update({'url': url,
-                                                     'tag': tag,
-                                                     'headers': extra_headers})
+            datastore.data['watching'][uuid].update(update_obj)
             datastore.needs_write = True
 
             messages.append({'class': 'ok', 'message': 'Updated watch.'})
@@ -183,8 +203,6 @@ def changedetection_app(config=None, datastore_o=None):
             return redirect(url_for('index'))
 
         else:
-
-            uuid = request.args.get('uuid')
             output = render_template("edit.html", uuid=uuid, watch=datastore.data['watching'][uuid], messages=messages)
 
         return output
diff --git a/backend/fetch_site_status.py b/backend/fetch_site_status.py
index c3e557e0..5c0d98ef 100644
--- a/backend/fetch_site_status.py
+++ b/backend/fetch_site_status.py
@@ -11,6 +11,15 @@ class perform_site_check():
         super().__init__(*args, **kwargs)
         self.datastore = datastore
 
+    def strip_ignore_text(self, content, list_ignore_text):
+
+        output=[]
+        for line in content.splitlines():
+            if not any(skip_text in line for skip_text in list_ignore_text):
+                output.append(line)
+
+        return "\n".join(output)
+
     def run(self, uuid):
         timestamp = int(time.time())  # used for storage etc too
         stripped_text_from_html = False
@@ -76,7 +85,16 @@ class perform_site_check():
             if not len(r.text):
                 update_obj["last_error"] = "Empty reply"
 
-            fetched_md5 = hashlib.md5(stripped_text_from_html.encode('utf-8')).hexdigest()
+            content = stripped_text_from_html.encode('utf-8')
+
+            # If there's text to skip
+            # @todo we could abstract out the get_text() to handle this cleaner
+            if len(self.datastore.data['watching'][uuid]['ignore_text']):
+                content = self.strip_ignore_text(content, self.datastore.data['watching'][uuid]['ignore_text'])
+
+            fetched_md5 = hashlib.md5(content).hexdigest()
+
+
 
             # could be None or False depending on JSON type
             if self.datastore.data['watching'][uuid]['previous_md5'] != fetched_md5:
diff --git a/backend/pytest.ini b/backend/pytest.ini
index 8b9ccf85..7716524b 100644
--- a/backend/pytest.ini
+++ b/backend/pytest.ini
@@ -1,2 +1,4 @@
 [pytest]
-addopts = --no-start-live-server --live-server-port=5005
\ No newline at end of file
+addopts = --no-start-live-server --live-server-port=5005
+live_server_scope = function
+
diff --git a/backend/store.py b/backend/store.py
index d36f4b25..b5e936ec 100644
--- a/backend/store.py
+++ b/backend/store.py
@@ -53,7 +53,8 @@ class ChangeDetectionStore:
             'previous_md5': "",
             'uuid': str(uuid_builder.uuid4()),
             'headers': {},  # Extra headers to send
-            'history': {}  # Dict of timestamp and output stripped filename
+            'history': {},  # Dict of timestamp and output stripped filename
+            'ignore_text': [] # List of text to ignore when calculating the comparison checksum
         }
 
         if path.isfile('/source.txt'):
diff --git a/backend/templates/edit.html b/backend/templates/edit.html
index 89c2bfd6..525d3b27 100644
--- a/backend/templates/edit.html
+++ b/backend/templates/edit.html
@@ -18,10 +18,26 @@
                 <span class="pure-form-message-inline">Grouping tags, can be a comma separated list.</span>
             </div>
 
+            <!-- @todo: move to tabs --->
+            <fieldset class="pure-group">
+                <label for="ignore-text">Ignore text</label>
+
+                <textarea id="ignore-text" name="ignore-text" class="pure-input-1-2" placeholder=""
+                          style="width: 100%;
+                            font-family:monospace;
+                            white-space: pre;
+                            overflow-wrap: normal;
+                            overflow-x: scroll;" rows="5">{% for value in watch.ignore_text %}{{ value }}
+{% endfor %}</textarea>
+                <span class="pure-form-message-inline">Each line will be processed separately as an ignore rule.</span>
+
+            </fieldset>
+
+            <!-- @todo: move to tabs --->
             <fieldset class="pure-group">
                 <label for="headers">Extra request headers</label>
 
-                <textarea id=headers name="headers" class="pure-input-1-2" placeholder="Example
+                <textarea id="headers" name="headers" class="pure-input-1-2" placeholder="Example
 Cookie: foobar
 User-Agent: wonderbra 1.0"
                           style="width: 100%;
@@ -33,6 +49,8 @@ User-Agent: wonderbra 1.0"
                 <br/>
 
             </fieldset>
+
+
             <div class="pure-control-group">
                 <button type="submit" class="pure-button pure-button-primary">Save</button>
             </div>
diff --git a/backend/templates/watch-overview.html b/backend/templates/watch-overview.html
index 568e9ae5..ba18d323 100644
--- a/backend/templates/watch-overview.html
+++ b/backend/templates/watch-overview.html
@@ -43,7 +43,8 @@
             <tr id="{{ watch.uuid }}"
                 class="{{ loop.cycle('pure-table-odd', 'pure-table-even') }}
                 {% if watch.last_error is defined and watch.last_error != False %}error{% endif %}
-                {% if watch.newest_history_key| int > watch.last_viewed| int %}unviewed{% endif %}">
+                {% if watch.newest_history_key| int > watch.last_viewed| int %}unviewed{% endif %}
+                ">
                 <td>{{ loop.index }}</td>
                 <td class="title-col">{{watch.title if watch.title is not none else watch.url}}
                     <a class="external" target=_blank href="{{ watch.url }}"></a>
@@ -64,7 +65,7 @@
                 <td>
                     <a href="/api/checknow?uuid={{ watch.uuid}}{% if request.args.get('tag') %}&tag={{request.args.get('tag')}}{% endif %}"
                        class="pure-button button-small pure-button-primary">Recheck</a>
-                    <a href="/edit?uuid={{ watch.uuid}}" class="pure-button button-small pure-button-primary">Edit</a>
+                    <a href="/edit/{{ watch.uuid}}" class="pure-button button-small pure-button-primary">Edit</a>
                     {% if watch.history|length >= 2 %}
                     <a href="/diff/{{ watch.uuid}}" class="pure-button button-small pure-button-primary">Diff</a>
                     {% endif %}
diff --git a/backend/tests/conftest.py b/backend/tests/conftest.py
index 2c2fa334..8cb28da4 100644
--- a/backend/tests/conftest.py
+++ b/backend/tests/conftest.py
@@ -13,7 +13,6 @@ import os
 
 global app
 
-
 @pytest.fixture(scope='session')
 def app(request):
     """Create application for the tests."""
@@ -37,7 +36,12 @@ def app(request):
     def teardown():
         datastore.stop_thread = True
         app.config['STOP_THREADS'] = True
+        try:
+            os.unlink("{}/url-watches.json".format(datastore_path))
+        except FileNotFoundError:
+            # This is fine in the case of a failure.
+            pass
 
-    request.addfinalizer(teardown)
 
+    request.addfinalizer(teardown)
     return app
diff --git a/backend/tests/test_ignore_text.py b/backend/tests/test_ignore_text.py
new file mode 100644
index 00000000..dbb25743
--- /dev/null
+++ b/backend/tests/test_ignore_text.py
@@ -0,0 +1,135 @@
+#!/usr/bin/python3
+
+import time
+from flask import url_for
+from urllib.request import urlopen
+
+
+# Unit test of the stripper
+def test_strip_text_func():
+    from backend import fetch_site_status
+
+    test_content = """
+    Some content
+    is listed here
+    
+    but sometimes we want to remove the lines.
+    
+    but not always."""
+
+    original_length = len(test_content.splitlines())
+
+    fetcher = fetch_site_status.perform_site_check(datastore=False)
+
+    ignore_lines = ["sometimes"]
+
+    stripped_content = fetcher.strip_ignore_text(test_content, ignore_lines)
+
+    # Should be one line shorter
+    assert len(stripped_content.splitlines()) == original_length - 1
+
+    assert "sometimes" not in stripped_content
+    assert "Some content" in stripped_content
+
+
+def set_original_response():
+    test_return_data = """<html>
+       <body>
+     Some initial text</br>
+     <p>Which is across multiple lines</p>
+     </br>
+     So let's see what happens.  </br>
+     </body>
+     </html>
+
+    """
+
+    with open("test-datastore/output.txt", "w") as f:
+        f.write(test_return_data)
+
+
+# Is the same but includes ZZZZZ, 'ZZZZZ' is the last line in ignore_text
+def set_modified_response():
+    test_return_data = """<html>
+       <body>
+     Some initial text</br>
+     <p>Which is across multiple lines</p>
+     <P>ZZZZZ</P>
+     </br>
+     So let's see what happens.  </br>
+     </body>
+     </html>
+
+    """
+
+    with open("test-datastore/output.txt", "w") as f:
+        f.write(test_return_data)
+
+
+def test_check_ignore_text_functionality(client, live_server):
+    sleep_time_for_fetch_thread = 5
+
+    ignore_text = "XXXXX\nYYYYY\nZZZZZ"
+    set_original_response()
+
+    @live_server.app.route('/test-ignore-endpoint')
+    def test_ignore_endpoint():
+        # Tried using a global var here but didn't seem to work, so reading from a file instead.
+        with open("test-datastore/output.txt", "r") as f:
+            return f.read()
+
+    live_server.start()
+
+    # Give the endpoint time to spin up
+    time.sleep(1)
+
+    # Add our URL to the import page
+    test_url = url_for('test_ignore_endpoint', _external=True)
+    res = client.post(
+        url_for("import_page"),
+        data={"urls": test_url},
+        follow_redirects=True
+    )
+    assert b"1 Imported" in res.data
+
+    # Goto the edit page, add our ignore text
+    # Add our URL to the import page
+    res = client.post(
+        url_for("edit_page", uuid="first"),
+        data={"ignore-text": ignore_text, "url": test_url, "tag": "", "headers": ""},
+        follow_redirects=True
+    )
+    assert b"Updated watch." in res.data
+
+    # Check it saved
+    res = client.get(
+        url_for("edit_page", uuid="first"),
+    )
+    assert bytes(ignore_text.encode('utf-8')) in res.data
+
+    # Trigger a check
+    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+
+    # Give the thread time to pick it up
+    time.sleep(sleep_time_for_fetch_thread)
+
+    # It should report nothing found (no new 'unviewed' class)
+    res = client.get(url_for("index"))
+    assert b'unviewed' not in res.data
+    assert b'/test-ignore-endpoint' in res.data
+
+    set_modified_response()
+
+    # Trigger a check
+    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+
+    # Give the thread time to pick it up
+    time.sleep(sleep_time_for_fetch_thread)
+
+    # It should report nothing found (no new 'unviewed' class)
+    res = client.get(url_for("index"))
+    assert b'unviewed' not in res.data
+    assert b'/test-ignore-endpoint' in res.data
+
+    live_server.stop()
+

From ec1ac300af10170717ff2a2d91e0eb542ac7e6b0 Mon Sep 17 00:00:00 2001
From: Leigh Morresi <275001+dgtlmoon@users.noreply.github.com>
Date: Sat, 27 Feb 2021 09:05:25 +0100
Subject: [PATCH 03/28] Activate workflow on all branches

---
 .github/workflows/python-app.yml  |  4 ++--
 backend/pytest.ini                | 10 ++++++++-
 backend/tests/conftest.py         | 14 +++++++++---
 backend/tests/test_backend.py     | 36 +++++++++++++++++--------------
 backend/tests/test_ignore_text.py | 28 +++++++++---------------
 5 files changed, 52 insertions(+), 40 deletions(-)

diff --git a/.github/workflows/python-app.yml b/.github/workflows/python-app.yml
index ae14bba8..53c1e1d2 100644
--- a/.github/workflows/python-app.yml
+++ b/.github/workflows/python-app.yml
@@ -6,9 +6,9 @@ name: changedetection.io
 
 on:
   push:
-    branches: [ master ]
+    branches: [ * ]
   pull_request:
-    branches: [ master ]
+    branches: [ * ]
 
 jobs:
   build:
diff --git a/backend/pytest.ini b/backend/pytest.ini
index 7716524b..2ac84422 100644
--- a/backend/pytest.ini
+++ b/backend/pytest.ini
@@ -1,4 +1,12 @@
 [pytest]
 addopts = --no-start-live-server --live-server-port=5005
-live_server_scope = function
+#testpaths = tests pytest_invenio
+#live_server_scope = session
 
+#filterwarnings =
+    #ignore::DeprecationWarning:sqlalchemy.*:
+
+; logging options
+#log_cli = 1
+#log_cli_level = DEBUG
+#log_cli_format = %(asctime)s %(name)s: %(levelname)s %(message)s
\ No newline at end of file
diff --git a/backend/tests/conftest.py b/backend/tests/conftest.py
index 8cb28da4..2edd3cd0 100644
--- a/backend/tests/conftest.py
+++ b/backend/tests/conftest.py
@@ -7,7 +7,6 @@ import os
 
 
 # https://github.com/pallets/flask/blob/1.1.2/examples/tutorial/tests/test_auth.py
-
 # Much better boilerplate than the docs
 # https://www.python-boilerplate.com/py3+flask+pytest/
 
@@ -16,7 +15,6 @@ global app
 @pytest.fixture(scope='session')
 def app(request):
     """Create application for the tests."""
-
     datastore_path = "./test-datastore"
 
     try:
@@ -42,6 +40,16 @@ def app(request):
             # This is fine in the case of a failure.
             pass
 
+        @live_server.app.route('/test-endpoint')
+        def test_endpoint():
+            # Tried using a global var here but didn't seem to work, so reading from a file instead.
+            with open("test-datastore/output.txt", "r") as f:
+                return f.read()
+
+        live_server.start()
+
+        assert 1 == 1
 
     request.addfinalizer(teardown)
-    return app
+    yield app
+
diff --git a/backend/tests/test_backend.py b/backend/tests/test_backend.py
index e2040da0..dac5aa5d 100644
--- a/backend/tests/test_backend.py
+++ b/backend/tests/test_backend.py
@@ -3,7 +3,23 @@
 import time
 from flask import url_for
 from urllib.request import urlopen
+import pytest
 
+sleep_time_for_fetch_thread = 3
+
+
+def test_setup_liveserver(live_server):
+
+
+    @live_server.app.route('/test-endpoint')
+    def test_endpoint():
+        # Tried using a global var here but didn't seem to work, so reading from a file instead.
+        with open("test-datastore/output.txt", "r") as f:
+            return f.read()
+    live_server.start()
+
+
+    assert 1 == 1
 
 def set_original_response():
     test_return_data = """<html>
@@ -14,7 +30,6 @@ def set_original_response():
      So let's see what happens.  </br>
      </body>
      </html>
-
     """
 
     with open("test-datastore/output.txt", "w") as f:
@@ -30,7 +45,6 @@ def set_modified_response():
      So let's see what happens.  </br>
      </body>
      </html>
-
     """
 
     with open("test-datastore/output.txt", "w") as f:
@@ -38,18 +52,8 @@ def set_modified_response():
 
 
 def test_check_basic_change_detection_functionality(client, live_server):
-    sleep_time_for_fetch_thread = 5
-
-    @live_server.app.route('/test-endpoint')
-    def test_endpoint():
-        # Tried using a global var here but didn't seem to work, so reading from a file instead.
-        with open("test-datastore/output.txt", "r") as f:
-            return f.read()
-
     set_original_response()
 
-    live_server.start()
-
     # Add our URL to the import page
     res = client.post(
         url_for("import_page"),
@@ -91,13 +95,13 @@ def test_check_basic_change_detection_functionality(client, live_server):
     assert b'unviewed' in res.data
 
     # Following the 'diff' link, it should no longer display as 'unviewed' even after we recheck it a few times
-    res = client.get(url_for("diff_history_page", uuid="first") )
+    res = client.get(url_for("diff_history_page", uuid="first"))
     assert b'Compare newest' in res.data
 
     time.sleep(2)
 
     # Do this a few times.. ensures we dont accidently set the status
-    for n in range(3):
+    for n in range(2):
         client.get(url_for("api_watch_checknow"), follow_redirects=True)
 
         # Give the thread time to pick it up
@@ -108,10 +112,10 @@ def test_check_basic_change_detection_functionality(client, live_server):
         assert b'unviewed' not in res.data
         assert b'test-endpoint' in res.data
 
-
     set_original_response()
 
     client.get(url_for("api_watch_checknow"), follow_redirects=True)
     time.sleep(sleep_time_for_fetch_thread)
     res = client.get(url_for("index"))
-    assert b'unviewed' in res.data
\ No newline at end of file
+    assert b'unviewed' in res.data
+
diff --git a/backend/tests/test_ignore_text.py b/backend/tests/test_ignore_text.py
index dbb25743..da72b71e 100644
--- a/backend/tests/test_ignore_text.py
+++ b/backend/tests/test_ignore_text.py
@@ -3,6 +3,7 @@
 import time
 from flask import url_for
 from urllib.request import urlopen
+import pytest
 
 
 # Unit test of the stripper
@@ -12,9 +13,9 @@ def test_strip_text_func():
     test_content = """
     Some content
     is listed here
-    
+
     but sometimes we want to remove the lines.
-    
+
     but not always."""
 
     original_length = len(test_content.splitlines())
@@ -32,7 +33,7 @@ def test_strip_text_func():
     assert "Some content" in stripped_content
 
 
-def set_original_response():
+def set_original_ignore_response():
     test_return_data = """<html>
        <body>
      Some initial text</br>
@@ -49,7 +50,7 @@ def set_original_response():
 
 
 # Is the same but includes ZZZZZ, 'ZZZZZ' is the last line in ignore_text
-def set_modified_response():
+def set_modified_ignore_response():
     test_return_data = """<html>
        <body>
      Some initial text</br>
@@ -70,21 +71,13 @@ def test_check_ignore_text_functionality(client, live_server):
     sleep_time_for_fetch_thread = 5
 
     ignore_text = "XXXXX\nYYYYY\nZZZZZ"
-    set_original_response()
-
-    @live_server.app.route('/test-ignore-endpoint')
-    def test_ignore_endpoint():
-        # Tried using a global var here but didn't seem to work, so reading from a file instead.
-        with open("test-datastore/output.txt", "r") as f:
-            return f.read()
-
-    live_server.start()
+    set_original_ignore_response()
 
     # Give the endpoint time to spin up
     time.sleep(1)
 
     # Add our URL to the import page
-    test_url = url_for('test_ignore_endpoint', _external=True)
+    test_url = url_for('test_endpoint', _external=True)
     res = client.post(
         url_for("import_page"),
         data={"urls": test_url},
@@ -116,9 +109,9 @@ def test_check_ignore_text_functionality(client, live_server):
     # It should report nothing found (no new 'unviewed' class)
     res = client.get(url_for("index"))
     assert b'unviewed' not in res.data
-    assert b'/test-ignore-endpoint' in res.data
+    assert b'/test-endpoint' in res.data
 
-    set_modified_response()
+    set_modified_ignore_response()
 
     # Trigger a check
     client.get(url_for("api_watch_checknow"), follow_redirects=True)
@@ -129,7 +122,6 @@ def test_check_ignore_text_functionality(client, live_server):
     # It should report nothing found (no new 'unviewed' class)
     res = client.get(url_for("index"))
     assert b'unviewed' not in res.data
-    assert b'/test-ignore-endpoint' in res.data
+    assert b'/test-endpoint' in res.data
 
-    live_server.stop()
 

From efe0356f37cd6e7ab5f1060771bcf9e83315c02e Mon Sep 17 00:00:00 2001
From: Leigh Morresi <275001+dgtlmoon@users.noreply.github.com>
Date: Sat, 27 Feb 2021 09:06:54 +0100
Subject: [PATCH 04/28] Fix syntax, Triggers the workflow on push or pull
 request events

---
 .github/workflows/python-app.yml | 6 +-----
 1 file changed, 1 insertion(+), 5 deletions(-)

diff --git a/.github/workflows/python-app.yml b/.github/workflows/python-app.yml
index 53c1e1d2..ee021775 100644
--- a/.github/workflows/python-app.yml
+++ b/.github/workflows/python-app.yml
@@ -4,11 +4,7 @@
 name: changedetection.io
 
 
-on:
-  push:
-    branches: [ * ]
-  pull_request:
-    branches: [ * ]
+on: [push, pull_request]
 
 jobs:
   build:

From 66be95ecc65c70c6ca225c59d6569e6c06782e8a Mon Sep 17 00:00:00 2001
From: Leigh Morresi <275001+dgtlmoon@users.noreply.github.com>
Date: Sat, 27 Feb 2021 09:08:25 +0100
Subject: [PATCH 05/28] Remove liveserver, doesnt belong here

---
 backend/tests/conftest.py | 8 --------
 1 file changed, 8 deletions(-)

diff --git a/backend/tests/conftest.py b/backend/tests/conftest.py
index 2edd3cd0..f026f22c 100644
--- a/backend/tests/conftest.py
+++ b/backend/tests/conftest.py
@@ -40,14 +40,6 @@ def app(request):
             # This is fine in the case of a failure.
             pass
 
-        @live_server.app.route('/test-endpoint')
-        def test_endpoint():
-            # Tried using a global var here but didn't seem to work, so reading from a file instead.
-            with open("test-datastore/output.txt", "r") as f:
-                return f.read()
-
-        live_server.start()
-
         assert 1 == 1
 
     request.addfinalizer(teardown)

From 013cbcabd44e6a283045ba57d4f87ab0132e46cc Mon Sep 17 00:00:00 2001
From: Leigh Morresi <275001+dgtlmoon@users.noreply.github.com>
Date: Sat, 27 Feb 2021 09:37:40 +0100
Subject: [PATCH 06/28] Clean up after test case

---
 backend/store.py                  | 5 ++++-
 backend/tests/test_backend.py     | 2 ++
 backend/tests/test_ignore_text.py | 2 ++
 3 files changed, 8 insertions(+), 1 deletion(-)

diff --git a/backend/store.py b/backend/store.py
index b5e936ec..752a39de 100644
--- a/backend/store.py
+++ b/backend/store.py
@@ -157,7 +157,10 @@ class ChangeDetectionStore:
 
     def delete(self, uuid):
         with self.lock:
-            del (self.__data['watching'][uuid])
+            if uuid == 'all':
+                self.__data['watching'] = {}
+            else:
+                del (self.__data['watching'][uuid])
             self.needs_write = True
 
     def url_exists(self, url):
diff --git a/backend/tests/test_backend.py b/backend/tests/test_backend.py
index dac5aa5d..9ccc049e 100644
--- a/backend/tests/test_backend.py
+++ b/backend/tests/test_backend.py
@@ -119,3 +119,5 @@ def test_check_basic_change_detection_functionality(client, live_server):
     res = client.get(url_for("index"))
     assert b'unviewed' in res.data
 
+    res = client.get(url_for("api_delete", uuid="all"), follow_redirects=True)
+    assert b'Deleted' in res.data
diff --git a/backend/tests/test_ignore_text.py b/backend/tests/test_ignore_text.py
index da72b71e..69489169 100644
--- a/backend/tests/test_ignore_text.py
+++ b/backend/tests/test_ignore_text.py
@@ -124,4 +124,6 @@ def test_check_ignore_text_functionality(client, live_server):
     assert b'unviewed' not in res.data
     assert b'/test-endpoint' in res.data
 
+    res = client.get(url_for("api_delete", uuid="all"), follow_redirects=True)
+    assert b'Deleted' in res.data
 

From 092f77f066700420a6a8f8463e79ecc4692b71ee Mon Sep 17 00:00:00 2001
From: Leigh Morresi <275001+dgtlmoon@users.noreply.github.com>
Date: Sat, 27 Feb 2021 09:38:51 +0100
Subject: [PATCH 07/28] Minor lint cleanup

---
 backend/tests/test_backend.py     | 5 ++---
 backend/tests/test_ignore_text.py | 1 -
 2 files changed, 2 insertions(+), 4 deletions(-)

diff --git a/backend/tests/test_backend.py b/backend/tests/test_backend.py
index 9ccc049e..75a01112 100644
--- a/backend/tests/test_backend.py
+++ b/backend/tests/test_backend.py
@@ -9,18 +9,17 @@ sleep_time_for_fetch_thread = 3
 
 
 def test_setup_liveserver(live_server):
-
-
     @live_server.app.route('/test-endpoint')
     def test_endpoint():
         # Tried using a global var here but didn't seem to work, so reading from a file instead.
         with open("test-datastore/output.txt", "r") as f:
             return f.read()
-    live_server.start()
 
+    live_server.start()
 
     assert 1 == 1
 
+
 def set_original_response():
     test_return_data = """<html>
        <body>
diff --git a/backend/tests/test_ignore_text.py b/backend/tests/test_ignore_text.py
index 69489169..f0bbfe76 100644
--- a/backend/tests/test_ignore_text.py
+++ b/backend/tests/test_ignore_text.py
@@ -126,4 +126,3 @@ def test_check_ignore_text_functionality(client, live_server):
 
     res = client.get(url_for("api_delete", uuid="all"), follow_redirects=True)
     assert b'Deleted' in res.data
-

From 022826493b2b9223335400883edf727fa1dc5c4c Mon Sep 17 00:00:00 2001
From: Leigh Morresi <275001+dgtlmoon@users.noreply.github.com>
Date: Sat, 27 Feb 2021 20:29:01 +0100
Subject: [PATCH 08/28] Fix edit action link

---
 backend/templates/edit.html | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/backend/templates/edit.html b/backend/templates/edit.html
index 525d3b27..3b032acb 100644
--- a/backend/templates/edit.html
+++ b/backend/templates/edit.html
@@ -4,7 +4,7 @@
 <div class="edit-form">
 
 
-    <form class="pure-form pure-form-stacked" action="/edit?uuid={{uuid}}" method="POST">
+    <form class="pure-form pure-form-stacked" action="/edit/{{uuid}}" method="POST">
         <fieldset>
             <div class="pure-control-group">
                 <label for="url">URL</label>

From 8bc7b5be408f3fe99a911b4e255eb5c9fe7d222b Mon Sep 17 00:00:00 2001
From: Leigh Morresi <275001+dgtlmoon@users.noreply.github.com>
Date: Sat, 27 Feb 2021 20:29:52 +0100
Subject: [PATCH 09/28] Adding filter and log output to pytest

---
 backend/pytest.ini | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/backend/pytest.ini b/backend/pytest.ini
index 2ac84422..883439b1 100644
--- a/backend/pytest.ini
+++ b/backend/pytest.ini
@@ -3,10 +3,10 @@ addopts = --no-start-live-server --live-server-port=5005
 #testpaths = tests pytest_invenio
 #live_server_scope = session
 
-#filterwarnings =
-    #ignore::DeprecationWarning:sqlalchemy.*:
+filterwarnings =
+    ignore::DeprecationWarning:urllib3.*:
 
 ; logging options
-#log_cli = 1
-#log_cli_level = DEBUG
-#log_cli_format = %(asctime)s %(name)s: %(levelname)s %(message)s
\ No newline at end of file
+log_cli = 1
+log_cli_level = DEBUG
+log_cli_format = %(asctime)s %(name)s: %(levelname)s %(message)s
\ No newline at end of file

From f1da8f96b6fabda77e454fd07aa7ae4a07fa6bf4 Mon Sep 17 00:00:00 2001
From: Leigh Morresi <275001+dgtlmoon@users.noreply.github.com>
Date: Sat, 27 Feb 2021 20:30:06 +0100
Subject: [PATCH 10/28] When new ignore text is specified, reprocess the
 checksum

---
 backend/__init__.py               |  2 +-
 backend/fetch_site_status.py      | 37 +++++++++++++++++-----
 backend/store.py                  |  3 +-
 backend/tests/test_ignore_text.py | 52 ++++++++++++++++++++++++-------
 4 files changed, 74 insertions(+), 20 deletions(-)

diff --git a/backend/__init__.py b/backend/__init__.py
index d46a06fb..7bc9cbb4 100644
--- a/backend/__init__.py
+++ b/backend/__init__.py
@@ -190,7 +190,7 @@ def changedetection_app(config=None, datastore_o=None):
                         ignore_text.append(text)
 
                 # Reset the previous_md5 so we process a new snapshot including stripping ignore text.
-                update_obj['previous_md5'] = ""
+                update_obj['previous_md5'] = "reprocess previous"
 
             update_obj['ignore_text'] = ignore_text
 
diff --git a/backend/fetch_site_status.py b/backend/fetch_site_status.py
index 5c0d98ef..0083d892 100644
--- a/backend/fetch_site_status.py
+++ b/backend/fetch_site_status.py
@@ -12,13 +12,20 @@ class perform_site_check():
         self.datastore = datastore
 
     def strip_ignore_text(self, content, list_ignore_text):
+        ignore = []
+        for k in list_ignore_text:
+            ignore.append(k.encode('utf8'))
 
-        output=[]
+        output = []
         for line in content.splitlines():
-            if not any(skip_text in line for skip_text in list_ignore_text):
-                output.append(line)
+            line = line.encode('utf8')
 
-        return "\n".join(output)
+            # Always ignore blank lines in this mode. (when this function gets called)
+            if len(line.strip()):
+                if not any(skip_text in line for skip_text in ignore):
+                    output.append(line)
+
+        return "\n".encode('utf8').join(output)
 
     def run(self, uuid):
         timestamp = int(time.time())  # used for storage etc too
@@ -85,16 +92,32 @@ class perform_site_check():
             if not len(r.text):
                 update_obj["last_error"] = "Empty reply"
 
-            content = stripped_text_from_html.encode('utf-8')
-
             # If there's text to skip
             # @todo we could abstract out the get_text() to handle this cleaner
             if len(self.datastore.data['watching'][uuid]['ignore_text']):
-                content = self.strip_ignore_text(content, self.datastore.data['watching'][uuid]['ignore_text'])
+                content = self.strip_ignore_text(stripped_text_from_html,
+                                                 self.datastore.data['watching'][uuid]['ignore_text'])
+            else:
+                content = stripped_text_from_html
 
             fetched_md5 = hashlib.md5(content).hexdigest()
 
+            # If they edited an existing watch, we need to know to reset the current/previous md5 to include
+            # the excluded text.
+
+            if self.datastore.data['watching'][uuid]['previous_md5'] == "reprocess previous":
+                # Get the most recent one
+                newest_history_key = self.datastore.get_newest_history_key(uuid)
+                if newest_history_key:
+                    with open(self.datastore.data['watching'][uuid]['history'][newest_history_key],
+                              encoding='utf-8') as file:
+                        raw_content = file.read()
+
+                        stripped_content = self.strip_ignore_text(raw_content,
+                                                                  self.datastore.data['watching'][uuid]['ignore_text'])
 
+                        checksum = hashlib.md5(stripped_content).hexdigest()
+                        self.datastore.data['watching'][uuid]['previous_md5'] = checksum
 
             # could be None or False depending on JSON type
             if self.datastore.data['watching'][uuid]['previous_md5'] != fetched_md5:
diff --git a/backend/store.py b/backend/store.py
index 752a39de..a3bdb964 100644
--- a/backend/store.py
+++ b/backend/store.py
@@ -64,6 +64,7 @@ class ChangeDetectionStore:
                 self.__data['build_sha'] = f.read()
 
         try:
+            # @todo retest with ", encoding='utf-8'"
             with open(self.json_store_path) as json_file:
                 from_disk = json.load(json_file)
 
@@ -139,7 +140,6 @@ class ChangeDetectionStore:
 
     @property
     def data(self):
-
         return self.__data
 
     def get_all_tags(self):
@@ -161,6 +161,7 @@ class ChangeDetectionStore:
                 self.__data['watching'] = {}
             else:
                 del (self.__data['watching'][uuid])
+
             self.needs_write = True
 
     def url_exists(self, url):
diff --git a/backend/tests/test_ignore_text.py b/backend/tests/test_ignore_text.py
index f0bbfe76..fe65be75 100644
--- a/backend/tests/test_ignore_text.py
+++ b/backend/tests/test_ignore_text.py
@@ -6,7 +6,20 @@ from urllib.request import urlopen
 import pytest
 
 
+def test_setup_liveserver(live_server):
+    @live_server.app.route('/test-endpoint')
+    def test_endpoint():
+        # Tried using a global var here but didn't seem to work, so reading from a file instead.
+        with open("test-datastore/output.txt", "r") as f:
+            return f.read()
+
+    live_server.start()
+
+    assert 1 == 1
+
+
 # Unit test of the stripper
+# Always we are dealing in utf-8
 def test_strip_text_func():
     from backend import fetch_site_status
 
@@ -18,19 +31,13 @@ def test_strip_text_func():
 
     but not always."""
 
-    original_length = len(test_content.splitlines())
-
-    fetcher = fetch_site_status.perform_site_check(datastore=False)
-
     ignore_lines = ["sometimes"]
 
+    fetcher = fetch_site_status.perform_site_check(datastore=False)
     stripped_content = fetcher.strip_ignore_text(test_content, ignore_lines)
 
-    # Should be one line shorter
-    assert len(stripped_content.splitlines()) == original_length - 1
-
-    assert "sometimes" not in stripped_content
-    assert "Some content" in stripped_content
+    assert b"sometimes" not in stripped_content
+    assert b"Some content" in stripped_content
 
 
 def set_original_ignore_response():
@@ -49,6 +56,22 @@ def set_original_ignore_response():
         f.write(test_return_data)
 
 
+def set_modified_original_ignore_response():
+    test_return_data = """<html>
+       <body>
+     Some NEW nice initial text</br>
+     <p>Which is across multiple lines</p>
+     </br>
+     So let's see what happens.  </br>
+     </body>
+     </html>
+
+    """
+
+    with open("test-datastore/output.txt", "w") as f:
+        f.write(test_return_data)
+
+
 # Is the same but includes ZZZZZ, 'ZZZZZ' is the last line in ignore_text
 def set_modified_ignore_response():
     test_return_data = """<html>
@@ -68,7 +91,7 @@ def set_modified_ignore_response():
 
 
 def test_check_ignore_text_functionality(client, live_server):
-    sleep_time_for_fetch_thread = 5
+    sleep_time_for_fetch_thread = 3
 
     ignore_text = "XXXXX\nYYYYY\nZZZZZ"
     set_original_ignore_response()
@@ -111,11 +134,11 @@ def test_check_ignore_text_functionality(client, live_server):
     assert b'unviewed' not in res.data
     assert b'/test-endpoint' in res.data
 
+    #  Make a change
     set_modified_ignore_response()
 
     # Trigger a check
     client.get(url_for("api_watch_checknow"), follow_redirects=True)
-
     # Give the thread time to pick it up
     time.sleep(sleep_time_for_fetch_thread)
 
@@ -124,5 +147,12 @@ def test_check_ignore_text_functionality(client, live_server):
     assert b'unviewed' not in res.data
     assert b'/test-endpoint' in res.data
 
+    # Just to be sure.. set a regular modified change..
+    set_modified_original_ignore_response()
+    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+    time.sleep(sleep_time_for_fetch_thread)
+    res = client.get(url_for("index"))
+    assert b'unviewed' in res.data
+
     res = client.get(url_for("api_delete", uuid="all"), follow_redirects=True)
     assert b'Deleted' in res.data

From 0ec9edb97175f64e269b87a91c29f5448b7bb905 Mon Sep 17 00:00:00 2001
From: Leigh Morresi <275001+dgtlmoon@users.noreply.github.com>
Date: Sat, 27 Feb 2021 20:30:36 +0100
Subject: [PATCH 11/28] Remove erroneous extra liveserver setup

---
 backend/tests/test_ignore_text.py | 12 ------------
 1 file changed, 12 deletions(-)

diff --git a/backend/tests/test_ignore_text.py b/backend/tests/test_ignore_text.py
index fe65be75..eaaf8429 100644
--- a/backend/tests/test_ignore_text.py
+++ b/backend/tests/test_ignore_text.py
@@ -6,18 +6,6 @@ from urllib.request import urlopen
 import pytest
 
 
-def test_setup_liveserver(live_server):
-    @live_server.app.route('/test-endpoint')
-    def test_endpoint():
-        # Tried using a global var here but didn't seem to work, so reading from a file instead.
-        with open("test-datastore/output.txt", "r") as f:
-            return f.read()
-
-    live_server.start()
-
-    assert 1 == 1
-
-
 # Unit test of the stripper
 # Always we are dealing in utf-8
 def test_strip_text_func():

From 47fcb8b4f87b358356da70c88abb58ad89e6905a Mon Sep 17 00:00:00 2001
From: Leigh Morresi <275001+dgtlmoon@users.noreply.github.com>
Date: Sat, 27 Feb 2021 22:01:42 +0100
Subject: [PATCH 12/28] Move logic

---
 backend/__init__.py           | 41 ++++++++++++++++++++++++++++++-----
 backend/fetch_site_status.py  | 21 +++---------------
 backend/tests/test_backend.py |  1 +
 3 files changed, 40 insertions(+), 23 deletions(-)

diff --git a/backend/__init__.py b/backend/__init__.py
index 7bc9cbb4..5e8adddd 100644
--- a/backend/__init__.py
+++ b/backend/__init__.py
@@ -151,6 +151,34 @@ def changedetection_app(config=None, datastore_o=None):
 
         return render_template("scrub.html")
 
+    # If they edited an existing watch, we need to know to reset the current/previous md5 to include
+    # the excluded text.
+    def get_current_checksum_include_ignore_text(uuid):
+
+        import hashlib
+        from backend import fetch_site_status
+
+        # Get the most recent one
+        newest_history_key = datastore.get_val(uuid, 'newest_history_key')
+
+        # 0 means that theres only one, so that there should be no 'unviewed' history availabe
+        if newest_history_key == 0:
+            newest_history_key = list(datastore.data['watching'][uuid]['history'].keys())[0]
+
+        if newest_history_key:
+            with open(datastore.data['watching'][uuid]['history'][newest_history_key],
+                      encoding='utf-8') as file:
+                raw_content = file.read()
+
+                handler = fetch_site_status.perform_site_check(datastore=datastore)
+                stripped_content = handler.strip_ignore_text(raw_content,
+                                                          datastore.data['watching'][uuid]['ignore_text'])
+
+                checksum = hashlib.md5(stripped_content).hexdigest()
+                return checksum
+
+        return datastore.data['watching'][uuid]['previous_md5']
+
     @app.route("/edit/<string:uuid>", methods=['GET', 'POST'])
     def edit_page(uuid):
         global messages
@@ -183,16 +211,19 @@ def changedetection_app(config=None, datastore_o=None):
             # Ignore text
             form_ignore_text = request.form.get('ignore-text').strip()
             ignore_text = []
-            if form_ignore_text:
+            if len(form_ignore_text):
                 for text in form_ignore_text.split("\n"):
                     text = text.strip()
                     if len(text):
                         ignore_text.append(text)
 
+                datastore.data['watching'][uuid]['ignore_text'] = ignore_text
+
                 # Reset the previous_md5 so we process a new snapshot including stripping ignore text.
-                update_obj['previous_md5'] = "reprocess previous"
+                if len( datastore.data['watching'][uuid]['history']):
+                    update_obj['previous_md5'] = get_current_checksum_include_ignore_text(uuid=uuid)
+
 
-            update_obj['ignore_text'] = ignore_text
 
             validators.url(url)  # @todo switch to prop/attr/observer
             datastore.data['watching'][uuid].update(update_obj)
@@ -255,8 +286,8 @@ def changedetection_app(config=None, datastore_o=None):
 
             messages.append({'class': 'ok', 'message': "{} Imported, {} Skipped.".format(good, len(remaining_urls))})
 
-        if len(remaining_urls) == 0:
-            return redirect(url_for('index'))
+            if len(remaining_urls) == 0:
+                return redirect(url_for('index'))
         else:
             output = render_template("import.html",
                                      messages=messages,
diff --git a/backend/fetch_site_status.py b/backend/fetch_site_status.py
index 0083d892..0370a754 100644
--- a/backend/fetch_site_status.py
+++ b/backend/fetch_site_status.py
@@ -27,6 +27,8 @@ class perform_site_check():
 
         return "\n".encode('utf8').join(output)
 
+
+
     def run(self, uuid):
         timestamp = int(time.time())  # used for storage etc too
         stripped_text_from_html = False
@@ -98,27 +100,10 @@ class perform_site_check():
                 content = self.strip_ignore_text(stripped_text_from_html,
                                                  self.datastore.data['watching'][uuid]['ignore_text'])
             else:
-                content = stripped_text_from_html
+                content = stripped_text_from_html.encode('utf8')
 
             fetched_md5 = hashlib.md5(content).hexdigest()
 
-            # If they edited an existing watch, we need to know to reset the current/previous md5 to include
-            # the excluded text.
-
-            if self.datastore.data['watching'][uuid]['previous_md5'] == "reprocess previous":
-                # Get the most recent one
-                newest_history_key = self.datastore.get_newest_history_key(uuid)
-                if newest_history_key:
-                    with open(self.datastore.data['watching'][uuid]['history'][newest_history_key],
-                              encoding='utf-8') as file:
-                        raw_content = file.read()
-
-                        stripped_content = self.strip_ignore_text(raw_content,
-                                                                  self.datastore.data['watching'][uuid]['ignore_text'])
-
-                        checksum = hashlib.md5(stripped_content).hexdigest()
-                        self.datastore.data['watching'][uuid]['previous_md5'] = checksum
-
             # could be None or False depending on JSON type
             if self.datastore.data['watching'][uuid]['previous_md5'] != fetched_md5:
                 changed_detected = True
diff --git a/backend/tests/test_backend.py b/backend/tests/test_backend.py
index 75a01112..c5fd46a9 100644
--- a/backend/tests/test_backend.py
+++ b/backend/tests/test_backend.py
@@ -118,5 +118,6 @@ def test_check_basic_change_detection_functionality(client, live_server):
     res = client.get(url_for("index"))
     assert b'unviewed' in res.data
 
+    # Cleanup everything
     res = client.get(url_for("api_delete", uuid="all"), follow_redirects=True)
     assert b'Deleted' in res.data

From 30d53c353f46a194e8f622c407a6394865a2b206 Mon Sep 17 00:00:00 2001
From: Leigh Morresi <275001+dgtlmoon@users.noreply.github.com>
Date: Sat, 27 Feb 2021 22:09:25 +0100
Subject: [PATCH 13/28] Tweak to tests

---
 backend/tests/test_ignore_text.py | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/backend/tests/test_ignore_text.py b/backend/tests/test_ignore_text.py
index eaaf8429..29ad7d06 100644
--- a/backend/tests/test_ignore_text.py
+++ b/backend/tests/test_ignore_text.py
@@ -96,6 +96,12 @@ def test_check_ignore_text_functionality(client, live_server):
     )
     assert b"1 Imported" in res.data
 
+    # Trigger a check
+    client.get(url_for("api_watch_checknow"), follow_redirects=True)
+
+    # Give the thread time to pick it up
+    time.sleep(sleep_time_for_fetch_thread)
+
     # Goto the edit page, add our ignore text
     # Add our URL to the import page
     res = client.post(

From cd6465f844e0325272a52b408b158fe5d4a7c681 Mon Sep 17 00:00:00 2001
From: Leigh Morresi <275001+dgtlmoon@users.noreply.github.com>
Date: Sat, 27 Feb 2021 22:49:56 +0100
Subject: [PATCH 14/28] next dev is 0.27

---
 backend/store.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/backend/store.py b/backend/store.py
index a3bdb964..d463fa6d 100644
--- a/backend/store.py
+++ b/backend/store.py
@@ -25,7 +25,7 @@ class ChangeDetectionStore:
         self.__data = {
             'note': "Hello! If you change this file manually, please be sure to restart your changedetection.io instance!",
             'watching': {},
-            'tag': '0.261',
+            'tag': '0.27',
             'settings': {
                 'headers': {
                     'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.66 Safari/537.36',

From e3e36b3cef612441d05be09ba1b830aab8475c59 Mon Sep 17 00:00:00 2001
From: Leigh Morresi <275001+dgtlmoon@users.noreply.github.com>
Date: Sat, 27 Feb 2021 23:20:40 +0100
Subject: [PATCH 15/28] Always override tag version (load from disk in future,
 so we can add it at build time)

---
 backend/store.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/backend/store.py b/backend/store.py
index d463fa6d..71a1867e 100644
--- a/backend/store.py
+++ b/backend/store.py
@@ -25,7 +25,6 @@ class ChangeDetectionStore:
         self.__data = {
             'note': "Hello! If you change this file manually, please be sure to restart your changedetection.io instance!",
             'watching': {},
-            'tag': '0.27',
             'settings': {
                 'headers': {
                     'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.66 Safari/537.36',
@@ -80,6 +79,8 @@ class ChangeDetectionStore:
                     if 'requests' in from_disk['settings']:
                         self.__data['settings']['requests'].update(from_disk['settings']['requests'])
 
+                self.__data['tag'] = "0.27"
+
                 # Reinitialise each `watching` with our generic_definition in the case that we add a new var in the future.
                 # @todo pretty sure theres a python we todo this with an abstracted(?) object!
 

From a82e9243a637c8c63703342908cd4049a05100bb Mon Sep 17 00:00:00 2001
From: Leigh Morresi <275001+dgtlmoon@users.noreply.github.com>
Date: Mon, 1 Mar 2021 11:25:04 +0100
Subject: [PATCH 16/28] Issue #7 - RSS feeds

---
 backend/__init__.py                   | 57 +++++++++++++++++++++++----
 backend/static/css/styles.css         | 11 ++++--
 backend/store.py                      | 16 +++++++-
 backend/templates/watch-overview.html | 17 +++++---
 4 files changed, 84 insertions(+), 17 deletions(-)

diff --git a/backend/__init__.py b/backend/__init__.py
index 5e8adddd..9b4ca4e2 100644
--- a/backend/__init__.py
+++ b/backend/__init__.py
@@ -23,6 +23,11 @@ import queue
 
 from flask import Flask, render_template, request, send_file, send_from_directory,  abort, redirect, url_for
 
+from feedgen.feed import FeedGenerator
+from flask import make_response
+import datetime
+import pytz
+
 datastore = None
 
 # Local
@@ -112,14 +117,41 @@ def changedetection_app(config=None, datastore_o=None):
         sorted_watches.sort(key=lambda x: x['last_changed'], reverse=True)
 
         existing_tags = datastore.get_all_tags()
-        output = render_template("watch-overview.html",
-                                 watches=sorted_watches,
-                                 messages=messages,
-                                 tags=existing_tags,
-                                 active_tag=limit_tag)
+        rss = request.args.get('rss')
+
+        if rss:
+            fg = FeedGenerator()
+            fg.title('changedetection.io')
+            fg.description('Feed description')
+            fg.link(href='https://changedetection.io')
+
+            for watch in sorted_watches:
+                if watch['unviewed']:
+                    fe = fg.add_entry()
+                    fe.title(watch['url'])
+                    fe.link(href=watch['url'])
+                    fe.description(watch['url'])
+                    fe.guid(watch['uuid'], permalink=False)
+                    dt = datetime.datetime.fromtimestamp(int(watch['newest_history_key']))
+                    dt = dt.replace(tzinfo=pytz.UTC)
+                    fe.pubDate(dt)
+
+            response = make_response(fg.rss_str())
+            response.headers.set('Content-Type', 'application/rss+xml')
+            return response
+
+        else:
+            output = render_template("watch-overview.html",
+                                     watches=sorted_watches,
+                                     messages=messages,
+                                     tags=existing_tags,
+                                     active_tag=limit_tag,
+                                     has_unviewed=datastore.data['has_unviewed'])
+
+            # Show messages but once.
+            messages = []
+
 
-        # Show messages but once.
-        messages = []
         return output
 
     @app.route("/scrub", methods=['GET', 'POST'])
@@ -296,6 +328,17 @@ def changedetection_app(config=None, datastore_o=None):
             messages = []
         return output
 
+    # Clear all statuses, so we do not see the 'unviewed' class
+    @app.route("/api/mark-all-viewed", methods=['GET'])
+    def mark_all_viewed():
+
+        # Save the current newest history as the most recently viewed
+        for watch_uuid, watch in datastore.data['watching'].items():
+            datastore.set_last_viewed(watch_uuid, watch['newest_history_key'])
+
+        messages.append({'class': 'ok', 'message': "Cleared all statuses."})
+        return redirect(url_for('index'))
+
     @app.route("/diff/<string:uuid>", methods=['GET'])
     def diff_history_page(uuid):
         global messages
diff --git a/backend/static/css/styles.css b/backend/static/css/styles.css
index 8f3c0dbf..9dd75dbb 100644
--- a/backend/static/css/styles.css
+++ b/backend/static/css/styles.css
@@ -88,11 +88,16 @@ section.content {
   margin: 0 3px 0 5px;
 }
 
-#check-all-button {
-  text-align:right;
+#post-list-buttons {
+  text-align: right;
+  padding: 0px;
+  margin: 0px;
+}
+#post-list-buttons li {
+  display: inline-block;
 }
 
-#check-all-button a {
+#post-list-buttons a {
    border-top-left-radius: initial;
    border-top-right-radius: initial;
    border-bottom-left-radius: 5px;
diff --git a/backend/store.py b/backend/store.py
index 71a1867e..0d1be2e9 100644
--- a/backend/store.py
+++ b/backend/store.py
@@ -120,7 +120,7 @@ class ChangeDetectionStore:
         return 0
 
     def set_last_viewed(self, uuid, timestamp):
-        self.data['watching'][uuid].update({'last_viewed': str(timestamp)})
+        self.data['watching'][uuid].update({'last_viewed': int(timestamp)})
         self.needs_write = True
 
     def update_watch(self, uuid, update_obj):
@@ -141,6 +141,20 @@ class ChangeDetectionStore:
 
     @property
     def data(self):
+
+        has_unviewed = False
+
+        for uuid, v in self.__data['watching'].items():
+            self.__data['watching'][uuid]['newest_history_key'] = self.get_newest_history_key(uuid)
+            if int(v['newest_history_key']) <= int(v['last_viewed']):
+                self.__data['watching'][uuid]['viewed'] = True
+
+            else:
+                self.__data['watching'][uuid]['viewed'] = False
+                has_unviewed = True
+
+        self.__data['has_unviewed'] = has_unviewed
+
         return self.__data
 
     def get_all_tags(self):
diff --git a/backend/templates/watch-overview.html b/backend/templates/watch-overview.html
index ba18d323..1cb044da 100644
--- a/backend/templates/watch-overview.html
+++ b/backend/templates/watch-overview.html
@@ -43,8 +43,7 @@
             <tr id="{{ watch.uuid }}"
                 class="{{ loop.cycle('pure-table-odd', 'pure-table-even') }}
                 {% if watch.last_error is defined and watch.last_error != False %}error{% endif %}
-                {% if watch.newest_history_key| int > watch.last_viewed| int %}unviewed{% endif %}
-                ">
+                {% if watch.newest_history_key| int > watch.last_viewed| int %}unviewed{% endif %}">
                 <td>{{ loop.index }}</td>
                 <td class="title-col">{{watch.title if watch.title is not none else watch.url}}
                     <a class="external" target=_blank href="{{ watch.url }}"></a>
@@ -76,11 +75,17 @@
 
             </tbody>
         </table>
-        <div id="check-all-button">
-
-            <a href="/api/checknow{% if active_tag%}?tag={{active_tag}}{%endif%}" class="pure-button button-tag ">Recheck
+        <ul id="post-list-buttons">
+            {% if has_unviewed %}
+            <li>
+                <a href="/api/mark-all-viewed" class="pure-button button-tag ">Mark all viewed</a>
+            </li>
+            {% endif %}
+            <li>
+               <a href="/api/checknow{% if active_tag%}?tag={{active_tag}}{%endif%}" class="pure-button button-tag ">Recheck
                 all {% if active_tag%}in "{{active_tag}}"{%endif%}</a>
-        </div>
+            </li>
+        </ul>
     </div>
 </div>
 {% endblock %}
\ No newline at end of file

From b1fb47e689706ca8825bddf60ac89ff5b5037db4 Mon Sep 17 00:00:00 2001
From: Leigh Morresi <275001+dgtlmoon@users.noreply.github.com>
Date: Mon, 1 Mar 2021 11:51:28 +0100
Subject: [PATCH 17/28] Add icon for RSS, RSS should show only unviewed entries

---
 backend/__init__.py                   |  2 +-
 backend/static/css/styles.css         |  4 ++++
 backend/templates/watch-overview.html | 11 ++++-------
 3 files changed, 9 insertions(+), 8 deletions(-)

diff --git a/backend/__init__.py b/backend/__init__.py
index 9b4ca4e2..791f4a83 100644
--- a/backend/__init__.py
+++ b/backend/__init__.py
@@ -126,7 +126,7 @@ def changedetection_app(config=None, datastore_o=None):
             fg.link(href='https://changedetection.io')
 
             for watch in sorted_watches:
-                if watch['unviewed']:
+                if not watch['viewed']:
                     fe = fg.add_entry()
                     fe.title(watch['url'])
                     fe.link(href=watch['url'])
diff --git a/backend/static/css/styles.css b/backend/static/css/styles.css
index 9dd75dbb..180ac5ef 100644
--- a/backend/static/css/styles.css
+++ b/backend/static/css/styles.css
@@ -248,4 +248,8 @@ footer {
     background: #fff;
     color: #444;
     text-align: center;
+}
+
+#feed-icon {
+    vertical-align: middle;
 }
\ No newline at end of file
diff --git a/backend/templates/watch-overview.html b/backend/templates/watch-overview.html
index 1cb044da..a2cd8c19 100644
--- a/backend/templates/watch-overview.html
+++ b/backend/templates/watch-overview.html
@@ -15,13 +15,9 @@
         <!-- user/pass r = requests.get('https://api.github.com/user', auth=('user', 'pass')) -->
     </form>
     <div>
-
+        <a href="/" class="pure-button button-tag {{'active' if not active_tag }}">All</a>
         {% for tag in tags %}
-        {% if tag == "" %}
-        <a href="/" class="pure-button button-tag {{'active' if active_tag == tag }}">All</a>
-        {% else %}
         <a href="/?tag={{ tag}}" class="pure-button button-tag {{'active' if active_tag == tag }}">{{ tag }}</a>
-        {% endif %}
         {% endfor %}
     </div>
 
@@ -71,8 +67,6 @@
                 </td>
             </tr>
             {% endfor %}
-
-
             </tbody>
         </table>
         <ul id="post-list-buttons">
@@ -85,6 +79,9 @@
                <a href="/api/checknow{% if active_tag%}?tag={{active_tag}}{%endif%}" class="pure-button button-tag ">Recheck
                 all {% if active_tag%}in "{{active_tag}}"{%endif%}</a>
             </li>
+            <li>
+                <a href="{{ url_for('index', tag=active_tag , rss=true)}}"><img id="feed-icon" src="/static/images/Generic_Feed-icon.svg" height="15px"></a>
+            </li>
         </ul>
     </div>
 </div>

From b0975694c82fdb2b0b95ce34f99ad27fbd7bd7a2 Mon Sep 17 00:00:00 2001
From: Leigh Morresi <275001+dgtlmoon@users.noreply.github.com>
Date: Mon, 1 Mar 2021 11:52:29 +0100
Subject: [PATCH 18/28] Remove todos

---
 backend/__init__.py | 2 --
 1 file changed, 2 deletions(-)

diff --git a/backend/__init__.py b/backend/__init__.py
index 791f4a83..95b915b6 100644
--- a/backend/__init__.py
+++ b/backend/__init__.py
@@ -2,10 +2,8 @@
 
 
 # @todo logging
-# @todo sort by last_changed
 # @todo extra options for url like , verify=False etc.
 # @todo enable https://urllib3.readthedocs.io/en/latest/user-guide.html#ssl as option?
-# @todo maybe a button to reset all 'last-changed'.. so you can see it clearly when something happens since your last visit
 # @todo option for interval day/6 hour/etc
 # @todo on change detected, config for calling some API
 # @todo make tables responsive!

From 187523d8d62d491c74ff3ac9c20f2bafe0b0a076 Mon Sep 17 00:00:00 2001
From: Leigh Morresi <275001+dgtlmoon@users.noreply.github.com>
Date: Mon, 1 Mar 2021 12:45:56 +0100
Subject: [PATCH 19/28] Add missing dep

---
 requirements.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/requirements.txt b/requirements.txt
index 34b693a9..60c8dbb6 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -8,4 +8,4 @@ requests
 validators
 timeago ~=1.0
 inscriptis ~= 1.1
-
+feedgen ~= 0.9
\ No newline at end of file

From e0578acca2d60b71f63ae87bb16a954374c8bf78 Mon Sep 17 00:00:00 2001
From: Leigh Morresi <275001+dgtlmoon@users.noreply.github.com>
Date: Mon, 1 Mar 2021 14:29:21 +0100
Subject: [PATCH 20/28] Tidy up thread logic and version check

---
 backend/__init__.py          | 60 +++++++++++++++++++++++++-----------
 backend/fetch_site_status.py |  3 +-
 backend/store.py             | 11 +++++--
 backend/templates/base.html  |  4 ++-
 backend/tests/conftest.py    |  3 +-
 5 files changed, 58 insertions(+), 23 deletions(-)

diff --git a/backend/__init__.py b/backend/__init__.py
index 95b915b6..bc25d412 100644
--- a/backend/__init__.py
+++ b/backend/__init__.py
@@ -17,6 +17,8 @@ import os
 import timeago
 
 import threading
+from threading import Event
+
 import queue
 
 from flask import Flask, render_template, request, send_file, send_from_directory,  abort, redirect, url_for
@@ -42,7 +44,9 @@ app = Flask(__name__, static_url_path="/var/www/change-detection/backen/static")
 # Stop browser caching of assets
 app.config['SEND_FILE_MAX_AGE_DEFAULT'] = 0
 
-app.config['STOP_THREADS'] = False
+app.config.exit = Event()
+
+app.config['NEW_VERSION_AVAILABLE'] = False
 
 # Disables caching of the templates
 app.config['TEMPLATES_AUTO_RELOAD'] = True
@@ -79,10 +83,12 @@ def _jinja2_filter_datetimestamp(timestamp, format="%Y-%m-%d %H:%M:%S"):
 def changedetection_app(config=None, datastore_o=None):
     global datastore
     datastore = datastore_o
-    # Hmm
+
     app.config.update(dict(DEBUG=True))
     app.config.update(config or {})
 
+
+
     # Setup cors headers to allow all domains
     # https://flask-cors.readthedocs.io/en/latest/
     #    CORS(app)
@@ -90,6 +96,7 @@ def changedetection_app(config=None, datastore_o=None):
     # https://github.com/pallets/flask/blob/93dd1709d05a1cf0e886df6223377bdab3b077fb/examples/tutorial/flaskr/__init__.py#L39
     # You can divide up the stuff like this
 
+
     @app.route("/", methods=['GET'])
     def index():
         global messages
@@ -318,12 +325,14 @@ def changedetection_app(config=None, datastore_o=None):
 
             if len(remaining_urls) == 0:
                 return redirect(url_for('index'))
+            #@todo repair
         else:
             output = render_template("import.html",
                                      messages=messages,
                                      remaining="\n".join(remaining_urls)
                                      )
             messages = []
+
         return output
 
     # Clear all statuses, so we do not see the 'unviewed' class
@@ -501,9 +510,27 @@ def changedetection_app(config=None, datastore_o=None):
     # @todo handle ctrl break
     ticker_thread = threading.Thread(target=ticker_thread_check_time_launch_checks).start()
 
+    # Check for new release version
+    threading.Thread(target=check_for_new_version).start()
     return app
 
 
+# Check for new version and anonymous stats
+def check_for_new_version():
+    import requests
+    app.config['NEW_VERSION_AVAILABLE'] = True
+    import urllib3
+    urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
+
+    while not app.config.exit.is_set():
+        r = requests.post("https://changedetection.io/check-ver.php",
+                          data={'version': datastore.data['version_tag'],
+                                'app_guid': datastore.data['app_guid']},
+
+                          verify=False)
+
+        app.config.exit.wait(10)
+
 # Requests for checking on the site use a pool of thread Workers managed by a Queue.
 class Worker(threading.Thread):
     current_uuid = None
@@ -517,16 +544,13 @@ class Worker(threading.Thread):
 
         update_handler = fetch_site_status.perform_site_check(datastore=datastore)
 
-        while True:
+        while not app.config.exit.is_set():
 
             try:
-                uuid = self.q.get(block=True, timeout=1)
+                uuid = self.q.get(block=False)
             except queue.Empty:
-                # We have a chance to kill this thread that needs to monitor for new jobs..
-                # Delays here would be caused by a current response object pending
-                # @todo switch to threaded response handler
-                if app.config['STOP_THREADS']:
-                    return
+                pass
+
             else:
                 self.current_uuid = uuid
 
@@ -549,9 +573,13 @@ class Worker(threading.Thread):
                 self.current_uuid = None  # Done
                 self.q.task_done()
 
+            app.config.exit.wait(1)
+
+
 
 # Thread runner to check every minute, look for new watches to feed into the Queue.
 def ticker_thread_check_time_launch_checks():
+
     # Spin up Workers.
     for _ in range(datastore.data['settings']['requests']['workers']):
         new_worker = Worker(update_q)
@@ -559,23 +587,19 @@ def ticker_thread_check_time_launch_checks():
         new_worker.start()
 
     # Every minute check for new UUIDs to follow up on
-    while True:
-
-        if app.config['STOP_THREADS']:
-            return
+    minutes = datastore.data['settings']['requests']['minutes_between_check']
 
+    while not app.config.exit.is_set():
         running_uuids = []
         for t in running_update_threads:
             running_uuids.append(t.current_uuid)
 
         # Look at the dataset, find a stale watch to process
-        minutes = datastore.data['settings']['requests']['minutes_between_check']
+        threshold = time.time() - (minutes * 60)
         for uuid, watch in datastore.data['watching'].items():
-            if watch['last_checked'] <= time.time() - (minutes * 60):
-
-                # @todo maybe update_q.queue is enough?
+            if watch['last_checked'] <= threshold:
                 if not uuid in running_uuids and uuid not in update_q.queue:
                     update_q.put(uuid)
 
         # Should be low so we can break this out in testing
-        time.sleep(1)
+        app.config.exit.wait(1)
diff --git a/backend/fetch_site_status.py b/backend/fetch_site_status.py
index 0370a754..9fb52c75 100644
--- a/backend/fetch_site_status.py
+++ b/backend/fetch_site_status.py
@@ -2,7 +2,8 @@ import time
 import requests
 import hashlib
 from inscriptis import get_text
-
+import urllib3
+urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
 
 # Some common stuff here that can be moved to a base class
 class perform_site_check():
diff --git a/backend/store.py b/backend/store.py
index 0d1be2e9..34d953bf 100644
--- a/backend/store.py
+++ b/backend/store.py
@@ -22,6 +22,7 @@ class ChangeDetectionStore:
         self.datastore_path = datastore_path
         self.json_store_path = "{}/url-watches.json".format(self.datastore_path)
         self.stop_thread = False
+
         self.__data = {
             'note': "Hello! If you change this file manually, please be sure to restart your changedetection.io instance!",
             'watching': {},
@@ -79,8 +80,6 @@ class ChangeDetectionStore:
                     if 'requests' in from_disk['settings']:
                         self.__data['settings']['requests'].update(from_disk['settings']['requests'])
 
-                self.__data['tag'] = "0.27"
-
                 # Reinitialise each `watching` with our generic_definition in the case that we add a new var in the future.
                 # @todo pretty sure theres a python we todo this with an abstracted(?) object!
 
@@ -101,6 +100,14 @@ class ChangeDetectionStore:
                 self.add_watch(url='https://www.gov.uk/coronavirus', tag='Covid')
                 self.add_watch(url='https://changedetection.io', tag='Tech news')
 
+
+        self.__data['version_tag'] = "0.27"
+
+        if not 'app_guid' in self.__data:
+            self.__data['app_guid'] = str(uuid_builder.uuid4())
+
+        self.needs_write = True
+
         # Finally start the thread that will manage periodic data saves to JSON
         save_data_thread = threading.Thread(target=self.save_datastore).start()
 
diff --git a/backend/templates/base.html b/backend/templates/base.html
index e5d258c7..f9e89e06 100644
--- a/backend/templates/base.html
+++ b/backend/templates/base.html
@@ -21,7 +21,9 @@
         {% if current_diff_url %}
             <a class=current-diff-url href="{{ current_diff_url }}"><span style="max-width: 30%; overflow: hidden;">{{ current_diff_url }}</a>
         {% else %}
-            <span id="version-text" class="pure-menu-heading">Version {{ version }}</span>
+            {% if new_version_available %}
+              <span id="version-text" class="pure-menu-heading">A new version is available</span>
+            {% endif %}
         {% endif %}
 
         <ul class="pure-menu-list">
diff --git a/backend/tests/conftest.py b/backend/tests/conftest.py
index f026f22c..836e0fd1 100644
--- a/backend/tests/conftest.py
+++ b/backend/tests/conftest.py
@@ -30,10 +30,11 @@ def app(request):
     app_config = {'datastore_path': datastore_path}
     datastore = store.ChangeDetectionStore(datastore_path=app_config['datastore_path'], include_default_watches=False)
     app = changedetection_app(app_config, datastore)
+    app.config['STOP_THREADS'] = True
 
     def teardown():
         datastore.stop_thread = True
-        app.config['STOP_THREADS'] = True
+        app.config.exit.set()
         try:
             os.unlink("{}/url-watches.json".format(datastore_path))
         except FileNotFoundError:

From fb38b06eae0de49b58e5b18f70dd4fa97123cfc9 Mon Sep 17 00:00:00 2001
From: Leigh Morresi <275001+dgtlmoon@users.noreply.github.com>
Date: Mon, 1 Mar 2021 14:31:45 +0100
Subject: [PATCH 21/28] Code tidy/lint

---
 backend/__init__.py | 24 ++++++++----------------
 1 file changed, 8 insertions(+), 16 deletions(-)

diff --git a/backend/__init__.py b/backend/__init__.py
index bc25d412..246fdc9d 100644
--- a/backend/__init__.py
+++ b/backend/__init__.py
@@ -21,7 +21,7 @@ from threading import Event
 
 import queue
 
-from flask import Flask, render_template, request, send_file, send_from_directory,  abort, redirect, url_for
+from flask import Flask, render_template, request, send_file, send_from_directory, abort, redirect, url_for
 
 from feedgen.feed import FeedGenerator
 from flask import make_response
@@ -87,8 +87,6 @@ def changedetection_app(config=None, datastore_o=None):
     app.config.update(dict(DEBUG=True))
     app.config.update(config or {})
 
-
-
     # Setup cors headers to allow all domains
     # https://flask-cors.readthedocs.io/en/latest/
     #    CORS(app)
@@ -96,7 +94,6 @@ def changedetection_app(config=None, datastore_o=None):
     # https://github.com/pallets/flask/blob/93dd1709d05a1cf0e886df6223377bdab3b077fb/examples/tutorial/flaskr/__init__.py#L39
     # You can divide up the stuff like this
 
-
     @app.route("/", methods=['GET'])
     def index():
         global messages
@@ -156,7 +153,6 @@ def changedetection_app(config=None, datastore_o=None):
             # Show messages but once.
             messages = []
 
-
         return output
 
     @app.route("/scrub", methods=['GET', 'POST'])
@@ -209,7 +205,7 @@ def changedetection_app(config=None, datastore_o=None):
 
                 handler = fetch_site_status.perform_site_check(datastore=datastore)
                 stripped_content = handler.strip_ignore_text(raw_content,
-                                                          datastore.data['watching'][uuid]['ignore_text'])
+                                                             datastore.data['watching'][uuid]['ignore_text'])
 
                 checksum = hashlib.md5(stripped_content).hexdigest()
                 return checksum
@@ -257,11 +253,9 @@ def changedetection_app(config=None, datastore_o=None):
                 datastore.data['watching'][uuid]['ignore_text'] = ignore_text
 
                 # Reset the previous_md5 so we process a new snapshot including stripping ignore text.
-                if len( datastore.data['watching'][uuid]['history']):
+                if len(datastore.data['watching'][uuid]['history']):
                     update_obj['previous_md5'] = get_current_checksum_include_ignore_text(uuid=uuid)
 
-
-
             validators.url(url)  # @todo switch to prop/attr/observer
             datastore.data['watching'][uuid].update(update_obj)
             datastore.needs_write = True
@@ -325,7 +319,7 @@ def changedetection_app(config=None, datastore_o=None):
 
             if len(remaining_urls) == 0:
                 return redirect(url_for('index'))
-            #@todo repair
+            # @todo repair
         else:
             output = render_template("import.html",
                                      messages=messages,
@@ -352,7 +346,7 @@ def changedetection_app(config=None, datastore_o=None):
 
         # More for testing, possible to return the first/only
         if uuid == 'first':
-            uuid= list(datastore.data['watching'].keys()).pop()
+            uuid = list(datastore.data['watching'].keys()).pop()
 
         extra_stylesheets = ['/static/css/diff.css']
         try:
@@ -367,9 +361,9 @@ def changedetection_app(config=None, datastore_o=None):
         dates.sort(reverse=True)
         dates = [str(i) for i in dates]
 
-
         if len(dates) < 2:
-            messages.append({'class': 'error', 'message': "Not enough saved change detection snapshots to produce a report."})
+            messages.append(
+                {'class': 'error', 'message': "Not enough saved change detection snapshots to produce a report."})
             return redirect(url_for('index'))
 
         # Save the current newest history as the most recently viewed
@@ -531,6 +525,7 @@ def check_for_new_version():
 
         app.config.exit.wait(10)
 
+
 # Requests for checking on the site use a pool of thread Workers managed by a Queue.
 class Worker(threading.Thread):
     current_uuid = None
@@ -569,17 +564,14 @@ class Worker(threading.Thread):
                                 # A change was detected
                                 datastore.save_history_text(uuid=uuid, contents=contents, result_obj=result)
 
-
                 self.current_uuid = None  # Done
                 self.q.task_done()
 
             app.config.exit.wait(1)
 
 
-
 # Thread runner to check every minute, look for new watches to feed into the Queue.
 def ticker_thread_check_time_launch_checks():
-
     # Spin up Workers.
     for _ in range(datastore.data['settings']['requests']['workers']):
         new_worker = Worker(update_q)

From 5b2fda1a6e4176d51dff886cd0fb1ab96797366b Mon Sep 17 00:00:00 2001
From: Leigh Morresi <275001+dgtlmoon@users.noreply.github.com>
Date: Mon, 1 Mar 2021 14:33:25 +0100
Subject: [PATCH 22/28] Fix import form flow logic

---
 backend/__init__.py | 15 ++++++++-------
 1 file changed, 8 insertions(+), 7 deletions(-)

diff --git a/backend/__init__.py b/backend/__init__.py
index 246fdc9d..06744202 100644
--- a/backend/__init__.py
+++ b/backend/__init__.py
@@ -318,14 +318,15 @@ def changedetection_app(config=None, datastore_o=None):
             messages.append({'class': 'ok', 'message': "{} Imported, {} Skipped.".format(good, len(remaining_urls))})
 
             if len(remaining_urls) == 0:
+                # Looking good, redirect to index.
                 return redirect(url_for('index'))
-            # @todo repair
-        else:
-            output = render_template("import.html",
-                                     messages=messages,
-                                     remaining="\n".join(remaining_urls)
-                                     )
-            messages = []
+
+        # Could be some remaining, or we could be on GET
+        output = render_template("import.html",
+                                 messages=messages,
+                                 remaining="\n".join(remaining_urls)
+                                 )
+        messages = []
 
         return output
 

From b31bf3489034ea3327f7a645bcda9a08a57f4712 Mon Sep 17 00:00:00 2001
From: Leigh Morresi <275001+dgtlmoon@users.noreply.github.com>
Date: Mon, 1 Mar 2021 15:09:37 +0100
Subject: [PATCH 23/28] Check for new version

---
 README.md                     |  2 +-
 backend/__init__.py           | 22 ++++++++++++++++------
 backend/static/css/styles.css | 15 ++++++++++++++-
 backend/templates/base.html   | 13 +++++++------
 changedetection.py            |  6 +++++-
 5 files changed, 43 insertions(+), 15 deletions(-)

diff --git a/README.md b/README.md
index 3dc4736d..c76996d9 100644
--- a/README.md
+++ b/README.md
@@ -4,7 +4,7 @@
   <img src="https://img.shields.io/docker/pulls/dgtlmoon/changedetection.io" alt="Docker Pulls"/>
 </a>
 <a href="https://hub.docker.com/r/dgtlmoon/changedetection.io" target="_blank" title="Change detection docker hub">
-  <img src="https://img.shields.io/docker/v/dgtlmoon/changedetection.io" alt="Change detection latest tag version"/> 
+  <img src="https://img.shields.io/docker/v/dgtlmoon/changedetection.io/0.27" alt="Change detection latest tag version"/> 
 </a>
 
 ## Self-hosted change monitoring of web pages.
diff --git a/backend/__init__.py b/backend/__init__.py
index 06744202..00b0ba81 100644
--- a/backend/__init__.py
+++ b/backend/__init__.py
@@ -513,18 +513,28 @@ def changedetection_app(config=None, datastore_o=None):
 # Check for new version and anonymous stats
 def check_for_new_version():
     import requests
-    app.config['NEW_VERSION_AVAILABLE'] = True
+
     import urllib3
     urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
 
     while not app.config.exit.is_set():
-        r = requests.post("https://changedetection.io/check-ver.php",
-                          data={'version': datastore.data['version_tag'],
-                                'app_guid': datastore.data['app_guid']},
+        try:
+            r = requests.post("https://changedetection.io/check-ver.php",
+                              data={'version': datastore.data['version_tag'],
+                                    'app_guid': datastore.data['app_guid']},
 
-                          verify=False)
+                              verify=False)
+        except:
+            pass
+
+        try:
+            if "new_version" in r.text:
+                app.config['NEW_VERSION_AVAILABLE'] = True
+        except:
+            pass
 
-        app.config.exit.wait(10)
+        # Check daily
+        app.config.exit.wait(86400)
 
 
 # Requests for checking on the site use a pool of thread Workers managed by a Queue.
diff --git a/backend/static/css/styles.css b/backend/static/css/styles.css
index 180ac5ef..da504eef 100644
--- a/backend/static/css/styles.css
+++ b/backend/static/css/styles.css
@@ -252,4 +252,17 @@ footer {
 
 #feed-icon {
     vertical-align: middle;
-}
\ No newline at end of file
+}
+
+#version {
+    position: absolute;
+    top: 80px;
+    right: 0px;
+    font-size: 8px;
+    background: #fff;
+    padding: 10px;
+}
+
+#new-version-text a{
+    color: #e07171;
+}
diff --git a/backend/templates/base.html b/backend/templates/base.html
index f9e89e06..03afb75a 100644
--- a/backend/templates/base.html
+++ b/backend/templates/base.html
@@ -19,11 +19,11 @@
     <div class="home-menu pure-menu pure-menu-horizontal pure-menu-fixed">
         <a class="pure-menu-heading" href="/"><strong>Change</strong>Detection.io</a>
         {% if current_diff_url %}
-            <a class=current-diff-url href="{{ current_diff_url }}"><span style="max-width: 30%; overflow: hidden;">{{ current_diff_url }}</a>
+        <a class=current-diff-url href="{{ current_diff_url }}"><span style="max-width: 30%; overflow: hidden;">{{ current_diff_url }}</span></a>
         {% else %}
-            {% if new_version_available %}
-              <span id="version-text" class="pure-menu-heading">A new version is available</span>
-            {% endif %}
+        {% if new_version_available %}
+        <span id="new-version-text" class="pure-menu-heading"><a href="https://github.com/dgtlmoon/changedetection.io">A new version is available</a></span>
+        {% endif %}
         {% endif %}
 
         <ul class="pure-menu-list">
@@ -38,7 +38,8 @@
                 <a href="/settings" class="pure-menu-link">SETTINGS</a>
             </li>
             <li class="pure-menu-item"><a class="github-link" href="https://github.com/dgtlmoon/changedetection.io">
-                <svg class="octicon octicon-mark-github v-align-middle" height="32" viewBox="0 0 16 16" version="1.1"
+                <svg class="octicon octicon-mark-github v-align-middle" height="32" viewBox="0 0 16 16"
+                     version="1.1"
                      width="32" aria-hidden="true">
                     <path fill-rule="evenodd"
                           d="M8 0C3.58 0 0 3.58 0 8c0 3.54 2.29 6.53 5.47 7.59.4.07.55-.17.55-.38 0-.19-.01-.82-.01-1.49-2.01.37-2.53-.49-2.69-.94-.09-.23-.48-.94-.82-1.13-.28-.15-.68-.52-.01-.53.63-.01 1.08.58 1.23.82.72 1.21 1.87.87 2.33.66.07-.52.28-.87.51-1.07-1.78-.2-3.64-.89-3.64-3.95 0-.87.31-1.59.82-2.15-.08-.2-.36-1.02.08-2.12 0 0 .67-.21 2.2.82.64-.18 1.32-.27 2-.27.68 0 1.36.09 2 .27 1.53-1.04 2.2-.82 2.2-.82.44 1.1.16 1.92.08 2.12.51.56.82 1.27.82 2.15 0 3.07-1.87 3.75-3.65 3.95.29.25.54.73.54 1.48 0 1.07-.01 1.93-.01 2.2 0 .21.15.46.55.38A8.013 8.013 0 0016 8c0-4.42-3.58-8-8-8z"></path>
@@ -51,7 +52,7 @@
         </ul>
     </div>
 </div>
-
+<div id="version">v{{ version }}</div>
 <section class="content">
     <header>
         {% block header %}{% endblock %}
diff --git a/changedetection.py b/changedetection.py
index f593548d..3a112c9c 100644
--- a/changedetection.py
+++ b/changedetection.py
@@ -52,7 +52,11 @@ def main(argv):
 
     @app.context_processor
     def inject_version():
-        return dict(version=datastore.data['tag'])
+        return dict(version=datastore.data['version_tag'])
+
+    @app.context_processor
+    def inject_new_version_available():
+        return dict(new_version_available=app.config['NEW_VERSION_AVAILABLE'])
 
     if ssl_mode:
         # @todo finalise SSL config, but this should get you in the right direction if you need it.

From 89134b5b6c1808ac782400dca7a3678f151930b4 Mon Sep 17 00:00:00 2001
From: Leigh Morresi <275001+dgtlmoon@users.noreply.github.com>
Date: Mon, 1 Mar 2021 15:11:03 +0100
Subject: [PATCH 24/28] Add missing pytz

---
 requirements.txt | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/requirements.txt b/requirements.txt
index 60c8dbb6..fa9d27c4 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -8,4 +8,5 @@ requests
 validators
 timeago ~=1.0
 inscriptis ~= 1.1
-feedgen ~= 0.9
\ No newline at end of file
+feedgen ~= 0.9
+pytz

From df96b8d76ca94dd5b21dd7bc3cb7bafeb3126568 Mon Sep 17 00:00:00 2001
From: Leigh Morresi <275001+dgtlmoon@users.noreply.github.com>
Date: Mon, 1 Mar 2021 15:21:15 +0100
Subject: [PATCH 25/28] Add missing urllib3

---
 requirements.txt | 1 +
 1 file changed, 1 insertion(+)

diff --git a/requirements.txt b/requirements.txt
index fa9d27c4..7121d639 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -10,3 +10,4 @@ timeago ~=1.0
 inscriptis ~= 1.1
 feedgen ~= 0.9
 pytz
+urllib3
\ No newline at end of file

From ff1b19cdb8fb72de86c922580d6c969310b7b381 Mon Sep 17 00:00:00 2001
From: Leigh Morresi <275001+dgtlmoon@users.noreply.github.com>
Date: Mon, 1 Mar 2021 15:32:59 +0100
Subject: [PATCH 26/28] Generic object sync should use private method

---
 backend/store.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/backend/store.py b/backend/store.py
index 34d953bf..499f2bce 100644
--- a/backend/store.py
+++ b/backend/store.py
@@ -82,8 +82,7 @@ class ChangeDetectionStore:
 
                 # Reinitialise each `watching` with our generic_definition in the case that we add a new var in the future.
                 # @todo pretty sure theres a python we todo this with an abstracted(?) object!
-
-                for uuid, watch in self.data['watching'].items():
+                for uuid, watch in self.__data['watching'].items():
                     _blank = deepcopy(self.generic_definition)
                     _blank.update(watch)
                     self.__data['watching'].update({uuid: _blank})

From c5c3e8c6c26af01114f430ef9af71c0daeda69cd Mon Sep 17 00:00:00 2001
From: Leigh Morresi <275001+dgtlmoon@users.noreply.github.com>
Date: Mon, 1 Mar 2021 15:39:36 +0100
Subject: [PATCH 27/28] Adding RSS feed icon

---
 backend/static/images/Generic_Feed-icon.svg | 18 ++++++++++++++++++
 1 file changed, 18 insertions(+)
 create mode 100644 backend/static/images/Generic_Feed-icon.svg

diff --git a/backend/static/images/Generic_Feed-icon.svg b/backend/static/images/Generic_Feed-icon.svg
new file mode 100644
index 00000000..a7f9cf19
--- /dev/null
+++ b/backend/static/images/Generic_Feed-icon.svg
@@ -0,0 +1,18 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<svg xmlns="http://www.w3.org/2000/svg"
+     id="RSSicon"
+     viewBox="0 0 8 8" width="256" height="256">
+
+  <title>RSS feed icon</title>
+
+  <style type="text/css">
+    .button {stroke: none; fill: orange;}
+    .symbol {stroke: none; fill: white;}
+  </style>
+
+  <rect   class="button" width="8" height="8" rx="1.5" />
+  <circle class="symbol" cx="2" cy="6" r="1" />
+  <path   class="symbol" d="m 1,4 a 3,3 0 0 1 3,3 h 1 a 4,4 0 0 0 -4,-4 z" />
+  <path   class="symbol" d="m 1,2 a 5,5 0 0 1 5,5 h 1 a 6,6 0 0 0 -6,-6 z" />
+
+</svg>
\ No newline at end of file

From 982a0d778187b57e5988b9c9baa9190016ea97bf Mon Sep 17 00:00:00 2001
From: Leigh Morresi <275001+dgtlmoon@users.noreply.github.com>
Date: Mon, 1 Mar 2021 15:44:34 +0100
Subject: [PATCH 28/28] Dont show 'empty' tag, it will be in the [ALL] list

---
 backend/templates/watch-overview.html | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/backend/templates/watch-overview.html b/backend/templates/watch-overview.html
index a2cd8c19..e17c4ffa 100644
--- a/backend/templates/watch-overview.html
+++ b/backend/templates/watch-overview.html
@@ -17,7 +17,9 @@
     <div>
         <a href="/" class="pure-button button-tag {{'active' if not active_tag }}">All</a>
         {% for tag in tags %}
-        <a href="/?tag={{ tag}}" class="pure-button button-tag {{'active' if active_tag == tag }}">{{ tag }}</a>
+            {% if tag != "" %}
+                <a href="/?tag={{ tag}}" class="pure-button button-tag {{'active' if active_tag == tag }}">{{ tag }}</a>
+            {% endif %}
         {% endfor %}
     </div>