From 148db6ea259d1b6466d1b5bddc35b142941f728a Mon Sep 17 00:00:00 2001
From: dgtlmoon <dgtlmoon@gmail.com>
Date: Mon, 24 Oct 2022 22:17:25 +0200
Subject: [PATCH] Re #1052 - Dynamic URLs

---
 changedetectionio/fetch_site_status.py |  8 ++++++-
 changedetectionio/templates/edit.html  |  3 ++-
 changedetectionio/tests/test_jinja2.py | 33 ++++++++++++++++++++++++++
 changedetectionio/tests/util.py        |  5 ++++
 requirements.txt                       |  4 ++++
 5 files changed, 51 insertions(+), 2 deletions(-)
 create mode 100644 changedetectionio/tests/test_jinja2.py

diff --git a/changedetectionio/fetch_site_status.py b/changedetectionio/fetch_site_status.py
index 0f84da16..76c821be 100644
--- a/changedetectionio/fetch_site_status.py
+++ b/changedetectionio/fetch_site_status.py
@@ -35,6 +35,8 @@ class perform_site_check():
 
 
     def run(self, uuid):
+        from jinja2 import Environment
+
         changed_detected = False
         screenshot = False  # as bytes
         stripped_text_from_html = ""
@@ -65,7 +67,11 @@ class perform_site_check():
             request_headers['Accept-Encoding'] = request_headers['Accept-Encoding'].replace(', br', '')
 
         timeout = self.datastore.data['settings']['requests'].get('timeout')
-        url = watch.get('url')
+
+        # Jinja2 available in URLs along with https://pypi.org/project/jinja2-time/
+        jinja2_env = Environment(extensions=['jinja2_time.TimeExtension'])
+        url = str(jinja2_env.from_string(watch.get('url')).render())
+
         request_body = self.datastore.data['watching'][uuid].get('body')
         request_method = self.datastore.data['watching'][uuid].get('method')
         ignore_status_codes = self.datastore.data['watching'][uuid].get('ignore_status_codes', False)
diff --git a/changedetectionio/templates/edit.html b/changedetectionio/templates/edit.html
index 59d95317..66286314 100644
--- a/changedetectionio/templates/edit.html
+++ b/changedetectionio/templates/edit.html
@@ -40,7 +40,8 @@
                 <fieldset>
                     <div class="pure-control-group">
                         {{ render_field(form.url, placeholder="https://...", required=true, class="m-d") }}
-                        <span class="pure-form-message-inline">Some sites use JavaScript to create the content, for this you should <a href="https://github.com/dgtlmoon/changedetection.io/wiki/Fetching-pages-with-WebDriver">use the Chrome/WebDriver Fetcher</a></span>
+                        <span class="pure-form-message-inline">Some sites use JavaScript to create the content, for this you should <a href="https://github.com/dgtlmoon/changedetection.io/wiki/Fetching-pages-with-WebDriver">use the Chrome/WebDriver Fetcher</a></span><br/>
+                        <span class="pure-form-message-inline">You can use variables in the URL, perfect for inserting the current date and other logic, <a href="https://github.com/dgtlmoon/changedetection.io/wiki/Handling-variables-in-the-watched-URL">help and examples here</a></span><br/>
                     </div>
                     <div class="pure-control-group">
                         {{ render_field(form.title, class="m-d") }}
diff --git a/changedetectionio/tests/test_jinja2.py b/changedetectionio/tests/test_jinja2.py
new file mode 100644
index 00000000..9c6baa9f
--- /dev/null
+++ b/changedetectionio/tests/test_jinja2.py
@@ -0,0 +1,33 @@
+#!/usr/bin/python3
+
+import time
+from flask import url_for
+from .util import live_server_setup
+
+
+# If there was only a change in the whitespacing, then we shouldnt have a change detected
+def test_jinja2_in_url_query(client, live_server):
+    live_server_setup(live_server)
+
+    # Give the endpoint time to spin up
+    time.sleep(1)
+
+    # Add our URL to the import page
+    test_url = url_for('test_return_query', _external=True)
+
+    # because url_for() will URL-encode the var, but we dont here
+    full_url = "{}?{}".format(test_url,
+                              "date={% now 'Europe/Berlin', '%Y' %}.{% now 'Europe/Berlin', '%m' %}.{% now 'Europe/Berlin', '%d' %}", )
+    res = client.post(
+        url_for("form_quick_watch_add"),
+        data={"url": full_url, "tag": "test"},
+        follow_redirects=True
+    )
+    assert b"Watch added" in res.data
+    time.sleep(3)
+    # It should report nothing found (no new 'unviewed' class)
+    res = client.get(
+        url_for("preview_page", uuid="first"),
+        follow_redirects=True
+    )
+    assert b'date=2' in res.data
diff --git a/changedetectionio/tests/util.py b/changedetectionio/tests/util.py
index e93d9a40..b943a606 100644
--- a/changedetectionio/tests/util.py
+++ b/changedetectionio/tests/util.py
@@ -159,5 +159,10 @@ def live_server_setup(live_server):
         ret = " ".join([auth.username, auth.password, auth.type])
         return ret
 
+    # Just return some GET var
+    @live_server.app.route('/test-return-query', methods=['GET'])
+    def test_return_query():
+        return request.query_string
+
     live_server.start()
 
diff --git a/requirements.txt b/requirements.txt
index 500f45f9..4bf29823 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -46,5 +46,9 @@ selenium ~= 4.1.0
 # need to revisit flask login versions
 werkzeug ~= 2.0.0
 
+# Templating, so far just in the URLs but in the future can be for the notifications also
+jinja2
+jinja2-time
+
 # playwright is installed at Dockerfile build time because it's not available on all platforms