changedetection.io/changedetectionio/tests/util.py

#!/usr/bin/python3

from flask import make_response, request
from flask import url_for
import logging
import time

def set_original_response():
    test_return_data = """<html>
    <head><title>head title</title></head>
    <body>
     Some initial text</br>
     <p>Which is across multiple lines</p>
     </br>
     So let's see what happens.  </br>
     <span class="foobar-detection" style='display:none'></span>
     </body>
     </html>
    """

    with open("test-datastore/endpoint-content.txt", "w") as f:
        f.write(test_return_data)
    return None

def set_modified_response():
    test_return_data = """<html>
    <head><title>modified head title</title></head>
    <body>
     Some initial text</br>
     <p>which has this one new line</p>
     </br>
     So let's see what happens.  </br>
     </body>
     </html>
    """

    with open("test-datastore/endpoint-content.txt", "w") as f:
        f.write(test_return_data)

    return None

def set_more_modified_response():
    test_return_data = """<html>
    <head><title>modified head title</title></head>
    <body>
     Some initial text</br>
     <p>which has this one new line</p>
     </br>
     So let's see what happens.  </br>
     Ohh yeah awesome<br/>
     </body>
     </html>
    """

    with open("test-datastore/endpoint-content.txt", "w") as f:
        f.write(test_return_data)

    return None


# kinda funky, but works for now
def extract_api_key_from_UI(client):
    import re
    res = client.get(
        url_for("settings_page"),
    )
    # <span id="api-key">{{api_key}}</span>

    m = re.search('<span id="api-key">(.+?)</span>', str(res.data))
    api_key = m.group(1)
    return api_key.strip()

# kinda funky, but works for now
def extract_rss_token_from_UI(client):
    import re
    res = client.get(
        url_for("index"),
    )
    m = re.search('token=(.+?)"', str(res.data))
    token_key = m.group(1)
    return token_key.strip()

# kinda funky, but works for now
def extract_UUID_from_client(client):
    import re
    res = client.get(
        url_for("index"),
    )
    # <span id="api-key">{{api_key}}</span>

    m = re.search('edit/(.+?)"', str(res.data))
    uuid = m.group(1)
    return uuid.strip()

def wait_for_all_checks(client):
    # Loop waiting until done..
    attempt=0
    time.sleep(0.1)
    while attempt < 60:
        time.sleep(1)
        res = client.get(url_for("index"))
        if not b'Checking now' in res.data:
            break
        logging.getLogger().info("Waiting for watch-list to not say 'Checking now'.. {}".format(attempt))

        attempt += 1

def live_server_setup(live_server):

    @live_server.app.route('/test-random-content-endpoint')
    def test_random_content_endpoint():
        import secrets
        return "Random content - {}\n".format(secrets.token_hex(64))


    @live_server.app.route('/test-endpoint')
    def test_endpoint():
        ctype = request.args.get('content_type')
        status_code = request.args.get('status_code')
        content = request.args.get('content') or None

        try:
            if content is not None:
                resp = make_response(content, status_code)
                resp.headers['Content-Type'] = ctype if ctype else 'text/html'
                return resp

            # Tried using a global var here but didn't seem to work, so reading from a file instead.
            with open("test-datastore/endpoint-content.txt", "r") as f:
                resp = make_response(f.read(), status_code)
                resp.headers['Content-Type'] = ctype if ctype else 'text/html'
                return resp
        except FileNotFoundError:
            return make_response('', status_code)

    # Just return the headers in the request
    @live_server.app.route('/test-headers')
    def test_headers():

        output= []

        for header in request.headers:
             output.append("{}:{}".format(str(header[0]),str(header[1])   ))

        return "\n".join(output)

    # Just return the body in the request
    @live_server.app.route('/test-body', methods=['POST', 'GET'])
    def test_body():
        print ("TEST-BODY GOT", request.data, "returning")
        return request.data

    # Just return the verb in the request
    @live_server.app.route('/test-method', methods=['POST', 'GET', 'PATCH'])
    def test_method():
        return request.method

    # Where we POST to as a notification
    @live_server.app.route('/test_notification_endpoint', methods=['POST', 'GET'])
    def test_notification_endpoint():
        with open("test-datastore/notification.txt", "wb") as f:
            # Debug method, dump all POST to file also, used to prove #65
            data = request.stream.read()
            if data != None:
                f.write(data)

        with open("test-datastore/notification-url.txt", "w") as f:
            f.write(request.url)

        print("\n>> Test notification endpoint was hit.\n", data)
        return "Text was set"


    # Just return the verb in the request
    @live_server.app.route('/test-basicauth', methods=['GET'])
    def test_basicauth_method():
        auth = request.authorization
        ret = " ".join([auth.username, auth.password, auth.type])
        return ret

    # Just return some GET var
    @live_server.app.route('/test-return-query', methods=['GET'])
    def test_return_query():
        return request.query_string


    @live_server.app.route('/endpoint-test.pdf')
    def test_pdf_endpoint():

        # Tried using a global var here but didn't seem to work, so reading from a file instead.
        with open("test-datastore/endpoint-test.pdf", "rb") as f:
            resp = make_response(f.read(), 200)
            resp.headers['Content-Type'] = 'application/pdf'
            return resp

    live_server.start()
Apprise notifications (#43) * issue #4 Adding settings screen for apprise URLS * Adding test notification mechanism * Move Worker module to own class file * Adding basic notification URL runner * Tests for notifications * Tweak readme with notification info * Move notification test to main test_backend.py * Fix spacing * Adding notifications screenshot * Cleanup more files from test * Offer send notification test on individual edits and main/default * Process global notifications * All branches test * Wrap worker notification process in try/catch, use global if nothing set * Fix syntax * Handle exception, increase wait time for liveserver to come up * Fixing test setup * remove debug * Split tests into their own totally isolated setups, if you know a better way to make live_server() work, MR :) * Tidying up lint/imports 4 years ago			`#!/usr/bin/python3`

Better text/plain detection and refactor tests (#443) 3 years ago			`from flask import make_response, request`
Move history data to a textfile, improves memory handling (#638) 3 years ago			`from flask import url_for`
Be sure visual-selector data is set when xPath/CSS filter is not yet found (#851) 2 years ago			`import logging`
			`import time`
Apprise notifications (#43) * issue #4 Adding settings screen for apprise URLS * Adding test notification mechanism * Move Worker module to own class file * Adding basic notification URL runner * Tests for notifications * Tweak readme with notification info * Move notification test to main test_backend.py * Fix spacing * Adding notifications screenshot * Cleanup more files from test * Offer send notification test on individual edits and main/default * Process global notifications * All branches test * Wrap worker notification process in try/catch, use global if nothing set * Fix syntax * Handle exception, increase wait time for liveserver to come up * Fixing test setup * remove debug * Split tests into their own totally isolated setups, if you know a better way to make live_server() work, MR :) * Tidying up lint/imports 4 years ago
			`def set_original_response():`
			`test_return_data = """<html>`
Auto extract html title as title (#102) * Auto extract <title> as watch title, Minor refactor for html tooling 4 years ago			`<head><title>head title</title></head>`
			`<body>`
Apprise notifications (#43) * issue #4 Adding settings screen for apprise URLS * Adding test notification mechanism * Move Worker module to own class file * Adding basic notification URL runner * Tests for notifications * Tweak readme with notification info * Move notification test to main test_backend.py * Fix spacing * Adding notifications screenshot * Cleanup more files from test * Offer send notification test on individual edits and main/default * Process global notifications * All branches test * Wrap worker notification process in try/catch, use global if nothing set * Fix syntax * Handle exception, increase wait time for liveserver to come up * Fixing test setup * remove debug * Split tests into their own totally isolated setups, if you know a better way to make live_server() work, MR :) * Tidying up lint/imports 4 years ago			`Some initial text</br>`
			`<p>Which is across multiple lines</p>`
			`</br>`
			`So let's see what happens. </br>`
Adding support for change detection of HTML source-code via "source:https://website.com" prefix (#540) 3 years ago			`<span class="foobar-detection" style='display:none'></span>`
Apprise notifications (#43) * issue #4 Adding settings screen for apprise URLS * Adding test notification mechanism * Move Worker module to own class file * Adding basic notification URL runner * Tests for notifications * Tweak readme with notification info * Move notification test to main test_backend.py * Fix spacing * Adding notifications screenshot * Cleanup more files from test * Offer send notification test on individual edits and main/default * Process global notifications * All branches test * Wrap worker notification process in try/catch, use global if nothing set * Fix syntax * Handle exception, increase wait time for liveserver to come up * Fixing test setup * remove debug * Split tests into their own totally isolated setups, if you know a better way to make live_server() work, MR :) * Tidying up lint/imports 4 years ago			`</body>`
			`</html>`
			`"""`

Chrome/Webdriver support for Javascript websites (#114) JS Support via fetching the page over WebDriver/Selenium network Refactor forms (Split into logical tabs) 3 years ago			`with open("test-datastore/endpoint-content.txt", "w") as f:`
Apprise notifications (#43) * issue #4 Adding settings screen for apprise URLS * Adding test notification mechanism * Move Worker module to own class file * Adding basic notification URL runner * Tests for notifications * Tweak readme with notification info * Move notification test to main test_backend.py * Fix spacing * Adding notifications screenshot * Cleanup more files from test * Offer send notification test on individual edits and main/default * Process global notifications * All branches test * Wrap worker notification process in try/catch, use global if nothing set * Fix syntax * Handle exception, increase wait time for liveserver to come up * Fixing test setup * remove debug * Split tests into their own totally isolated setups, if you know a better way to make live_server() work, MR :) * Tidying up lint/imports 4 years ago			`f.write(test_return_data)`
			`return None`

			`def set_modified_response():`
			`test_return_data = """<html>`
Auto extract html title as title (#102) * Auto extract <title> as watch title, Minor refactor for html tooling 4 years ago			`<head><title>modified head title</title></head>`
			`<body>`
Apprise notifications (#43) * issue #4 Adding settings screen for apprise URLS * Adding test notification mechanism * Move Worker module to own class file * Adding basic notification URL runner * Tests for notifications * Tweak readme with notification info * Move notification test to main test_backend.py * Fix spacing * Adding notifications screenshot * Cleanup more files from test * Offer send notification test on individual edits and main/default * Process global notifications * All branches test * Wrap worker notification process in try/catch, use global if nothing set * Fix syntax * Handle exception, increase wait time for liveserver to come up * Fixing test setup * remove debug * Split tests into their own totally isolated setups, if you know a better way to make live_server() work, MR :) * Tidying up lint/imports 4 years ago			`Some initial text</br>`
			`<p>which has this one new line</p>`
			`</br>`
			`So let's see what happens. </br>`
			`</body>`
			`</html>`
			`"""`

Chrome/Webdriver support for Javascript websites (#114) JS Support via fetching the page over WebDriver/Selenium network Refactor forms (Split into logical tabs) 3 years ago			`with open("test-datastore/endpoint-content.txt", "w") as f:`
Apprise notifications (#43) * issue #4 Adding settings screen for apprise URLS * Adding test notification mechanism * Move Worker module to own class file * Adding basic notification URL runner * Tests for notifications * Tweak readme with notification info * Move notification test to main test_backend.py * Fix spacing * Adding notifications screenshot * Cleanup more files from test * Offer send notification test on individual edits and main/default * Process global notifications * All branches test * Wrap worker notification process in try/catch, use global if nothing set * Fix syntax * Handle exception, increase wait time for liveserver to come up * Fixing test setup * remove debug * Split tests into their own totally isolated setups, if you know a better way to make live_server() work, MR :) * Tidying up lint/imports 4 years ago			`f.write(test_return_data)`

			`return None`

Introduce an AJAX button for sending test notifications instead of the checkbox (#519) 3 years ago			`def set_more_modified_response():`
			`test_return_data = """<html>`
			`<head><title>modified head title</title></head>`
			`<body>`
			`Some initial text</br>`
			`<p>which has this one new line</p>`
			`</br>`
			`So let's see what happens. </br>`
			`Ohh yeah awesome<br/>`
			`</body>`
			`</html>`
			`"""`

			`with open("test-datastore/endpoint-content.txt", "w") as f:`
			`f.write(test_return_data)`

			`return None`

Apprise notifications (#43) * issue #4 Adding settings screen for apprise URLS * Adding test notification mechanism * Move Worker module to own class file * Adding basic notification URL runner * Tests for notifications * Tweak readme with notification info * Move notification test to main test_backend.py * Fix spacing * Adding notifications screenshot * Cleanup more files from test * Offer send notification test on individual edits and main/default * Process global notifications * All branches test * Wrap worker notification process in try/catch, use global if nothing set * Fix syntax * Handle exception, increase wait time for liveserver to come up * Fixing test setup * remove debug * Split tests into their own totally isolated setups, if you know a better way to make live_server() work, MR :) * Tidying up lint/imports 4 years ago
Move history data to a textfile, improves memory handling (#638) 3 years ago			`# kinda funky, but works for now`
			`def extract_api_key_from_UI(client):`
			`import re`
			`res = client.get(`
			`url_for("settings_page"),`
			`)`
			`# <span id="api-key">{{api_key}}</span>`

			`m = re.search('<span id="api-key">(.+?)</span>', str(res.data))`
			`api_key = m.group(1)`
			`return api_key.strip()`

tidying up rss token test 2 years ago			`# kinda funky, but works for now`
			`def extract_rss_token_from_UI(client):`
			`import re`
			`res = client.get(`
			`url_for("index"),`
			`)`
			`m = re.search('token=(.+?)"', str(res.data))`
			`token_key = m.group(1)`
			`return token_key.strip()`
Be sure visual-selector data is set when xPath/CSS filter is not yet found (#851) 2 years ago
			`# kinda funky, but works for now`
			`def extract_UUID_from_client(client):`
			`import re`
			`res = client.get(`
			`url_for("index"),`
			`)`
			`# <span id="api-key">{{api_key}}</span>`

			`m = re.search('edit/(.+?)"', str(res.data))`
			`uuid = m.group(1)`
			`return uuid.strip()`

			`def wait_for_all_checks(client):`
			`# Loop waiting until done..`
			`attempt=0`
Removing unused code (#1070) 2 years ago			`time.sleep(0.1)`
Be sure visual-selector data is set when xPath/CSS filter is not yet found (#851) 2 years ago			`while attempt < 60:`
			`time.sleep(1)`
			`res = client.get(url_for("index"))`
			`if not b'Checking now' in res.data:`
			`break`
			`logging.getLogger().info("Waiting for watch-list to not say 'Checking now'.. {}".format(attempt))`

			`attempt += 1`

Apprise notifications (#43) * issue #4 Adding settings screen for apprise URLS * Adding test notification mechanism * Move Worker module to own class file * Adding basic notification URL runner * Tests for notifications * Tweak readme with notification info * Move notification test to main test_backend.py * Fix spacing * Adding notifications screenshot * Cleanup more files from test * Offer send notification test on individual edits and main/default * Process global notifications * All branches test * Wrap worker notification process in try/catch, use global if nothing set * Fix syntax * Handle exception, increase wait time for liveserver to come up * Fixing test setup * remove debug * Split tests into their own totally isolated setups, if you know a better way to make live_server() work, MR :) * Tidying up lint/imports 4 years ago			`def live_server_setup(live_server):`

Adding test 2 years ago			`@live_server.app.route('/test-random-content-endpoint')`
			`def test_random_content_endpoint():`
			`import secrets`
			`return "Random content - {}\n".format(secrets.token_hex(64))`


Apprise notifications (#43) * issue #4 Adding settings screen for apprise URLS * Adding test notification mechanism * Move Worker module to own class file * Adding basic notification URL runner * Tests for notifications * Tweak readme with notification info * Move notification test to main test_backend.py * Fix spacing * Adding notifications screenshot * Cleanup more files from test * Offer send notification test on individual edits and main/default * Process global notifications * All branches test * Wrap worker notification process in try/catch, use global if nothing set * Fix syntax * Handle exception, increase wait time for liveserver to come up * Fixing test setup * remove debug * Split tests into their own totally isolated setups, if you know a better way to make live_server() work, MR :) * Tidying up lint/imports 4 years ago			`@live_server.app.route('/test-endpoint')`
			`def test_endpoint():`
Better text/plain detection and refactor tests (#443) 3 years ago			`ctype = request.args.get('content_type')`
Allow changedetector to ignore status codes as a per-site setting (#479) (#485) Co-authored-by: Ara Hayrabedian <ara.hayrabedian@gmail.com> 3 years ago			`status_code = request.args.get('status_code')`
Move history data to a textfile, improves memory handling (#638) 3 years ago			`content = request.args.get('content') or None`
Allow changedetector to ignore status codes as a per-site setting (#479) (#485) Co-authored-by: Ara Hayrabedian <ara.hayrabedian@gmail.com> 3 years ago
			`try:`
Move history data to a textfile, improves memory handling (#638) 3 years ago			`if content is not None:`
			`resp = make_response(content, status_code)`
			`resp.headers['Content-Type'] = ctype if ctype else 'text/html'`
			`return resp`

Allow changedetector to ignore status codes as a per-site setting (#479) (#485) Co-authored-by: Ara Hayrabedian <ara.hayrabedian@gmail.com> 3 years ago			`# Tried using a global var here but didn't seem to work, so reading from a file instead.`
			`with open("test-datastore/endpoint-content.txt", "r") as f:`
			`resp = make_response(f.read(), status_code)`
			`resp.headers['Content-Type'] = ctype if ctype else 'text/html'`
			`return resp`
			`except FileNotFoundError:`
			`return make_response('', status_code)`
Better handling of EmptyReply exception, always bump 'last_checked' in the case of an error (#354) * Better handling of EmptyReply exception, always bump 'last_checked' in the case of an error, adds test 3 years ago
Re #137 - Adding test to confirm that headers are not repeated 3 years ago			`# Just return the headers in the request`
			`@live_server.app.route('/test-headers')`
			`def test_headers():`

			`output= []`

			`for header in request.headers:`
			`output.append("{}:{}".format(str(header[0]),str(header[1]) ))`

			`return "\n".join(output)`

Extend Request Parameters to add Body & Method (#325) 3 years ago			`# Just return the body in the request`
			`@live_server.app.route('/test-body', methods=['POST', 'GET'])`
			`def test_body():`
Refactor form handling (#548) 3 years ago			`print ("TEST-BODY GOT", request.data, "returning")`
Extend Request Parameters to add Body & Method (#325) 3 years ago			`return request.data`

			`# Just return the verb in the request`
			`@live_server.app.route('/test-method', methods=['POST', 'GET', 'PATCH'])`
			`def test_method():`
			`return request.method`
Re #137 - Adding test to confirm that headers are not repeated 3 years ago
Customisable notifications (#123) * Customisable notifications (#121) * Test improvements * Setup BASE_URL environment in test Co-authored-by: dtomlinson91 <53234158+dtomlinson91@users.noreply.github.com> 4 years ago			`# Where we POST to as a notification`
Re #185 - [feature] Custom notifications templates per watch (#226) * Re #185 - [feature] Custom text templates for the notification per monitored entry as override. Bonus points: Adding validation for apprise URLs 3 years ago			`@live_server.app.route('/test_notification_endpoint', methods=['POST', 'GET'])`
Apprise notifications (#43) * issue #4 Adding settings screen for apprise URLS * Adding test notification mechanism * Move Worker module to own class file * Adding basic notification URL runner * Tests for notifications * Tweak readme with notification info * Move notification test to main test_backend.py * Fix spacing * Adding notifications screenshot * Cleanup more files from test * Offer send notification test on individual edits and main/default * Process global notifications * All branches test * Wrap worker notification process in try/catch, use global if nothing set * Fix syntax * Handle exception, increase wait time for liveserver to come up * Fixing test setup * remove debug * Split tests into their own totally isolated setups, if you know a better way to make live_server() work, MR :) * Tidying up lint/imports 4 years ago			`def test_notification_endpoint():`
Customisable notifications (#123) * Customisable notifications (#121) * Test improvements * Setup BASE_URL environment in test Co-authored-by: dtomlinson91 <53234158+dtomlinson91@users.noreply.github.com> 4 years ago			`with open("test-datastore/notification.txt", "wb") as f:`
Re #65 - Append BASE_URL env var to the notification if it is set (#66) * Re #65 - Append BASE_URL env var to the notification if it is set 4 years ago			`# Debug method, dump all POST to file also, used to prove #65`
			`data = request.stream.read()`
			`if data != None:`
Customisable notifications (#123) * Customisable notifications (#121) * Test improvements * Setup BASE_URL environment in test Co-authored-by: dtomlinson91 <53234158+dtomlinson91@users.noreply.github.com> 4 years ago			`f.write(data)`
Re #65 - Append BASE_URL env var to the notification if it is set (#66) * Re #65 - Append BASE_URL env var to the notification if it is set 4 years ago
Notifications - tokens/jinja2 templating (#1184) 2 years ago			`with open("test-datastore/notification-url.txt", "w") as f:`
			`f.write(request.url)`

Introduce an AJAX button for sending test notifications instead of the checkbox (#519) 3 years ago			`print("\n>> Test notification endpoint was hit.\n", data)`
Apprise notifications (#43) * issue #4 Adding settings screen for apprise URLS * Adding test notification mechanism * Move Worker module to own class file * Adding basic notification URL runner * Tests for notifications * Tweak readme with notification info * Move notification test to main test_backend.py * Fix spacing * Adding notifications screenshot * Cleanup more files from test * Offer send notification test on individual edits and main/default * Process global notifications * All branches test * Wrap worker notification process in try/catch, use global if nothing set * Fix syntax * Handle exception, increase wait time for liveserver to come up * Fixing test setup * remove debug * Split tests into their own totally isolated setups, if you know a better way to make live_server() work, MR :) * Tidying up lint/imports 4 years ago			`return "Text was set"`

Simple HTTP auth (#372) HTTP Basic Auth form validation 3 years ago
			`# Just return the verb in the request`
			`@live_server.app.route('/test-basicauth', methods=['GET'])`
			`def test_basicauth_method():`
			`auth = request.authorization`
			`ret = " ".join([auth.username, auth.password, auth.type])`
			`return ret`

Re #1052 - Dynamic URLs, use variables in the URL (such as the current date, the date in a month, and other logic see https://github.com/dgtlmoon/changedetection.io/wiki/Handling-variables-in-the-watched-URL ) (#1057) 2 years ago			`# Just return some GET var`
			`@live_server.app.route('/test-return-query', methods=['GET'])`
			`def test_return_query():`
			`return request.query_string`

PDF File change detection - Initial PDF fetcher support with basic text extraction (#1244) 2 years ago
			`@live_server.app.route('/endpoint-test.pdf')`
			`def test_pdf_endpoint():`

			`# Tried using a global var here but didn't seem to work, so reading from a file instead.`
			`with open("test-datastore/endpoint-test.pdf", "rb") as f:`
			`resp = make_response(f.read(), 200)`
			`resp.headers['Content-Type'] = 'application/pdf'`
			`return resp`

Customisable notifications (#123) * Customisable notifications (#121) * Test improvements * Setup BASE_URL environment in test Co-authored-by: dtomlinson91 <53234158+dtomlinson91@users.noreply.github.com> 4 years ago			`live_server.start()`
Be sure visual-selector data is set when xPath/CSS filter is not yet found (#851) 2 years ago