changedetection.io/changedetectionio/tests/test_ignore_text.py

#!/usr/bin/python3

import time
from flask import url_for
from . util import live_server_setup
from changedetectionio import html_tools

def test_setup(live_server):
    live_server_setup(live_server)

# Unit test of the stripper
# Always we are dealing in utf-8
def test_strip_text_func():
    from ..fetchers import text_json_diff as fetch_site_status

    test_content = """
    Some content
    is listed here

    but sometimes we want to remove the lines.

    but not always."""

    ignore_lines = ["sometimes"]

    fetcher = fetch_site_status.perform_site_check(datastore=False)
    stripped_content = html_tools.strip_ignore_text(test_content, ignore_lines)

    assert b"sometimes" not in stripped_content
    assert b"Some content" in stripped_content


def set_original_ignore_response():
    test_return_data = """<html>
       <body>
     Some initial text<br>
     <p>Which is across multiple lines</p>
     <br>
     So let's see what happens.  <br>
     </body>
     </html>

    """

    with open("test-datastore/endpoint-content.txt", "w") as f:
        f.write(test_return_data)


def set_modified_original_ignore_response():
    test_return_data = """<html>
       <body>
     Some NEW nice initial text<br>
     <p>Which is across multiple lines</p>
     <br>
     So let's see what happens.  <br>
     <p>new ignore stuff</p>
     <p>blah</p>
     </body>
     </html>

    """

    with open("test-datastore/endpoint-content.txt", "w") as f:
        f.write(test_return_data)


# Is the same but includes ZZZZZ, 'ZZZZZ' is the last line in ignore_text
def set_modified_ignore_response():
    test_return_data = """<html>
       <body>
     Some initial text<br>
     <p>Which is across multiple lines</p>
     <P>ZZZZz</P>
     <br>
     So let's see what happens.  <br>
     </body>
     </html>

    """

    with open("test-datastore/endpoint-content.txt", "w") as f:
        f.write(test_return_data)


def test_check_ignore_text_functionality(client, live_server):
    sleep_time_for_fetch_thread = 3

    # Use a mix of case in ZzZ to prove it works case-insensitive.
    ignore_text = "XXXXX\r\nYYYYY\r\nzZzZZ\r\nnew ignore stuff"
    set_original_ignore_response()

    # Give the endpoint time to spin up
    time.sleep(1)

    # Add our URL to the import page
    test_url = url_for('test_endpoint', _external=True)
    res = client.post(
        url_for("import_page"),
        data={"urls": test_url},
        follow_redirects=True
    )
    assert b"1 Imported" in res.data

    # Give the thread time to pick it up
    time.sleep(sleep_time_for_fetch_thread)

    # Goto the edit page, add our ignore text
    # Add our URL to the import page
    res = client.post(
        url_for("edit_page", uuid="first"),
        data={"ignore_text": ignore_text, "url": test_url, 'fetch_backend': "html_requests"},
        follow_redirects=True
    )
    assert b"Updated watch." in res.data

    # Check it saved
    res = client.get(
        url_for("edit_page", uuid="first"),
    )
    assert bytes(ignore_text.encode('utf-8')) in res.data

    # Trigger a check
    client.get(url_for("form_watch_checknow"), follow_redirects=True)

    # Give the thread time to pick it up
    time.sleep(sleep_time_for_fetch_thread)

    # It should report nothing found (no new 'unviewed' class)
    res = client.get(url_for("index"))
    assert b'unviewed' not in res.data
    assert b'/test-endpoint' in res.data

    #  Make a change
    set_modified_ignore_response()

    # Trigger a check
    client.get(url_for("form_watch_checknow"), follow_redirects=True)
    # Give the thread time to pick it up
    time.sleep(sleep_time_for_fetch_thread)

    # It should report nothing found (no new 'unviewed' class)
    res = client.get(url_for("index"))
    assert b'unviewed' not in res.data
    assert b'/test-endpoint' in res.data


    # Just to be sure.. set a regular modified change..
    set_modified_original_ignore_response()
    client.get(url_for("form_watch_checknow"), follow_redirects=True)
    time.sleep(sleep_time_for_fetch_thread)

    res = client.get(url_for("index"))
    assert b'unviewed' in res.data

    # Check the preview/highlighter, we should be able to see what we ignored, but it should be highlighted
    # We only introduce the "modified" content that includes what we ignore so we can prove the newest version also displays
    # at /preview
    res = client.get(url_for("preview_page", uuid="first"))
    # We should be able to see what we ignored
    assert b'<div class="ignored">new ignore stuff' in res.data

    res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
    assert b'Deleted' in res.data

def test_check_global_ignore_text_functionality(client, live_server):
    sleep_time_for_fetch_thread = 3

    # Give the endpoint time to spin up
    time.sleep(1)

    ignore_text = "XXXXX\r\nYYYYY\r\nZZZZZ"
    set_original_ignore_response()

    # Goto the settings page, add our ignore text
    res = client.post(
        url_for("settings_page"),
        data={
            "requests-time_between_check-minutes": 180,
            "application-global_ignore_text": ignore_text,
            'application-fetch_backend': "html_requests"
        },
        follow_redirects=True
    )
    assert b"Settings updated." in res.data


    # Add our URL to the import page
    test_url = url_for('test_endpoint', _external=True)
    res = client.post(
        url_for("import_page"),
        data={"urls": test_url},
        follow_redirects=True
    )
    assert b"1 Imported" in res.data

    # Give the thread time to pick it up
    time.sleep(sleep_time_for_fetch_thread)


    # Goto the edit page of the item, add our ignore text
    # Add our URL to the import page
    res = client.post(
        url_for("edit_page", uuid="first"),
        data={"ignore_text": "something irrelevent but just to check", "url": test_url, 'fetch_backend': "html_requests"},
        follow_redirects=True
    )
    assert b"Updated watch." in res.data

    # Check it saved
    res = client.get(
        url_for("settings_page"),
    )
    assert bytes(ignore_text.encode('utf-8')) in res.data

    # Trigger a check
    client.get(url_for("form_watch_checknow"), follow_redirects=True)

    # Give the thread time to pick it up
    time.sleep(sleep_time_for_fetch_thread)

    # so that we are sure everything is viewed and in a known 'nothing changed' state
    res = client.get(url_for("diff_history_page", uuid="first"))

    # It should report nothing found (no new 'unviewed' class)
    res = client.get(url_for("index"))
    assert b'unviewed' not in res.data
    assert b'/test-endpoint' in res.data


    #  Make a change which includes the ignore text
    set_modified_ignore_response()

    # Trigger a check
    client.get(url_for("form_watch_checknow"), follow_redirects=True)
    # Give the thread time to pick it up
    time.sleep(sleep_time_for_fetch_thread)

    # It should report nothing found (no new 'unviewed' class)
    res = client.get(url_for("index"))
    assert b'unviewed' not in res.data
    assert b'/test-endpoint' in res.data

    # Just to be sure.. set a regular modified change that will trigger it
    set_modified_original_ignore_response()
    client.get(url_for("form_watch_checknow"), follow_redirects=True)
    time.sleep(sleep_time_for_fetch_thread)
    res = client.get(url_for("index"))
    assert b'unviewed' in res.data

    res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
    assert b'Deleted' in res.data
Issue #14 - Tweaks to edit, create ignore text, tests for ignore text, integrate ignore text 4 years ago			`#!/usr/bin/python3`

			`import time`
			`from flask import url_for`
Apprise notifications (#43) * issue #4 Adding settings screen for apprise URLS * Adding test notification mechanism * Move Worker module to own class file * Adding basic notification URL runner * Tests for notifications * Tweak readme with notification info * Move notification test to main test_backend.py * Fix spacing * Adding notifications screenshot * Cleanup more files from test * Offer send notification test on individual edits and main/default * Process global notifications * All branches test * Wrap worker notification process in try/catch, use global if nothing set * Fix syntax * Handle exception, increase wait time for liveserver to come up * Fixing test setup * remove debug * Split tests into their own totally isolated setups, if you know a better way to make live_server() work, MR :) * Tidying up lint/imports 4 years ago			`from . util import live_server_setup`
Ability to visualise trigger and filter rules against the current snapshot on the preview page 3 years ago			`from changedetectionio import html_tools`
Issue #14 - Tweaks to edit, create ignore text, tests for ignore text, integrate ignore text 4 years ago
Apprise notifications (#43) * issue #4 Adding settings screen for apprise URLS * Adding test notification mechanism * Move Worker module to own class file * Adding basic notification URL runner * Tests for notifications * Tweak readme with notification info * Move notification test to main test_backend.py * Fix spacing * Adding notifications screenshot * Cleanup more files from test * Offer send notification test on individual edits and main/default * Process global notifications * All branches test * Wrap worker notification process in try/catch, use global if nothing set * Fix syntax * Handle exception, increase wait time for liveserver to come up * Fixing test setup * remove debug * Split tests into their own totally isolated setups, if you know a better way to make live_server() work, MR :) * Tidying up lint/imports 4 years ago			`def test_setup(live_server):`
			`live_server_setup(live_server)`
Issue #14 - Tweaks to edit, create ignore text, tests for ignore text, integrate ignore text 4 years ago
			`# Unit test of the stripper`
When new ignore text is specified, reprocess the checksum 4 years ago			`# Always we are dealing in utf-8`
Issue #14 - Tweaks to edit, create ignore text, tests for ignore text, integrate ignore text 4 years ago			`def test_strip_text_func():`
Code - Abstract out the diff fetch types to make it easier to integrate new ones (#1467) 2 years ago			`from ..fetchers import text_json_diff as fetch_site_status`
Issue #14 - Tweaks to edit, create ignore text, tests for ignore text, integrate ignore text 4 years ago
			`test_content = """`
			`Some content`
			`is listed here`
Activate workflow on all branches 4 years ago
Issue #14 - Tweaks to edit, create ignore text, tests for ignore text, integrate ignore text 4 years ago			`but sometimes we want to remove the lines.`
Activate workflow on all branches 4 years ago
Issue #14 - Tweaks to edit, create ignore text, tests for ignore text, integrate ignore text 4 years ago			`but not always."""`

			`ignore_lines = ["sometimes"]`

When new ignore text is specified, reprocess the checksum 4 years ago			`fetcher = fetch_site_status.perform_site_check(datastore=False)`
Ability to visualise trigger and filter rules against the current snapshot on the preview page 3 years ago			`stripped_content = html_tools.strip_ignore_text(test_content, ignore_lines)`
Issue #14 - Tweaks to edit, create ignore text, tests for ignore text, integrate ignore text 4 years ago
When new ignore text is specified, reprocess the checksum 4 years ago			`assert b"sometimes" not in stripped_content`
			`assert b"Some content" in stripped_content`
Issue #14 - Tweaks to edit, create ignore text, tests for ignore text, integrate ignore text 4 years ago

Activate workflow on all branches 4 years ago			`def set_original_ignore_response():`
Issue #14 - Tweaks to edit, create ignore text, tests for ignore text, integrate ignore text 4 years ago			`test_return_data = """<html>`
			`<body>`
Fixing bad linebreak definition `</br>` in notifications and UI (#1465) 2 years ago			`Some initial text<br>`
Issue #14 - Tweaks to edit, create ignore text, tests for ignore text, integrate ignore text 4 years ago			`<p>Which is across multiple lines</p>`
Fixing bad linebreak definition `</br>` in notifications and UI (#1465) 2 years ago			`<br>`
			`So let's see what happens. <br>`
Issue #14 - Tweaks to edit, create ignore text, tests for ignore text, integrate ignore text 4 years ago			`</body>`
			`</html>`

			`"""`

Chrome/Webdriver support for Javascript websites (#114) JS Support via fetching the page over WebDriver/Selenium network Refactor forms (Split into logical tabs) 3 years ago			`with open("test-datastore/endpoint-content.txt", "w") as f:`
Issue #14 - Tweaks to edit, create ignore text, tests for ignore text, integrate ignore text 4 years ago			`f.write(test_return_data)`


When new ignore text is specified, reprocess the checksum 4 years ago			`def set_modified_original_ignore_response():`
			`test_return_data = """<html>`
			`<body>`
Fixing bad linebreak definition `</br>` in notifications and UI (#1465) 2 years ago			`Some NEW nice initial text<br>`
When new ignore text is specified, reprocess the checksum 4 years ago			`<p>Which is across multiple lines</p>`
Fixing bad linebreak definition `</br>` in notifications and UI (#1465) 2 years ago			`<br>`
			`So let's see what happens. <br>`
Ability to visualise trigger and filter rules against the current snapshot on the preview page 3 years ago			`<p>new ignore stuff</p>`
			`<p>blah</p>`
When new ignore text is specified, reprocess the checksum 4 years ago			`</body>`
			`</html>`

			`"""`

Chrome/Webdriver support for Javascript websites (#114) JS Support via fetching the page over WebDriver/Selenium network Refactor forms (Split into logical tabs) 3 years ago			`with open("test-datastore/endpoint-content.txt", "w") as f:`
When new ignore text is specified, reprocess the checksum 4 years ago			`f.write(test_return_data)`


Issue #14 - Tweaks to edit, create ignore text, tests for ignore text, integrate ignore text 4 years ago			`# Is the same but includes ZZZZZ, 'ZZZZZ' is the last line in ignore_text`
Activate workflow on all branches 4 years ago			`def set_modified_ignore_response():`
Issue #14 - Tweaks to edit, create ignore text, tests for ignore text, integrate ignore text 4 years ago			`test_return_data = """<html>`
			`<body>`
Fixing bad linebreak definition `</br>` in notifications and UI (#1465) 2 years ago			`Some initial text<br>`
Issue #14 - Tweaks to edit, create ignore text, tests for ignore text, integrate ignore text 4 years ago			`<p>Which is across multiple lines</p>`
Ensure string matching on the ignore filter is always case-INsensitive 3 years ago			`<P>ZZZZz</P>`
Fixing bad linebreak definition `</br>` in notifications and UI (#1465) 2 years ago			`<br>`
			`So let's see what happens. <br>`
Issue #14 - Tweaks to edit, create ignore text, tests for ignore text, integrate ignore text 4 years ago			`</body>`
			`</html>`

			`"""`

Chrome/Webdriver support for Javascript websites (#114) JS Support via fetching the page over WebDriver/Selenium network Refactor forms (Split into logical tabs) 3 years ago			`with open("test-datastore/endpoint-content.txt", "w") as f:`
Issue #14 - Tweaks to edit, create ignore text, tests for ignore text, integrate ignore text 4 years ago			`f.write(test_return_data)`


			`def test_check_ignore_text_functionality(client, live_server):`
When new ignore text is specified, reprocess the checksum 4 years ago			`sleep_time_for_fetch_thread = 3`
Issue #14 - Tweaks to edit, create ignore text, tests for ignore text, integrate ignore text 4 years ago
Ensure string matching on the ignore filter is always case-INsensitive 3 years ago			`# Use a mix of case in ZzZ to prove it works case-insensitive.`
			`ignore_text = "XXXXX\r\nYYYYY\r\nzZzZZ\r\nnew ignore stuff"`
Activate workflow on all branches 4 years ago			`set_original_ignore_response()`
Issue #14 - Tweaks to edit, create ignore text, tests for ignore text, integrate ignore text 4 years ago
			`# Give the endpoint time to spin up`
			`time.sleep(1)`

			`# Add our URL to the import page`
Activate workflow on all branches 4 years ago			`test_url = url_for('test_endpoint', _external=True)`
Issue #14 - Tweaks to edit, create ignore text, tests for ignore text, integrate ignore text 4 years ago			`res = client.post(`
			`url_for("import_page"),`
			`data={"urls": test_url},`
			`follow_redirects=True`
			`)`
			`assert b"1 Imported" in res.data`

Tweak to tests 4 years ago			`# Give the thread time to pick it up`
			`time.sleep(sleep_time_for_fetch_thread)`

Issue #14 - Tweaks to edit, create ignore text, tests for ignore text, integrate ignore text 4 years ago			`# Goto the edit page, add our ignore text`
			`# Add our URL to the import page`
			`res = client.post(`
			`url_for("edit_page", uuid="first"),`
Chrome/Webdriver support for Javascript websites (#114) JS Support via fetching the page over WebDriver/Selenium network Refactor forms (Split into logical tabs) 3 years ago			`data={"ignore_text": ignore_text, "url": test_url, 'fetch_backend': "html_requests"},`
Issue #14 - Tweaks to edit, create ignore text, tests for ignore text, integrate ignore text 4 years ago			`follow_redirects=True`
			`)`
			`assert b"Updated watch." in res.data`

			`# Check it saved`
			`res = client.get(`
			`url_for("edit_page", uuid="first"),`
			`)`
			`assert bytes(ignore_text.encode('utf-8')) in res.data`

			`# Trigger a check`
API Interface (#617) 2 years ago			`client.get(url_for("form_watch_checknow"), follow_redirects=True)`
Issue #14 - Tweaks to edit, create ignore text, tests for ignore text, integrate ignore text 4 years ago
			`# Give the thread time to pick it up`
			`time.sleep(sleep_time_for_fetch_thread)`

			`# It should report nothing found (no new 'unviewed' class)`
			`res = client.get(url_for("index"))`
			`assert b'unviewed' not in res.data`
Activate workflow on all branches 4 years ago			`assert b'/test-endpoint' in res.data`
Issue #14 - Tweaks to edit, create ignore text, tests for ignore text, integrate ignore text 4 years ago
When new ignore text is specified, reprocess the checksum 4 years ago			`# Make a change`
Activate workflow on all branches 4 years ago			`set_modified_ignore_response()`
Issue #14 - Tweaks to edit, create ignore text, tests for ignore text, integrate ignore text 4 years ago
			`# Trigger a check`
API Interface (#617) 2 years ago			`client.get(url_for("form_watch_checknow"), follow_redirects=True)`
Issue #14 - Tweaks to edit, create ignore text, tests for ignore text, integrate ignore text 4 years ago			`# Give the thread time to pick it up`
			`time.sleep(sleep_time_for_fetch_thread)`

			`# It should report nothing found (no new 'unviewed' class)`
			`res = client.get(url_for("index"))`
			`assert b'unviewed' not in res.data`
Activate workflow on all branches 4 years ago			`assert b'/test-endpoint' in res.data`
Issue #14 - Tweaks to edit, create ignore text, tests for ignore text, integrate ignore text 4 years ago
Ability to visualise trigger and filter rules against the current snapshot on the preview page 3 years ago



When new ignore text is specified, reprocess the checksum 4 years ago			`# Just to be sure.. set a regular modified change..`
			`set_modified_original_ignore_response()`
API Interface (#617) 2 years ago			`client.get(url_for("form_watch_checknow"), follow_redirects=True)`
When new ignore text is specified, reprocess the checksum 4 years ago			`time.sleep(sleep_time_for_fetch_thread)`
Ability to visualise trigger and filter rules against the current snapshot on the preview page 3 years ago
When new ignore text is specified, reprocess the checksum 4 years ago			`res = client.get(url_for("index"))`
			`assert b'unviewed' in res.data`

Ability to visualise trigger and filter rules against the current snapshot on the preview page 3 years ago			`# Check the preview/highlighter, we should be able to see what we ignored, but it should be highlighted`
			`# We only introduce the "modified" content that includes what we ignore so we can prove the newest version also displays`
			`# at /preview`
			`res = client.get(url_for("preview_page", uuid="first"))`
			`# We should be able to see what we ignored`
			`assert b'<div class="ignored">new ignore stuff' in res.data`

API Interface (#617) 2 years ago			`res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)`
Clean up after test case 4 years ago			`assert b'Deleted' in res.data`
Adding global ignore text (#339) 3 years ago
			`def test_check_global_ignore_text_functionality(client, live_server):`
			`sleep_time_for_fetch_thread = 3`

Seconds/minutes/hours/days between checks form field upgrade from 'minutes' only (#512) 3 years ago			`# Give the endpoint time to spin up`
			`time.sleep(1)`

Adding global ignore text (#339) 3 years ago			`ignore_text = "XXXXX\r\nYYYYY\r\nZZZZZ"`
			`set_original_ignore_response()`

Seconds/minutes/hours/days between checks form field upgrade from 'minutes' only (#512) 3 years ago			`# Goto the settings page, add our ignore text`
			`res = client.post(`
			`url_for("settings_page"),`
			`data={`
			`"requests-time_between_check-minutes": 180,`
			`"application-global_ignore_text": ignore_text,`
			`'application-fetch_backend': "html_requests"`
			`},`
			`follow_redirects=True`
			`)`
			`assert b"Settings updated." in res.data`

Adding global ignore text (#339) 3 years ago
			`# Add our URL to the import page`
			`test_url = url_for('test_endpoint', _external=True)`
			`res = client.post(`
			`url_for("import_page"),`
			`data={"urls": test_url},`
			`follow_redirects=True`
			`)`
			`assert b"1 Imported" in res.data`

			`# Give the thread time to pick it up`
			`time.sleep(sleep_time_for_fetch_thread)`


			`# Goto the edit page of the item, add our ignore text`
			`# Add our URL to the import page`
			`res = client.post(`
			`url_for("edit_page", uuid="first"),`
			`data={"ignore_text": "something irrelevent but just to check", "url": test_url, 'fetch_backend': "html_requests"},`
			`follow_redirects=True`
			`)`
			`assert b"Updated watch." in res.data`

			`# Check it saved`
			`res = client.get(`
			`url_for("settings_page"),`
			`)`
			`assert bytes(ignore_text.encode('utf-8')) in res.data`

			`# Trigger a check`
API Interface (#617) 2 years ago			`client.get(url_for("form_watch_checknow"), follow_redirects=True)`
Adding global ignore text (#339) 3 years ago
			`# Give the thread time to pick it up`
			`time.sleep(sleep_time_for_fetch_thread)`

Seconds/minutes/hours/days between checks form field upgrade from 'minutes' only (#512) 3 years ago			`# so that we are sure everything is viewed and in a known 'nothing changed' state`
			`res = client.get(url_for("diff_history_page", uuid="first"))`

Adding global ignore text (#339) 3 years ago			`# It should report nothing found (no new 'unviewed' class)`
			`res = client.get(url_for("index"))`
			`assert b'unviewed' not in res.data`
			`assert b'/test-endpoint' in res.data`

Seconds/minutes/hours/days between checks form field upgrade from 'minutes' only (#512) 3 years ago
			`# Make a change which includes the ignore text`
Adding global ignore text (#339) 3 years ago			`set_modified_ignore_response()`

			`# Trigger a check`
API Interface (#617) 2 years ago			`client.get(url_for("form_watch_checknow"), follow_redirects=True)`
Adding global ignore text (#339) 3 years ago			`# Give the thread time to pick it up`
			`time.sleep(sleep_time_for_fetch_thread)`

			`# It should report nothing found (no new 'unviewed' class)`
			`res = client.get(url_for("index"))`
			`assert b'unviewed' not in res.data`
			`assert b'/test-endpoint' in res.data`

Seconds/minutes/hours/days between checks form field upgrade from 'minutes' only (#512) 3 years ago			`# Just to be sure.. set a regular modified change that will trigger it`
Adding global ignore text (#339) 3 years ago			`set_modified_original_ignore_response()`
API Interface (#617) 2 years ago			`client.get(url_for("form_watch_checknow"), follow_redirects=True)`
Adding global ignore text (#339) 3 years ago			`time.sleep(sleep_time_for_fetch_thread)`
			`res = client.get(url_for("index"))`
			`assert b'unviewed' in res.data`

API Interface (#617) 2 years ago			`res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)`
Re #133 Option for ignoring whitespacing (#345) * Global setting option to ignore whitespace when detecting a change 3 years ago			`assert b'Deleted' in res.data`