changedetection.io/changedetectionio/tests/test_jsonpath_jq_selector.py

#!/usr/bin/python3
# coding=utf-8

import time
from flask import url_for, escape
from . util import live_server_setup
import pytest


def test_setup(live_server):
    live_server_setup(live_server)

def test_unittest_inline_html_extract():
    # So lets pretend that the JSON we want is inside some HTML
    content="""
    <html>
    
    food and stuff and more
    <script>
    alert('nothing really good here');
    </script>
    
    <script type="application/ld+json">
  xx {"@context":"http://schema.org","@type":"Product","name":"Nan Optipro Stage 1 Baby Formula  800g","description":"During the first year of life, nutrition is critical for your baby. NAN OPTIPRO 1 is tailored to ensure your formula fed infant receives balanced, high quality nutrition.<br />Starter infant formula. The age optimised protein source (whey dominant) is from cow’s milk.<br />Backed by more than 150 years of Nestlé expertise.<br />For hygiene and convenience, it is available in an innovative packaging format with a separate storage area for the scoop, and a semi-transparent window which allows you to see how much powder is left in the can without having to open it.","image":"https://cdn0.woolworths.media/content/wowproductimages/large/155536.jpg","brand":{"@context":"http://schema.org","@type":"Organization","name":"Nan"},"gtin13":"7613287517388","offers":{"@context":"http://schema.org","@type":"Offer","potentialAction":{"@context":"http://schema.org","@type":"BuyAction"},"availability":"http://schema.org/InStock","itemCondition":"http://schema.org/NewCondition","price":23.5,"priceCurrency":"AUD"},"review":[],"sku":"155536"}
</script>
<body>
and it can also be repeated
<script type="application/ld+json">
  {"@context":"http://schema.org","@type":"Product","name":"Nan Optipro Stage 1 Baby Formula  800g","description":"During the first year of life, nutrition is critical for your baby. NAN OPTIPRO 1 is tailored to ensure your formula fed infant receives balanced, high quality nutrition.<br />Starter infant formula. The age optimised protein source (whey dominant) is from cow’s milk.<br />Backed by more than 150 years of Nestlé expertise.<br />For hygiene and convenience, it is available in an innovative packaging format with a separate storage area for the scoop, and a semi-transparent window which allows you to see how much powder is left in the can without having to open it.","image":"https://cdn0.woolworths.media/content/wowproductimages/large/155536.jpg","brand":{"@context":"http://schema.org","@type":"Organization","name":"Nan"},"gtin13":"7613287517388","offers":{"@context":"http://schema.org","@type":"Offer","potentialAction":{"@context":"http://schema.org","@type":"BuyAction"},"availability":"http://schema.org/InStock","itemCondition":"http://schema.org/NewCondition","price":23.5,"priceCurrency":"AUD"},"review":[],"sku":"155536"}
</script>
<h4>ok</h4>
</body>
</html>

    """
    from .. import html_tools

    # See that we can find the second <script> one, which is not broken, and matches our filter
    text = html_tools.extract_json_as_string(content, "json:$.offers.price")
    assert text == "23.5"

    # also check for jq
    text = html_tools.extract_json_as_string(content, "jq:.offers.price")
    assert text == "23.5"

    text = html_tools.extract_json_as_string('{"id":5}', "json:$.id")
    assert text == "5"

    text = html_tools.extract_json_as_string('{"id":5}', "jq:.id")
    assert text == "5"

    # When nothing at all is found, it should throw JSONNOTFound
    # Which is caught and shown to the user in the watch-overview table
    with pytest.raises(html_tools.JSONNotFound) as e_info:
        html_tools.extract_json_as_string('COMPLETE GIBBERISH, NO JSON!', "json:$.id")

    with pytest.raises(html_tools.JSONNotFound) as e_info:
        html_tools.extract_json_as_string('COMPLETE GIBBERISH, NO JSON!', "jq:.id")

def set_original_ext_response():
    data = """
        [
        {
            "isPriceLowered": false,
            "status": "ForSale",
            "statusOrig": "for sale"
        },
        {
            "_id": "5e7b3e1fb3262d306323ff1e",
            "listingsType": "consumer",
            "status": "ForSale",
            "statusOrig": "for sale"
        }
    ]
        """

    with open("test-datastore/endpoint-content.txt", "w") as f:
        f.write(data)
    return None

def set_modified_ext_response():
    data = """
    [
    {
        "isPriceLowered": false,
        "status": "Sold",
        "statusOrig": "sold"
    },
    {
        "_id": "5e7b3e1fb3262d306323ff1e",
        "listingsType": "consumer",
        "isPriceLowered": false,
        "status": "Sold"
    }
]
    """

    with open("test-datastore/endpoint-content.txt", "w") as f:
        f.write(data)
    return None

def set_original_response():
    test_return_data = """
    {
      "employees": [
        {
          "id": 1,
          "name": "Pankaj",
          "salary": "10000"
        },
        {
          "name": "David",
          "salary": "5000",
          "id": 2
        }
      ],
      "boss": {
        "name": "Fat guy"
      },
      "available": true
    }
    """
    with open("test-datastore/endpoint-content.txt", "w") as f:
        f.write(test_return_data)
    return None


def set_response_with_html():
    test_return_data = """
    {
      "test": [
        {
          "html": "<b>"
        }
      ]
    }
    """
    with open("test-datastore/endpoint-content.txt", "w") as f:
        f.write(test_return_data)
    return None

def set_modified_response():
    test_return_data = """
    {
      "employees": [
        {
          "id": 1,
          "name": "Pankaj",
          "salary": "10000"
        },
        {
          "name": "David",
          "salary": "5000",
          "id": 2
        }
      ],
      "boss": {
        "name": "Örnsköldsvik"
      },
      "available": false
    }
        """

    with open("test-datastore/endpoint-content.txt", "w") as f:
        f.write(test_return_data)

    return None

def test_check_json_without_filter(client, live_server):
    # Request a JSON document from a application/json source containing HTML
    # and be sure it doesn't get chewed up by instriptis
    set_response_with_html()

    # Give the endpoint time to spin up
    time.sleep(1)

    # Add our URL to the import page
    test_url = url_for('test_endpoint', content_type="application/json", _external=True)
    client.post(
        url_for("import_page"),
        data={"urls": test_url},
        follow_redirects=True
    )

    # Trigger a check
    client.get(url_for("form_watch_checknow"), follow_redirects=True)

    # Give the thread time to pick it up
    time.sleep(3)

    res = client.get(
        url_for("preview_page", uuid="first"),
        follow_redirects=True
    )

    assert b'&#34;&lt;b&gt;' in res.data
    assert res.data.count(b'{\n') >= 2

    res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
    assert b'Deleted' in res.data

def check_json_filter(json_filter, client, live_server):
    set_original_response()

    # Give the endpoint time to spin up
    time.sleep(1)

    # Add our URL to the import page
    test_url = url_for('test_endpoint', content_type="application/json", _external=True)
    res = client.post(
        url_for("import_page"),
        data={"urls": test_url},
        follow_redirects=True
    )
    assert b"1 Imported" in res.data

    # Trigger a check
    client.get(url_for("form_watch_checknow"), follow_redirects=True)

    # Give the thread time to pick it up
    time.sleep(3)

    # Goto the edit page, add our ignore text
    # Add our URL to the import page
    res = client.post(
        url_for("edit_page", uuid="first"),
        data={"css_filter": json_filter,
              "url": test_url,
              "tag": "",
              "headers": "",
              "fetch_backend": "html_requests"
              },
        follow_redirects=True
    )
    assert b"Updated watch." in res.data

    # Check it saved
    res = client.get(
        url_for("edit_page", uuid="first"),
    )
    assert bytes(escape(json_filter).encode('utf-8')) in res.data

    # Trigger a check
    client.get(url_for("form_watch_checknow"), follow_redirects=True)

    # Give the thread time to pick it up
    time.sleep(3)
    #  Make a change
    set_modified_response()

    # Trigger a check
    client.get(url_for("form_watch_checknow"), follow_redirects=True)
    # Give the thread time to pick it up
    time.sleep(4)

    # It should have 'unviewed' still
    res = client.get(url_for("index"))
    assert b'unviewed' in res.data

    # Should not see this, because its not in the JSONPath we entered
    res = client.get(url_for("diff_history_page", uuid="first"))

    # But the change should be there, tho its hard to test the change was detected because it will show old and new versions
    # And #462 - check we see the proper utf-8 string there
    assert "Örnsköldsvik".encode('utf-8') in res.data

    res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
    assert b'Deleted' in res.data

def test_check_jsonpath_filter(client, live_server):
    check_json_filter('json:boss.name', client, live_server)

def test_check_jq_filter(client, live_server):
    check_json_filter('jq:.boss.name', client, live_server)

def check_json_filter_bool_val(json_filter, client, live_server):
    set_original_response()

    # Give the endpoint time to spin up
    time.sleep(1)

    test_url = url_for('test_endpoint', content_type="application/json", _external=True)

    res = client.post(
        url_for("import_page"),
        data={"urls": test_url},
        follow_redirects=True
    )
    assert b"1 Imported" in res.data

    time.sleep(3)
    # Goto the edit page, add our ignore text
    # Add our URL to the import page
    res = client.post(
        url_for("edit_page", uuid="first"),
        data={"css_filter": json_filter,
              "url": test_url,
              "tag": "",
              "headers": "",
              "fetch_backend": "html_requests"
              },
        follow_redirects=True
    )
    assert b"Updated watch." in res.data

    time.sleep(3)

    # Trigger a check
    client.get(url_for("form_watch_checknow"), follow_redirects=True)

    # Give the thread time to pick it up
    time.sleep(3)
    #  Make a change
    set_modified_response()

    # Trigger a check
    client.get(url_for("form_watch_checknow"), follow_redirects=True)
    # Give the thread time to pick it up
    time.sleep(3)

    res = client.get(url_for("diff_history_page", uuid="first"))
    # But the change should be there, tho its hard to test the change was detected because it will show old and new versions
    assert b'false' in res.data

    res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
    assert b'Deleted' in res.data

def test_check_jsonpath_filter_bool_val(client, live_server):
    check_json_filter_bool_val("json:$['available']", client, live_server)

def test_check_jq_filter_bool_val(client, live_server):
    check_json_filter_bool_val("jq:.available", client, live_server)

# Re #265 - Extended JSON selector test
# Stuff to consider here
# - Selector should be allowed to return empty when it doesnt match (people might wait for some condition)
# - The 'diff' tab could show the old and new content
# - Form should let us enter a selector that doesnt (yet) match anything
def check_json_ext_filter(json_filter, client, live_server):
    set_original_ext_response()

    # Give the endpoint time to spin up
    time.sleep(1)

    # Add our URL to the import page
    test_url = url_for('test_endpoint', content_type="application/json", _external=True)
    res = client.post(
        url_for("import_page"),
        data={"urls": test_url},
        follow_redirects=True
    )
    assert b"1 Imported" in res.data

    # Trigger a check
    client.get(url_for("form_watch_checknow"), follow_redirects=True)

    # Give the thread time to pick it up
    time.sleep(3)

    # Goto the edit page, add our ignore text
    # Add our URL to the import page
    res = client.post(
        url_for("edit_page", uuid="first"),
        data={"css_filter": json_filter,
              "url": test_url,
              "tag": "",
              "headers": "",
              "fetch_backend": "html_requests"
              },
        follow_redirects=True
    )
    assert b"Updated watch." in res.data

    # Check it saved
    res = client.get(
        url_for("edit_page", uuid="first"),
    )
    assert bytes(escape(json_filter).encode('utf-8')) in res.data

    # Trigger a check
    client.get(url_for("form_watch_checknow"), follow_redirects=True)

    # Give the thread time to pick it up
    time.sleep(3)
    #  Make a change
    set_modified_ext_response()

    # Trigger a check
    client.get(url_for("form_watch_checknow"), follow_redirects=True)
    # Give the thread time to pick it up
    time.sleep(4)

    # It should have 'unviewed'
    res = client.get(url_for("index"))
    assert b'unviewed' in res.data

    res = client.get(url_for("diff_history_page", uuid="first"))

    # We should never see 'ForSale' because we are selecting on 'Sold' in the rule,
    # But we should know it triggered ('unviewed' assert above)
    assert b'ForSale' not in res.data
    assert b'Sold' in res.data

    res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
    assert b'Deleted' in res.data

def test_check_jsonpath_ext_filter(client, live_server):
    check_json_ext_filter('json:$[?(@.status==Sold)]', client, live_server)

def test_check_jq_ext_filter(client, live_server):
    check_json_ext_filter('jq:.[] | select(.status | contains("Sold"))', client, live_server)
-												Re #117 Jsonpath based JSON change detection filter (#125)

* Re #117 - Experimental JSON selector support by using 'json:' prefix and any JSONpath rule
											
										
										
											4 years ago
+								#!/usr/bin/python3
-												JSON diff/preview should use utf-8 encoding where possible (#465)


											
										
										
											3 years ago
+								# coding=utf-8
-												Re #117 Jsonpath based JSON change detection filter (#125)

* Re #117 - Experimental JSON selector support by using 'json:' prefix and any JSONpath rule
											
										
										
											4 years ago
 								import time
-												Adds support for jq JSON path querying engine (#1001)


											
										
										
											2 years ago
+								from flask import url_for, escape
-												Re #117 Jsonpath based JSON change detection filter (#125)

* Re #117 - Experimental JSON selector support by using 'json:' prefix and any JSONpath rule
											
										
										
											4 years ago
+								from . util import live_server_setup
-												Assert that html_tools.JSONNotFound is correctly raised

											
										
										
											3 years ago
+								import pytest
-												Re #117 Jsonpath based JSON change detection filter (#125)

* Re #117 - Experimental JSON selector support by using 'json:' prefix and any JSONpath rule
											
										
										
											4 years ago
-												Re #257 - Handle bool val of json path better (#263)

* Re #257 - Handle bool val of json path better, with test
											
										
										
											3 years ago
 								def test_setup(live_server):
 								    live_server_setup(live_server)
-												Re #154 Ldjson extract parse (#158)

* Use parsable JSON hiding in <script type="application/ld+json"> where possible, if it matches the filter rule, use it.
* Update README.md
											
										
										
											3 years ago
+								def test_unittest_inline_html_extract():
 								    # So lets pretend that the JSON we want is inside some HTML
 								    content="""
 								    <html>
 								    food and stuff and more
 								    <script>
 								    alert('nothing really good here');
 								    </script>
 								    <script type="application/ld+json">
 								  xx {"@context":"http://schema.org","@type":"Product","name":"Nan Optipro Stage 1 Baby Formula  800g","description":"During the first year of life, nutrition is critical for your baby. NAN OPTIPRO 1 is tailored to ensure your formula fed infant receives balanced, high quality nutrition.<br />Starter infant formula. The age optimised protein source (whey dominant) is from cow’s milk.<br />Backed by more than 150 years of Nestlé expertise.<br />For hygiene and convenience, it is available in an innovative packaging format with a separate storage area for the scoop, and a semi-transparent window which allows you to see how much powder is left in the can without having to open it.","image":"https://cdn0.woolworths.media/content/wowproductimages/large/155536.jpg","brand":{"@context":"http://schema.org","@type":"Organization","name":"Nan"},"gtin13":"7613287517388","offers":{"@context":"http://schema.org","@type":"Offer","potentialAction":{"@context":"http://schema.org","@type":"BuyAction"},"availability":"http://schema.org/InStock","itemCondition":"http://schema.org/NewCondition","price":23.5,"priceCurrency":"AUD"},"review":[],"sku":"155536"}
 								</script>
 								<body>
 								and it can also be repeated
 								<script type="application/ld+json">
 								  {"@context":"http://schema.org","@type":"Product","name":"Nan Optipro Stage 1 Baby Formula  800g","description":"During the first year of life, nutrition is critical for your baby. NAN OPTIPRO 1 is tailored to ensure your formula fed infant receives balanced, high quality nutrition.<br />Starter infant formula. The age optimised protein source (whey dominant) is from cow’s milk.<br />Backed by more than 150 years of Nestlé expertise.<br />For hygiene and convenience, it is available in an innovative packaging format with a separate storage area for the scoop, and a semi-transparent window which allows you to see how much powder is left in the can without having to open it.","image":"https://cdn0.woolworths.media/content/wowproductimages/large/155536.jpg","brand":{"@context":"http://schema.org","@type":"Organization","name":"Nan"},"gtin13":"7613287517388","offers":{"@context":"http://schema.org","@type":"Offer","potentialAction":{"@context":"http://schema.org","@type":"BuyAction"},"availability":"http://schema.org/InStock","itemCondition":"http://schema.org/NewCondition","price":23.5,"priceCurrency":"AUD"},"review":[],"sku":"155536"}
 								</script>
 								<h4>ok</h4>
 								</body>
 								</html>
 								    """
 								    from .. import html_tools
 								    # See that we can find the second <script> one, which is not broken, and matches our filter
-												Adds support for jq JSON path querying engine (#1001)


											
										
										
											2 years ago
+								    text = html_tools.extract_json_as_string(content, "json:$.offers.price")
-												Re #154 Ldjson extract parse (#158)

* Use parsable JSON hiding in <script type="application/ld+json"> where possible, if it matches the filter rule, use it.
* Update README.md
											
										
										
											3 years ago
+								    assert text == "23.5"
-												Adds support for jq JSON path querying engine (#1001)


											
										
										
											2 years ago
+								    # also check for jq
 								    text = html_tools.extract_json_as_string(content, "jq:.offers.price")
 								    assert text == "23.5"
 								    text = html_tools.extract_json_as_string('{"id":5}', "json:$.id")
 								    assert text == "5"
 								    text = html_tools.extract_json_as_string('{"id":5}', "jq:.id")
-												Re #154 Ldjson extract parse (#158)

* Use parsable JSON hiding in <script type="application/ld+json"> where possible, if it matches the filter rule, use it.
* Update README.md
											
										
										
											3 years ago
+								    assert text == "5"
-												Assert that html_tools.JSONNotFound is correctly raised

											
										
										
											3 years ago
+								    # When nothing at all is found, it should throw JSONNOTFound
 								    # Which is caught and shown to the user in the watch-overview table
 								    with pytest.raises(html_tools.JSONNotFound) as e_info:
-												Adds support for jq JSON path querying engine (#1001)


											
										
										
											2 years ago
+								        html_tools.extract_json_as_string('COMPLETE GIBBERISH, NO JSON!', "json:$.id")
 								    with pytest.raises(html_tools.JSONNotFound) as e_info:
 								        html_tools.extract_json_as_string('COMPLETE GIBBERISH, NO JSON!', "jq:.id")
-												Re #154 Ldjson extract parse (#158)

* Use parsable JSON hiding in <script type="application/ld+json"> where possible, if it matches the filter rule, use it.
* Update README.md
											
										
										
											3 years ago
-												Re #265 - extended jsonpath support (#266)

* Re #265 - Use extended JSONpath support,
Allow a JSONPath selector to not match anything (yet)
Adding test
Correctly capture invalid JSONPath query error
											
										
										
											3 years ago
+								def set_original_ext_response():
 								    data = """
 								        [
 								        {
 								            "isPriceLowered": false,
 								            "status": "ForSale",
 								            "statusOrig": "for sale"
 								        },
 								        {
 								            "_id": "5e7b3e1fb3262d306323ff1e",
 								            "listingsType": "consumer",
 								            "status": "ForSale",
 								            "statusOrig": "for sale"
 								        }
 								    ]
 								        """
 								    with open("test-datastore/endpoint-content.txt", "w") as f:
 								        f.write(data)
-												Adds support for jq JSON path querying engine (#1001)


											
										
										
											2 years ago
+								    return None
-												Re #265 - extended jsonpath support (#266)

* Re #265 - Use extended JSONpath support,
Allow a JSONPath selector to not match anything (yet)
Adding test
Correctly capture invalid JSONPath query error
											
										
										
											3 years ago
 								def set_modified_ext_response():
 								    data = """
 								    [
 								    {
 								        "isPriceLowered": false,
 								        "status": "Sold",
 								        "statusOrig": "sold"
 								    },
 								    {
 								        "_id": "5e7b3e1fb3262d306323ff1e",
 								        "listingsType": "consumer",
 								        "isPriceLowered": false,
 								        "status": "Sold"
 								    }
 								]
 								    """
 								    with open("test-datastore/endpoint-content.txt", "w") as f:
 								        f.write(data)
-												Adds support for jq JSON path querying engine (#1001)


											
										
										
											2 years ago
+								    return None
-												Re #154 Ldjson extract parse (#158)

* Use parsable JSON hiding in <script type="application/ld+json"> where possible, if it matches the filter rule, use it.
* Update README.md
											
										
										
											3 years ago
-												Re #117 Jsonpath based JSON change detection filter (#125)

* Re #117 - Experimental JSON selector support by using 'json:' prefix and any JSONpath rule
											
										
										
											4 years ago
+								def set_original_response():
 								    test_return_data = """
 								    {
 								      "employees": [
 								        {
 								          "id": 1,
 								          "name": "Pankaj",
 								          "salary": "10000"
 								        },
 								        {
 								          "name": "David",
 								          "salary": "5000",
 								          "id": 2
 								        }
 								      ],
 								      "boss": {
 								        "name": "Fat guy"
-												Re #257 - Handle bool val of json path better (#263)

* Re #257 - Handle bool val of json path better, with test
											
										
										
											3 years ago
+								      },
 								      "available": true
-												Re #117 Jsonpath based JSON change detection filter (#125)

* Re #117 - Experimental JSON selector support by using 'json:' prefix and any JSONpath rule
											
										
										
											4 years ago
+								    }
 								    """
-												Chrome/Webdriver support for Javascript websites (#114)

JS Support via fetching the page over WebDriver/Selenium network
Refactor forms (Split into logical tabs)
											
										
										
											3 years ago
+								    with open("test-datastore/endpoint-content.txt", "w") as f:
-												Re #117 Jsonpath based JSON change detection filter (#125)

* Re #117 - Experimental JSON selector support by using 'json:' prefix and any JSONpath rule
											
										
										
											4 years ago
+								        f.write(test_return_data)
 								    return None
-												Be sure that documents returned with a application/json header are not parsed with inscriptis (#337)

* Auto-detect JSON by Content-Type header
* Add test to not parse JSON responses with inscriptis
											
										
										
											3 years ago
 								def set_response_with_html():
 								    test_return_data = """
 								    {
 								      "test": [
 								        {
 								          "html": "<b>"
 								        }
 								      ]
 								    }
 								    """
 								    with open("test-datastore/endpoint-content.txt", "w") as f:
 								        f.write(test_return_data)
 								    return None
-												Re #117 Jsonpath based JSON change detection filter (#125)

* Re #117 - Experimental JSON selector support by using 'json:' prefix and any JSONpath rule
											
										
										
											4 years ago
+								def set_modified_response():
 								    test_return_data = """
 								    {
 								      "employees": [
 								        {
 								          "id": 1,
 								          "name": "Pankaj",
 								          "salary": "10000"
 								        },
 								        {
 								          "name": "David",
 								          "salary": "5000",
 								          "id": 2
 								        }
 								      ],
 								      "boss": {
-												JSON diff/preview should use utf-8 encoding where possible (#465)


											
										
										
											3 years ago
+								        "name": "Örnsköldsvik"
-												Re #257 - Handle bool val of json path better (#263)

* Re #257 - Handle bool val of json path better, with test
											
										
										
											3 years ago
+								      },
 								      "available": false
-												Re #117 Jsonpath based JSON change detection filter (#125)

* Re #117 - Experimental JSON selector support by using 'json:' prefix and any JSONpath rule
											
										
										
											4 years ago
+								    }
 								        """
-												Chrome/Webdriver support for Javascript websites (#114)

JS Support via fetching the page over WebDriver/Selenium network
Refactor forms (Split into logical tabs)
											
										
										
											3 years ago
+								    with open("test-datastore/endpoint-content.txt", "w") as f:
-												Re #117 Jsonpath based JSON change detection filter (#125)

* Re #117 - Experimental JSON selector support by using 'json:' prefix and any JSONpath rule
											
										
										
											4 years ago
+								        f.write(test_return_data)
 								    return None
-												Be sure that documents returned with a application/json header are not parsed with inscriptis (#337)

* Auto-detect JSON by Content-Type header
* Add test to not parse JSON responses with inscriptis
											
										
										
											3 years ago
+								def test_check_json_without_filter(client, live_server):
 								    # Request a JSON document from a application/json source containing HTML
 								    # and be sure it doesn't get chewed up by instriptis
 								    set_response_with_html()
 								    # Give the endpoint time to spin up
 								    time.sleep(1)
 								    # Add our URL to the import page
-												Better text/plain detection and refactor tests (#443)


											
										
										
											3 years ago
+								    test_url = url_for('test_endpoint', content_type="application/json", _external=True)
-												Be sure that documents returned with a application/json header are not parsed with inscriptis (#337)

* Auto-detect JSON by Content-Type header
* Add test to not parse JSON responses with inscriptis
											
										
										
											3 years ago
+								    client.post(
 								        url_for("import_page"),
 								        data={"urls": test_url},
 								        follow_redirects=True
 								    )
 								    # Trigger a check
-												API Interface (#617)


											
										
										
											3 years ago
+								    client.get(url_for("form_watch_checknow"), follow_redirects=True)
-												Be sure that documents returned with a application/json header are not parsed with inscriptis (#337)

* Auto-detect JSON by Content-Type header
* Add test to not parse JSON responses with inscriptis
											
										
										
											3 years ago
 								    # Give the thread time to pick it up
 								    time.sleep(3)
 								    res = client.get(
 								        url_for("preview_page", uuid="first"),
 								        follow_redirects=True
 								    )
 								    assert b'&#34;&lt;b&gt;' in res.data
 								    assert res.data.count(b'{\n') >= 2
-												Adds support for jq JSON path querying engine (#1001)


											
										
										
											2 years ago
+								    res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
 								    assert b'Deleted' in res.data
-												Be sure that documents returned with a application/json header are not parsed with inscriptis (#337)

* Auto-detect JSON by Content-Type header
* Add test to not parse JSON responses with inscriptis
											
										
										
											3 years ago
-												Adds support for jq JSON path querying engine (#1001)


											
										
										
											2 years ago
+								def check_json_filter(json_filter, client, live_server):
-												Re #117 Jsonpath based JSON change detection filter (#125)

* Re #117 - Experimental JSON selector support by using 'json:' prefix and any JSONpath rule
											
										
										
											4 years ago
+								    set_original_response()
 								    # Give the endpoint time to spin up
 								    time.sleep(1)
 								    # Add our URL to the import page
-												Better text/plain detection and refactor tests (#443)


											
										
										
											3 years ago
+								    test_url = url_for('test_endpoint', content_type="application/json", _external=True)
-												Re #117 Jsonpath based JSON change detection filter (#125)

* Re #117 - Experimental JSON selector support by using 'json:' prefix and any JSONpath rule
											
										
										
											4 years ago
+								    res = client.post(
 								        url_for("import_page"),
 								        data={"urls": test_url},
 								        follow_redirects=True
 								    )
 								    assert b"1 Imported" in res.data
 								    # Trigger a check
-												API Interface (#617)


											
										
										
											3 years ago
+								    client.get(url_for("form_watch_checknow"), follow_redirects=True)
-												Re #117 Jsonpath based JSON change detection filter (#125)

* Re #117 - Experimental JSON selector support by using 'json:' prefix and any JSONpath rule
											
										
										
											4 years ago
 								    # Give the thread time to pick it up
 								    time.sleep(3)
 								    # Goto the edit page, add our ignore text
 								    # Add our URL to the import page
 								    res = client.post(
 								        url_for("edit_page", uuid="first"),
-												Chrome/Webdriver support for Javascript websites (#114)

JS Support via fetching the page over WebDriver/Selenium network
Refactor forms (Split into logical tabs)
											
										
										
											3 years ago
+								        data={"css_filter": json_filter,
 								              "url": test_url,
 								              "tag": "",
 								              "headers": "",
 								              "fetch_backend": "html_requests"
 								              },
-												Re #117 Jsonpath based JSON change detection filter (#125)

* Re #117 - Experimental JSON selector support by using 'json:' prefix and any JSONpath rule
											
										
										
											4 years ago
+								        follow_redirects=True
 								    )
 								    assert b"Updated watch." in res.data
 								    # Check it saved
 								    res = client.get(
 								        url_for("edit_page", uuid="first"),
 								    )
-												Adds support for jq JSON path querying engine (#1001)


											
										
										
											2 years ago
+								    assert bytes(escape(json_filter).encode('utf-8')) in res.data
-												Re #117 Jsonpath based JSON change detection filter (#125)

* Re #117 - Experimental JSON selector support by using 'json:' prefix and any JSONpath rule
											
										
										
											4 years ago
 								    # Trigger a check
-												API Interface (#617)


											
										
										
											3 years ago
+								    client.get(url_for("form_watch_checknow"), follow_redirects=True)
-												Re #117 Jsonpath based JSON change detection filter (#125)

* Re #117 - Experimental JSON selector support by using 'json:' prefix and any JSONpath rule
											
										
										
											4 years ago
 								    # Give the thread time to pick it up
 								    time.sleep(3)
 								    #  Make a change
 								    set_modified_response()
 								    # Trigger a check
-												API Interface (#617)


											
										
										
											3 years ago
+								    client.get(url_for("form_watch_checknow"), follow_redirects=True)
-												Re #117 Jsonpath based JSON change detection filter (#125)

* Re #117 - Experimental JSON selector support by using 'json:' prefix and any JSONpath rule
											
										
										
											4 years ago
+								    # Give the thread time to pick it up
-												Chrome/Webdriver support for Javascript websites (#114)

JS Support via fetching the page over WebDriver/Selenium network
Refactor forms (Split into logical tabs)
											
										
										
											3 years ago
+								    time.sleep(4)
-												Re #117 Jsonpath based JSON change detection filter (#125)

* Re #117 - Experimental JSON selector support by using 'json:' prefix and any JSONpath rule
											
										
										
											4 years ago
 								    # It should have 'unviewed' still
 								    res = client.get(url_for("index"))
 								    assert b'unviewed' in res.data
 								    # Should not see this, because its not in the JSONPath we entered
 								    res = client.get(url_for("diff_history_page", uuid="first"))
-												JSON diff/preview should use utf-8 encoding where possible (#465)


											
										
										
											3 years ago
-												Re #117 Jsonpath based JSON change detection filter (#125)

* Re #117 - Experimental JSON selector support by using 'json:' prefix and any JSONpath rule
											
										
										
											4 years ago
+								    # But the change should be there, tho its hard to test the change was detected because it will show old and new versions
-												JSON diff/preview should use utf-8 encoding where possible (#465)


											
										
										
											3 years ago
+								    # And #462 - check we see the proper utf-8 string there
 								    assert "Örnsköldsvik".encode('utf-8') in res.data
-												Re #257 - Handle bool val of json path better (#263)

* Re #257 - Handle bool val of json path better, with test
											
										
										
											3 years ago
-												Adds support for jq JSON path querying engine (#1001)


											
										
										
											2 years ago
+								    res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
 								    assert b'Deleted' in res.data
 								def test_check_jsonpath_filter(client, live_server):
 								    check_json_filter('json:boss.name', client, live_server)
-												Re #257 - Handle bool val of json path better (#263)

* Re #257 - Handle bool val of json path better, with test
											
										
										
											3 years ago
-												Adds support for jq JSON path querying engine (#1001)


											
										
										
											2 years ago
+								def test_check_jq_filter(client, live_server):
 								    check_json_filter('jq:.boss.name', client, live_server)
-												Re #257 - Handle bool val of json path better (#263)

* Re #257 - Handle bool val of json path better, with test
											
										
										
											3 years ago
-												Adds support for jq JSON path querying engine (#1001)


											
										
										
											2 years ago
+								def check_json_filter_bool_val(json_filter, client, live_server):
-												Re #257 - Handle bool val of json path better (#263)

* Re #257 - Handle bool val of json path better, with test
											
										
										
											3 years ago
+								    set_original_response()
 								    # Give the endpoint time to spin up
 								    time.sleep(1)
-												Better text/plain detection and refactor tests (#443)


											
										
										
											3 years ago
+								    test_url = url_for('test_endpoint', content_type="application/json", _external=True)
-												Re #257 - Handle bool val of json path better (#263)

* Re #257 - Handle bool val of json path better, with test
											
										
										
											3 years ago
 								    res = client.post(
 								        url_for("import_page"),
 								        data={"urls": test_url},
 								        follow_redirects=True
 								    )
 								    assert b"1 Imported" in res.data
-												Seconds/minutes/hours/days between checks form field upgrade from 'minutes' only (#512)


											
										
										
											3 years ago
+								    time.sleep(3)
-												Re #257 - Handle bool val of json path better (#263)

* Re #257 - Handle bool val of json path better, with test
											
										
										
											3 years ago
+								    # Goto the edit page, add our ignore text
 								    # Add our URL to the import page
 								    res = client.post(
 								        url_for("edit_page", uuid="first"),
 								        data={"css_filter": json_filter,
 								              "url": test_url,
 								              "tag": "",
 								              "headers": "",
 								              "fetch_backend": "html_requests"
 								              },
 								        follow_redirects=True
 								    )
 								    assert b"Updated watch." in res.data
-												Seconds/minutes/hours/days between checks form field upgrade from 'minutes' only (#512)


											
										
										
											3 years ago
+								    time.sleep(3)
-												Re #257 - Handle bool val of json path better (#263)

* Re #257 - Handle bool val of json path better, with test
											
										
										
											3 years ago
 								    # Trigger a check
-												API Interface (#617)


											
										
										
											3 years ago
+								    client.get(url_for("form_watch_checknow"), follow_redirects=True)
-												Re #257 - Handle bool val of json path better (#263)

* Re #257 - Handle bool val of json path better, with test
											
										
										
											3 years ago
 								    # Give the thread time to pick it up
 								    time.sleep(3)
 								    #  Make a change
 								    set_modified_response()
 								    # Trigger a check
-												API Interface (#617)


											
										
										
											3 years ago
+								    client.get(url_for("form_watch_checknow"), follow_redirects=True)
-												Re #257 - Handle bool val of json path better (#263)

* Re #257 - Handle bool val of json path better, with test
											
										
										
											3 years ago
+								    # Give the thread time to pick it up
 								    time.sleep(3)
 								    res = client.get(url_for("diff_history_page", uuid="first"))
 								    # But the change should be there, tho its hard to test the change was detected because it will show old and new versions
 								    assert b'false' in res.data
-												Re #265 - extended jsonpath support (#266)

* Re #265 - Use extended JSONpath support,
Allow a JSONPath selector to not match anything (yet)
Adding test
Correctly capture invalid JSONPath query error
											
										
										
											3 years ago
-												Adds support for jq JSON path querying engine (#1001)


											
										
										
											2 years ago
+								    res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
 								    assert b'Deleted' in res.data
 								def test_check_jsonpath_filter_bool_val(client, live_server):
 								    check_json_filter_bool_val("json:$['available']", client, live_server)
 								def test_check_jq_filter_bool_val(client, live_server):
 								    check_json_filter_bool_val("jq:.available", client, live_server)
-												Re #265 - extended jsonpath support (#266)

* Re #265 - Use extended JSONpath support,
Allow a JSONPath selector to not match anything (yet)
Adding test
Correctly capture invalid JSONPath query error
											
										
										
											3 years ago
+								# Re #265 - Extended JSON selector test
 								# Stuff to consider here
 								# - Selector should be allowed to return empty when it doesnt match (people might wait for some condition)
 								# - The 'diff' tab could show the old and new content
 								# - Form should let us enter a selector that doesnt (yet) match anything
-												Adds support for jq JSON path querying engine (#1001)


											
										
										
											2 years ago
+								def check_json_ext_filter(json_filter, client, live_server):
-												Re #265 - extended jsonpath support (#266)

* Re #265 - Use extended JSONpath support,
Allow a JSONPath selector to not match anything (yet)
Adding test
Correctly capture invalid JSONPath query error
											
										
										
											3 years ago
+								    set_original_ext_response()
 								    # Give the endpoint time to spin up
 								    time.sleep(1)
 								    # Add our URL to the import page
-												Better text/plain detection and refactor tests (#443)


											
										
										
											3 years ago
+								    test_url = url_for('test_endpoint', content_type="application/json", _external=True)
-												Re #265 - extended jsonpath support (#266)

* Re #265 - Use extended JSONpath support,
Allow a JSONPath selector to not match anything (yet)
Adding test
Correctly capture invalid JSONPath query error
											
										
										
											3 years ago
+								    res = client.post(
 								        url_for("import_page"),
 								        data={"urls": test_url},
 								        follow_redirects=True
 								    )
 								    assert b"1 Imported" in res.data
 								    # Trigger a check
-												API Interface (#617)


											
										
										
											3 years ago
+								    client.get(url_for("form_watch_checknow"), follow_redirects=True)
-												Re #265 - extended jsonpath support (#266)

* Re #265 - Use extended JSONpath support,
Allow a JSONPath selector to not match anything (yet)
Adding test
Correctly capture invalid JSONPath query error
											
										
										
											3 years ago
 								    # Give the thread time to pick it up
 								    time.sleep(3)
 								    # Goto the edit page, add our ignore text
 								    # Add our URL to the import page
 								    res = client.post(
 								        url_for("edit_page", uuid="first"),
 								        data={"css_filter": json_filter,
 								              "url": test_url,
 								              "tag": "",
 								              "headers": "",
 								              "fetch_backend": "html_requests"
 								              },
 								        follow_redirects=True
 								    )
 								    assert b"Updated watch." in res.data
 								    # Check it saved
 								    res = client.get(
 								        url_for("edit_page", uuid="first"),
 								    )
-												Adds support for jq JSON path querying engine (#1001)


											
										
										
											2 years ago
+								    assert bytes(escape(json_filter).encode('utf-8')) in res.data
-												Re #265 - extended jsonpath support (#266)

* Re #265 - Use extended JSONpath support,
Allow a JSONPath selector to not match anything (yet)
Adding test
Correctly capture invalid JSONPath query error
											
										
										
											3 years ago
 								    # Trigger a check
-												API Interface (#617)


											
										
										
											3 years ago
+								    client.get(url_for("form_watch_checknow"), follow_redirects=True)
-												Re #265 - extended jsonpath support (#266)

* Re #265 - Use extended JSONpath support,
Allow a JSONPath selector to not match anything (yet)
Adding test
Correctly capture invalid JSONPath query error
											
										
										
											3 years ago
 								    # Give the thread time to pick it up
 								    time.sleep(3)
 								    #  Make a change
 								    set_modified_ext_response()
 								    # Trigger a check
-												API Interface (#617)


											
										
										
											3 years ago
+								    client.get(url_for("form_watch_checknow"), follow_redirects=True)
-												Re #265 - extended jsonpath support (#266)

* Re #265 - Use extended JSONpath support,
Allow a JSONPath selector to not match anything (yet)
Adding test
Correctly capture invalid JSONPath query error
											
										
										
											3 years ago
+								    # Give the thread time to pick it up
 								    time.sleep(4)
 								    # It should have 'unviewed'
 								    res = client.get(url_for("index"))
 								    assert b'unviewed' in res.data
 								    res = client.get(url_for("diff_history_page", uuid="first"))
 								    # We should never see 'ForSale' because we are selecting on 'Sold' in the rule,
 								    # But we should know it triggered ('unviewed' assert above)
 								    assert b'ForSale' not in res.data
 								    assert b'Sold' in res.data
-												Adds support for jq JSON path querying engine (#1001)


											
										
										
											2 years ago
+								    res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True)
 								    assert b'Deleted' in res.data
 								def test_check_jsonpath_ext_filter(client, live_server):
 								    check_json_ext_filter('json:$[?(@.status==Sold)]', client, live_server)
 								def test_check_jq_ext_filter(client, live_server):
 								    check_json_ext_filter('jq:.[] | select(.status | contains("Sold"))', client, live_server)