diff --git a/README.md b/README.md index 67de6c11..e0a941e4 100644 --- a/README.md +++ b/README.md @@ -268,3 +268,7 @@ I offer commercial support, this software is depended on by network security, ae [license-shield]: https://img.shields.io/github/license/dgtlmoon/changedetection.io.svg?style=for-the-badge [release-link]: https://github.com/dgtlmoon/changedetection.io/releases [docker-link]: https://hub.docker.com/r/dgtlmoon/changedetection.io + +## Third-party licenses + +changedetectionio.html_tools.elementpath_tostring: Copyright (c), 2018-2021, SISSA (Scuola Internazionale Superiore di Studi Avanzati), Licensed under [MIT license](https://github.com/sissaschool/elementpath/blob/master/LICENSE) diff --git a/changedetectionio/blueprint/tags/templates/edit-tag.html b/changedetectionio/blueprint/tags/templates/edit-tag.html index 6372156d..449ba382 100644 --- a/changedetectionio/blueprint/tags/templates/edit-tag.html +++ b/changedetectionio/blueprint/tags/templates/edit-tag.html @@ -69,11 +69,12 @@ xpath://body/div/span[contains(@class, 'example-class')]", {% endif %} -
  • XPath - Limit text to this XPath rule, simply start with a forward-slash, +
  • XPath - Limit text to this XPath rule, simply start with a forward-slash. To specify XPath to be used explicitly or the XPath rule starts with an XPath function: Prefix with xpath:
    • diff --git a/changedetectionio/forms.py b/changedetectionio/forms.py index b3de842b..c640b218 100644 --- a/changedetectionio/forms.py +++ b/changedetectionio/forms.py @@ -328,11 +328,30 @@ class ValidateCSSJSONXPATHInput(object): return # Does it look like XPath? - if line.strip()[0] == '/': + if line.strip()[0] == '/' or line.strip().startswith('xpath:'): if not self.allow_xpath: raise ValidationError("XPath not permitted in this field!") from lxml import etree, html + import elementpath + # xpath 2.0-3.1 + from elementpath.xpath3 import XPath3Parser tree = html.fromstring("") + line = line.replace('xpath:', '') + + try: + elementpath.select(tree, line.strip(), parser=XPath3Parser) + except elementpath.ElementPathError as e: + message = field.gettext('\'%s\' is not a valid XPath expression. (%s)') + raise ValidationError(message % (line, str(e))) + except: + raise ValidationError("A system-error occurred when validating your XPath expression") + + if line.strip().startswith('xpath1:'): + if not self.allow_xpath: + raise ValidationError("XPath not permitted in this field!") + from lxml import etree, html + tree = html.fromstring("") + line = line.replace('xpath1:', '') try: tree.xpath(line.strip()) diff --git a/changedetectionio/html_tools.py b/changedetectionio/html_tools.py index 19ca653b..7c9844c8 100644 --- a/changedetectionio/html_tools.py +++ b/changedetectionio/html_tools.py @@ -69,10 +69,89 @@ def element_removal(selectors: List[str], html_content): selector = ",".join(selectors) return subtractive_css_selector(selector, html_content) +def elementpath_tostring(obj): + """ + change elementpath.select results to string type + # The MIT License (MIT), Copyright (c), 2018-2021, SISSA (Scuola Internazionale Superiore di Studi Avanzati) + # https://github.com/sissaschool/elementpath/blob/dfcc2fd3d6011b16e02bf30459a7924f547b47d0/elementpath/xpath_tokens.py#L1038 + """ + + import elementpath + from decimal import Decimal + import math + + if obj is None: + return '' + # https://elementpath.readthedocs.io/en/latest/xpath_api.html#elementpath.select + elif isinstance(obj, elementpath.XPathNode): + return obj.string_value + elif isinstance(obj, bool): + return 'true' if obj else 'false' + elif isinstance(obj, Decimal): + value = format(obj, 'f') + if '.' in value: + return value.rstrip('0').rstrip('.') + return value + + elif isinstance(obj, float): + if math.isnan(obj): + return 'NaN' + elif math.isinf(obj): + return str(obj).upper() + + value = str(obj) + if '.' in value: + value = value.rstrip('0').rstrip('.') + if '+' in value: + value = value.replace('+', '') + if 'e' in value: + return value.upper() + return value + + return str(obj) # Return str Utf-8 of matched rules def xpath_filter(xpath_filter, html_content, append_pretty_line_formatting=False, is_rss=False): from lxml import etree, html + import elementpath + # xpath 2.0-3.1 + from elementpath.xpath3 import XPath3Parser + + parser = etree.HTMLParser() + if is_rss: + # So that we can keep CDATA for cdata_in_document_to_text() to process + parser = etree.XMLParser(strip_cdata=False) + + tree = html.fromstring(bytes(html_content, encoding='utf-8'), parser=parser) + html_block = "" + + r = elementpath.select(tree, xpath_filter.strip(), namespaces={'re': 'http://exslt.org/regular-expressions'}, parser=XPath3Parser) + #@note: //title/text() wont work where CDATA.. + + if type(r) != list: + r = [r] + + for element in r: + # When there's more than 1 match, then add the suffix to separate each line + # And where the matched result doesn't include something that will cause Inscriptis to add a newline + # (This way each 'match' reliably has a new-line in the diff) + # Divs are converted to 4 whitespaces by inscriptis + if append_pretty_line_formatting and len(html_block) and (not hasattr( element, 'tag' ) or not element.tag in (['br', 'hr', 'div', 'p'])): + html_block += TEXT_FILTER_LIST_LINE_SUFFIX + + if type(element) == str: + html_block += element + elif issubclass(type(element), etree._Element) or issubclass(type(element), etree._ElementTree): + html_block += etree.tostring(element, pretty_print=True).decode('utf-8') + else: + html_block += elementpath_tostring(element) + + return html_block + +# Return str Utf-8 of matched rules +# 'xpath1:' +def xpath1_filter(xpath_filter, html_content, append_pretty_line_formatting=False, is_rss=False): + from lxml import etree, html parser = None if is_rss: diff --git a/changedetectionio/processors/text_json_diff.py b/changedetectionio/processors/text_json_diff.py index fc35b135..b8cf8a9e 100644 --- a/changedetectionio/processors/text_json_diff.py +++ b/changedetectionio/processors/text_json_diff.py @@ -173,6 +173,11 @@ class perform_site_check(difference_detection_processor): html_content=self.fetcher.content, append_pretty_line_formatting=not watch.is_source_type_url, is_rss=is_rss) + elif filter_rule.startswith('xpath1:'): + html_content += html_tools.xpath1_filter(xpath_filter=filter_rule.replace('xpath1:', ''), + html_content=fetcher.content, + append_pretty_line_formatting=not is_source, + is_rss=is_rss) else: # CSS Filter, extract the HTML that matches and feed that into the existing inscriptis::get_text html_content += html_tools.include_filters(include_filters=filter_rule, diff --git a/changedetectionio/store.py b/changedetectionio/store.py index c00018c4..9522d582 100644 --- a/changedetectionio/store.py +++ b/changedetectionio/store.py @@ -847,4 +847,14 @@ class ChangeDetectionStore: if not watch.get('date_created'): self.data['watching'][uuid]['date_created'] = i i+=1 - return \ No newline at end of file + return + + # #1774 - protect xpath1 against migration + def update_14(self): + for awatch in self.__data["watching"]: + if self.__data["watching"][awatch]['include_filters']: + for num, selector in enumerate(self.__data["watching"][awatch]['include_filters']): + if selector.startswith('/'): + self.__data["watching"][awatch]['include_filters'][num] = 'xpath1:' + selector + if selector.startswith('xpath:'): + self.__data["watching"][awatch]['include_filters'][num] = selector.replace('xpath:', 'xpath1:', 1) diff --git a/changedetectionio/templates/edit.html b/changedetectionio/templates/edit.html index 270cdbce..e6882280 100644 --- a/changedetectionio/templates/edit.html +++ b/changedetectionio/templates/edit.html @@ -290,11 +290,12 @@ xpath://body/div/span[contains(@class, 'example-class')]", {% endif %} </ul> </li> - <li>XPath - Limit text to this XPath rule, simply start with a forward-slash, + <li>XPath - Limit text to this XPath rule, simply start with a forward-slash. To specify XPath to be used explicitly or the XPath rule starts with an XPath function: Prefix with <code>xpath:</code> <ul> - <li>Example: <code>//*[contains(@class, 'sametext')]</code> or <code>xpath://*[contains(@class, 'sametext')]</code>, <a + <li>Example: <code>//*[contains(@class, 'sametext')]</code> or <code>xpath:count(//*[contains(@class, 'sametext')])</code>, <a href="http://xpather.com/" target="new">test your XPath here</a></li> <li>Example: Get all titles from an RSS feed <code>//title/text()</code></li> + <li>To use XPath1.0: Prefix with <code>xpath1:</code></li> </ul> </li> </ul> diff --git a/changedetectionio/tests/test_xpath_selector.py b/changedetectionio/tests/test_xpath_selector.py index ba263ecf..836dd5b5 100644 --- a/changedetectionio/tests/test_xpath_selector.py +++ b/changedetectionio/tests/test_xpath_selector.py @@ -6,9 +6,11 @@ from .util import live_server_setup, wait_for_all_checks from ..html_tools import * + def test_setup(live_server): live_server_setup(live_server) + def set_original_response(): test_return_data = """<html> <body> @@ -26,6 +28,7 @@ def set_original_response(): f.write(test_return_data) return None + def set_modified_response(): test_return_data = """<html> <body> @@ -44,11 +47,12 @@ def set_modified_response(): return None + # Handle utf-8 charset replies https://github.com/dgtlmoon/changedetection.io/pull/613 def test_check_xpath_filter_utf8(client, live_server): - filter='//item/*[self::description]' + filter = '//item/*[self::description]' - d='''<?xml version="1.0" encoding="UTF-8"?> + d = '''<?xml version="1.0" encoding="UTF-8"?> <rss xmlns:taxo="http://purl.org/rss/1.0/modules/taxonomy/" xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#" xmlns:itunes="http://www.itunes.com/dtds/podcast-1.0.dtd" xmlns:dc="http://purl.org/dc/elements/1.1/" version="2.0"> <channel> <title>rpilocator.com @@ -102,9 +106,9 @@ def test_check_xpath_filter_utf8(client, live_server): # Handle utf-8 charset replies https://github.com/dgtlmoon/changedetection.io/pull/613 def test_check_xpath_text_function_utf8(client, live_server): - filter='//item/title/text()' + filter = '//item/title/text()' - d=''' + d = ''' rpilocator.com @@ -163,15 +167,12 @@ def test_check_xpath_text_function_utf8(client, live_server): res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True) assert b'Deleted' in res.data -def test_check_markup_xpath_filter_restriction(client, live_server): +def test_check_markup_xpath_filter_restriction(client, live_server): xpath_filter = "//*[contains(@class, 'sametext')]" set_original_response() - # Give the endpoint time to spin up - time.sleep(1) - # Add our URL to the import page test_url = url_for('test_endpoint', _external=True) res = client.post( @@ -214,7 +215,6 @@ def test_check_markup_xpath_filter_restriction(client, live_server): def test_xpath_validation(client, live_server): - # Add our URL to the import page test_url = url_for('test_endpoint', _external=True) res = client.post( @@ -235,6 +235,48 @@ def test_xpath_validation(client, live_server): assert b'Deleted' in res.data +def test_xpath23_prefix_validation(client, live_server): + # Add our URL to the import page + test_url = url_for('test_endpoint', _external=True) + res = client.post( + url_for("import_page"), + data={"urls": test_url}, + follow_redirects=True + ) + assert b"1 Imported" in res.data + wait_for_all_checks(client) + + res = client.post( + url_for("edit_page", uuid="first"), + data={"include_filters": "xpath:/something horrible", "url": test_url, "tags": "", "headers": "", 'fetch_backend': "html_requests"}, + follow_redirects=True + ) + assert b"is not a valid XPath expression" in res.data + res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True) + assert b'Deleted' in res.data + + +def test_xpath1_validation(client, live_server): + # Add our URL to the import page + test_url = url_for('test_endpoint', _external=True) + res = client.post( + url_for("import_page"), + data={"urls": test_url}, + follow_redirects=True + ) + assert b"1 Imported" in res.data + wait_for_all_checks(client) + + res = client.post( + url_for("edit_page", uuid="first"), + data={"include_filters": "xpath1:/something horrible", "url": test_url, "tags": "", "headers": "", 'fetch_backend': "html_requests"}, + follow_redirects=True + ) + assert b"is not a valid XPath expression" in res.data + res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True) + assert b'Deleted' in res.data + + # actually only really used by the distll.io importer, but could be handy too def test_check_with_prefix_include_filters(client, live_server): res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True) @@ -254,7 +296,8 @@ def test_check_with_prefix_include_filters(client, live_server): res = client.post( url_for("edit_page", uuid="first"), - data={"include_filters": "xpath://*[contains(@class, 'sametext')]", "url": test_url, "tags": "", "headers": "", 'fetch_backend': "html_requests"}, + data={"include_filters": "xpath://*[contains(@class, 'sametext')]", "url": test_url, "tags": "", "headers": "", + 'fetch_backend': "html_requests"}, follow_redirects=True ) @@ -266,13 +309,15 @@ def test_check_with_prefix_include_filters(client, live_server): follow_redirects=True ) - assert b"Some text thats the same" in res.data #in selector - assert b"Some text that will change" not in res.data #not in selector + assert b"Some text thats the same" in res.data # in selector + assert b"Some text that will change" not in res.data # not in selector client.get(url_for("form_delete", uuid="all"), follow_redirects=True) + + def test_various_rules(client, live_server): # Just check these don't error - #live_server_setup(live_server) + # live_server_setup(live_server) with open("test-datastore/endpoint-content.txt", "w") as f: f.write(""" @@ -285,10 +330,11 @@ def test_various_rules(client, live_server): some linky another some linky - + """) + test_url = url_for('test_endpoint', _external=True) res = client.post( url_for("import_page"), @@ -298,7 +344,6 @@ def test_various_rules(client, live_server): assert b"1 Imported" in res.data wait_for_all_checks(client) - for r in ['//div', '//a', 'xpath://div', 'xpath://a']: res = client.post( url_for("edit_page", uuid="first"), @@ -313,3 +358,153 @@ def test_various_rules(client, live_server): assert b"Updated watch." in res.data res = client.get(url_for("index")) assert b'fetch-error' not in res.data, f"Should not see errors after '{r} filter" + + res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True) + assert b'Deleted' in res.data + + +def test_xpath_20(client, live_server): + test_url = url_for('test_endpoint', _external=True) + res = client.post( + url_for("import_page"), + data={"urls": test_url}, + follow_redirects=True + ) + assert b"1 Imported" in res.data + wait_for_all_checks(client) + + set_original_response() + + test_url = url_for('test_endpoint', _external=True) + res = client.post( + url_for("edit_page", uuid="first"), + data={"include_filters": "//*[contains(@class, 'sametext')]|//*[contains(@class, 'changetext')]", + "url": test_url, + "tags": "", + "headers": "", + 'fetch_backend': "html_requests"}, + follow_redirects=True + ) + + assert b"Updated watch." in res.data + wait_for_all_checks(client) + + res = client.get( + url_for("preview_page", uuid="first"), + follow_redirects=True + ) + + assert b"Some text thats the same" in res.data # in selector + assert b"Some text that will change" in res.data # in selector + + client.get(url_for("form_delete", uuid="all"), follow_redirects=True) + + +def test_xpath_20_function_count(client, live_server): + set_original_response() + + # Add our URL to the import page + test_url = url_for('test_endpoint', _external=True) + res = client.post( + url_for("import_page"), + data={"urls": test_url}, + follow_redirects=True + ) + assert b"1 Imported" in res.data + wait_for_all_checks(client) + + res = client.post( + url_for("edit_page", uuid="first"), + data={"include_filters": "xpath:count(//div) * 123456789987654321", + "url": test_url, + "tags": "", + "headers": "", + 'fetch_backend': "html_requests"}, + follow_redirects=True + ) + + assert b"Updated watch." in res.data + wait_for_all_checks(client) + + res = client.get( + url_for("preview_page", uuid="first"), + follow_redirects=True + ) + + assert b"246913579975308642" in res.data # in selector + + client.get(url_for("form_delete", uuid="all"), follow_redirects=True) + + +def test_xpath_20_function_count2(client, live_server): + set_original_response() + + # Add our URL to the import page + test_url = url_for('test_endpoint', _external=True) + res = client.post( + url_for("import_page"), + data={"urls": test_url}, + follow_redirects=True + ) + assert b"1 Imported" in res.data + wait_for_all_checks(client) + + res = client.post( + url_for("edit_page", uuid="first"), + data={"include_filters": "/html/body/count(div) * 123456789987654321", + "url": test_url, + "tags": "", + "headers": "", + 'fetch_backend': "html_requests"}, + follow_redirects=True + ) + + assert b"Updated watch." in res.data + wait_for_all_checks(client) + + res = client.get( + url_for("preview_page", uuid="first"), + follow_redirects=True + ) + + assert b"246913579975308642" in res.data # in selector + + client.get(url_for("form_delete", uuid="all"), follow_redirects=True) + + +def test_xpath_20_function_string_join_matches(client, live_server): + set_original_response() + + # Add our URL to the import page + test_url = url_for('test_endpoint', _external=True) + res = client.post( + url_for("import_page"), + data={"urls": test_url}, + follow_redirects=True + ) + assert b"1 Imported" in res.data + wait_for_all_checks(client) + + res = client.post( + url_for("edit_page", uuid="first"), + data={ + "include_filters": "xpath:string-join(//*[contains(@class, 'sametext')]|//*[matches(@class, 'changetext')], 'specialconjunction')", + "url": test_url, + "tags": "", + "headers": "", + 'fetch_backend': "html_requests"}, + follow_redirects=True + ) + + assert b"Updated watch." in res.data + wait_for_all_checks(client) + + res = client.get( + url_for("preview_page", uuid="first"), + follow_redirects=True + ) + + assert b"Some text thats the samespecialconjunctionSome text that will change" in res.data # in selector + + client.get(url_for("form_delete", uuid="all"), follow_redirects=True) + diff --git a/changedetectionio/tests/test_xpath_selector_unit.py b/changedetectionio/tests/test_xpath_selector_unit.py new file mode 100644 index 00000000..b4dda080 --- /dev/null +++ b/changedetectionio/tests/test_xpath_selector_unit.py @@ -0,0 +1,203 @@ +import sys +import os +import pytest +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +import html_tools + +# test generation guide. +# 1. Do not include encoding in the xml declaration if the test object is a str type. +# 2. Always paraphrase test. + +hotels = """ + + + + Christopher + Anderson + 25 + + + Christopher + Carter + 30 + + + + + Lisa + Walker + 60 + + + Jessica + Walker + 32 + + + Jennifer + Roberts + 50 + + +""" + +@pytest.mark.parametrize("html_content", [hotels]) +@pytest.mark.parametrize("xpath, answer", [('(//staff/given_name, //staff/age)', '25'), + ("xs:date('2023-10-10')", '2023-10-10'), + ("if (/hotel/branch[@location = 'California']/staff[1]/age = 25) then 'is 25' else 'is not 25'", 'is 25'), + ("if (//hotel/branch[@location = 'California']/staff[1]/age = 25) then 'is 25' else 'is not 25'", 'is 25'), + ("if (count(/hotel/branch/staff) = 5) then true() else false()", 'true'), + ("if (count(//hotel/branch/staff) = 5) then true() else false()", 'true'), + ("for $i in /hotel/branch/staff return if ($i/age >= 40) then upper-case($i/surname) else lower-case($i/surname)", 'anderson'), + ("given_name = 'Christopher' and age = 40", 'false'), + ("//given_name = 'Christopher' and //age = 40", 'false'), + #("(staff/given_name, staff/age)", 'Lisa'), + ("(//staff/given_name, //staff/age)", 'Lisa'), + #("hotel/branch[@location = 'California']/staff/age union hotel/branch[@location = 'Las Vegas']/staff/age", ''), + ("(//hotel/branch[@location = 'California']/staff/age union //hotel/branch[@location = 'Las Vegas']/staff/age)", '60'), + ("(200 to 210)", "205"), + ("(//hotel/branch[@location = 'California']/staff/age union //hotel/branch[@location = 'Las Vegas']/staff/age)", "50"), + ("(1, 9, 9, 5)", "5"), + ("(3, (), (14, 15), 92, 653)", "653"), + ("for $i in /hotel/branch/staff return $i/given_name", "Christopher"), + ("for $i in //hotel/branch/staff return $i/given_name", "Christopher"), + ("distinct-values(for $i in /hotel/branch/staff return $i/given_name)", "Jessica"), + ("distinct-values(for $i in //hotel/branch/staff return $i/given_name)", "Jessica"), + ("for $i in (7 to 15) return $i*10", "130"), + ("some $i in /hotel/branch/staff satisfies $i/age < 20", "false"), + ("some $i in //hotel/branch/staff satisfies $i/age < 20", "false"), + ("every $i in /hotel/branch/staff satisfies $i/age > 20", "true"), + ("every $i in //hotel/branch/staff satisfies $i/age > 20 ", "true"), + ("let $x := branch[@location = 'California'], $y := branch[@location = 'Las Vegas'] return (avg($x/staff/age), avg($y/staff/age))", "27.5"), + ("let $x := //branch[@location = 'California'], $y := //branch[@location = 'Las Vegas'] return (avg($x/staff/age), avg($y/staff/age))", "27.5"), + ("let $nu := 1, $de := 1000 return 'probability = ' || $nu div $de * 100 || '%'", "0.1%"), + ("let $nu := 2, $probability := function ($argument) { 'probability = ' || $nu div $argument * 100 || '%'}, $de := 5 return $probability($de)", "40%"), + ("'XPATH2.0-3.1 dissemination' instance of xs:string ", "true"), + ("'new stackoverflow question incoming' instance of xs:integer ", "false"), + ("'50000' cast as xs:integer", "50000"), + ("//branch[@location = 'California']/staff[1]/surname eq 'Anderson'", "true"), + ("fn:false()", "false")]) +def test_hotels(html_content, xpath, answer): + html_content = html_tools.xpath_filter(xpath, html_content, append_pretty_line_formatting=True) + assert type(html_content) == str + assert answer in html_content + + + +branches_to_visit = """ + + + Area 51 + A place with no name + Stalsk12 + + + Stalsk12 + Barcelona + Paris + + """ +@pytest.mark.parametrize("html_content", [branches_to_visit]) +@pytest.mark.parametrize("xpath, answer", [ + ("manager[@name = 'Godot']/branch union manager[@name = 'Freya']/branch", "Area 51"), + ("//manager[@name = 'Godot']/branch union //manager[@name = 'Freya']/branch", "Stalsk12"), + ("manager[@name = 'Godot']/branch | manager[@name = 'Freya']/branch", "Stalsk12"), + ("//manager[@name = 'Godot']/branch | //manager[@name = 'Freya']/branch", "Stalsk12"), + ("manager/branch intersect manager[@name = 'Godot']/branch", "A place with no name"), + ("//manager/branch intersect //manager[@name = 'Godot']/branch", "A place with no name"), + ("manager[@name = 'Godot']/branch intersect manager[@name = 'Freya']/branch", ""), + ("manager/branch except manager[@name = 'Godot']/branch", "Barcelona"), + ("manager[@name = 'Godot']/branch[1] eq 'Area 51'", "true"), + ("//manager[@name = 'Godot']/branch[1] eq 'Area 51'", "true"), + ("manager[@name = 'Godot']/branch[1] eq 'Seoul'", "false"), + ("//manager[@name = 'Godot']/branch[1] eq 'Seoul'", "false"), + ("manager[@name = 'Godot']/branch[2] eq manager[@name = 'Freya']/branch[2]", "false"), + ("//manager[@name = 'Godot']/branch[2] eq //manager[@name = 'Freya']/branch[2]", "false"), + ("manager[1]/@room_no lt manager[2]/@room_no", "false"), + ("//manager[1]/@room_no lt //manager[2]/@room_no", "false"), + ("manager[1]/@room_no gt manager[2]/@room_no", "true"), + ("//manager[1]/@room_no gt //manager[2]/@room_no", "true"), + ("manager[@name = 'Godot']/branch[1] = 'Area 51'", "true"), + ("//manager[@name = 'Godot']/branch[1] = 'Area 51'", "true"), + ("manager[@name = 'Godot']/branch[1] = 'Seoul'", "false"), + ("//manager[@name = 'Godot']/branch[1] = 'Seoul'", "false"), + ("manager[@name = 'Godot']/branch = 'Area 51'", "true"), + ("//manager[@name = 'Godot']/branch = 'Area 51'", "true"), + ("manager[@name = 'Godot']/branch = 'Barcelona'", "false"), + ("//manager[@name = 'Godot']/branch = 'Barcelona'", "false"), + ("manager[1]/@room_no > manager[2]/@room_no", "true"), + ("//manager[1]/@room_no > //manager[2]/@room_no", "true"), + ("manager[@name = 'Godot']/branch[ . = 'Stalsk12'] is manager[1]/branch[1]", "false"), + ("//manager[@name = 'Godot']/branch[ . = 'Stalsk12'] is //manager[1]/branch[1]", "false"), + ("manager[@name = 'Godot']/branch[ . = 'Stalsk12'] is manager[1]/branch[3]", "true"), + ("//manager[@name = 'Godot']/branch[ . = 'Stalsk12'] is //manager[1]/branch[3]", "true"), + ("manager[@name = 'Godot']/branch[ . = 'Stalsk12'] << manager[1]/branch[1]", "false"), + ("//manager[@name = 'Godot']/branch[ . = 'Stalsk12'] << //manager[1]/branch[1]", "false"), + ("manager[@name = 'Godot']/branch[ . = 'Stalsk12'] >> manager[1]/branch[1]", "true"), + ("//manager[@name = 'Godot']/branch[ . = 'Stalsk12'] >> //manager[1]/branch[1]", "true"), + ("manager[@name = 'Godot']/branch[ . = 'Stalsk12'] is manager[@name = 'Freya']/branch[ . = 'Stalsk12']", "false"), + ("//manager[@name = 'Godot']/branch[ . = 'Stalsk12'] is //manager[@name = 'Freya']/branch[ . = 'Stalsk12']", "false"), + ("manager[1]/@name || manager[2]/@name", "GodotFreya"), + ("//manager[1]/@name || //manager[2]/@name", "GodotFreya"), + ]) +def test_branches_to_visit(html_content, xpath, answer): + html_content = html_tools.xpath_filter(xpath, html_content, append_pretty_line_formatting=True) + assert type(html_content) == str + assert answer in html_content + +trips = """ + + + 2023-10-06 + 2023-10-10 + + 4 + 2000.00 + + + + 2023-10-06 + 2023-10-12 + + 6 + 3500.34 + + +""" +@pytest.mark.parametrize("html_content", [trips]) +@pytest.mark.parametrize("xpath, answer", [ + ("1 + 9 * 9 + 5 div 5", "83"), + ("(1 + 9 * 9 + 5) div 6", "14.5"), + ("23 idiv 3", "7"), + ("23 div 3", "7.66666666"), + ("for $i in ./trip return $i/traveler/duration * $i/traveler/price", "21002.04"), + ("for $i in ./trip return $i/traveler/duration ", "4"), + ("for $i in .//trip return $i/traveler/duration * $i/traveler/price", "21002.04"), + ("sum(for $i in ./trip return $i/traveler/duration * $i/traveler/price)", "29002.04"), + ("sum(for $i in .//trip return $i/traveler/duration * $i/traveler/price)", "29002.04"), + #("trip[1]/depart - trip[1]/arrive", "fail_to_get_answer"), + #("//trip[1]/depart - //trip[1]/arrive", "fail_to_get_answer"), + #("trip[1]/depart + trip[1]/arrive", "fail_to_get_answer"), + #("xs:date(trip[1]/depart) + xs:date(trip[1]/arrive)", "fail_to_get_answer"), + ("(//trip[1]/arrive cast as xs:date) - (//trip[1]/depart cast as xs:date)", "P4D"), + ("(//trip[1]/depart cast as xs:date) - (//trip[1]/arrive cast as xs:date)", "-P4D"), + ("(//trip[1]/depart cast as xs:date) + xs:dayTimeDuration('P3D')", "2023-10-09"), + ("(//trip[1]/depart cast as xs:date) - xs:dayTimeDuration('P3D')", "2023-10-03"), + ("(456, 623) instance of xs:integer", "false"), + ("(456, 623) instance of xs:integer*", "true"), + ("/trips/trip instance of element()", "false"), + ("/trips/trip instance of element()*", "true"), + ("/trips/trip[1]/arrive instance of xs:date", "false"), + ("date(/trips/trip[1]/arrive) instance of xs:date", "true"), + ("'8' cast as xs:integer", "8"), + ("'11.1E3' cast as xs:double", "11100"), + ("6.5 cast as xs:integer", "6"), + #("/trips/trip[1]/arrive cast as xs:dateTime", "fail_to_get_answer"), + ("/trips/trip[1]/arrive cast as xs:date", "2023-10-10"), + ("('2023-10-12') cast as xs:date", "2023-10-12"), + ("for $i in //trip return concat($i/depart, ' ', $i/arrive)", "2023-10-06 2023-10-10"), + ]) +def test_trips(html_content, xpath, answer): + html_content = html_tools.xpath_filter(xpath, html_content, append_pretty_line_formatting=True) + assert type(html_content) == str + assert answer in html_content diff --git a/requirements.txt b/requirements.txt index 089e7b39..3599c50e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -46,6 +46,9 @@ beautifulsoup4 # XPath filtering, lxml is required by bs4 anyway, but put it here to be safe. lxml +# XPath 2.0-3.1 support +elementpath + selenium~=4.14.0 werkzeug~=3.0