From 52c895b2e8009977ad60dfc2b715f3ecbeeac738 Mon Sep 17 00:00:00 2001 From: Constantin Hong Date: Wed, 21 Feb 2024 19:46:23 +0900 Subject: [PATCH] text_json_diff/fix: Keep an order of filter and remove duplicated filters. 2 (#2178) --- .../processors/text_json_diff.py | 2 +- changedetectionio/tests/test_group.py | 151 ++++++++++++++++++ 2 files changed, 152 insertions(+), 1 deletion(-) diff --git a/changedetectionio/processors/text_json_diff.py b/changedetectionio/processors/text_json_diff.py index 8e341432..e89e469d 100644 --- a/changedetectionio/processors/text_json_diff.py +++ b/changedetectionio/processors/text_json_diff.py @@ -119,7 +119,7 @@ class perform_site_check(difference_detection_processor): include_filters_from_tags = self.datastore.get_tag_overrides_for_watch(uuid=uuid, attr='include_filters') # 1845 - remove duplicated filters in both group and watch include filter - include_filters_rule = list({*watch.get('include_filters', []), *include_filters_from_tags}) + include_filters_rule = list(dict.fromkeys(watch.get('include_filters', []) + include_filters_from_tags)) subtractive_selectors = [*self.datastore.get_tag_overrides_for_watch(uuid=uuid, attr='subtractive_selectors'), *watch.get("subtractive_selectors", []), diff --git a/changedetectionio/tests/test_group.py b/changedetectionio/tests/test_group.py index ed38cb98..d9912a06 100644 --- a/changedetectionio/tests/test_group.py +++ b/changedetectionio/tests/test_group.py @@ -321,3 +321,154 @@ def test_clone_tag_on_quickwatchform_add(client, live_server): res = client.get(url_for("tags.delete_all"), follow_redirects=True) assert b'All tags deleted' in res.data + +def test_order_of_filters_tag_filter_and_watch_filter(client, live_server): + + # Add a tag with some config, import a tag and it should roughly work + res = client.post( + url_for("tags.form_tag_add"), + data={"name": "test-tag-keep-order"}, + follow_redirects=True + ) + assert b"Tag added" in res.data + assert b"test-tag-keep-order" in res.data + tag_filters = [ + '#only-this', # duplicated filters + '#only-this', + '#only-this', + '#only-this', + ] + + res = client.post( + url_for("tags.form_tag_edit_submit", uuid="first"), + data={"name": "test-tag-keep-order", + "include_filters": '\n'.join(tag_filters) }, + follow_redirects=True + ) + assert b"Updated" in res.data + tag_uuid = get_UUID_for_tag_name(client, name="test-tag-keep-order") + res = client.get( + url_for("tags.form_tag_edit", uuid="first") + ) + assert b"#only-this" in res.data + + + d = """ + + Some initial text
+

And 1 this

+
+

And 2 this

+

And 3 this

+

And 4 this

+

And 5 this

+

And 6 this

+

And 7 this

+

And 8 this

+

And 9 this

+

And 10 this

+

And 11 this

+

And 12 this

+

And 13 this

+

And 14 this

+

And 15 this

+ + + """ + + with open("test-datastore/endpoint-content.txt", "w") as f: + f.write(d) + + test_url = url_for('test_endpoint', _external=True) + res = client.post( + url_for("import_page"), + data={"urls": test_url}, + follow_redirects=True + ) + assert b"1 Imported" in res.data + wait_for_all_checks(client) + + filters = [ + '/html/body/p[3]', + '/html/body/p[4]', + '/html/body/p[5]', + '/html/body/p[6]', + '/html/body/p[7]', + '/html/body/p[8]', + '/html/body/p[9]', + '/html/body/p[10]', + '/html/body/p[11]', + '/html/body/p[12]', + '/html/body/p[13]', # duplicated tags + '/html/body/p[13]', + '/html/body/p[13]', + '/html/body/p[13]', + '/html/body/p[13]', + '/html/body/p[14]', + ] + + res = client.post( + url_for("edit_page", uuid="first"), + data={"include_filters": '\n'.join(filters), + "url": test_url, + "tags": "test-tag-keep-order", + "headers": "", + 'fetch_backend': "html_requests"}, + follow_redirects=True + ) + assert b"Updated watch." in res.data + wait_for_all_checks(client) + + res = client.get( + url_for("preview_page", uuid="first"), + follow_redirects=True + ) + + assert b"And 1 this" in res.data # test-tag-keep-order + + a_tag_filter_check = b'And 1 this' #'#only-this' of tag_filters + # check there is no duplication of tag_filters + assert res.data.count(a_tag_filter_check) == 1, f"duplicated filters didn't removed {res.data.count(a_tag_filter_check)} of {a_tag_filter_check} in {res.data=}" + + a_filter_check = b"And 13 this" # '/html/body/p[13]' + # check there is no duplication of filters + assert res.data.count(a_filter_check) == 1, f"duplicated filters didn't removed. {res.data.count(a_filter_check)} of {a_filter_check} in {res.data=}" + + a_filter_check_not_include = b"And 2 this" # '/html/body/p[2]' + assert a_filter_check_not_include not in res.data + + checklist = [ + b"And 3 this", + b"And 4 this", + b"And 5 this", + b"And 6 this", + b"And 7 this", + b"And 8 this", + b"And 9 this", + b"And 10 this", + b"And 11 this", + b"And 12 this", + b"And 13 this", + b"And 14 this", + b"And 1 this", # result of filter from tag. + ] + # check whether everything a user requested is there + for test in checklist: + assert test in res.data + + # check whether everything a user requested is in order of filters. + n = 0 + for test in checklist: + t_index = res.data[n:].find(test) + # if the text is not searched, return -1. + assert t_index >= 0, f"""failed because {test=} not in {res.data[n:]=} +##################### +Looks like some feature changed the order of result of filters. +##################### +the {test} appeared before. {test in res.data[:n]=} +{res.data[:n]=} + """ + n += t_index + len(test) + + res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True) + assert b'Deleted' in res.data