diff --git a/changedetectionio/importer.py b/changedetectionio/importer.py index 963482de..c8f8d475 100644 --- a/changedetectionio/importer.py +++ b/changedetectionio/importer.py @@ -144,6 +144,7 @@ class import_xlsx_wachete(Importer): flash, datastore, ): + good = 0 now = time.time() self.new_uuids = [] @@ -157,25 +158,24 @@ class import_xlsx_wachete(Importer): flash("Unable to read export XLSX file, something wrong with the file?", 'error') return - sheet_obj = wb.active - - i = 1 - row = 2 - while sheet_obj.cell(row=row, column=1).value: + for row in wb.active.iter_rows(min_row=2): + extras = {} data = {} - while sheet_obj.cell(row=row, column=i).value: - column_title = sheet_obj.cell(row=1, column=i).value.strip().lower() - column_row_value = sheet_obj.cell(row=row, column=i).value - data[column_title] = column_row_value + for cell in row: + column_title = wb.active.cell(row=1, column=cell.column).value.strip().lower() + data[column_title] = cell.value - i += 1 + # Forced switch to webdriver/playwright/etc + dynamic_wachet = str(data.get('dynamic wachet')).strip().lower() # Convert bool to str to cover all cases + # libreoffice and others can have it as =FALSE() =TRUE(), or bool(true) + if 'true' in dynamic_wachet or dynamic_wachet == '1': + extras['fetch_backend'] = 'html_webdriver' - extras = {} if data.get('xpath'): #@todo split by || ? extras['include_filters'] = [data.get('xpath')] if data.get('name'): - extras['title'] = [data.get('name').strip()] + extras['title'] = data.get('name').strip() if data.get('interval (min)'): minutes = int(data.get('interval (min)')) hours, minutes = divmod(minutes, 60) @@ -183,7 +183,6 @@ class import_xlsx_wachete(Importer): weeks, days = divmod(days, 7) extras['time_between_check'] = {'weeks': weeks, 'days': days, 'hours': hours, 'minutes': minutes, 'seconds': 0} - # At minimum a URL is required. if data.get('url'): try: @@ -202,10 +201,6 @@ class import_xlsx_wachete(Importer): self.new_uuids.append(new_uuid) good += 1 - row += 1 - i = 1 - - flash( "{} imported from Wachete .xlsx in {:.2f}s".format(len(self.new_uuids), time.time() - now)) @@ -216,6 +211,7 @@ class import_xlsx_custom(Importer): flash, datastore, ): + good = 0 now = time.time() self.new_uuids = [] @@ -231,37 +227,44 @@ class import_xlsx_custom(Importer): # @todo cehck atleast 2 rows, same in other method - sheet_obj = wb.active from .forms import validate_url - row = 2 - while sheet_obj.cell(row=row, column=1).value: + + for row in wb.active.iter_rows(): url = None tags = None extras = {} - for col_i, cell_map in self.import_profile.items(): - cell_val = sheet_obj.cell(row=row, column=col_i).value + + for cell in row: + if not self.import_profile.get(cell.col_idx): + continue + if not cell.value: + continue + + cell_map = self.import_profile.get(cell.col_idx) + + cell_val = str(cell.value).strip() # could be bool + if cell_map == 'url': - url = cell_val.strip() + url = cell.value.strip() try: validate_url(url) except ValidationError as e: - print (">> Import URL error",url, str(e)) + print(">> Import URL error", url, str(e)) # Don't bother processing anything else on this row url = None break - elif cell_map == 'tag': - tags = cell_val.strip() + tags = cell.value.strip() elif cell_map == 'include_filters': # @todo validate? - extras['include_filters'] = [cell_val.strip()] + extras['include_filters'] = [cell.value.strip()] elif cell_map == 'interval_minutes': hours, minutes = divmod(int(cell_val), 60) days, hours = divmod(hours, 24) weeks, days = divmod(days, 7) extras['time_between_check'] = {'weeks': weeks, 'days': days, 'hours': hours, 'minutes': minutes, 'seconds': 0} else: - extras[cell_map] = cell_val.strip() + extras[cell_map] = cell_val # At minimum a URL is required. if url: @@ -274,7 +277,5 @@ class import_xlsx_custom(Importer): self.new_uuids.append(new_uuid) good += 1 - row += 1 - flash( "{} imported from custom .xlsx in {:.2f}s".format(len(self.new_uuids), time.time() - now)) diff --git a/changedetectionio/tests/import/spreadsheet.xlsx b/changedetectionio/tests/import/spreadsheet.xlsx index 600847b8..a0c02c65 100644 Binary files a/changedetectionio/tests/import/spreadsheet.xlsx and b/changedetectionio/tests/import/spreadsheet.xlsx differ diff --git a/changedetectionio/tests/test_import.py b/changedetectionio/tests/test_import.py index 4772226c..04e9d086 100644 --- a/changedetectionio/tests/test_import.py +++ b/changedetectionio/tests/test_import.py @@ -127,6 +127,7 @@ def test_import_custom_xlsx(client, live_server): """Test can upload a excel spreadsheet and the watches are created correctly""" #live_server_setup(live_server) + dirname = os.path.dirname(__file__) filename = os.path.join(dirname, 'import/spreadsheet.xlsx') with open(filename, 'rb') as f: @@ -150,13 +151,12 @@ def test_import_custom_xlsx(client, live_server): follow_redirects=True, ) - assert b'2 imported from custom .xlsx' in res.data + assert b'3 imported from custom .xlsx' in res.data res = client.get( url_for("index") ) - assert b'Somesite results ABC' in res.data assert b'City news results' in res.data @@ -167,6 +167,9 @@ def test_import_custom_xlsx(client, live_server): assert filters[0] == '/html[1]/body[1]/div[4]/div[1]/div[1]/div[1]||//*[@id=\'content\']/div[3]/div[1]/div[1]||//*[@id=\'content\']/div[1]' assert watch.get('time_between_check') == {'weeks': 0, 'days': 1, 'hours': 6, 'minutes': 24, 'seconds': 0} + res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True) + assert b'Deleted' in res.data + def test_import_watchete_xlsx(client, live_server): """Test can upload a excel spreadsheet and the watches are created correctly""" @@ -186,7 +189,7 @@ def test_import_watchete_xlsx(client, live_server): follow_redirects=True, ) - assert b'2 imported from Wachete .xlsx' in res.data + assert b'3 imported from Wachete .xlsx' in res.data res = client.get( url_for("index") @@ -201,3 +204,10 @@ def test_import_watchete_xlsx(client, live_server): filters = watch.get('include_filters') assert filters[0] == '/html[1]/body[1]/div[4]/div[1]/div[1]/div[1]||//*[@id=\'content\']/div[3]/div[1]/div[1]||//*[@id=\'content\']/div[1]' assert watch.get('time_between_check') == {'weeks': 0, 'days': 1, 'hours': 6, 'minutes': 24, 'seconds': 0} + assert watch.get('fetch_backend') == 'system' # always uses default + + if watch.get('title') == 'JS website': + assert watch.get('fetch_backend') == 'html_webdriver' # Has active 'dynamic wachet' + + res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True) + assert b'Deleted' in res.data