From 3b43da35ec463ca2f698dcb7e6190106a8b14212 Mon Sep 17 00:00:00 2001 From: dgtlmoon Date: Tue, 31 Oct 2023 10:31:34 +0100 Subject: [PATCH 1/2] Docker build - upgrade image to "bookworm" debian version - fix glibc mismatch (#1918) --- Dockerfile | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index 7d4865d0..d0f4ea75 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,5 @@ # pip dependencies install stage -FROM python:3.11-slim-bullseye as builder +FROM python:3.11-slim-bookworm as builder # See `cryptography` pin comment in requirements.txt ARG CRYPTOGRAPHY_DONT_BUILD_RUST=1 @@ -34,10 +34,9 @@ RUN pip install --target=/dependencies playwright~=1.27.1 \ || echo "WARN: Failed to install Playwright. The application can still run, but the Playwright option will be disabled." # Final image stage -FROM python:3.11-slim-bullseye +FROM python:3.11-slim-bookworm RUN apt-get update && apt-get install -y --no-install-recommends \ - libssl1.1 \ libxslt1.1 \ # For pdftohtml poppler-utils \ From e209d9fba018c0f7c5893f4b25054e9a8599370e Mon Sep 17 00:00:00 2001 From: dgtlmoon Date: Wed, 1 Nov 2023 15:36:49 +0100 Subject: [PATCH 2/2] Ability to Import from Wachete XLSX (or any XLSX) - Wachete alternative made easy (#1921) --- changedetectionio/__init__.py | 28 +++- changedetectionio/forms.py | 40 +++-- changedetectionio/importer.py | 146 ++++++++++++++++++ changedetectionio/store.py | 2 + changedetectionio/templates/import.html | 39 ++++- .../tests/import/spreadsheet.xlsx | Bin 0 -> 29444 bytes changedetectionio/tests/test_import.py | 88 ++++++++++- requirements.txt | 2 +- 8 files changed, 323 insertions(+), 22 deletions(-) create mode 100644 changedetectionio/tests/import/spreadsheet.xlsx diff --git a/changedetectionio/__init__.py b/changedetectionio/__init__.py index 850e9966..54c02804 100644 --- a/changedetectionio/__init__.py +++ b/changedetectionio/__init__.py @@ -822,6 +822,7 @@ def changedetection_app(config=None, datastore_o=None): from . import forms if request.method == 'POST': + from .importer import import_url_list, import_distill_io_json # URL List import @@ -845,11 +846,32 @@ def changedetection_app(config=None, datastore_o=None): for uuid in d_importer.new_uuids: update_q.put(queuedWatchMetaData.PrioritizedItem(priority=1, item={'uuid': uuid, 'skip_when_checksum_same': True})) + # XLSX importer + if request.files and request.files.get('xlsx_file'): + file = request.files['xlsx_file'] + from .importer import import_xlsx_wachete, import_xlsx_custom + + if request.values.get('file_mapping') == 'wachete': + w_importer = import_xlsx_wachete() + w_importer.run(data=file, flash=flash, datastore=datastore) + else: + w_importer = import_xlsx_custom() + # Building mapping of col # to col # type + map = {} + for i in range(10): + c = request.values.get(f"custom_xlsx[col_{i}]") + v = request.values.get(f"custom_xlsx[col_type_{i}]") + if c and v: + map[int(c)] = v + + w_importer.import_profile = map + w_importer.run(data=file, flash=flash, datastore=datastore) + + for uuid in w_importer.new_uuids: + update_q.put(queuedWatchMetaData.PrioritizedItem(priority=1, item={'uuid': uuid, 'skip_when_checksum_same': True})) - form = forms.importForm(formdata=request.form if request.method == 'POST' else None, -# data=default, - ) # Could be some remaining, or we could be on GET + form = forms.importForm(formdata=request.form if request.method == 'POST' else None) output = render_template("import.html", form=form, import_url_list_remaining="\n".join(remaining_urls), diff --git a/changedetectionio/forms.py b/changedetectionio/forms.py index 4177ed80..e8c35cb8 100644 --- a/changedetectionio/forms.py +++ b/changedetectionio/forms.py @@ -15,9 +15,14 @@ from wtforms import ( validators, widgets ) +from flask_wtf.file import FileField, FileAllowed from wtforms.fields import FieldList + from wtforms.validators import ValidationError +from validators.url import url as url_validator + + # default # each select