diff --git a/.github/workflows/image-javascript.yml b/.github/workflows/image-javascript.yml index a6b4d8ee..99eaba8a 100644 --- a/.github/workflows/image-javascript.yml +++ b/.github/workflows/image-javascript.yml @@ -20,6 +20,7 @@ jobs: python -m pip install --upgrade pip pip install flake8 pytest if [ -f requirements.txt ]; then pip install -r requirements.txt; fi + if [ -f requirements-dev.txt ]; then pip install -r requirements-dev.txt; fi - name: Lint with flake8 run: | @@ -30,14 +31,14 @@ jobs: - name: Create release metadata run: | - # COPY'ed by Dockerfile into backend/ of the image, then read by the server in store.py - echo ${{ github.sha }} > backend/source.txt - echo ${{ github.ref }} > backend/tag.txt + # COPY'ed by Dockerfile into changedetectionio/ of the image, then read by the server in store.py + echo ${{ github.sha }} > changedetectionio/source.txt + echo ${{ github.ref }} > changedetectionio/tag.txt - name: Test with pytest run: | # Each test is totally isolated and performs its own cleanup/reset - cd backend; ./run_all_tests.sh + cd changedetectionio; ./run_all_tests.sh - name: Set up QEMU uses: docker/setup-qemu-action@v1 diff --git a/.github/workflows/image-tag.yml b/.github/workflows/image-tag.yml index 21050f80..67f1e3b3 100644 --- a/.github/workflows/image-tag.yml +++ b/.github/workflows/image-tag.yml @@ -29,6 +29,7 @@ jobs: python -m pip install --upgrade pip pip install flake8 pytest if [ -f requirements.txt ]; then pip install -r requirements.txt; fi + if [ -f requirements-dev.txt ]; then pip install -r requirements-dev.txt; fi - name: Lint with flake8 run: | @@ -39,14 +40,14 @@ jobs: - name: Create release metadata run: | - # COPY'ed by Dockerfile into backend/ of the image, then read by the server in store.py - echo ${{ github.sha }} > backend/source.txt - echo ${{ github.ref }} > backend/tag.txt + # COPY'ed by Dockerfile into changedetectionio/ of the image, then read by the server in store.py + echo ${{ github.sha }} > changedetectionio/source.txt + echo ${{ github.ref }} > changedetectionio/tag.txt - name: Test with pytest run: | # Each test is totally isolated and performs its own cleanup/reset - cd backend; ./run_all_tests.sh + cd changedetectionio; ./run_all_tests.sh - name: Set up QEMU uses: docker/setup-qemu-action@v1 diff --git a/.github/workflows/image.yml b/.github/workflows/image.yml index 7abf493c..10288ffc 100644 --- a/.github/workflows/image.yml +++ b/.github/workflows/image.yml @@ -20,6 +20,7 @@ jobs: python -m pip install --upgrade pip pip install flake8 pytest if [ -f requirements.txt ]; then pip install -r requirements.txt; fi + if [ -f requirements-dev.txt ]; then pip install -r requirements-dev.txt; fi - name: Lint with flake8 run: | @@ -30,14 +31,14 @@ jobs: - name: Create release metadata run: | - # COPY'ed by Dockerfile into backend/ of the image, then read by the server in store.py - echo ${{ github.sha }} > backend/source.txt - echo ${{ github.ref }} > backend/tag.txt + # COPY'ed by Dockerfile into changedetectionio/ of the image, then read by the server in store.py + echo ${{ github.sha }} > changedetectionio/source.txt + echo ${{ github.ref }} > changedetectionio/tag.txt - name: Test with pytest run: | # Each test is totally isolated and performs its own cleanup/reset - cd backend; ./run_all_tests.sh + cd changedetectionio; ./run_all_tests.sh - name: Set up QEMU uses: docker/setup-qemu-action@v1 diff --git a/.github/workflows/test-only.yml b/.github/workflows/test-only.yml index 2481b709..000f2e8a 100644 --- a/.github/workflows/test-only.yml +++ b/.github/workflows/test-only.yml @@ -19,15 +19,28 @@ jobs: python -m pip install --upgrade pip pip install flake8 pytest if [ -f requirements.txt ]; then pip install -r requirements.txt; fi - + if [ -f requirements-dev.txt ]; then pip install -r requirements-dev.txt; fi - name: Lint with flake8 run: | # stop the build if there are Python syntax errors or undefined names flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics + - name: Test with pytest run: | # Each test is totally isolated and performs its own cleanup/reset - cd backend; ./run_all_tests.sh + cd changedetectionio; ./run_all_tests.sh + + - name: Test that pip builds without error + run: | + pip3 --version + python3 -m pip install wheel + python3 setup.py bdist_wheel + python3 -m pip install dist/changedetection.io-*-none-any.whl --force + changedetection.io -d /tmp -p 10000 & + sleep 3 + curl http://127.0.0.1:10000/static/styles/pure-min.css >/dev/null + killall -9 changedetection.io + diff --git a/.gitignore b/.gitignore index df3a8817..cdaa9072 100644 --- a/.gitignore +++ b/.gitignore @@ -5,3 +5,5 @@ datastore/url-watches.json datastore/* __pycache__ .pytest_cache +build +dist diff --git a/Dockerfile b/Dockerfile index 8a0401b4..cd340acf 100644 --- a/Dockerfile +++ b/Dockerfile @@ -47,7 +47,7 @@ COPY --from=builder /dependencies /usr/local ENV PYTHONPATH=/usr/local # The actual flask app -COPY backend /app/backend +COPY changedetectionio /app/changedetectionio # The eventlet server wrapper COPY changedetection.py /app/changedetection.py diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 00000000..ace5309f --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1,4 @@ +recursive-include changedetectionio/templates * +recursive-include changedetectionio/static * +include changedetection.py +global-exclude *.pyc \ No newline at end of file diff --git a/README-pip.md b/README-pip.md new file mode 100644 index 00000000..d770ad09 --- /dev/null +++ b/README-pip.md @@ -0,0 +1,71 @@ +# changedetection.io +![changedetection.io](https://github.com/dgtlmoon/changedetection.io/actions/workflows/test-only.yml/badge.svg?branch=master) + + Docker Pulls + + + Change detection latest tag version + + +## Self-hosted open source change monitoring of web pages. + +_Know when web pages change! Stay ontop of new information!_ + +Live your data-life *pro-actively* instead of *re-actively*, do not rely on manipulative social media for consuming important information. + + +Self-hosted web page change monitoring + +#### Example use cases + +Know when ... + +- Government department updates (changes are often only on their websites) +- Local government news (changes are often only on their websites) +- New software releases, security advisories when you're not on their mailing list. +- Festivals with changes +- Realestate listing changes +- COVID related news from government websites +- Detect and monitor changes in JSON API responses +- API monitoring and alerting + +**Get monitoring now!** + +```bash +$ pip3 install changedetection.io +``` + +Specify a target for the *datastore path* with `-d` (required) and a *listening port* with `-p` (defaults to `5000`) + +```bash +$ changedetection.io -d /path/to/empty/data/dir -p 5000 +``` + + +Then visit http://127.0.0.1:5000 , You should now be able to access the UI. + +### Features +- Website monitoring +- Change detection of content and analyses +- Filters on change (Select by CSS or JSON) +- Triggers (Wait for text, wait for regex) +- Notification support +- JSON API Monitoring +- Parse JSON embedded in HTML +- (Reverse) Proxy support +- Javascript support via WebDriver +- RaspberriPi (arm v6/v7/64 support) + +See https://github.com/dgtlmoon/changedetection.io for more information. + + + +### Support us + +Do you use changedetection.io to make money? does it save you time or money? Does it make your life easier? less stressful? Remember, we write this software when we should be doing actual paid work, we have to buy food and pay rent just like you. + +Please support us, even small amounts help a LOT. + +BTC `1PLFN327GyUarpJd7nVe7Reqg9qHx5frNn` + +Support us! diff --git a/README.md b/README.md index ac8e0f45..dd1c3486 100644 --- a/README.md +++ b/README.md @@ -13,6 +13,8 @@ _Know when web pages change! Stay ontop of new information!_ Live your data-life *pro-actively* instead of *re-actively*, do not rely on manipulative social media for consuming important information. +Open source web page monitoring, notification and change detection. + Self-hosted web page change monitoring @@ -29,32 +31,34 @@ Know when ... - Detect and monitor changes in JSON API responses - API monitoring and alerting -_Need an actual Chrome runner with Javascript support? see the experimental Javascript/Chrome support changedetection.io branch!_ +_Need an actual Chrome runner with Javascript support? We support fetching via WebDriver!_ **Get monitoring now! super simple, one command!** Run the python code on your own machine by cloning this repository, or with docker and/or docker-compose -With one docker-compose command +**Docker** + +With Docker composer, just clone this repository and +```bash +$ docker-compose up -d +``` +Docker standalone +```bash +$ docker run -d --restart always -p "127.0.0.1:5000:5000" -v datastore-volume:/datastore --name changedetection.io dgtlmoon/changedetection.io +``` +**Python PIP** ```bash -docker-compose up -d +$ pip3 install changedetection.io +$ changedetection.io -d /path/to/empty/data/dir -p 5000 ``` Then visit http://127.0.0.1:5000 , You should now be able to access the UI. _Now with per-site configurable support for using a fast built in HTTP fetcher or use a Chrome based fetcher for monitoring of JavaScript websites!_ - -#### Updating to the latest version - -Highly recommended :) - -```bash -docker pull dgtlmoon/changedetection.io -docker-compose up -d -``` - + ### Screenshots Examining differences in content. @@ -131,15 +135,6 @@ For more information see https://docs.python-requests.org/en/master/user/advance This proxy support also extends to the notifications https://github.com/caronc/apprise/issues/387#issuecomment-841718867 -### Notes - -- ~~Does not yet support Javascript~~ -- ~~Wont work with Cloudfare type "Please turn on javascript" protected pages~~ -- You can use the 'headers' section to monitor password protected web page changes - -See the experimental Javascript/Chrome browser support! - - ### RaspberriPi support? RaspberriPi and linux/arm/v6 linux/arm/v7 arm64 devices are supported! diff --git a/changedetection.py b/changedetection.py index 4188ba4a..5814f8fe 100755 --- a/changedetection.py +++ b/changedetection.py @@ -8,11 +8,11 @@ import sys import eventlet import eventlet.wsgi -import backend +import changedetectionio -from backend import store +from changedetectionio import store -def main(argv): +def main(): ssl_mode = False port = os.environ.get('PORT') or 5000 do_cleanup = False @@ -21,7 +21,7 @@ def main(argv): datastore_path = os.path.join(os.getcwd(), "datastore") try: - opts, args = getopt.getopt(argv, "csd:p:", "port") + opts, args = getopt.getopt(sys.argv[1:], "csd:p:", "port") except getopt.GetoptError: print('backend.py -s SSL enable -p [port] -d [datastore path]') sys.exit(2) @@ -48,8 +48,13 @@ def main(argv): # isnt there some @thingy to attach to each route to tell it, that this route needs a datastore app_config = {'datastore_path': datastore_path} - datastore = store.ChangeDetectionStore(datastore_path=app_config['datastore_path']) - app = backend.changedetection_app(app_config, datastore) + if not os.path.isdir(app_config['datastore_path']): + print ("ERROR: Directory path for the datastore '{}' does not exist, cannot start, please make sure the directory exists.\n" + "Alternatively, use the -d parameter.".format(app_config['datastore_path']),file=sys.stderr) + sys.exit(2) + + datastore = store.ChangeDetectionStore(datastore_path=app_config['datastore_path'], version_tag=changedetectionio.__version__) + app = changedetectionio.changedetection_app(app_config, datastore) # Go into cleanup mode if do_cleanup: @@ -89,4 +94,4 @@ def main(argv): if __name__ == '__main__': - main(sys.argv[1:]) + main() diff --git a/backend/__init__.py b/changedetectionio/__init__.py similarity index 97% rename from backend/__init__.py rename to changedetectionio/__init__.py index e42d44d3..21684fe9 100644 --- a/backend/__init__.py +++ b/changedetectionio/__init__.py @@ -29,6 +29,8 @@ from flask import make_response import datetime import pytz +__version__ = '0.39' + datastore = None # Local @@ -41,7 +43,11 @@ update_q = queue.Queue() notification_q = queue.Queue() -app = Flask(__name__, static_url_path="/var/www/change-detection/backend/static") +# Needs to be set this way because we also build and publish via pip +base_path = os.path.dirname(os.path.realpath(__file__)) +app = Flask(__name__, + static_url_path="{}/static".format(base_path), + template_folder="{}/templates".format(base_path)) # Stop browser caching of assets app.config['SEND_FILE_MAX_AGE_DEFAULT'] = 0 @@ -157,7 +163,6 @@ def changedetection_app(config=None, datastore_o=None): global datastore datastore = datastore_o - app.config.update(dict(DEBUG=True)) #app.config.update(config or {}) login_manager = flask_login.LoginManager(app) @@ -278,7 +283,7 @@ def changedetection_app(config=None, datastore_o=None): return response else: - from backend import forms + from changedetectionio import forms form = forms.quickWatchForm(request.form) output = render_template("watch-overview.html", @@ -344,7 +349,7 @@ def changedetection_app(config=None, datastore_o=None): def get_current_checksum_include_ignore_text(uuid): import hashlib - from backend import fetch_site_status + from changedetectionio import fetch_site_status # Get the most recent one newest_history_key = datastore.get_val(uuid, 'newest_history_key') @@ -371,7 +376,7 @@ def changedetection_app(config=None, datastore_o=None): @app.route("/edit/", methods=['GET', 'POST']) @login_required def edit_page(uuid): - from backend import forms + from changedetectionio import forms form = forms.watchForm(request.form) # More for testing, possible to return the first/only @@ -473,8 +478,8 @@ def changedetection_app(config=None, datastore_o=None): @login_required def settings_page(): - from backend import forms - from backend import content_fetcher + from changedetectionio import forms + from changedetectionio import content_fetcher form = forms.globalSettingsForm(request.form) @@ -722,18 +727,15 @@ def changedetection_app(config=None, datastore_o=None): @app.route("/static//", methods=['GET']) def static_content(group, filename): # These files should be in our subdirectory - full_path = os.path.realpath(__file__) - p = os.path.dirname(full_path) - try: - return send_from_directory("{}/static/{}".format(p, group), filename=filename) + return send_from_directory("static/{}".format(group), filename=filename) except FileNotFoundError: abort(404) @app.route("/api/add", methods=['POST']) @login_required def api_watch_add(): - from backend import forms + from changedetectionio import forms form = forms.quickWatchForm(request.form) if form.validate(): @@ -821,7 +823,7 @@ def check_for_new_version(): while not app.config.exit.is_set(): try: r = requests.post("https://changedetection.io/check-ver.php", - data={'version': datastore.data['version_tag'], + data={'version': __version__, 'app_guid': datastore.data['app_guid'], 'watch_count': len(datastore.data['watching']) }, @@ -850,7 +852,7 @@ def notification_runner(): else: # Process notifications try: - from backend import notification + from changedetectionio import notification notification.process_notification(n_object, datastore) except Exception as e: @@ -860,7 +862,7 @@ def notification_runner(): # Thread runner to check every minute, look for new watches to feed into the Queue. def ticker_thread_check_time_launch_checks(): - from backend import update_worker + from changedetectionio import update_worker # Spin up Workers. for _ in range(datastore.data['settings']['requests']['workers']): diff --git a/backend/content_fetcher.py b/changedetectionio/content_fetcher.py similarity index 98% rename from backend/content_fetcher.py rename to changedetectionio/content_fetcher.py index 50d649ec..d8221096 100644 --- a/backend/content_fetcher.py +++ b/changedetectionio/content_fetcher.py @@ -45,7 +45,7 @@ class Fetcher(): def available_fetchers(): import inspect - from backend import content_fetcher + from changedetectionio import content_fetcher p=[] for name, obj in inspect.getmembers(content_fetcher): if inspect.isclass(obj): diff --git a/backend/dev-docker/Dockerfile b/changedetectionio/dev-docker/Dockerfile similarity index 100% rename from backend/dev-docker/Dockerfile rename to changedetectionio/dev-docker/Dockerfile diff --git a/backend/dev-docker/sleep.py b/changedetectionio/dev-docker/sleep.py similarity index 100% rename from backend/dev-docker/sleep.py rename to changedetectionio/dev-docker/sleep.py diff --git a/backend/fetch_site_status.py b/changedetectionio/fetch_site_status.py similarity index 99% rename from backend/fetch_site_status.py rename to changedetectionio/fetch_site_status.py index 61ff7746..18f2b0bd 100644 --- a/backend/fetch_site_status.py +++ b/changedetectionio/fetch_site_status.py @@ -1,5 +1,5 @@ import time -from backend import content_fetcher +from changedetectionio import content_fetcher import hashlib from inscriptis import get_text import urllib3 diff --git a/backend/forms.py b/changedetectionio/forms.py similarity index 98% rename from backend/forms.py rename to changedetectionio/forms.py index ae0847e0..e8f82bdc 100644 --- a/backend/forms.py +++ b/changedetectionio/forms.py @@ -3,7 +3,7 @@ from wtforms import Form, SelectField, RadioField, BooleanField, StringField, Pa from wtforms import widgets from wtforms.validators import ValidationError from wtforms.fields import html5 -from backend import content_fetcher +from changedetectionio import content_fetcher import re class StringListField(StringField): @@ -91,7 +91,7 @@ class ValidateContentFetcherIsReady(object): self.message = message def __call__(self, form, field): - from backend import content_fetcher + from changedetectionio import content_fetcher import urllib3.exceptions # Better would be a radiohandler that keeps a reference to each class diff --git a/backend/html_tools.py b/changedetectionio/html_tools.py similarity index 100% rename from backend/html_tools.py rename to changedetectionio/html_tools.py diff --git a/backend/notification.py b/changedetectionio/notification.py similarity index 100% rename from backend/notification.py rename to changedetectionio/notification.py diff --git a/backend/pytest.ini b/changedetectionio/pytest.ini similarity index 100% rename from backend/pytest.ini rename to changedetectionio/pytest.ini diff --git a/backend/run_all_tests.sh b/changedetectionio/run_all_tests.sh similarity index 100% rename from backend/run_all_tests.sh rename to changedetectionio/run_all_tests.sh diff --git a/backend/static/images/Generic_Feed-icon.svg b/changedetectionio/static/images/Generic_Feed-icon.svg similarity index 100% rename from backend/static/images/Generic_Feed-icon.svg rename to changedetectionio/static/images/Generic_Feed-icon.svg diff --git a/backend/static/images/Google-Chrome-icon.png b/changedetectionio/static/images/Google-Chrome-icon.png similarity index 100% rename from backend/static/images/Google-Chrome-icon.png rename to changedetectionio/static/images/Google-Chrome-icon.png diff --git a/backend/static/images/favicon.ico b/changedetectionio/static/images/favicon.ico similarity index 100% rename from backend/static/images/favicon.ico rename to changedetectionio/static/images/favicon.ico diff --git a/backend/static/images/gradient-border.png b/changedetectionio/static/images/gradient-border.png similarity index 100% rename from backend/static/images/gradient-border.png rename to changedetectionio/static/images/gradient-border.png diff --git a/backend/static/images/pause.svg b/changedetectionio/static/images/pause.svg similarity index 100% rename from backend/static/images/pause.svg rename to changedetectionio/static/images/pause.svg diff --git a/backend/static/js/diff.js b/changedetectionio/static/js/diff.js similarity index 100% rename from backend/static/js/diff.js rename to changedetectionio/static/js/diff.js diff --git a/backend/static/js/settings.js b/changedetectionio/static/js/settings.js similarity index 100% rename from backend/static/js/settings.js rename to changedetectionio/static/js/settings.js diff --git a/backend/static/js/tabs.js b/changedetectionio/static/js/tabs.js similarity index 100% rename from backend/static/js/tabs.js rename to changedetectionio/static/js/tabs.js diff --git a/backend/static/styles/.gitignore b/changedetectionio/static/styles/.gitignore similarity index 100% rename from backend/static/styles/.gitignore rename to changedetectionio/static/styles/.gitignore diff --git a/backend/static/styles/diff.css b/changedetectionio/static/styles/diff.css similarity index 100% rename from backend/static/styles/diff.css rename to changedetectionio/static/styles/diff.css diff --git a/backend/static/styles/diff.scss b/changedetectionio/static/styles/diff.scss similarity index 100% rename from backend/static/styles/diff.scss rename to changedetectionio/static/styles/diff.scss diff --git a/backend/static/styles/package-lock.json b/changedetectionio/static/styles/package-lock.json similarity index 100% rename from backend/static/styles/package-lock.json rename to changedetectionio/static/styles/package-lock.json diff --git a/backend/static/styles/package.json b/changedetectionio/static/styles/package.json similarity index 100% rename from backend/static/styles/package.json rename to changedetectionio/static/styles/package.json diff --git a/backend/static/styles/pure-min.css b/changedetectionio/static/styles/pure-min.css similarity index 100% rename from backend/static/styles/pure-min.css rename to changedetectionio/static/styles/pure-min.css diff --git a/backend/static/styles/styles.css b/changedetectionio/static/styles/styles.css similarity index 100% rename from backend/static/styles/styles.css rename to changedetectionio/static/styles/styles.css diff --git a/backend/static/styles/styles.scss b/changedetectionio/static/styles/styles.scss similarity index 100% rename from backend/static/styles/styles.scss rename to changedetectionio/static/styles/styles.scss diff --git a/backend/store.py b/changedetectionio/store.py similarity index 98% rename from backend/store.py rename to changedetectionio/store.py index 17668717..15861b58 100644 --- a/backend/store.py +++ b/changedetectionio/store.py @@ -15,7 +15,7 @@ import threading class ChangeDetectionStore: lock = Lock() - def __init__(self, datastore_path="/datastore", include_default_watches=True): + def __init__(self, datastore_path="/datastore", include_default_watches=True, version_tag="0.0.0"): self.needs_write = False self.datastore_path = datastore_path self.json_store_path = "{}/url-watches.json".format(self.datastore_path) @@ -72,8 +72,8 @@ class ChangeDetectionStore: 'fetch_backend': None, } - if path.isfile('backend/source.txt'): - with open('backend/source.txt') as f: + if path.isfile('changedetectionio/source.txt'): + with open('changedetectionio/source.txt') as f: # Should be set in Dockerfile to look for /source.txt , this will give us the git commit # # So when someone gives us a backup file to examine, we know exactly what code they were running. self.__data['build_sha'] = f.read() @@ -120,7 +120,7 @@ class ChangeDetectionStore: self.add_watch(url='https://www.gov.uk/coronavirus', tag='Covid') self.add_watch(url='https://changedetection.io', tag='Tech news') - self.__data['version_tag'] = "0.38.2" + self.__data['version_tag'] = version_tag # Helper to remove password protection password_reset_lockfile = "{}/removepassword.lock".format(self.datastore_path) diff --git a/backend/templates/_helpers.jinja b/changedetectionio/templates/_helpers.jinja similarity index 100% rename from backend/templates/_helpers.jinja rename to changedetectionio/templates/_helpers.jinja diff --git a/backend/templates/base.html b/changedetectionio/templates/base.html similarity index 100% rename from backend/templates/base.html rename to changedetectionio/templates/base.html diff --git a/backend/templates/diff.html b/changedetectionio/templates/diff.html similarity index 100% rename from backend/templates/diff.html rename to changedetectionio/templates/diff.html diff --git a/backend/templates/edit.html b/changedetectionio/templates/edit.html similarity index 100% rename from backend/templates/edit.html rename to changedetectionio/templates/edit.html diff --git a/backend/templates/import.html b/changedetectionio/templates/import.html similarity index 100% rename from backend/templates/import.html rename to changedetectionio/templates/import.html diff --git a/backend/templates/login.html b/changedetectionio/templates/login.html similarity index 100% rename from backend/templates/login.html rename to changedetectionio/templates/login.html diff --git a/backend/templates/preview.html b/changedetectionio/templates/preview.html similarity index 100% rename from backend/templates/preview.html rename to changedetectionio/templates/preview.html diff --git a/backend/templates/scrub.html b/changedetectionio/templates/scrub.html similarity index 100% rename from backend/templates/scrub.html rename to changedetectionio/templates/scrub.html diff --git a/backend/templates/settings.html b/changedetectionio/templates/settings.html similarity index 100% rename from backend/templates/settings.html rename to changedetectionio/templates/settings.html diff --git a/backend/templates/watch-overview.html b/changedetectionio/templates/watch-overview.html similarity index 100% rename from backend/templates/watch-overview.html rename to changedetectionio/templates/watch-overview.html diff --git a/backend/tests/__init__.py b/changedetectionio/tests/__init__.py similarity index 100% rename from backend/tests/__init__.py rename to changedetectionio/tests/__init__.py diff --git a/backend/tests/conftest.py b/changedetectionio/tests/conftest.py similarity index 94% rename from backend/tests/conftest.py rename to changedetectionio/tests/conftest.py index ef5cdc42..f9b9de7c 100644 --- a/backend/tests/conftest.py +++ b/changedetectionio/tests/conftest.py @@ -1,8 +1,8 @@ #!/usr/bin/python3 import pytest -from backend import changedetection_app -from backend import store +from changedetectionio import changedetection_app +from changedetectionio import store import os # https://github.com/pallets/flask/blob/1.1.2/examples/tutorial/tests/test_auth.py diff --git a/backend/tests/test_access_control.py b/changedetectionio/tests/test_access_control.py similarity index 100% rename from backend/tests/test_access_control.py rename to changedetectionio/tests/test_access_control.py diff --git a/backend/tests/test_backend.py b/changedetectionio/tests/test_backend.py similarity index 100% rename from backend/tests/test_backend.py rename to changedetectionio/tests/test_backend.py diff --git a/backend/tests/test_css_selector.py b/changedetectionio/tests/test_css_selector.py similarity index 98% rename from backend/tests/test_css_selector.py rename to changedetectionio/tests/test_css_selector.py index 21183b06..cf8eed72 100644 --- a/backend/tests/test_css_selector.py +++ b/changedetectionio/tests/test_css_selector.py @@ -47,7 +47,7 @@ def set_modified_response(): # Test that the CSS extraction works how we expect, important here is the right placing of new lines \n's def test_css_filter_output(): - from backend import fetch_site_status + from changedetectionio import fetch_site_status from inscriptis import get_text # Check text with sub-parts renders correctly diff --git a/backend/tests/test_headers.py b/changedetectionio/tests/test_headers.py similarity index 100% rename from backend/tests/test_headers.py rename to changedetectionio/tests/test_headers.py diff --git a/backend/tests/test_ignore_regex_text.py b/changedetectionio/tests/test_ignore_regex_text.py similarity index 94% rename from backend/tests/test_ignore_regex_text.py rename to changedetectionio/tests/test_ignore_regex_text.py index 482434a2..e70d16a4 100644 --- a/backend/tests/test_ignore_regex_text.py +++ b/changedetectionio/tests/test_ignore_regex_text.py @@ -10,7 +10,7 @@ def test_setup(live_server): # Unit test of the stripper # Always we are dealing in utf-8 def test_strip_regex_text_func(): - from backend import fetch_site_status + from changedetectionio import fetch_site_status test_content = """ but sometimes we want to remove the lines. diff --git a/backend/tests/test_ignore_text.py b/changedetectionio/tests/test_ignore_text.py similarity index 98% rename from backend/tests/test_ignore_text.py rename to changedetectionio/tests/test_ignore_text.py index cdcb9bbb..119f26eb 100644 --- a/backend/tests/test_ignore_text.py +++ b/changedetectionio/tests/test_ignore_text.py @@ -10,7 +10,7 @@ def test_setup(live_server): # Unit test of the stripper # Always we are dealing in utf-8 def test_strip_text_func(): - from backend import fetch_site_status + from changedetectionio import fetch_site_status test_content = """ Some content diff --git a/backend/tests/test_jsonpath_selector.py b/changedetectionio/tests/test_jsonpath_selector.py similarity index 100% rename from backend/tests/test_jsonpath_selector.py rename to changedetectionio/tests/test_jsonpath_selector.py diff --git a/backend/tests/test_notification.py b/changedetectionio/tests/test_notification.py similarity index 100% rename from backend/tests/test_notification.py rename to changedetectionio/tests/test_notification.py diff --git a/backend/tests/test_trigger.py b/changedetectionio/tests/test_trigger.py similarity index 100% rename from backend/tests/test_trigger.py rename to changedetectionio/tests/test_trigger.py diff --git a/backend/tests/test_trigger_regex.py b/changedetectionio/tests/test_trigger_regex.py similarity index 100% rename from backend/tests/test_trigger_regex.py rename to changedetectionio/tests/test_trigger_regex.py diff --git a/backend/tests/test_trigger_regex_with_filter.py b/changedetectionio/tests/test_trigger_regex_with_filter.py similarity index 100% rename from backend/tests/test_trigger_regex_with_filter.py rename to changedetectionio/tests/test_trigger_regex_with_filter.py diff --git a/backend/tests/test_watch_fields_storage.py b/changedetectionio/tests/test_watch_fields_storage.py similarity index 100% rename from backend/tests/test_watch_fields_storage.py rename to changedetectionio/tests/test_watch_fields_storage.py diff --git a/backend/tests/util.py b/changedetectionio/tests/util.py similarity index 100% rename from backend/tests/util.py rename to changedetectionio/tests/util.py diff --git a/backend/update_worker.py b/changedetectionio/update_worker.py similarity index 97% rename from backend/update_worker.py rename to changedetectionio/update_worker.py index 49eabd29..a5f36bda 100644 --- a/backend/update_worker.py +++ b/changedetectionio/update_worker.py @@ -14,7 +14,7 @@ class update_worker(threading.Thread): super().__init__(*args, **kwargs) def run(self): - from backend import fetch_site_status + from changedetectionio import fetch_site_status update_handler = fetch_site_status.perform_site_check(datastore=self.datastore) @@ -27,7 +27,7 @@ class update_worker(threading.Thread): else: self.current_uuid = uuid - from backend import content_fetcher + from changedetectionio import content_fetcher if uuid in list(self.datastore.data['watching'].keys()): diff --git a/requirements-dev.txt b/requirements-dev.txt new file mode 100644 index 00000000..146e954c --- /dev/null +++ b/requirements-dev.txt @@ -0,0 +1,2 @@ +pytest ~=6.2 +pytest-flask ~=1.2 diff --git a/requirements.txt b/requirements.txt index 18bb875a..f37bfff6 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,6 @@ chardet==2.3.0 flask~= 1.0 -pytest ~=6.2 -pytest-flask ~=1.2 + eventlet>=0.31.0 requests[socks] ~= 2.15 validators @@ -20,4 +19,4 @@ apprise ~= 0.9 # Used for CSS filtering, replace with soupsieve and lxml for xpath bs4 -selenium ~= 3.141 \ No newline at end of file +selenium ~= 3.141 diff --git a/setup.py b/setup.py new file mode 100644 index 00000000..3b73c93d --- /dev/null +++ b/setup.py @@ -0,0 +1,72 @@ +#!/usr/bin/env python +import codecs +import os.path +import re +import sys + +from setuptools import setup, find_packages + +here = os.path.abspath(os.path.dirname(__file__)) + + +def read(*parts): + return codecs.open(os.path.join(here, *parts), 'r').read() + + +def find_version(*file_paths): + version_file = read(*file_paths) + version_match = re.search(r"^__version__ = ['\"]([^'\"]*)['\"]", + version_file, re.M) + if version_match: + return version_match.group(1) + raise RuntimeError("Unable to find version string.") + + +install_requires = open('requirements.txt').readlines() + +setup( + name='changedetection.io', + version=find_version("changedetectionio", "__init__.py"), + description='Website change detection and monitoring service', + long_description=open('README-pip.md').read(), + long_description_content_type='text/markdown', + keywords='website change monitor for changes notification change detection ' + 'alerts tracking website tracker change alert website and monitoring', + zip_safe=False, + entry_points={"console_scripts": ["changedetection.io=changedetection:main"]}, + author='dgtlmoon', + url='https://changedetection.io', + scripts=['changedetection.py'], + packages=['changedetectionio'], + include_package_data=True, + install_requires=install_requires, + license="Apache License 2.0", + python_requires=">= 3.6", + classifiers=['Intended Audience :: Customer Service', + 'Intended Audience :: Developers', + 'Intended Audience :: Education', + 'Intended Audience :: End Users/Desktop', + 'Intended Audience :: Financial and Insurance Industry', + 'Intended Audience :: Healthcare Industry', + 'Intended Audience :: Information Technology', + 'Intended Audience :: Legal Industry', + 'Intended Audience :: Manufacturing', + 'Intended Audience :: Other Audience', + 'Intended Audience :: Religion', + 'Intended Audience :: Science/Research', + 'Intended Audience :: System Administrators', + 'Intended Audience :: Telecommunications Industry', + 'Topic :: Education', + 'Topic :: Internet', + 'Topic :: Internet :: WWW/HTTP :: Indexing/Search', + 'Topic :: Internet :: WWW/HTTP :: Site Management', + 'Topic :: Internet :: WWW/HTTP :: Site Management :: Link Checking', + 'Topic :: Internet :: WWW/HTTP :: Browsers', + 'Topic :: Internet :: WWW/HTTP :: Dynamic Content', + 'Topic :: Office/Business', + 'Topic :: Other/Nonlisted Topic', + 'Topic :: Scientific/Engineering :: Information Analysis', + 'Topic :: Text Processing :: Markup :: HTML', + 'Topic :: Utilities' + ], +)