From 9c5588c79100f0fc4dfaa1518d86d197681b5c22 Mon Sep 17 00:00:00 2001 From: Entepotenz <19738301+Entepotenz@users.noreply.github.com> Date: Sun, 23 Oct 2022 11:25:29 +0200 Subject: [PATCH 1/6] update path for validation in the CONTRIBUTING.md (#1046) --- CONTRIBUTING.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 9641dd16..8478a7ab 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -6,7 +6,7 @@ Otherwise, it's always best to PR into the `dev` branch. Please be sure that all new functionality has a matching test! -Use `pytest` to validate/test, you can run the existing tests as `pytest tests/test_notifications.py` for example +Use `pytest` to validate/test, you can run the existing tests as `pytest tests/test_notification.py` for example ``` pip3 install -r requirements-dev From 7839551d6b69af6d16cfff4a18a745de70de96e1 Mon Sep 17 00:00:00 2001 From: Entepotenz <19738301+Entepotenz@users.noreply.github.com> Date: Sun, 23 Oct 2022 11:26:32 +0200 Subject: [PATCH 2/6] Testing - Use same version of playwright while running tests as in production builds (#1047) --- changedetectionio/run_all_tests.sh | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/changedetectionio/run_all_tests.sh b/changedetectionio/run_all_tests.sh index 28dd85c6..4eff9e93 100755 --- a/changedetectionio/run_all_tests.sh +++ b/changedetectionio/run_all_tests.sh @@ -9,6 +9,8 @@ # exit when any command fails set -e +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) + find tests/test_*py -type f|while read test_name do echo "TEST RUNNING $test_name" @@ -45,7 +47,9 @@ docker kill $$-test_selenium echo "TESTING WEBDRIVER FETCH > PLAYWRIGHT/BROWSERLESS..." # Not all platforms support playwright (not ARM/rPI), so it's not packaged in requirements.txt -pip3 install playwright~=1.24 +PLAYWRIGHT_VERSION=$(grep -i -E "RUN pip install.+" "$SCRIPT_DIR/../Dockerfile" | grep --only-matching -i -E "playwright[=><~+]+[0-9\.]+") +echo "using $PLAYWRIGHT_VERSION" +pip3 install "$PLAYWRIGHT_VERSION" docker run -d --name $$-test_browserless -e "DEFAULT_LAUNCH_ARGS=[\"--window-size=1920,1080\"]" --rm -p 3000:3000 --shm-size="2g" browserless/chrome:1.53-chrome-stable # takes a while to spin up sleep 5 From 0394a56be57f6278f884908fc38fffcb6ed4f685 Mon Sep 17 00:00:00 2001 From: dgtlmoon <dgtlmoon@gmail.com> Date: Sun, 23 Oct 2022 15:54:19 +0200 Subject: [PATCH 3/6] Building - Test container build on PR --- .github/workflows/test-container-build.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/test-container-build.yml b/.github/workflows/test-container-build.yml index dc6ab712..cc457286 100644 --- a/.github/workflows/test-container-build.yml +++ b/.github/workflows/test-container-build.yml @@ -1,8 +1,7 @@ name: ChangeDetection.io Container Build Test # Triggers the workflow on push or pull request events -on: - push: +on: [push, pull_request] paths: - requirements.txt - Dockerfile From 492bbce6b67908d1b7557f58642746d1cb3519bc Mon Sep 17 00:00:00 2001 From: dgtlmoon <dgtlmoon@gmail.com> Date: Sun, 23 Oct 2022 16:02:13 +0200 Subject: [PATCH 4/6] Build - Fix syntax in container build test (#1050) --- .github/workflows/test-container-build.yml | 12 +++++++++++- Dockerfile | 1 + requirements.txt | 7 ++++--- 3 files changed, 16 insertions(+), 4 deletions(-) diff --git a/.github/workflows/test-container-build.yml b/.github/workflows/test-container-build.yml index cc457286..7d59ad0b 100644 --- a/.github/workflows/test-container-build.yml +++ b/.github/workflows/test-container-build.yml @@ -1,7 +1,17 @@ name: ChangeDetection.io Container Build Test # Triggers the workflow on push or pull request events -on: [push, pull_request] + +# This line doesnt work, even tho it is the documented one +#on: [push, pull_request] + +on: + push: + paths: + - requirements.txt + - Dockerfile + + pull_request: paths: - requirements.txt - Dockerfile diff --git a/Dockerfile b/Dockerfile index d422918e..978a912c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -64,6 +64,7 @@ EXPOSE 5000 # The actual flask app COPY changedetectionio /app/changedetectionio + # The eventlet server wrapper COPY changedetection.py /app/changedetection.py diff --git a/requirements.txt b/requirements.txt index bffc2a7f..500f45f9 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,8 +1,8 @@ -flask~= 2.0 +flask ~= 2.0 flask_wtf -eventlet>=0.31.0 +eventlet >= 0.31.0 validators -timeago ~=1.0 +timeago ~= 1.0 inscriptis ~= 2.2 feedgen ~= 0.9 flask-login ~= 0.5 @@ -47,3 +47,4 @@ selenium ~= 4.1.0 werkzeug ~= 2.0.0 # playwright is installed at Dockerfile build time because it's not available on all platforms + From 5d40e16c73b74888a19abecb911e01156d0172de Mon Sep 17 00:00:00 2001 From: dgtlmoon <dgtlmoon@gmail.com> Date: Sun, 23 Oct 2022 19:15:11 +0200 Subject: [PATCH 5/6] API - Adding basic system info/system state API (#1051) --- changedetectionio/__init__.py | 3 +++ changedetectionio/api/api_v1.py | 30 +++++++++++++++++++++++++++++ changedetectionio/store.py | 6 +++--- changedetectionio/tests/test_api.py | 10 ++++++++++ 4 files changed, 46 insertions(+), 3 deletions(-) diff --git a/changedetectionio/__init__.py b/changedetectionio/__init__.py index c6f95f1e..8bbb747d 100644 --- a/changedetectionio/__init__.py +++ b/changedetectionio/__init__.py @@ -194,6 +194,9 @@ def changedetection_app(config=None, datastore_o=None): watch_api.add_resource(api_v1.Watch, '/api/v1/watch/<string:uuid>', resource_class_kwargs={'datastore': datastore, 'update_q': update_q}) + watch_api.add_resource(api_v1.SystemInfo, '/api/v1/systeminfo', + resource_class_kwargs={'datastore': datastore, 'update_q': update_q}) + diff --git a/changedetectionio/api/api_v1.py b/changedetectionio/api/api_v1.py index a432bc67..d44c990e 100644 --- a/changedetectionio/api/api_v1.py +++ b/changedetectionio/api/api_v1.py @@ -122,3 +122,33 @@ class CreateWatch(Resource): return {'status': "OK"}, 200 return list, 200 + +class SystemInfo(Resource): + def __init__(self, **kwargs): + # datastore is a black box dependency + self.datastore = kwargs['datastore'] + self.update_q = kwargs['update_q'] + + @auth.check_token + def get(self): + import time + overdue_watches = [] + + # Check all watches and report which have not been checked but should have been + + for uuid, watch in self.datastore.data.get('watching', {}).items(): + # see if now - last_checked is greater than the time that should have been + # this is not super accurate (maybe they just edited it) but better than nothing + t = watch.threshold_seconds() + if not t: + t = self.datastore.threshold_seconds + time_since_check = time.time() - watch.get('last_checked') + if time_since_check > t: + overdue_watches.append(uuid) + + return { + 'queue_size': self.update_q.qsize(), + 'overdue_watches': overdue_watches, + 'uptime': round(time.time() - self.datastore.start_time, 2), + 'watch_count': len(self.datastore.data.get('watching', {})) + }, 200 diff --git a/changedetectionio/store.py b/changedetectionio/store.py index bd86039a..6182aef8 100644 --- a/changedetectionio/store.py +++ b/changedetectionio/store.py @@ -30,14 +30,14 @@ class ChangeDetectionStore: def __init__(self, datastore_path="/datastore", include_default_watches=True, version_tag="0.0.0"): # Should only be active for docker # logging.basicConfig(filename='/dev/stdout', level=logging.INFO) - self.needs_write = False + self.__data = App.model() self.datastore_path = datastore_path self.json_store_path = "{}/url-watches.json".format(self.datastore_path) + self.needs_write = False self.proxy_list = None + self.start_time = time.time() self.stop_thread = False - self.__data = App.model() - # Base definition for all watchers # deepcopy part of #569 - not sure why its needed exactly self.generic_definition = deepcopy(Watch.model(datastore_path = datastore_path, default={})) diff --git a/changedetectionio/tests/test_api.py b/changedetectionio/tests/test_api.py index dd66012e..504a9554 100644 --- a/changedetectionio/tests/test_api.py +++ b/changedetectionio/tests/test_api.py @@ -147,6 +147,16 @@ def test_api_simple(client, live_server): # @todo how to handle None/default global values? assert watch['history_n'] == 2, "Found replacement history section, which is in its own API" + # basic systeminfo check + res = client.get( + url_for("systeminfo"), + headers={'x-api-key': api_key}, + ) + info = json.loads(res.data) + assert info.get('watch_count') == 1 + assert info.get('uptime') > 0.5 + + # Finally delete the watch res = client.delete( url_for("watch", uuid=watch_uuid), From 4eb4b401a1891c6af359abc0a58b243cf58acc19 Mon Sep 17 00:00:00 2001 From: dgtlmoon <dgtlmoon@gmail.com> Date: Sun, 23 Oct 2022 23:12:28 +0200 Subject: [PATCH 6/6] API - system info - allow 5 minutes grace before watch is considered 'overdue' --- changedetectionio/api/api_v1.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/changedetectionio/api/api_v1.py b/changedetectionio/api/api_v1.py index d44c990e..40131ca5 100644 --- a/changedetectionio/api/api_v1.py +++ b/changedetectionio/api/api_v1.py @@ -141,9 +141,13 @@ class SystemInfo(Resource): # this is not super accurate (maybe they just edited it) but better than nothing t = watch.threshold_seconds() if not t: + # Use the system wide default t = self.datastore.threshold_seconds + time_since_check = time.time() - watch.get('last_checked') - if time_since_check > t: + + # Allow 5 minutes of grace time before we decide it's overdue + if time_since_check - (5 * 60) > t: overdue_watches.append(uuid) return {