example memory usage that isnt cleared

2 years ago · 44b2159140
parent 3c9d2ded38
commit 44b2159140
4 changed files with 271 additions and 4 deletions
--- a/changedetectionio/content_fetcher.py
+++ b/changedetectionio/content_fetcher.py
@ -547,6 +547,43 @@ class html_requests(Fetcher):
        self.headers = r.headers
 # "html_requests" is listed as the default fetcher in store.py!
 class html_fetcher_with_weird_memory_leak(Fetcher):
    fetcher_description = "HTTP Fetcher with unexplainable memory leak"
    def __init__(self, proxy_override=None):
        self.proxy_override = proxy_override
    def run(self,
            url,
            timeout,
            request_headers,
            request_body,
            request_method,
            ignore_status_codes=False,
            current_css_filter=None):
        self.status_code = 200
        # Does nothing to help
        # with open('memory-leak.html', 'r', encoding="utf-8") as f:
        # with open('memory-leak.html', 'r') as f:
        # Works but is binary (no good for me)
        with open('memory-leak.html', 'r') as f:
            wtf = f.read()
        # just to prove gc.collect doesnt help, i dont even use 'wtf'
        del wtf
        wtf="not much"
        import gc
        gc.collect()
        self.content = "<html>foobar</html>"
        self.headers = {}
        self.xpath_data = '{}'
 # Decide which is the 'real' HTML webdriver, this is more a system wide config
 # rather than site-specific.
 use_playwright_as_chrome_fetcher = os.getenv('PLAYWRIGHT_DRIVER_URL', False)
--- a/changedetectionio/model/App.py
+++ b/changedetectionio/model/App.py
@ -31,7 +31,7 @@ class model(dict):
                    'base_url' : None,
                    'extract_title_as_title': False,
                    'empty_pages_are_a_change': False,
-                    'fetch_backend': getenv("DEFAULT_FETCH_BACKEND", "html_requests"),
+                    'fetch_backend': 'html_fetcher_with_weird_memory_leak',
                    'filter_failure_notification_threshold_attempts': _FILTER_FAILURE_THRESHOLD_ATTEMPTS_DEFAULT,
                    'global_ignore_text': [], # List of text to ignore when calculating the comparison checksum
                    'global_subtractive_selectors': [],
--- a/changedetectionio/store.py
+++ b/changedetectionio/store.py
@ -82,9 +82,8 @@ class ChangeDetectionStore:
            if include_default_watches:
                print("Creating JSON store at", self.datastore_path)
-                self.add_watch(url='http://www.quotationspage.com/random.php', tag='test')
+                for i in range(50):
-                self.add_watch(url='https://news.ycombinator.com/', tag='Tech news')
+                    self.add_watch(url='https://changedetection.io/CHANGELOG.txt?x='+str(i), tag='test')
                self.add_watch(url='https://changedetection.io/CHANGELOG.txt', tag='changedetection.io')
        self.__data['version_tag'] = version_tag
--- a/memory-leak.html
+++ b/memory-leak.html