small fixes

backend/backend.py

@@ -2,6 +2,14 @@
 
 
 # @todo logging
+# @todo sort by last_changed
+# @todo extra options for url like , verify=False etc.
+# @todo enable https://urllib3.readthedocs.io/en/latest/user-guide.html#ssl as option?
+# @todo maybe a button to reset all 'last-changed'.. so you can see it clearly when something happens since your last visit
+# @todo option for interval day/6 hour/etc
+# @todo on change detected, config for calling some API
+# @todo make tables responsive!
+# @todo fetch title into json
 
 import json
 import eventlet
@@ -12,6 +20,7 @@ import os
 import getopt
 import sys
 import datetime
+import timeago
 
 import threading
 
@@ -46,16 +55,23 @@ def _jinja2_filter_datetime(watch_obj, format="%Y-%m-%d %H:%M:%S"):
             return "Checking now.."
 
     if watch_obj['last_checked'] == 0:
-        return 'Never'
+        return 'Not yet'
 
     return datetime.datetime.utcfromtimestamp(int(watch_obj['last_checked'])).strftime(format)
 
+#@app.context_processor
+#def timeago():
+#    def _timeago(lower_time, now):
+#        return timeago.format(lower_time, now)
+#    return dict(timeago=_timeago)
 
-@app.template_filter('format_timestamp')
+@app.template_filter('format_timestamp_timeago')
 def _jinja2_filter_datetimestamp(timestamp, format="%Y-%m-%d %H:%M:%S"):
     if timestamp == 0:
-        return 'Never'
+        return 'Not yet'
-    return datetime.datetime.utcfromtimestamp(timestamp).strftime(format)
+    return timeago.format(timestamp, time.time())
+    #return timeago.format(timestamp, time.time())
+    #return datetime.datetime.utcfromtimestamp(timestamp).strftime(format)
 
 
 @app.route("/", methods=['GET'])
@@ -111,7 +127,7 @@ def launch_checks():
     global running_update_threads
 
     for watch in datastore.data['watching']:
-        if watch['last_checked'] <= time.time() - 86400:
+        if watch['last_checked'] <= time.time() - 3*60*60:
             running_update_threads[watch['uuid']] = fetch_site_status.perform_site_check(uuid=watch['uuid'],
                                                                                          datastore=datastore)
             running_update_threads[watch['uuid']].start()

backend/fetch_site_status.py

@@ -45,19 +45,20 @@ class perform_site_check(Thread):
         import html2text
 
         try:
-            r = requests.get(self.url, headers=headers, timeout=15)
+            r = requests.get(self.url, headers=headers, timeout=15, verify=False)
-
             stripped_text_from_html = html2text.html2text(r.content.decode('utf-8'))
+
         # Usually from networkIO/requests level
         except (requests.exceptions.ConnectionError,requests.exceptions.ReadTimeout) as e:
             self.datastore.update_watch(self.uuid, 'last_error', str(e))
             print(str(e))
 
-
         # Usually from html2text level
         except UnicodeDecodeError as e:
             self.datastore.update_watch(self.uuid, 'last_error', str(e))
             print(str(e))
+            # figure out how to deal with this cleaner..
+            # 'utf-8' codec can't decode byte 0xe9 in position 480: invalid continuation byte
 
         else:
 
@@ -72,9 +73,14 @@ class perform_site_check(Thread):
             fetched_md5 = hashlib.md5(stripped_text_from_html.encode('utf-8')).hexdigest()
 
             if self.current_md5 != fetched_md5:
+
+                # Dont confuse people by putting last-changed, when it actually just changed from nothing..
+                if self.datastore.get_val(self.uuid, 'previous_md5') is not None:
+                    self.datastore.update_watch(self.uuid, 'last_changed', self.timestamp)
+
                 self.datastore.update_watch(self.uuid, 'previous_md5', fetched_md5)
                 self.save_response_output(r.text)
-                self.datastore.update_watch(self.uuid, 'last_changed', self.timestamp)
+
 
         self.datastore.update_watch(self.uuid, 'last_checked', int(time.time()))
         pass

backend/static/css/styles.css

@@ -4,9 +4,10 @@
  */
 body {
     color: #333;
+    background: #262626;
 }
 
-
+/* Some styles from https://css-tricks.com/ */
 
 a {
     text-decoration: none;
@@ -18,13 +19,61 @@ a.github-link {
 
 section.content {
 
-    padding-top: 3em;
+    padding-top: 5em;
+    padding-bottom: 5em;
 flex-direction: column;
   display: flex;
   align-items: center;
   justify-content: center;
 }
 
+.watch-table {
+font-size: 90%;
+
+}
+
 .watch-table .error {
 color: #aa0000;
 }
+
+.home-menu {
+background: #fff;
+padding: 5px;
+    display: flex;
+    justify-content: space-between;
+}
+
+.pure-table-even  {
+/* missing */
+background: #fff;
+}
+body:after {
+    content: "";
+    background: linear-gradient(130deg,#ff7a18,#af002d 41.07%,#319197 76.05%)
+}
+
+body:after,body:before {
+    display: block;
+    height: 600px;
+    position: absolute;
+    top: 0;
+    left: 0;
+    width: 100%;
+    z-index: -1;
+}
+    body::after {
+      opacity: 0.91;
+    }
+    body::before {
+      content: "";
+      background-image:
+        url(/static/images/gradient-border.png);
+    }
+body:before {
+    background-size: cover
+}
+
+body:after,body:before {
+    -webkit-clip-path: polygon(100% 0,0 0,0 77.5%,1% 77.4%,2% 77.1%,3% 76.6%,4% 75.9%,5% 75.05%,6% 74.05%,7% 72.95%,8% 71.75%,9% 70.55%,10% 69.3%,11% 68.05%,12% 66.9%,13% 65.8%,14% 64.8%,15% 64%,16% 63.35%,17% 62.85%,18% 62.6%,19% 62.5%,20% 62.65%,21% 63%,22% 63.5%,23% 64.2%,24% 65.1%,25% 66.1%,26% 67.2%,27% 68.4%,28% 69.65%,29% 70.9%,30% 72.15%,31% 73.3%,32% 74.35%,33% 75.3%,34% 76.1%,35% 76.75%,36% 77.2%,37% 77.45%,38% 77.5%,39% 77.3%,40% 76.95%,41% 76.4%,42% 75.65%,43% 74.75%,44% 73.75%,45% 72.6%,46% 71.4%,47% 70.15%,48% 68.9%,49% 67.7%,50% 66.55%,51% 65.5%,52% 64.55%,53% 63.75%,54% 63.15%,55% 62.75%,56% 62.55%,57% 62.5%,58% 62.7%,59% 63.1%,60% 63.7%,61% 64.45%,62% 65.4%,63% 66.45%,64% 67.6%,65% 68.8%,66% 70.05%,67% 71.3%,68% 72.5%,69% 73.6%,70% 74.65%,71% 75.55%,72% 76.35%,73% 76.9%,74% 77.3%,75% 77.5%,76% 77.45%,77% 77.25%,78% 76.8%,79% 76.2%,80% 75.4%,81% 74.45%,82% 73.4%,83% 72.25%,84% 71.05%,85% 69.8%,86% 68.55%,87% 67.35%,88% 66.2%,89% 65.2%,90% 64.3%,91% 63.55%,92% 63%,93% 62.65%,94% 62.5%,95% 62.55%,96% 62.8%,97% 63.3%,98% 63.9%,99% 64.75%,100% 65.7%);
+    clip-path: polygon(100% 0,0 0,0 77.5%,1% 77.4%,2% 77.1%,3% 76.6%,4% 75.9%,5% 75.05%,6% 74.05%,7% 72.95%,8% 71.75%,9% 70.55%,10% 69.3%,11% 68.05%,12% 66.9%,13% 65.8%,14% 64.8%,15% 64%,16% 63.35%,17% 62.85%,18% 62.6%,19% 62.5%,20% 62.65%,21% 63%,22% 63.5%,23% 64.2%,24% 65.1%,25% 66.1%,26% 67.2%,27% 68.4%,28% 69.65%,29% 70.9%,30% 72.15%,31% 73.3%,32% 74.35%,33% 75.3%,34% 76.1%,35% 76.75%,36% 77.2%,37% 77.45%,38% 77.5%,39% 77.3%,40% 76.95%,41% 76.4%,42% 75.65%,43% 74.75%,44% 73.75%,45% 72.6%,46% 71.4%,47% 70.15%,48% 68.9%,49% 67.7%,50% 66.55%,51% 65.5%,52% 64.55%,53% 63.75%,54% 63.15%,55% 62.75%,56% 62.55%,57% 62.5%,58% 62.7%,59% 63.1%,60% 63.7%,61% 64.45%,62% 65.4%,63% 66.45%,64% 67.6%,65% 68.8%,66% 70.05%,67% 71.3%,68% 72.5%,69% 73.6%,70% 74.65%,71% 75.55%,72% 76.35%,73% 76.9%,74% 77.3%,75% 77.5%,76% 77.45%,77% 77.25%,78% 76.8%,79% 76.2%,80% 75.4%,81% 74.45%,82% 73.4%,83% 72.25%,84% 71.05%,85% 69.8%,86% 68.55%,87% 67.35%,88% 66.2%,89% 65.2%,90% 64.3%,91% 63.55%,92% 63%,93% 62.65%,94% 62.5%,95% 62.55%,96% 62.8%,97% 63.3%,98% 63.9%,99% 64.75%,100% 65.7%)
+}

backend/store.py

@@ -13,9 +13,9 @@ class ChangeDetectionStore:
         try:
             with open('/datastore/url-watches.json') as json_file:
                 self.data = json.load(json_file)
+                self.data['watching'].reverse()
                 for p in self.data['watching']:
-                    print('url: ' + p['url'])
+                    print("Watching:", p['url'])
-                    print('')
 
         # First time ran, doesnt exist.
         except (FileNotFoundError, json.decoder.JSONDecodeError):
@@ -51,6 +51,15 @@ class ChangeDetectionStore:
                 self.sync_to_json()
 
 
+    def url_exists(self, url):
+
+        # Probably their should be dict...
+        for watch in self.data['watching']:
+            if watch['url'] == url:
+                    return True
+
+        return False
+
     def get_val(self, uuid, val):
         # Probably their should be dict...
         for watch in self.data['watching']:
@@ -78,6 +87,6 @@ class ChangeDetectionStore:
 
     def sync_to_json(self):
         with open('/datastore/url-watches.json', 'w') as json_file:
-            json.dump(self.data, json_file)
+            json.dump(self.data, json_file, indent=4)
 
 # body of the constructor

backend/templates/base.html

@@ -12,10 +12,10 @@
 
 <div class="header">
     <div class="home-menu pure-menu pure-menu-horizontal pure-menu-fixed">
-        <a class="pure-menu-heading" href="">ChangeDetection</a>
+        <a class="pure-menu-heading" href=""><strong>Change</strong>Detection.io</a>
 
         <ul class="pure-menu-list">
-            <li class="pure-menu-item pure-menu-selected"><a class="github-link " href="https://github.com/"
+            <li class="pure-menu-item pure-menu-selected"><a class="github-link " href="https://github.com/dgtlmoon/changedetection.io"
                                                              data-hotkey="g d" aria-label="Homepage "
                                                              data-ga-click="Header, go to dashboard, icon:logo">
                 <svg class="octicon octicon-mark-github v-align-middle" height="32" viewBox="0 0 16 16" version="1.1"
@@ -24,8 +24,10 @@
                           d="M8 0C3.58 0 0 3.58 0 8c0 3.54 2.29 6.53 5.47 7.59.4.07.55-.17.55-.38 0-.19-.01-.82-.01-1.49-2.01.37-2.53-.49-2.69-.94-.09-.23-.48-.94-.82-1.13-.28-.15-.68-.52-.01-.53.63-.01 1.08.58 1.23.82.72 1.21 1.87.87 2.33.66.07-.52.28-.87.51-1.07-1.78-.2-3.64-.89-3.64-3.95 0-.87.31-1.59.82-2.15-.08-.2-.36-1.02.08-2.12 0 0 .67-.21 2.2.82.64-.18 1.32-.27 2-.27.68 0 1.36.09 2 .27 1.53-1.04 2.2-.82 2.2-.82.44 1.1.16 1.92.08 2.12.51.56.82 1.27.82 2.15 0 3.07-1.87 3.75-3.65 3.95.29.25.54.73.54 1.48 0 1.07-.01 1.93-.01 2.2 0 .21.15.46.55.38A8.013 8.013 0 0016 8c0-4.42-3.58-8-8-8z"></path>
                 </svg>
             </a></li>
+            <!--
             <li class="pure-menu-item"><a href="#" class="pure-menu-link">Tour</a></li>
             <li class="pure-menu-item"><a href="#" class="pure-menu-link">Sign Up</a></li>
+            -->
         </ul>
     </div>
 </div>

backend/templates/watch-overview.html

@@ -15,7 +15,7 @@
 </form>
 
 <!-- make a nice list of tags here to click on -->
-
+    <i>Note: Times are in UTC for now - todo - JS front end format<br/></i>
 <table class="pure-table pure-table-striped watch-table">
     <thead>
         <tr>
@@ -23,7 +23,7 @@
             <th>URL</th>
             <th>Last Checked</th>
             <th>Last Changed</th>
-            <th>op</th>
+            <th></th>
         </tr>
     </thead>
     <tbody>
@@ -36,7 +36,7 @@
             <td>{{watch|format_last_checked_time}}
             {% if watch.last_error is defined and watch.last_error != False %} !! {% endif %}
             </td>
-            <td>{{watch.last_changed|format_timestamp}}</td>
+            <td>{{watch.last_changed|format_timestamp_timeago}}</td>
             <td><a href="/api/checknow?uuid={{ watch.uuid}}" class="pure-button pure-button-primary">Recheck</a> <button type="submit" class="pure-button pure-button-primary">Delete</button></td>
         </tr>
   {% endfor %}