Python 3 updates

.gitignore

@@ -8,6 +8,7 @@ __pycache__/
 # Distribution / packaging
 .Python
 env/
+venv/
 bin/
 build/
 develop-eggs/
@@ -59,3 +60,6 @@ GeoHealthCheck/static/docs
 GeoHealthCheck/static/lib
 GeoHealthCheck.wsgi
 GeoHealthCheck.conf
+
+# Data
+GeoHealthCheck/data.db

.travis.yml

@@ -1,7 +1,7 @@
 language: python
 
 python:
-  - "2.7"
+  - "3.7"
 
 sudo: false
 

Dockerfile

@@ -1,4 +1,4 @@
-FROM python:2.7.15-alpine3.8
+FROM python:3.7.4-alpine3.10
 
 # Thanks to http://www.sandtable.com/reduce-docker-image-sizes-using-alpine
 # FROM debian:jessie
@@ -63,8 +63,7 @@ WSGI_WORKER_CLASS='eventlet' \
 GHC_USER_PLUGINS=''
 
 RUN apk add --no-cache --virtual .build-deps gcc build-base libxslt-dev libxml2-dev linux-headers postgresql-dev \
-    && apk add --no-cache bash postgresql-client libxslt libxml2 tzdata openntpd \
-    && pip install virtualenv \
+    && apk add --no-cache bash postgresql-client libxslt libxml2 tzdata openntpd python3 python3-dev \
     && rm -rf /var/cache/apk/* /tmp/* /var/tmp/*
 
 # Add standard files and Add/override Plugins

GeoHealthCheck/app.py

@@ -1,4 +1,3 @@
-# coding=utf-8
 # =================================================================
 #
 # Authors: Tom Kralidis <tomkralidis@gmail.com>
@@ -33,7 +32,7 @@
 import csv
 import json
 import logging
-from StringIO import StringIO
+from io import StringIO
 
 from flask import (abort, flash, g, jsonify, redirect,
                    render_template, request, url_for)
@@ -90,7 +89,7 @@ def db_commit():
     err = None
     try:
         DB.session.commit()
-    except Exception as err:
+    except Exception:
         DB.session.rollback()
     # finally:
     #     DB.session.close()
@@ -240,7 +239,8 @@ def context_processors():
         'resource_types_counts': rtc['counts'],
         'resources_total': rtc['total'],
         'languages': LANGUAGES,
-        'tags': tags
+        'tags': tags,
+        'tagnames': list(tags.keys())
     }
 
 
@@ -278,7 +278,7 @@ def export():
 
                 json_dict['resources'].append({
                     'resource_type': r.resource_type,
-                    'title': r.title.encode('utf-8'),
+                    'title': r.title,
                     'url': r.url,
                     'ghc_url': ghc_url,
                     'ghc_json': '%s/json' % ghc_url,
@@ -314,7 +314,7 @@ def export():
 
                 writer.writerow([
                     r.resource_type,
-                    r.title.encode('utf-8'),
+                    r.title,
                     r.url,
                     ghc_url,
                     '%s/json' % ghc_url,
@@ -361,7 +361,7 @@ def export_resource(identifier):
 
         json_dict = {
             'identifier': resource.identifier,
-            'title': resource.title.encode('utf-8'),
+            'title': resource.title,
             'url': resource.url,
             'resource_type': resource.resource_type,
             'owner': resource.owner.username,
@@ -390,7 +390,7 @@ def export_resource(identifier):
         writer.writerow(header)
         writer.writerow([
             resource.identifier,
-            resource.title.encode('utf-8'),
+            resource.title,
             resource.url,
             resource.resource_type,
             resource.owner.username,
@@ -424,7 +424,7 @@ def export_resource_history(identifier):
                 'owner': resource.owner.username,
                 'resource_type': resource.resource_type,
                 'checked_datetime': format_checked_datetime(run),
-                'title': resource.title.encode('utf-8'),
+                'title': resource.title,
                 'url': resource.url,
                 'response_time': round(run.response_time, 2),
                 'status': format_run_status(run)
@@ -443,7 +443,7 @@ def export_resource_history(identifier):
                 resource.owner.username,
                 resource.resource_type,
                 format_checked_datetime(run),
-                resource.title.encode('utf-8'),
+                resource.title,
                 resource.url,
                 round(run.response_time, 2),
                 format_run_status(run),
@@ -852,7 +852,6 @@ def get_check_edit_form(check_class):
     check_vars = CheckVars(
         None, check_class, check_obj.get_default_parameter_values())
 
-    # print(str(check_info))
     return render_template('includes/check_edit_form.html',
                            lang=g.current_lang,
                            check=check_vars, check_info=check_info)

GeoHealthCheck/factory.py

@@ -46,7 +46,7 @@ def create_class(class_string):
                 raise ValueError('Class name must contain module part.')
             class_obj = getattr(
                 __import__(module_name, globals(), locals(),
-                           [class_name], -1), class_name)
+                           [class_name]), class_name)
         except Exception as e:
             LOGGER.error("cannot create class '%s'" % class_string)
             raise e

GeoHealthCheck/healthcheck.py

@@ -30,8 +30,8 @@
 from datetime import datetime
 import logging
 import json
-from urllib2 import urlopen
-from urlparse import urlparse
+from urllib.request import urlopen
+from urllib.parse import urlparse
 from functools import partial
 from flask_babel import gettext
 
@@ -62,6 +62,7 @@ def db_commit():
     try:
         DB.session.commit()
     except Exception as err:
+        LOGGER.warning("Cannot commit to database %s".format(err))
         DB.session.rollback()
     # finally:
     #     DB.session.close()
@@ -173,7 +174,7 @@ def sniff_test_resource(config, resource_type, url):
             try:
                 ows = ows_handler(url)
                 break
-            except Exception, err:
+            except Exception as err:
                 LOGGER.warning("Cannot use %s on %s: %s",
                                ows_handler, url, err, exc_info=err)
         if ows is None:
@@ -244,7 +245,6 @@ def sniff_test_resource(config, resource_type, url):
                 title = ows.identification.title
         if title is None:
             title = '%s %s %s' % (resource_type, gettext('for'), url)
-        title = title.decode('utf-8')
         success = True
     except Exception as err:
         title = 'Untitled'
@@ -285,7 +285,7 @@ def geonode_get_ows(base_url):
 
     try:
         data = json.load(r)
-    except (TypeError, ValueError,), err:
+    except (TypeError, ValueError,) as err:
         msg = "Cannot decode response from GeoNode at {}: {}".format(base_url,
                                                                      err)
         raise ValueError(msg)

GeoHealthCheck/models.py

@@ -94,6 +94,24 @@ def __init__(self, resource, result,
         self.message = result.message
         self.report = result.get_report()
 
+    def __lt__(self, other):
+        return self.identifier < other.identifier
+
+    def __le__(self, other):
+        return self.identifier <= other.identifier
+
+    def __eq__(self, other):
+        return self.identifier == other.identifier
+
+    def __gt__(self, other):
+        return self.identifier > other.identifier
+
+    def __ge__(self, other):
+        return self.identifief >= other.identifier
+
+    def __hash__(self):
+        return hash(f"{self.identifier}{self.checked_datetime}{self.resource}")
+
     # JSON string object specifying report for the Run
     # See http://docs.sqlalchemy.org/en/latest/orm/mapped_attributes.html
     _report = DB.Column("report", DB.Text, default={})
@@ -222,7 +240,7 @@ def _validate_webhook(value):
     from GeoHealthCheck.notifications import _parse_webhook_location
     try:
         _parse_webhook_location(value)
-    except ValueError, err:
+    except ValueError as err:
         raise ValidationError('{}: {}'.format(value, err))
     return value
 
@@ -290,7 +308,7 @@ def validate(cls, channel, value):
         for v in validators:
             try:
                 v(value)
-            except (ValidationError, TypeError), err:
+            except (ValidationError, TypeError) as err:
                 raise ValueError("Bad value: {}".format(err), err)
 
     def is_email(self):
@@ -314,7 +332,7 @@ def burry_dead(cls):
     def get_or_create(cls, channel, location):
         try:
             cls.validate(channel, location)
-        except ValidationError, err:
+        except ValidationError as err:
             raise ValueError("invalid value {}: {}".format(location, err))
 
         try:
@@ -690,7 +708,7 @@ def is_anonymous(self):
         return False
 
     def get_id(self):
-        return unicode(self.identifier)
+        return self.identifier
 
     def set_password(self, password):
         self.password = self.encrypt(password)
@@ -884,13 +902,13 @@ def db_commit():
                 password1 = sys.argv[3]
                 email1 = sys.argv[4]
             else:
-                username = raw_input('Enter your username: ').strip()
-                password1 = raw_input('Enter your password: ').strip()
-                password2 = raw_input('Enter your password again: ').strip()
+                username = input('Enter your username: ').strip()
+                password1 = input('Enter your password: ').strip()
+                password2 = input('Enter your password again: ').strip()
                 if password1 != password2:
                     raise ValueError('Passwords must match')
-                email1 = raw_input('Enter your email: ').strip()
-                email2 = raw_input('Enter your email again: ').strip()
+                email1 = input('Enter your email: ').strip()
+                email2 = input('Enter your email again: ').strip()
                 if email1 != email2:
                     raise ValueError('Emails must match')
 
@@ -908,7 +926,7 @@ def db_commit():
                 yesno = 'n'
                 if len(sys.argv) == 3:
                     print('WARNING: all DB data will be lost! Proceed?')
-                    yesno = raw_input(
+                    yesno = input(
                         'Enter y (proceed) or n (abort): ').strip()
                 elif len(sys.argv) == 4:
                     yesno = sys.argv[3]

GeoHealthCheck/notifications.py

@@ -1,6 +1,3 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-
 # =================================================================
 #
 # Authors: Tom Kralidis <tomkralidis@gmail.com>
@@ -102,7 +99,7 @@ def do_email(config, resource, run, status_changed, result):
     try:
         if config['GHC_SMTP']['tls']:
             server.starttls()
-    except Exception, err:
+    except Exception as err:
         LOGGER.exception("Cannot connect to smtp: %s[:%s]: %s",
                          config['GHC_SMTP']['server'],
                          config['GHC_SMTP']['port'],
@@ -112,7 +109,7 @@ def do_email(config, resource, run, status_changed, result):
     try:
         server.login(config['GHC_SMTP']['username'],
                      config['GHC_SMTP']['password'])
-    except Exception, err:
+    except Exception as err:
         LOGGER.exception("Cannot log in to smtp: %s", err,
                          exc_info=err)
     try:
@@ -205,7 +202,7 @@ def do_webhook(config, resource, run, status_changed, result):
     for rcp in recipients:
         try:
             url, params = _parse_webhook_location(rcp)
-        except ValueError, err:
+        except ValueError as err:
             LOGGER.warning("Cannot send to {}: {}"
                            .format(rcp, err), exc_info=err)
 
@@ -223,7 +220,7 @@ def do_webhook(config, resource, run, status_changed, result):
             r = requests.post(url, params)
             LOGGER.info("webhook deployed, got %s as reposnse",
                         r)
-        except requests.exceptions.RequestException, err:
+        except requests.exceptions.RequestException as err:
             LOGGER.warning("cannot deploy webhook %s: %s",
                            rcp, err, exc_info=err)
 
@@ -262,6 +259,6 @@ def notify(config, resource, run, last_run_success):
     for chann_handler in (do_email, do_webhook,):
         try:
             chann_handler(config, resource, run, status_changed, result)
-        except Exception, err:
+        except Exception as err:
             LOGGER.warning("couldn't run notification for %s: %s",
                            chann_handler.func_name, err, exc_info=err)

GeoHealthCheck/plugin.py

@@ -1,8 +1,7 @@
-# -*- coding: utf-8 -*-
 from factory import Factory
 import logging
 import inspect
-import collections
+from collections.abc import Mapping
 import copy
 from init import App
 
@@ -126,9 +125,9 @@ def dict_merge(dct, merge_dct):
             :param merge_dct: dict merged into dct
             :return: None
             """
-            for k, v in merge_dct.iteritems():
+            for k, v in merge_dct.items():
                 if k in dct and isinstance(dct[k], dict) \
-                        and isinstance(merge_dct[k], collections.Mapping):
+                        and isinstance(merge_dct[k], Mapping):
                     dict_merge(dct[k], merge_dct[k])
                 else:
                     dct[k] = merge_dct[k]

GeoHealthCheck/plugins/check/checks.py

@@ -2,10 +2,7 @@
 from owslib.etree import etree
 from GeoHealthCheck.plugin import Plugin
 from GeoHealthCheck.check import Check
-try:
-    from html import escape  # python 3.x
-except ImportError:
-    from cgi import escape  # python 2.x
+from html import escape
 
 
 """ Contains basic Check classes for a Probe object."""
@@ -25,7 +22,7 @@ def __init__(self):
     def perform(self):
         """Default check: Resource should at least give no error"""
         status = self.probe.response.status_code
-        overall_status = status / 100
+        overall_status = status // 100
         if overall_status in [4, 5]:
             self.set_result(False, 'HTTP Error status=%d' % status)
 

GeoHealthCheck/plugins/probe/wfs.py

@@ -142,7 +142,7 @@ def expand_params(self, resource):
 
             ft_namespaces = set([name.split(':')[0] if ':' in name else None
                                  for name in feature_type_names])
-            ft_namespaces = filter(None, list(ft_namespaces))
+            ft_namespaces = list(filter(None, list(ft_namespaces)))
 
             # In some cases default NS is used: no FT NSs
             nsmap = None
@@ -151,7 +151,7 @@ def expand_params(self, resource):
                     # issue #243 this depends if lxml etree present
                     # and used by OWSLib ! Otherwise fall-back.
                     nsmap = wfs._capabilities.nsmap
-                except Exception as err:
+                except Exception:
                     # Fall-back
                     pass
 

GeoHealthCheck/plugins/probe/wfs3.py

@@ -114,7 +114,7 @@ def perform_request(self):
         try:
             for collection in collections:
                 coll_id = collection['id']
-                coll_id = coll_id.encode('utf-8')
+                coll_id = coll_id
 
                 try:
                     coll = wfs3.collection(coll_id)