py3k deps update

.travis.yml

@@ -1,7 +1,7 @@
 language: python
 
 python:
-  - "2.7"
+  - "3.5"
 
 sudo: false
 

GeoHealthCheck/__init__.py

@@ -27,7 +27,7 @@
 #
 # =================================================================
 
-from util import read
+from .util import read
 
 
 def get_package_version(file_):

GeoHealthCheck/app.py

@@ -32,7 +32,7 @@
 import csv
 import logging
 import json
-from StringIO import StringIO
+from io import BytesIO
 from itertools import chain
 
 from flask import (flash, g, jsonify, redirect,
@@ -90,6 +90,7 @@ def db_commit():
     try:
         DB.session.commit()
     except Exception as err:
+        LOGGER.warning(err)
         DB.session.rollback()
     # finally:
     #     DB.session.close()
@@ -245,7 +246,7 @@ def export():
 
         return jsonify(json_dict)
     elif request.url_rule.rule == '/csv':
-        output = StringIO()
+        output = BytesIO()
         writer = csv.writer(output)
         header = [
             'resource_type', 'title', 'url', 'ghc_url', 'ghc_json', 'ghc_csv',
@@ -325,7 +326,7 @@ def export_resource(identifier):
         }
         return jsonify(json_dict)
     elif 'csv' in request.url_rule.rule:
-        output = StringIO()
+        output = BytesIO()
         writer = csv.writer(output)
         header = [
             'identifier', 'title', 'url', 'resource_type', 'owner',
@@ -378,7 +379,7 @@ def export_resource_history(identifier):
             })
         return jsonify(json_dict)
     elif 'csv' in request.url_rule.rule:
-        output = StringIO()
+        output = BytesIO()
         writer = csv.writer(output)
         header = [
             'owner', 'resource_type', 'checked_datetime', 'title', 'url',

GeoHealthCheck/healthcheck.py

@@ -30,8 +30,8 @@
 from datetime import datetime
 import logging
 import json
-from urllib2 import urlopen
-from urlparse import urlparse
+from urllib.request import urlopen
+from urllib.parse import urlparse
 from functools import partial
 from flask_babel import gettext
 
@@ -62,6 +62,7 @@ def db_commit():
     try:
         DB.session.commit()
     except Exception as err:
+        LOGGER.warning(err)
         DB.session.rollback()
     # finally:
     #     DB.session.close()
@@ -171,7 +172,7 @@ def sniff_test_resource(config, resource_type, url):
             try:
                 ows = ows_handler(url)
                 break
-            except Exception, err:
+            except Exception as err:
                 LOGGER.warning("Cannot use %s on %s: %s",
                                ows_handler, url, err, exc_info=err)
         if ows is None:
@@ -281,7 +282,7 @@ def geonode_get_ows(base_url):
 
     try:
         data = json.load(r)
-    except (TypeError, ValueError,), err:
+    except (TypeError, ValueError,) as err:
         msg = "Cannot decode response from GeoNode at {}: {}".format(base_url,
                                                                      err)
         raise ValueError(msg)

GeoHealthCheck/models.py

@@ -38,7 +38,7 @@
 from sqlalchemy.orm import deferred
 from sqlalchemy.orm.exc import MultipleResultsFound, NoResultFound
 
-import util
+from . import util
 from enums import RESOURCE_TYPES
 from factory import Factory
 from init import App
@@ -213,7 +213,7 @@ def _validate_webhook(value):
     from GeoHealthCheck.notifications import _parse_webhook_location
     try:
         _parse_webhook_location(value)
-    except ValueError, err:
+    except ValueError as err:
         raise ValidationError('{}: {}'.format(value, err))
     return value
 
@@ -281,7 +281,7 @@ def validate(cls, channel, value):
         for v in validators:
             try:
                 v(value)
-            except (ValidationError, TypeError), err:
+            except (ValidationError, TypeError) as err:
                 raise ValueError("Bad value: {}".format(err), err)
 
     def is_email(self):
@@ -305,7 +305,7 @@ def burry_dead(cls):
     def get_or_create(cls, channel, location):
         try:
             cls.validate(channel, location)
-        except ValidationError, err:
+        except ValidationError as err:
             raise ValueError("invalid value {}: {}".format(location, err))
 
         try:
@@ -631,7 +631,7 @@ def is_anonymous(self):
         return False
 
     def get_id(self):
-        return unicode(self.identifier)
+        return self.identifier
 
     def set_password(self, password):
         self.password = self.encrypt(password)
@@ -825,13 +825,13 @@ def db_commit():
                 password1 = sys.argv[3]
                 email1 = sys.argv[4]
             else:
-                username = raw_input('Enter your username: ').strip()
-                password1 = raw_input('Enter your password: ').strip()
-                password2 = raw_input('Enter your password again: ').strip()
+                username = input('Enter your username: ').strip()
+                password1 = input('Enter your password: ').strip()
+                password2 = input('Enter your password again: ').strip()
                 if password1 != password2:
                     raise ValueError('Passwords must match')
-                email1 = raw_input('Enter your email: ').strip()
-                email2 = raw_input('Enter your email again: ').strip()
+                email1 = input('Enter your email: ').strip()
+                email2 = input('Enter your email again: ').strip()
                 if email1 != email2:
                     raise ValueError('Emails must match')
 
@@ -849,7 +849,7 @@ def db_commit():
                 yesno = 'n'
                 if len(sys.argv) == 3:
                     print('WARNING: all DB data will be lost! Proceed?')
-                    yesno = raw_input(
+                    yesno = input(
                         'Enter y (proceed) or n (abort): ').strip()
                 elif len(sys.argv) == 4:
                     yesno = sys.argv[3]

GeoHealthCheck/notifications.py

@@ -102,7 +102,7 @@ def do_email(config, resource, run, status_changed, result):
     try:
         if config['GHC_SMTP']['tls']:
             server.starttls()
-    except Exception, err:
+    except Exception as err:
         LOGGER.exception("Cannot connect to smtp: %s[:%s]: %s",
                          config['GHC_SMTP']['server'],
                          config['GHC_SMTP']['port'],
@@ -112,7 +112,7 @@ def do_email(config, resource, run, status_changed, result):
     try:
         server.login(config['GHC_SMTP']['username'],
                      config['GHC_SMTP']['password'])
-    except Exception, err:
+    except Exception as err:
         LOGGER.exception("Cannot log in to smtp: %s", err,
                          exc_info=err)
     try:
@@ -205,7 +205,7 @@ def do_webhook(config, resource, run, status_changed, result):
     for rcp in recipients:
         try:
             url, params = _parse_webhook_location(rcp)
-        except ValueError, err:
+        except ValueError as err:
             LOGGER.warning("Cannot send to {}: {}"
                            .format(rcp, err), exc_info=err)
 
@@ -223,7 +223,7 @@ def do_webhook(config, resource, run, status_changed, result):
             r = requests.post(url, params)
             LOGGER.info("webhook deployed, got %s as reposnse",
                         r)
-        except requests.exceptions.RequestException, err:
+        except requests.exceptions.RequestException as err:
             LOGGER.warning("cannot deploy webhook %s: %s",
                            rcp, err, exc_info=err)
 
@@ -262,6 +262,6 @@ def notify(config, resource, run, last_run_success):
     for chann_handler in (do_email, do_webhook,):
         try:
             chann_handler(config, resource, run, status_changed, result)
-        except Exception, err:
+        except Exception as err:
             LOGGER.warning("couldn't run notification for %s: %s",
                            chann_handler.func_name, err, exc_info=err)

GeoHealthCheck/plugin.py

@@ -123,7 +123,7 @@ def dict_merge(dct, merge_dct):
             :param merge_dct: dict merged into dct
             :return: None
             """
-            for k, v in merge_dct.iteritems():
+            for k, v in merge_dct.items():
                 if k in dct and isinstance(dct[k], dict) \
                         and isinstance(merge_dct[k], collections.Mapping):
                     dict_merge(dct[k], merge_dct[k])

GeoHealthCheck/plugins/probe/wfs.py

@@ -1,8 +1,12 @@
+import logging
+
 from GeoHealthCheck.probe import Probe
 from GeoHealthCheck.plugin import Plugin
 from GeoHealthCheck.util import transform_bbox
 from owslib.wfs import WebFeatureService
 
+LOGGER = logging.getLogger(__name__)
+
 
 class WfsGetFeatureBbox(Probe):
     """
@@ -149,6 +153,7 @@ def expand_params(self, resource):
                     # and used by OWSLib ! Otherwise fall-back.
                     nsmap = wfs._capabilities.nsmap
                 except Exception as err:
+                    LOGGER.warning(err)
                     # Fall-back
                     pass
 

GeoHealthCheck/scheduler.py

@@ -56,6 +56,7 @@ def db_commit():
     try:
         DB.session.commit()
     except Exception as err:
+        LOGGER.warning(err)
         DB.session.rollback()
     # finally:
     #     DB.session.close()

GeoHealthCheck/templates/edit_resource.html

@@ -109,7 +109,7 @@ <h1 class="page-header">[{{ _('Edit') }}] <span id="resource_title_h1">{{ resour
                 <th>Probes<br/>Available</th>
 
                 <td>
-                    {% for probe_class, probe_avail in probes_avail.iteritems() %}
+                    {% for probe_class, probe_avail in probes_avail.items() %}
                         {% include 'includes/probe_info.html' %}
                     {% endfor %}
                 </td>

GeoHealthCheck/templates/includes/check_info.html

@@ -16,7 +16,7 @@
                         <br/>This Check has no parameters.
                         {% else %}
                         <table class="table table-condensed table-bordered">
-                            {% for param, param_def in check_info.PARAM_DEFS.iteritems() %}
+                            {% for param, param_def in check_info.PARAM_DEFS.items() %}
                             <tr>
                                 <td width="30%">{{ param }}</td>
                                 <td>

GeoHealthCheck/templates/includes/probe_edit_form.html

@@ -98,7 +98,7 @@
                           {% else %}
 
                           <table class="table">
-                         {% for check_class, check in probe_info.CHECKS_AVAIL.iteritems() %}
+                         {% for check_class, check in probe_info.CHECKS_AVAIL.items() %}
                            <tr>
                                <td>
                                    <strong>{{ check.NAME }}</strong>

GeoHealthCheck/templates/includes/probe_info.html

@@ -77,7 +77,7 @@
                             {% if not probe_avail.CHECKS_AVAIL %}
                             This Probe has no checks.
                             {% else %}
-                            {% for check_class, check_info in probe_avail.CHECKS_AVAIL.iteritems() %}
+                            {% for check_class, check_info in probe_avail.CHECKS_AVAIL.items() %}
                                 {% include 'includes/check_info.html' %}
                             {% endfor %}
                             {% endif %}

GeoHealthCheck/util.py

@@ -32,8 +32,8 @@
 import logging
 import os
 import smtplib
-from urllib2 import urlopen
-from urlparse import urlparse
+from urllib.request import urlopen
+from urllib.parse import urlparse
 from gettext import translation
 from passlib.hash import pbkdf2_sha256
 
@@ -105,7 +105,7 @@ def get_python_snippet(resource):
         lines.append('# testing via OWSLib')
         lines.append('# test GetCapabilities')
     else:
-        lines.append('# testing via urllib2 and urlparse')
+        lines.append('# testing via urllib')
 
     if resource.resource_type == 'OGC:WMS':
         lines.append('from owslib.wms import WebMapService')
@@ -130,19 +130,19 @@ def get_python_snippet(resource):
         lines.append('myows = SensorObservationService(\'%s\')' % resource.url)
     elif resource.resource_type == 'WWW:LINK':
         lines.append('import re')
-        lines.append('from urllib2 import urlopen')
+        lines.append('from urllib.request import urlopen')
         lines.append('ows = urlopen(\'%s\')' % resource.url)
         lines.append('try:')
         lines.append('    title_re = re.compile("<title>(.+?)</title>")')
         lines.append('    title = title_re.search(ows.read()).group(1)')
     elif resource.resource_type == 'urn:geoss:waf':
-        lines.append('from urllib2 import urlopen')
-        lines.append('from urlparse import urlparse')
+        lines.append('from urllib.request import urlopen')
+        lines.append('from urllib.parse import urlparse')
         lines.append('ows = urlopen(\'%s\')' % resource.url)
         lines.append('title = urlparse(url).hostname')
     elif resource.resource_type == 'FTP':
-        lines.append('from urllib2 import urlopen')
-        lines.append('from urlparse import urlparse')
+        lines.append('from urllib.request import urlopen')
+        lines.append('from urllib.parse import urlparse')
         lines.append('ows = urlopen(\'%s\')' % resource.url)
         lines.append('title = urlparse(url).hostname')
 

pavement.py

@@ -31,8 +31,8 @@
 import os
 import shutil
 import tempfile
-from StringIO import StringIO
-from urllib2 import urlopen
+from io import BytesIO
+from urllib.request import urlopen
 import zipfile
 
 from paver.easy import (Bunch, call_task, cmdopts, info, options,
@@ -89,7 +89,7 @@ def setup():
             need_to_fetch = True
 
     if need_to_fetch:
-        zipstr = StringIO(urlopen(skin).read())
+        zipstr = BytesIO(urlopen(skin).read())
         zipfile_obj = zipfile.ZipFile(zipstr)
         zipfile_obj.extractall(options.base.static_lib)
 
@@ -106,16 +106,16 @@ def setup():
                            'startbootstrap-sb-admin-2-3.3.7-1'))
 
     # install sparklines to static/site/js
-    with open(path(options.base.static_lib / 'jspark.js'), 'w') as f:
+    with open(path(options.base.static_lib / 'jspark.js'), 'wb') as f:
         content = urlopen('http://ejohn.org/files/jspark.js').read()
-        content.replace('red', 'green')
+        content.replace(b'red', b'green')
         f.write(content)
 
     # install bootstrap-tagsinput to static/lib
     info('Getting select2')
     select2 = 'https://github.com/select2/select2/archive/4.0.3.zip'
 
-    zipstr = StringIO(urlopen(select2).read())
+    zipstr = BytesIO(urlopen(select2).read())
     zipfile_obj = zipfile.ZipFile(zipstr)
     zipfile_obj.extractall(options.base.static_lib)
     dirname = glob.glob(options.base.static_lib / 'select2-*')[0]
@@ -130,19 +130,19 @@ def setup():
     info('Getting leaflet')
     leafletjs = 'http://cdn.leafletjs.com/downloads/leaflet-0.7.5.zip'
 
-    zipstr = StringIO(urlopen(leafletjs).read())
+    zipstr = BytesIO(urlopen(leafletjs).read())
     zipfile_obj = zipfile.ZipFile(zipstr)
     zipfile_obj.extractall(options.base.static_lib / 'leaflet')
 
     # install html5shiv to static/lib
-    with open(path(options.base.static_lib / 'html5shiv.min.js'), 'w') as f:
-        url = 'http://oss.maxcdn.com/html5shiv/3.7.2/html5shiv.min.js'
+    with open(path(options.base.static_lib / 'html5shiv.min.js'), 'wb') as f:
+        url = 'https://oss.maxcdn.com/html5shiv/3.7.2/html5shiv.min.js'
         content = urlopen(url).read()
         f.write(content)
 
     # install respond to static/lib
-    with open(path(options.base.static_lib / 'respond.min.js'), 'w') as f:
-        url = 'http://oss.maxcdn.com/respond/1.4.2/respond.min.js'
+    with open(path(options.base.static_lib / 'respond.min.js'), 'wb') as f:
+        url = 'https://oss.maxcdn.com/respond/1.4.2/respond.min.js'
         content = urlopen(url).read()
         f.write(content)