diff --git a/.travis.yml b/.travis.yml index c17dcbf9..f5e7aaec 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,7 +1,7 @@ language: python python: - - "2.7" + - "3.5" sudo: false diff --git a/GeoHealthCheck/__init__.py b/GeoHealthCheck/__init__.py index e1614da1..2246184a 100644 --- a/GeoHealthCheck/__init__.py +++ b/GeoHealthCheck/__init__.py @@ -27,7 +27,7 @@ # # ================================================================= -from util import read +from .util import read def get_package_version(file_): diff --git a/GeoHealthCheck/app.py b/GeoHealthCheck/app.py index 6dbcfcff..e46b3599 100644 --- a/GeoHealthCheck/app.py +++ b/GeoHealthCheck/app.py @@ -32,7 +32,7 @@ import csv import logging import json -from StringIO import StringIO +from io import BytesIO from itertools import chain from flask import (flash, g, jsonify, redirect, @@ -90,6 +90,7 @@ def db_commit(): try: DB.session.commit() except Exception as err: + LOGGER.warning(err) DB.session.rollback() # finally: # DB.session.close() @@ -245,7 +246,7 @@ def export(): return jsonify(json_dict) elif request.url_rule.rule == '/csv': - output = StringIO() + output = BytesIO() writer = csv.writer(output) header = [ 'resource_type', 'title', 'url', 'ghc_url', 'ghc_json', 'ghc_csv', @@ -325,7 +326,7 @@ def export_resource(identifier): } return jsonify(json_dict) elif 'csv' in request.url_rule.rule: - output = StringIO() + output = BytesIO() writer = csv.writer(output) header = [ 'identifier', 'title', 'url', 'resource_type', 'owner', @@ -378,7 +379,7 @@ def export_resource_history(identifier): }) return jsonify(json_dict) elif 'csv' in request.url_rule.rule: - output = StringIO() + output = BytesIO() writer = csv.writer(output) header = [ 'owner', 'resource_type', 'checked_datetime', 'title', 'url', diff --git a/GeoHealthCheck/healthcheck.py b/GeoHealthCheck/healthcheck.py index 0985eca6..76b6e1f2 100644 --- a/GeoHealthCheck/healthcheck.py +++ b/GeoHealthCheck/healthcheck.py @@ -30,8 +30,8 @@ from datetime import datetime import logging import json -from urllib2 import urlopen -from urlparse import urlparse +from urllib.request import urlopen +from urllib.parse import urlparse from functools import partial from flask_babel import gettext @@ -62,6 +62,7 @@ def db_commit(): try: DB.session.commit() except Exception as err: + LOGGER.warning(err) DB.session.rollback() # finally: # DB.session.close() @@ -171,7 +172,7 @@ def sniff_test_resource(config, resource_type, url): try: ows = ows_handler(url) break - except Exception, err: + except Exception as err: LOGGER.warning("Cannot use %s on %s: %s", ows_handler, url, err, exc_info=err) if ows is None: @@ -281,7 +282,7 @@ def geonode_get_ows(base_url): try: data = json.load(r) - except (TypeError, ValueError,), err: + except (TypeError, ValueError,) as err: msg = "Cannot decode response from GeoNode at {}: {}".format(base_url, err) raise ValueError(msg) diff --git a/GeoHealthCheck/models.py b/GeoHealthCheck/models.py index 1ce0efd3..447a352b 100644 --- a/GeoHealthCheck/models.py +++ b/GeoHealthCheck/models.py @@ -38,7 +38,7 @@ from sqlalchemy.orm import deferred from sqlalchemy.orm.exc import MultipleResultsFound, NoResultFound -import util +from . import util from enums import RESOURCE_TYPES from factory import Factory from init import App @@ -213,7 +213,7 @@ def _validate_webhook(value): from GeoHealthCheck.notifications import _parse_webhook_location try: _parse_webhook_location(value) - except ValueError, err: + except ValueError as err: raise ValidationError('{}: {}'.format(value, err)) return value @@ -281,7 +281,7 @@ def validate(cls, channel, value): for v in validators: try: v(value) - except (ValidationError, TypeError), err: + except (ValidationError, TypeError) as err: raise ValueError("Bad value: {}".format(err), err) def is_email(self): @@ -305,7 +305,7 @@ def burry_dead(cls): def get_or_create(cls, channel, location): try: cls.validate(channel, location) - except ValidationError, err: + except ValidationError as err: raise ValueError("invalid value {}: {}".format(location, err)) try: @@ -631,7 +631,7 @@ def is_anonymous(self): return False def get_id(self): - return unicode(self.identifier) + return self.identifier def set_password(self, password): self.password = self.encrypt(password) @@ -825,13 +825,13 @@ def db_commit(): password1 = sys.argv[3] email1 = sys.argv[4] else: - username = raw_input('Enter your username: ').strip() - password1 = raw_input('Enter your password: ').strip() - password2 = raw_input('Enter your password again: ').strip() + username = input('Enter your username: ').strip() + password1 = input('Enter your password: ').strip() + password2 = input('Enter your password again: ').strip() if password1 != password2: raise ValueError('Passwords must match') - email1 = raw_input('Enter your email: ').strip() - email2 = raw_input('Enter your email again: ').strip() + email1 = input('Enter your email: ').strip() + email2 = input('Enter your email again: ').strip() if email1 != email2: raise ValueError('Emails must match') @@ -849,7 +849,7 @@ def db_commit(): yesno = 'n' if len(sys.argv) == 3: print('WARNING: all DB data will be lost! Proceed?') - yesno = raw_input( + yesno = input( 'Enter y (proceed) or n (abort): ').strip() elif len(sys.argv) == 4: yesno = sys.argv[3] diff --git a/GeoHealthCheck/notifications.py b/GeoHealthCheck/notifications.py index 71190282..e729b229 100644 --- a/GeoHealthCheck/notifications.py +++ b/GeoHealthCheck/notifications.py @@ -102,7 +102,7 @@ def do_email(config, resource, run, status_changed, result): try: if config['GHC_SMTP']['tls']: server.starttls() - except Exception, err: + except Exception as err: LOGGER.exception("Cannot connect to smtp: %s[:%s]: %s", config['GHC_SMTP']['server'], config['GHC_SMTP']['port'], @@ -112,7 +112,7 @@ def do_email(config, resource, run, status_changed, result): try: server.login(config['GHC_SMTP']['username'], config['GHC_SMTP']['password']) - except Exception, err: + except Exception as err: LOGGER.exception("Cannot log in to smtp: %s", err, exc_info=err) try: @@ -205,7 +205,7 @@ def do_webhook(config, resource, run, status_changed, result): for rcp in recipients: try: url, params = _parse_webhook_location(rcp) - except ValueError, err: + except ValueError as err: LOGGER.warning("Cannot send to {}: {}" .format(rcp, err), exc_info=err) @@ -223,7 +223,7 @@ def do_webhook(config, resource, run, status_changed, result): r = requests.post(url, params) LOGGER.info("webhook deployed, got %s as reposnse", r) - except requests.exceptions.RequestException, err: + except requests.exceptions.RequestException as err: LOGGER.warning("cannot deploy webhook %s: %s", rcp, err, exc_info=err) @@ -262,6 +262,6 @@ def notify(config, resource, run, last_run_success): for chann_handler in (do_email, do_webhook,): try: chann_handler(config, resource, run, status_changed, result) - except Exception, err: + except Exception as err: LOGGER.warning("couldn't run notification for %s: %s", chann_handler.func_name, err, exc_info=err) diff --git a/GeoHealthCheck/plugin.py b/GeoHealthCheck/plugin.py index 2b36a7da..da28253c 100644 --- a/GeoHealthCheck/plugin.py +++ b/GeoHealthCheck/plugin.py @@ -123,7 +123,7 @@ def dict_merge(dct, merge_dct): :param merge_dct: dict merged into dct :return: None """ - for k, v in merge_dct.iteritems(): + for k, v in merge_dct.items(): if k in dct and isinstance(dct[k], dict) \ and isinstance(merge_dct[k], collections.Mapping): dict_merge(dct[k], merge_dct[k]) diff --git a/GeoHealthCheck/plugins/probe/wfs.py b/GeoHealthCheck/plugins/probe/wfs.py index 1a92f18a..513c2f76 100644 --- a/GeoHealthCheck/plugins/probe/wfs.py +++ b/GeoHealthCheck/plugins/probe/wfs.py @@ -1,8 +1,12 @@ +import logging + from GeoHealthCheck.probe import Probe from GeoHealthCheck.plugin import Plugin from GeoHealthCheck.util import transform_bbox from owslib.wfs import WebFeatureService +LOGGER = logging.getLogger(__name__) + class WfsGetFeatureBbox(Probe): """ @@ -149,6 +153,7 @@ def expand_params(self, resource): # and used by OWSLib ! Otherwise fall-back. nsmap = wfs._capabilities.nsmap except Exception as err: + LOGGER.warning(err) # Fall-back pass diff --git a/GeoHealthCheck/scheduler.py b/GeoHealthCheck/scheduler.py index ebfa6310..0e343f2a 100644 --- a/GeoHealthCheck/scheduler.py +++ b/GeoHealthCheck/scheduler.py @@ -56,6 +56,7 @@ def db_commit(): try: DB.session.commit() except Exception as err: + LOGGER.warning(err) DB.session.rollback() # finally: # DB.session.close() diff --git a/GeoHealthCheck/templates/edit_resource.html b/GeoHealthCheck/templates/edit_resource.html index 92d024ff..b1957f98 100644 --- a/GeoHealthCheck/templates/edit_resource.html +++ b/GeoHealthCheck/templates/edit_resource.html @@ -109,7 +109,7 @@

[{{ _('Edit') }}] {{ resour Probes
Available - {% for probe_class, probe_avail in probes_avail.iteritems() %} + {% for probe_class, probe_avail in probes_avail.items() %} {% include 'includes/probe_info.html' %} {% endfor %} diff --git a/GeoHealthCheck/templates/includes/check_info.html b/GeoHealthCheck/templates/includes/check_info.html index 83149cc2..60daa401 100644 --- a/GeoHealthCheck/templates/includes/check_info.html +++ b/GeoHealthCheck/templates/includes/check_info.html @@ -16,7 +16,7 @@
This Check has no parameters. {% else %} - {% for param, param_def in check_info.PARAM_DEFS.iteritems() %} + {% for param, param_def in check_info.PARAM_DEFS.items() %}
{{ param }} diff --git a/GeoHealthCheck/templates/includes/probe_edit_form.html b/GeoHealthCheck/templates/includes/probe_edit_form.html index 0fcf10c3..00198f94 100644 --- a/GeoHealthCheck/templates/includes/probe_edit_form.html +++ b/GeoHealthCheck/templates/includes/probe_edit_form.html @@ -98,7 +98,7 @@ {% else %} - {% for check_class, check in probe_info.CHECKS_AVAIL.iteritems() %} + {% for check_class, check in probe_info.CHECKS_AVAIL.items() %}
{{ check.NAME }} diff --git a/GeoHealthCheck/templates/includes/probe_info.html b/GeoHealthCheck/templates/includes/probe_info.html index 8b614224..0cfaa1bf 100644 --- a/GeoHealthCheck/templates/includes/probe_info.html +++ b/GeoHealthCheck/templates/includes/probe_info.html @@ -77,7 +77,7 @@ {% if not probe_avail.CHECKS_AVAIL %} This Probe has no checks. {% else %} - {% for check_class, check_info in probe_avail.CHECKS_AVAIL.iteritems() %} + {% for check_class, check_info in probe_avail.CHECKS_AVAIL.items() %} {% include 'includes/check_info.html' %} {% endfor %} {% endif %} diff --git a/GeoHealthCheck/util.py b/GeoHealthCheck/util.py index 539b18c4..1bc6e5ad 100644 --- a/GeoHealthCheck/util.py +++ b/GeoHealthCheck/util.py @@ -32,8 +32,8 @@ import logging import os import smtplib -from urllib2 import urlopen -from urlparse import urlparse +from urllib.request import urlopen +from urllib.parse import urlparse from gettext import translation from passlib.hash import pbkdf2_sha256 @@ -105,7 +105,7 @@ def get_python_snippet(resource): lines.append('# testing via OWSLib') lines.append('# test GetCapabilities') else: - lines.append('# testing via urllib2 and urlparse') + lines.append('# testing via urllib') if resource.resource_type == 'OGC:WMS': lines.append('from owslib.wms import WebMapService') @@ -130,19 +130,19 @@ def get_python_snippet(resource): lines.append('myows = SensorObservationService(\'%s\')' % resource.url) elif resource.resource_type == 'WWW:LINK': lines.append('import re') - lines.append('from urllib2 import urlopen') + lines.append('from urllib.request import urlopen') lines.append('ows = urlopen(\'%s\')' % resource.url) lines.append('try:') lines.append(' title_re = re.compile("(.+?)")') lines.append(' title = title_re.search(ows.read()).group(1)') elif resource.resource_type == 'urn:geoss:waf': - lines.append('from urllib2 import urlopen') - lines.append('from urlparse import urlparse') + lines.append('from urllib.request import urlopen') + lines.append('from urllib.parse import urlparse') lines.append('ows = urlopen(\'%s\')' % resource.url) lines.append('title = urlparse(url).hostname') elif resource.resource_type == 'FTP': - lines.append('from urllib2 import urlopen') - lines.append('from urlparse import urlparse') + lines.append('from urllib.request import urlopen') + lines.append('from urllib.parse import urlparse') lines.append('ows = urlopen(\'%s\')' % resource.url) lines.append('title = urlparse(url).hostname') diff --git a/pavement.py b/pavement.py index dd35c618..4dcb2382 100644 --- a/pavement.py +++ b/pavement.py @@ -31,8 +31,8 @@ import os import shutil import tempfile -from StringIO import StringIO -from urllib2 import urlopen +from io import BytesIO +from urllib.request import urlopen import zipfile from paver.easy import (Bunch, call_task, cmdopts, info, options, @@ -89,7 +89,7 @@ def setup(): need_to_fetch = True if need_to_fetch: - zipstr = StringIO(urlopen(skin).read()) + zipstr = BytesIO(urlopen(skin).read()) zipfile_obj = zipfile.ZipFile(zipstr) zipfile_obj.extractall(options.base.static_lib) @@ -106,16 +106,16 @@ def setup(): 'startbootstrap-sb-admin-2-3.3.7-1')) # install sparklines to static/site/js - with open(path(options.base.static_lib / 'jspark.js'), 'w') as f: + with open(path(options.base.static_lib / 'jspark.js'), 'wb') as f: content = urlopen('http://ejohn.org/files/jspark.js').read() - content.replace('red', 'green') + content.replace(b'red', b'green') f.write(content) # install bootstrap-tagsinput to static/lib info('Getting select2') select2 = 'https://github.com/select2/select2/archive/4.0.3.zip' - zipstr = StringIO(urlopen(select2).read()) + zipstr = BytesIO(urlopen(select2).read()) zipfile_obj = zipfile.ZipFile(zipstr) zipfile_obj.extractall(options.base.static_lib) dirname = glob.glob(options.base.static_lib / 'select2-*')[0] @@ -130,19 +130,19 @@ def setup(): info('Getting leaflet') leafletjs = 'http://cdn.leafletjs.com/downloads/leaflet-0.7.5.zip' - zipstr = StringIO(urlopen(leafletjs).read()) + zipstr = BytesIO(urlopen(leafletjs).read()) zipfile_obj = zipfile.ZipFile(zipstr) zipfile_obj.extractall(options.base.static_lib / 'leaflet') # install html5shiv to static/lib - with open(path(options.base.static_lib / 'html5shiv.min.js'), 'w') as f: - url = 'http://oss.maxcdn.com/html5shiv/3.7.2/html5shiv.min.js' + with open(path(options.base.static_lib / 'html5shiv.min.js'), 'wb') as f: + url = 'https://oss.maxcdn.com/html5shiv/3.7.2/html5shiv.min.js' content = urlopen(url).read() f.write(content) # install respond to static/lib - with open(path(options.base.static_lib / 'respond.min.js'), 'w') as f: - url = 'http://oss.maxcdn.com/respond/1.4.2/respond.min.js' + with open(path(options.base.static_lib / 'respond.min.js'), 'wb') as f: + url = 'https://oss.maxcdn.com/respond/1.4.2/respond.min.js' content = urlopen(url).read() f.write(content) diff --git a/requirements-dev.txt b/requirements-dev.txt index bd869515..db1d01d3 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,3 +1,2 @@ flake8 Paver==1.3.4 -pylint diff --git a/tests/data/fixtures.json b/tests/data/fixtures.json index f6f9f434..98db2ffa 100644 --- a/tests/data/fixtures.json +++ b/tests/data/fixtures.json @@ -29,7 +29,7 @@ "resource_type": "OGC:WFS", "active": true, "title": "WOUDC Web Feature Service", - "url": "http://geo.woudc.org/ows", + "url": "https://geo.woudc.org/ows", "tags": [ "ows" ] @@ -50,7 +50,7 @@ "resource_type": "OGC:CSW", "active": true, "title": "WOUDC Catalogue Service", - "url": "http://geo.woudc.org/csw", + "url": "https://geo.woudc.org/csw", "tags": [ "ows" ] @@ -71,7 +71,7 @@ "resource_type": "WWW:LINK", "active": true, "title": "WOUDC Definitions Service", - "url": "http://geo.woudc.org/def", + "url": "https://geo.woudc.org/def", "tags": [] }, "PDOK TMS": { diff --git a/tests/test_resources.py b/tests/test_resources.py index 636130fe..03340604 100644 --- a/tests/test_resources.py +++ b/tests/test_resources.py @@ -130,7 +130,7 @@ def testWebhookNotifications(self): self.assertTrue(success) self.assertEqual(test_url, url) self.assertEqual(test_params, params) - except Exception, err: + except Exception as err: self.assertFalse(success, str(err))