diff --git a/.gitignore b/.gitignore index 501ffbba..a8186829 100644 --- a/.gitignore +++ b/.gitignore @@ -8,6 +8,7 @@ __pycache__/ # Distribution / packaging .Python env/ +venv/ bin/ build/ develop-eggs/ @@ -59,3 +60,6 @@ GeoHealthCheck/static/docs GeoHealthCheck/static/lib GeoHealthCheck.wsgi GeoHealthCheck.conf + +# Data +GeoHealthCheck/data.db diff --git a/.travis.yml b/.travis.yml index c17dcbf9..3b43f7be 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,7 +1,7 @@ language: python python: - - "2.7" + - "3.7" sudo: false diff --git a/Dockerfile b/Dockerfile index d29370fe..04ed6203 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM python:2.7.15-alpine3.8 +FROM python:3.7.4-alpine3.10 # Thanks to http://www.sandtable.com/reduce-docker-image-sizes-using-alpine # FROM debian:jessie @@ -63,8 +63,7 @@ WSGI_WORKER_CLASS='eventlet' \ GHC_USER_PLUGINS='' RUN apk add --no-cache --virtual .build-deps gcc build-base libxslt-dev libxml2-dev linux-headers postgresql-dev \ - && apk add --no-cache bash postgresql-client libxslt libxml2 tzdata openntpd \ - && pip install virtualenv \ + && apk add --no-cache bash postgresql-client libxslt libxml2 tzdata openntpd python3 python3-dev \ && rm -rf /var/cache/apk/* /tmp/* /var/tmp/* # Add standard files and Add/override Plugins diff --git a/GeoHealthCheck/app.py b/GeoHealthCheck/app.py index 13665a81..f7feba37 100644 --- a/GeoHealthCheck/app.py +++ b/GeoHealthCheck/app.py @@ -1,4 +1,3 @@ -# coding=utf-8 # ================================================================= # # Authors: Tom Kralidis @@ -33,7 +32,7 @@ import csv import json import logging -from StringIO import StringIO +from io import StringIO from flask import (abort, flash, g, jsonify, redirect, render_template, request, url_for) @@ -90,7 +89,7 @@ def db_commit(): err = None try: DB.session.commit() - except Exception as err: + except Exception: DB.session.rollback() # finally: # DB.session.close() @@ -240,7 +239,8 @@ def context_processors(): 'resource_types_counts': rtc['counts'], 'resources_total': rtc['total'], 'languages': LANGUAGES, - 'tags': tags + 'tags': tags, + 'tagnames': list(tags.keys()) } @@ -278,7 +278,7 @@ def export(): json_dict['resources'].append({ 'resource_type': r.resource_type, - 'title': r.title.encode('utf-8'), + 'title': r.title, 'url': r.url, 'ghc_url': ghc_url, 'ghc_json': '%s/json' % ghc_url, @@ -314,7 +314,7 @@ def export(): writer.writerow([ r.resource_type, - r.title.encode('utf-8'), + r.title, r.url, ghc_url, '%s/json' % ghc_url, @@ -361,7 +361,7 @@ def export_resource(identifier): json_dict = { 'identifier': resource.identifier, - 'title': resource.title.encode('utf-8'), + 'title': resource.title, 'url': resource.url, 'resource_type': resource.resource_type, 'owner': resource.owner.username, @@ -390,7 +390,7 @@ def export_resource(identifier): writer.writerow(header) writer.writerow([ resource.identifier, - resource.title.encode('utf-8'), + resource.title, resource.url, resource.resource_type, resource.owner.username, @@ -424,7 +424,7 @@ def export_resource_history(identifier): 'owner': resource.owner.username, 'resource_type': resource.resource_type, 'checked_datetime': format_checked_datetime(run), - 'title': resource.title.encode('utf-8'), + 'title': resource.title, 'url': resource.url, 'response_time': round(run.response_time, 2), 'status': format_run_status(run) @@ -443,7 +443,7 @@ def export_resource_history(identifier): resource.owner.username, resource.resource_type, format_checked_datetime(run), - resource.title.encode('utf-8'), + resource.title, resource.url, round(run.response_time, 2), format_run_status(run), @@ -852,7 +852,6 @@ def get_check_edit_form(check_class): check_vars = CheckVars( None, check_class, check_obj.get_default_parameter_values()) - # print(str(check_info)) return render_template('includes/check_edit_form.html', lang=g.current_lang, check=check_vars, check_info=check_info) diff --git a/GeoHealthCheck/factory.py b/GeoHealthCheck/factory.py index 9b659494..d7f49592 100644 --- a/GeoHealthCheck/factory.py +++ b/GeoHealthCheck/factory.py @@ -46,7 +46,7 @@ def create_class(class_string): raise ValueError('Class name must contain module part.') class_obj = getattr( __import__(module_name, globals(), locals(), - [class_name], -1), class_name) + [class_name]), class_name) except Exception as e: LOGGER.error("cannot create class '%s'" % class_string) raise e diff --git a/GeoHealthCheck/healthcheck.py b/GeoHealthCheck/healthcheck.py index 34ec5808..8897fa56 100644 --- a/GeoHealthCheck/healthcheck.py +++ b/GeoHealthCheck/healthcheck.py @@ -30,8 +30,8 @@ from datetime import datetime import logging import json -from urllib2 import urlopen -from urlparse import urlparse +from urllib.request import urlopen +from urllib.parse import urlparse from functools import partial from flask_babel import gettext @@ -62,6 +62,7 @@ def db_commit(): try: DB.session.commit() except Exception as err: + LOGGER.warning("Cannot commit to database %s".format(err)) DB.session.rollback() # finally: # DB.session.close() @@ -173,7 +174,7 @@ def sniff_test_resource(config, resource_type, url): try: ows = ows_handler(url) break - except Exception, err: + except Exception as err: LOGGER.warning("Cannot use %s on %s: %s", ows_handler, url, err, exc_info=err) if ows is None: @@ -244,7 +245,6 @@ def sniff_test_resource(config, resource_type, url): title = ows.identification.title if title is None: title = '%s %s %s' % (resource_type, gettext('for'), url) - title = title.decode('utf-8') success = True except Exception as err: title = 'Untitled' @@ -285,7 +285,7 @@ def geonode_get_ows(base_url): try: data = json.load(r) - except (TypeError, ValueError,), err: + except (TypeError, ValueError,) as err: msg = "Cannot decode response from GeoNode at {}: {}".format(base_url, err) raise ValueError(msg) diff --git a/GeoHealthCheck/models.py b/GeoHealthCheck/models.py index a5b33890..1ec7d7e5 100644 --- a/GeoHealthCheck/models.py +++ b/GeoHealthCheck/models.py @@ -94,6 +94,24 @@ def __init__(self, resource, result, self.message = result.message self.report = result.get_report() + def __lt__(self, other): + return self.identifier < other.identifier + + def __le__(self, other): + return self.identifier <= other.identifier + + def __eq__(self, other): + return self.identifier == other.identifier + + def __gt__(self, other): + return self.identifier > other.identifier + + def __ge__(self, other): + return self.identifief >= other.identifier + + def __hash__(self): + return hash(f"{self.identifier}{self.checked_datetime}{self.resource}") + # JSON string object specifying report for the Run # See http://docs.sqlalchemy.org/en/latest/orm/mapped_attributes.html _report = DB.Column("report", DB.Text, default={}) @@ -222,7 +240,7 @@ def _validate_webhook(value): from GeoHealthCheck.notifications import _parse_webhook_location try: _parse_webhook_location(value) - except ValueError, err: + except ValueError as err: raise ValidationError('{}: {}'.format(value, err)) return value @@ -290,7 +308,7 @@ def validate(cls, channel, value): for v in validators: try: v(value) - except (ValidationError, TypeError), err: + except (ValidationError, TypeError) as err: raise ValueError("Bad value: {}".format(err), err) def is_email(self): @@ -314,7 +332,7 @@ def burry_dead(cls): def get_or_create(cls, channel, location): try: cls.validate(channel, location) - except ValidationError, err: + except ValidationError as err: raise ValueError("invalid value {}: {}".format(location, err)) try: @@ -690,7 +708,7 @@ def is_anonymous(self): return False def get_id(self): - return unicode(self.identifier) + return self.identifier def set_password(self, password): self.password = self.encrypt(password) @@ -884,13 +902,13 @@ def db_commit(): password1 = sys.argv[3] email1 = sys.argv[4] else: - username = raw_input('Enter your username: ').strip() - password1 = raw_input('Enter your password: ').strip() - password2 = raw_input('Enter your password again: ').strip() + username = input('Enter your username: ').strip() + password1 = input('Enter your password: ').strip() + password2 = input('Enter your password again: ').strip() if password1 != password2: raise ValueError('Passwords must match') - email1 = raw_input('Enter your email: ').strip() - email2 = raw_input('Enter your email again: ').strip() + email1 = input('Enter your email: ').strip() + email2 = input('Enter your email again: ').strip() if email1 != email2: raise ValueError('Emails must match') @@ -908,7 +926,7 @@ def db_commit(): yesno = 'n' if len(sys.argv) == 3: print('WARNING: all DB data will be lost! Proceed?') - yesno = raw_input( + yesno = input( 'Enter y (proceed) or n (abort): ').strip() elif len(sys.argv) == 4: yesno = sys.argv[3] diff --git a/GeoHealthCheck/notifications.py b/GeoHealthCheck/notifications.py index 71190282..562cd1d1 100644 --- a/GeoHealthCheck/notifications.py +++ b/GeoHealthCheck/notifications.py @@ -1,6 +1,3 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - # ================================================================= # # Authors: Tom Kralidis @@ -102,7 +99,7 @@ def do_email(config, resource, run, status_changed, result): try: if config['GHC_SMTP']['tls']: server.starttls() - except Exception, err: + except Exception as err: LOGGER.exception("Cannot connect to smtp: %s[:%s]: %s", config['GHC_SMTP']['server'], config['GHC_SMTP']['port'], @@ -112,7 +109,7 @@ def do_email(config, resource, run, status_changed, result): try: server.login(config['GHC_SMTP']['username'], config['GHC_SMTP']['password']) - except Exception, err: + except Exception as err: LOGGER.exception("Cannot log in to smtp: %s", err, exc_info=err) try: @@ -205,7 +202,7 @@ def do_webhook(config, resource, run, status_changed, result): for rcp in recipients: try: url, params = _parse_webhook_location(rcp) - except ValueError, err: + except ValueError as err: LOGGER.warning("Cannot send to {}: {}" .format(rcp, err), exc_info=err) @@ -223,7 +220,7 @@ def do_webhook(config, resource, run, status_changed, result): r = requests.post(url, params) LOGGER.info("webhook deployed, got %s as reposnse", r) - except requests.exceptions.RequestException, err: + except requests.exceptions.RequestException as err: LOGGER.warning("cannot deploy webhook %s: %s", rcp, err, exc_info=err) @@ -262,6 +259,6 @@ def notify(config, resource, run, last_run_success): for chann_handler in (do_email, do_webhook,): try: chann_handler(config, resource, run, status_changed, result) - except Exception, err: + except Exception as err: LOGGER.warning("couldn't run notification for %s: %s", chann_handler.func_name, err, exc_info=err) diff --git a/GeoHealthCheck/plugin.py b/GeoHealthCheck/plugin.py index ca20fa6c..3534c4b3 100644 --- a/GeoHealthCheck/plugin.py +++ b/GeoHealthCheck/plugin.py @@ -1,8 +1,7 @@ -# -*- coding: utf-8 -*- from factory import Factory import logging import inspect -import collections +from collections.abc import Mapping import copy from init import App @@ -126,9 +125,9 @@ def dict_merge(dct, merge_dct): :param merge_dct: dict merged into dct :return: None """ - for k, v in merge_dct.iteritems(): + for k, v in merge_dct.items(): if k in dct and isinstance(dct[k], dict) \ - and isinstance(merge_dct[k], collections.Mapping): + and isinstance(merge_dct[k], Mapping): dict_merge(dct[k], merge_dct[k]) else: dct[k] = merge_dct[k] diff --git a/GeoHealthCheck/plugins/check/checks.py b/GeoHealthCheck/plugins/check/checks.py index a37ca794..0cb784b6 100644 --- a/GeoHealthCheck/plugins/check/checks.py +++ b/GeoHealthCheck/plugins/check/checks.py @@ -2,10 +2,7 @@ from owslib.etree import etree from GeoHealthCheck.plugin import Plugin from GeoHealthCheck.check import Check -try: - from html import escape # python 3.x -except ImportError: - from cgi import escape # python 2.x +from html import escape """ Contains basic Check classes for a Probe object.""" @@ -25,7 +22,7 @@ def __init__(self): def perform(self): """Default check: Resource should at least give no error""" status = self.probe.response.status_code - overall_status = status / 100 + overall_status = status // 100 if overall_status in [4, 5]: self.set_result(False, 'HTTP Error status=%d' % status) diff --git a/GeoHealthCheck/plugins/probe/wfs.py b/GeoHealthCheck/plugins/probe/wfs.py index 088619f1..edf8ef79 100644 --- a/GeoHealthCheck/plugins/probe/wfs.py +++ b/GeoHealthCheck/plugins/probe/wfs.py @@ -142,7 +142,7 @@ def expand_params(self, resource): ft_namespaces = set([name.split(':')[0] if ':' in name else None for name in feature_type_names]) - ft_namespaces = filter(None, list(ft_namespaces)) + ft_namespaces = list(filter(None, list(ft_namespaces))) # In some cases default NS is used: no FT NSs nsmap = None @@ -151,7 +151,7 @@ def expand_params(self, resource): # issue #243 this depends if lxml etree present # and used by OWSLib ! Otherwise fall-back. nsmap = wfs._capabilities.nsmap - except Exception as err: + except Exception: # Fall-back pass diff --git a/GeoHealthCheck/plugins/probe/wfs3.py b/GeoHealthCheck/plugins/probe/wfs3.py index 23cd08bd..225d8437 100644 --- a/GeoHealthCheck/plugins/probe/wfs3.py +++ b/GeoHealthCheck/plugins/probe/wfs3.py @@ -114,7 +114,7 @@ def perform_request(self): try: for collection in collections: coll_id = collection['id'] - coll_id = coll_id.encode('utf-8') + coll_id = coll_id try: coll = wfs3.collection(coll_id) diff --git a/GeoHealthCheck/plugins/resourceauth/resourceauths.py b/GeoHealthCheck/plugins/resourceauth/resourceauths.py index 604eb6ff..24c823a6 100644 --- a/GeoHealthCheck/plugins/resourceauth/resourceauths.py +++ b/GeoHealthCheck/plugins/resourceauth/resourceauths.py @@ -102,8 +102,9 @@ def encode_auth_header_val(self): # like: 'Basic base64encode(username + ':' + password) auth_creds = self.auth_dict['data'] auth_val = base64.encodestring( - '%s:%s' % (auth_creds['username'], auth_creds['password'])) - auth_val = "Basic %s" % auth_val + '{}:{}'.format(auth_creds['username'], auth_creds['password']). + encode()) + auth_val = 'Basic {}'.format(auth_val.decode()) return auth_val diff --git a/GeoHealthCheck/resourceauth.py b/GeoHealthCheck/resourceauth.py index e8d5f5a8..ba1982eb 100644 --- a/GeoHealthCheck/resourceauth.py +++ b/GeoHealthCheck/resourceauth.py @@ -80,7 +80,7 @@ def decode(encoded): return None try: - s = decode(APP.config['SECRET_KEY'], str(encoded)) + s = decode(APP.config['SECRET_KEY'], encoded) return json.loads(s) except Exception as err: LOGGER.error('Error decoding auth: %s' % str(err)) diff --git a/GeoHealthCheck/result.py b/GeoHealthCheck/result.py index b17dfb5a..ae890ef2 100644 --- a/GeoHealthCheck/result.py +++ b/GeoHealthCheck/result.py @@ -47,7 +47,7 @@ def stop(self): def __str__(self): if self.message: - self.message = self.message.encode('utf-8') + self.message = self.message return "success=%s msg=%s response_time=%s" % \ (self.success, self.message, self.response_time_str) diff --git a/GeoHealthCheck/scheduler.py b/GeoHealthCheck/scheduler.py index ebfa6310..62fa9c2c 100644 --- a/GeoHealthCheck/scheduler.py +++ b/GeoHealthCheck/scheduler.py @@ -1,4 +1,3 @@ -# coding=utf-8 # ================================================================= # # Authors: Tom Kralidis @@ -55,7 +54,7 @@ def db_commit(): err = None try: DB.session.commit() - except Exception as err: + except Exception: DB.session.rollback() # finally: # DB.session.close() @@ -249,7 +248,7 @@ def add_job(resource): scheduler.add_job( run_job, 'interval', args=[resource.identifier, freq], minutes=freq, next_run_time=next_run_time, max_instances=1, - misfire_grace_time=(freq * 60) / 2, coalesce=True, + misfire_grace_time=round((freq * 60) / 2), coalesce=True, id=str(resource.identifier)) diff --git a/GeoHealthCheck/templates/add.html b/GeoHealthCheck/templates/add.html index a899073c..bf3e9249 100644 --- a/GeoHealthCheck/templates/add.html +++ b/GeoHealthCheck/templates/add.html @@ -34,7 +34,7 @@

{{ _('Add Resource') }}

{% block extrafoot %}