From b6729fe683dc834f5e783c7fcd98910105fc633d Mon Sep 17 00:00:00 2001 From: Rob van Loon Date: Sat, 31 Aug 2019 09:45:14 +0200 Subject: [PATCH 01/47] Update gitignore with database and venv --- .gitignore | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.gitignore b/.gitignore index 501ffbba..a8186829 100644 --- a/.gitignore +++ b/.gitignore @@ -8,6 +8,7 @@ __pycache__/ # Distribution / packaging .Python env/ +venv/ bin/ build/ develop-eggs/ @@ -59,3 +60,6 @@ GeoHealthCheck/static/docs GeoHealthCheck/static/lib GeoHealthCheck.wsgi GeoHealthCheck.conf + +# Data +GeoHealthCheck/data.db From ec8ae21cb1b6023d99693f064401d8cc4c7715f8 Mon Sep 17 00:00:00 2001 From: Rob van Loon Date: Sat, 31 Aug 2019 11:19:11 +0200 Subject: [PATCH 02/47] Keep track of work file --- py3changes.txt | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 py3changes.txt diff --git a/py3changes.txt b/py3changes.txt new file mode 100644 index 00000000..f0599e76 --- /dev/null +++ b/py3changes.txt @@ -0,0 +1,9 @@ +update import stringio -> io +update import urllib2 -> urllib.parse +update catching exceptions to py3-style +delete str.decode('utf-8') +changed if-condition + +TODO: +iteritems() -> items() +search code for utf-8 From 74750e198060e63aa87a7cc163e04be9d2cb2504 Mon Sep 17 00:00:00 2001 From: Rob van Loon Date: Sat, 31 Aug 2019 11:19:36 +0200 Subject: [PATCH 03/47] WIP: first updates on python3 --- GeoHealthCheck/app.py | 5 ++--- GeoHealthCheck/factory.py | 2 +- GeoHealthCheck/healthcheck.py | 8 ++++---- GeoHealthCheck/models.py | 6 +++--- GeoHealthCheck/notifications.py | 10 +++++----- GeoHealthCheck/scheduler.py | 2 +- GeoHealthCheck/templates/layout.html | 2 +- GeoHealthCheck/util.py | 4 ++-- GeoHealthCheck/views.py | 4 ++-- 9 files changed, 21 insertions(+), 22 deletions(-) diff --git a/GeoHealthCheck/app.py b/GeoHealthCheck/app.py index 031abe1f..9b1c63bc 100644 --- a/GeoHealthCheck/app.py +++ b/GeoHealthCheck/app.py @@ -33,7 +33,7 @@ import csv import json import logging -from StringIO import StringIO +from io import StringIO from flask import (abort, flash, g, jsonify, redirect, render_template, request, url_for) @@ -278,7 +278,7 @@ def export(): json_dict['resources'].append({ 'resource_type': r.resource_type, - 'title': r.title.encode('utf-8'), + 'title': r.title, 'url': r.url, 'ghc_url': ghc_url, 'ghc_json': '%s/json' % ghc_url, @@ -863,7 +863,6 @@ def get_check_edit_form(check_class): check_vars = CheckVars( None, check_class, check_obj.get_default_parameter_values()) - # print(str(check_info)) return render_template('includes/check_edit_form.html', lang=g.current_lang, check=check_vars, check_info=check_info) diff --git a/GeoHealthCheck/factory.py b/GeoHealthCheck/factory.py index 9b659494..d7f49592 100644 --- a/GeoHealthCheck/factory.py +++ b/GeoHealthCheck/factory.py @@ -46,7 +46,7 @@ def create_class(class_string): raise ValueError('Class name must contain module part.') class_obj = getattr( __import__(module_name, globals(), locals(), - [class_name], -1), class_name) + [class_name]), class_name) except Exception as e: LOGGER.error("cannot create class '%s'" % class_string) raise e diff --git a/GeoHealthCheck/healthcheck.py b/GeoHealthCheck/healthcheck.py index 34ec5808..a19d69ec 100644 --- a/GeoHealthCheck/healthcheck.py +++ b/GeoHealthCheck/healthcheck.py @@ -30,8 +30,8 @@ from datetime import datetime import logging import json -from urllib2 import urlopen -from urlparse import urlparse +from urllib.request import urlopen +from urllib.parse import urlparse from functools import partial from flask_babel import gettext @@ -173,7 +173,7 @@ def sniff_test_resource(config, resource_type, url): try: ows = ows_handler(url) break - except Exception, err: + except Exception as err: LOGGER.warning("Cannot use %s on %s: %s", ows_handler, url, err, exc_info=err) if ows is None: @@ -285,7 +285,7 @@ def geonode_get_ows(base_url): try: data = json.load(r) - except (TypeError, ValueError,), err: + except (TypeError, ValueError,) as err: msg = "Cannot decode response from GeoNode at {}: {}".format(base_url, err) raise ValueError(msg) diff --git a/GeoHealthCheck/models.py b/GeoHealthCheck/models.py index a5b33890..1507d427 100644 --- a/GeoHealthCheck/models.py +++ b/GeoHealthCheck/models.py @@ -222,7 +222,7 @@ def _validate_webhook(value): from GeoHealthCheck.notifications import _parse_webhook_location try: _parse_webhook_location(value) - except ValueError, err: + except ValueError as err: raise ValidationError('{}: {}'.format(value, err)) return value @@ -290,7 +290,7 @@ def validate(cls, channel, value): for v in validators: try: v(value) - except (ValidationError, TypeError), err: + except (ValidationError, TypeError) as err: raise ValueError("Bad value: {}".format(err), err) def is_email(self): @@ -314,7 +314,7 @@ def burry_dead(cls): def get_or_create(cls, channel, location): try: cls.validate(channel, location) - except ValidationError, err: + except ValidationError as err: raise ValueError("invalid value {}: {}".format(location, err)) try: diff --git a/GeoHealthCheck/notifications.py b/GeoHealthCheck/notifications.py index 71190282..42a0a998 100644 --- a/GeoHealthCheck/notifications.py +++ b/GeoHealthCheck/notifications.py @@ -102,7 +102,7 @@ def do_email(config, resource, run, status_changed, result): try: if config['GHC_SMTP']['tls']: server.starttls() - except Exception, err: + except Exception as err: LOGGER.exception("Cannot connect to smtp: %s[:%s]: %s", config['GHC_SMTP']['server'], config['GHC_SMTP']['port'], @@ -112,7 +112,7 @@ def do_email(config, resource, run, status_changed, result): try: server.login(config['GHC_SMTP']['username'], config['GHC_SMTP']['password']) - except Exception, err: + except Exception as err: LOGGER.exception("Cannot log in to smtp: %s", err, exc_info=err) try: @@ -205,7 +205,7 @@ def do_webhook(config, resource, run, status_changed, result): for rcp in recipients: try: url, params = _parse_webhook_location(rcp) - except ValueError, err: + except ValueError as err: LOGGER.warning("Cannot send to {}: {}" .format(rcp, err), exc_info=err) @@ -223,7 +223,7 @@ def do_webhook(config, resource, run, status_changed, result): r = requests.post(url, params) LOGGER.info("webhook deployed, got %s as reposnse", r) - except requests.exceptions.RequestException, err: + except requests.exceptions.RequestException as err: LOGGER.warning("cannot deploy webhook %s: %s", rcp, err, exc_info=err) @@ -262,6 +262,6 @@ def notify(config, resource, run, last_run_success): for chann_handler in (do_email, do_webhook,): try: chann_handler(config, resource, run, status_changed, result) - except Exception, err: + except Exception as err: LOGGER.warning("couldn't run notification for %s: %s", chann_handler.func_name, err, exc_info=err) diff --git a/GeoHealthCheck/scheduler.py b/GeoHealthCheck/scheduler.py index ebfa6310..cc28d3ff 100644 --- a/GeoHealthCheck/scheduler.py +++ b/GeoHealthCheck/scheduler.py @@ -249,7 +249,7 @@ def add_job(resource): scheduler.add_job( run_job, 'interval', args=[resource.identifier, freq], minutes=freq, next_run_time=next_run_time, max_instances=1, - misfire_grace_time=(freq * 60) / 2, coalesce=True, + misfire_grace_time=round((freq * 60) / 2), coalesce=True, id=str(resource.identifier)) diff --git a/GeoHealthCheck/templates/layout.html b/GeoHealthCheck/templates/layout.html index 84af625b..afbecfd5 100644 --- a/GeoHealthCheck/templates/layout.html +++ b/GeoHealthCheck/templates/layout.html @@ -58,7 +58,7 @@ {{ _('Language') }} diff --git a/GeoHealthCheck/util.py b/GeoHealthCheck/util.py index fbefea65..fc448a7c 100644 --- a/GeoHealthCheck/util.py +++ b/GeoHealthCheck/util.py @@ -34,8 +34,8 @@ import smtplib import six import base64 -from urllib2 import urlopen -from urlparse import urlparse +from urllib.request import urlopen +from urllib.parse import urlparse from gettext import translation from passlib.hash import pbkdf2_sha256 diff --git a/GeoHealthCheck/views.py b/GeoHealthCheck/views.py index a2ba0f4a..4a9989d0 100644 --- a/GeoHealthCheck/views.py +++ b/GeoHealthCheck/views.py @@ -83,9 +83,9 @@ def list_resources(resource_type=None, query=None, tag=None): for resource in response['resources']: if resource.runs.count() > 0: # View should work even without Runs - if resource.first_run < first_run or first_run is None: + if first_run is None or resource.first_run < first_run: first_run = resource.first_run - if resource.last_run < last_run or last_run is None: + if last_run is None or resource.last_run < last_run: last_run = resource.last_run response['first_run'] = first_run response['last_run'] = last_run From 8a0ffe91ab94cde4b59cbdf23d60558dc47d7e37 Mon Sep 17 00:00:00 2001 From: Rob van Loon Date: Sat, 31 Aug 2019 11:34:54 +0200 Subject: [PATCH 04/47] WIP: fix iteritems() to items() --- GeoHealthCheck/plugin.py | 2 +- GeoHealthCheck/templates/edit_resource.html | 2 +- GeoHealthCheck/templates/includes/check_info.html | 2 +- GeoHealthCheck/templates/includes/probe_edit_form.html | 2 +- GeoHealthCheck/templates/includes/probe_info.html | 2 +- py3changes.txt | 1 - 6 files changed, 5 insertions(+), 6 deletions(-) diff --git a/GeoHealthCheck/plugin.py b/GeoHealthCheck/plugin.py index ca20fa6c..c4bf0f87 100644 --- a/GeoHealthCheck/plugin.py +++ b/GeoHealthCheck/plugin.py @@ -126,7 +126,7 @@ def dict_merge(dct, merge_dct): :param merge_dct: dict merged into dct :return: None """ - for k, v in merge_dct.iteritems(): + for k, v in merge_dct.items(): if k in dct and isinstance(dct[k], dict) \ and isinstance(merge_dct[k], collections.Mapping): dict_merge(dct[k], merge_dct[k]) diff --git a/GeoHealthCheck/templates/edit_resource.html b/GeoHealthCheck/templates/edit_resource.html index adf36403..8594f88f 100644 --- a/GeoHealthCheck/templates/edit_resource.html +++ b/GeoHealthCheck/templates/edit_resource.html @@ -153,7 +153,7 @@

[{{ _('Edit') }}] {{ resour Probes
Available - {% for probe_class, probe_avail in probes_avail.iteritems() %} + {% for probe_class, probe_avail in probes_avail.items() %} {% include 'includes/probe_info.html' %} {% endfor %} diff --git a/GeoHealthCheck/templates/includes/check_info.html b/GeoHealthCheck/templates/includes/check_info.html index 83149cc2..60daa401 100644 --- a/GeoHealthCheck/templates/includes/check_info.html +++ b/GeoHealthCheck/templates/includes/check_info.html @@ -16,7 +16,7 @@
This Check has no parameters. {% else %} - {% for param, param_def in check_info.PARAM_DEFS.iteritems() %} + {% for param, param_def in check_info.PARAM_DEFS.items() %}
{{ param }} diff --git a/GeoHealthCheck/templates/includes/probe_edit_form.html b/GeoHealthCheck/templates/includes/probe_edit_form.html index 0fcf10c3..00198f94 100644 --- a/GeoHealthCheck/templates/includes/probe_edit_form.html +++ b/GeoHealthCheck/templates/includes/probe_edit_form.html @@ -98,7 +98,7 @@ {% else %} - {% for check_class, check in probe_info.CHECKS_AVAIL.iteritems() %} + {% for check_class, check in probe_info.CHECKS_AVAIL.items() %}
{{ check.NAME }} diff --git a/GeoHealthCheck/templates/includes/probe_info.html b/GeoHealthCheck/templates/includes/probe_info.html index 8b614224..0cfaa1bf 100644 --- a/GeoHealthCheck/templates/includes/probe_info.html +++ b/GeoHealthCheck/templates/includes/probe_info.html @@ -77,7 +77,7 @@ {% if not probe_avail.CHECKS_AVAIL %} This Probe has no checks. {% else %} - {% for check_class, check_info in probe_avail.CHECKS_AVAIL.iteritems() %} + {% for check_class, check_info in probe_avail.CHECKS_AVAIL.items() %} {% include 'includes/check_info.html' %} {% endfor %} {% endif %} diff --git a/py3changes.txt b/py3changes.txt index f0599e76..0d0cf6d7 100644 --- a/py3changes.txt +++ b/py3changes.txt @@ -5,5 +5,4 @@ delete str.decode('utf-8') changed if-condition TODO: -iteritems() -> items() search code for utf-8 From cac0495b302cd56d12a8d328ac39d33b9a92f7b6 Mon Sep 17 00:00:00 2001 From: Rob van Loon Date: Sat, 31 Aug 2019 12:18:20 +0200 Subject: [PATCH 05/47] Fix empty tags in template --- GeoHealthCheck/templates/add.html | 6 +++++- GeoHealthCheck/templates/edit_resource.html | 6 +++++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/GeoHealthCheck/templates/add.html b/GeoHealthCheck/templates/add.html index a899073c..68c3604f 100644 --- a/GeoHealthCheck/templates/add.html +++ b/GeoHealthCheck/templates/add.html @@ -34,7 +34,11 @@

{{ _('Add Resource') }}

{% block extrafoot %}