From b53c02e76689b21f2e991f0ca0c55a190c022168 Mon Sep 17 00:00:00 2001 From: Just van den Broecke Date: Tue, 20 Mar 2018 18:18:44 +0100 Subject: [PATCH 01/20] #132 first stab at healthcheck scheduler using daemon process --- GeoHealthCheck/app.py | 11 +- GeoHealthCheck/healthcheck.py | 65 +++++- GeoHealthCheck/init.py | 63 +++--- .../migrations/versions/34531bfd7cab_.py | 47 ++++ GeoHealthCheck/models.py | 125 ++++++++--- GeoHealthCheck/scheduler.py | 207 ++++++++++++++++++ GeoHealthCheck/templates/edit_resource.html | 10 + GeoHealthCheck/templates/resource.html | 6 + .../translations/en/LC_MESSAGES/messages.po | 10 + .../nl_NL/LC_MESSAGES/messages.po | 10 + pavement.py | 6 + 11 files changed, 485 insertions(+), 75 deletions(-) create mode 100644 GeoHealthCheck/migrations/versions/34531bfd7cab_.py create mode 100644 GeoHealthCheck/scheduler.py diff --git a/GeoHealthCheck/app.py b/GeoHealthCheck/app.py index a8be709a..aaebe5b6 100644 --- a/GeoHealthCheck/app.py +++ b/GeoHealthCheck/app.py @@ -161,11 +161,11 @@ def cssize_reliability(value, css_type=None): score = 'danger' panel = 'red' elif (CONFIG['GHC_RELIABILITY_MATRIX']['orange']['min'] <= number <= - CONFIG['GHC_RELIABILITY_MATRIX']['orange']['max']): + CONFIG['GHC_RELIABILITY_MATRIX']['orange']['max']): score = 'warning' panel = 'yellow' elif (CONFIG['GHC_RELIABILITY_MATRIX']['green']['min'] <= number <= - CONFIG['GHC_RELIABILITY_MATRIX']['green']['max']): + CONFIG['GHC_RELIABILITY_MATRIX']['green']['max']): score = 'success' panel = 'green' else: # should never really get here @@ -556,7 +556,7 @@ def add(): param_vals = {} for param in param_defs: if param_defs[param]['value']: - param_vals[param] =\ + param_vals[param] = \ param_defs[param]['value'] check_vars = CheckVars( probe_to_add, check_class, param_vals) @@ -744,10 +744,6 @@ def delete(resource_identifier): flash(gettext('Resource not found'), 'danger') return redirect(url_for('home', lang=g.current_lang)) - runs = Run.query.filter_by(resource_identifier=resource_identifier).all() - - for run in runs: - DB.session.delete(run) resource.clear_recipients() DB.session.delete(resource) @@ -898,6 +894,7 @@ def api_probes_avail(resource_type=None, resource_id=None): if __name__ == '__main__': # run locally, for fun import sys + logging.basicConfig() HOST = '0.0.0.0' PORT = 8000 diff --git a/GeoHealthCheck/healthcheck.py b/GeoHealthCheck/healthcheck.py index ae0c1df6..cb5624a8 100644 --- a/GeoHealthCheck/healthcheck.py +++ b/GeoHealthCheck/healthcheck.py @@ -27,13 +27,14 @@ # # ================================================================= -import datetime +from datetime import datetime import logging import json from urllib2 import urlopen from urlparse import urlparse from functools import partial from pprint import pprint +from flask_babel import gettext from owslib.wms import WebMapService from owslib.wmts import WebMapTileService @@ -44,12 +45,66 @@ from owslib.csw import CatalogueServiceWeb from owslib.sos import SensorObservationService -from flask_babel import gettext +from init import App from enums import RESOURCE_TYPES +from models import Resource, Run from probe import Probe from result import ResourceResult +from notifications import notify LOGGER = logging.getLogger(__name__) +APP = App.get_app() +DB = App.get_db() + + +# commit or rollback shorthand +def db_commit(): + err = None + try: + DB.session.commit() + except Exception as err: + DB.session.rollback() + # finally: + # DB.session.close() + return err + + +# complete handle of resource test +def run_resource(resourceid): + resource = Resource.query.filter_by(identifier=resourceid).first() + + if not resource.active: + # Exit test of resource if it's not active + return + + # Get the status of the last run, + # assume success if there is none + last_run_success = True + last_run = resource.last_run + if last_run: + last_run_success = last_run.success + + # Run test + result = run_test_resource(resource) + + run1 = Run(resource, result, datetime.utcnow()) + + DB.session.add(run1) + + # commit or rollback each run to avoid long-lived transactions + # see https://github.com/geopython/GeoHealthCheck/issues/14 + db_commit() + + if APP.config['GHC_NOTIFICATIONS']: + # Attempt notification + try: + notify(APP.config, resource, run1, last_run_success) + except Exception as err: + # Don't bail out on failure in order to commit the Run + msg = str(err) + logging.warn('error notifying: %s' % msg) + if not __name__ == '__main__': + DB.session.remove() def run_test_resource(resource): @@ -81,7 +136,7 @@ def sniff_test_resource(config, resource_type, url): raise RuntimeError(msg2) title = None - start_time = datetime.datetime.utcnow() + start_time = datetime.utcnow() message = None resource_type_map = {'OGC:WMS': [partial(WebMapService, version='1.3.0'), partial(WebMapService, version='1.1.1')], @@ -153,7 +208,7 @@ def sniff_test_resource(config, resource_type, url): title = urlparse(url).hostname elif resource_type == 'OSGeo:GeoNode': endpoints = ows - end_time = datetime.datetime.utcnow() + end_time = datetime.utcnow() delta = end_time - start_time response_time = '%s.%s' % (delta.seconds, delta.microseconds) base_tags = geonode_make_tags(url) @@ -184,7 +239,7 @@ def sniff_test_resource(config, resource_type, url): message = msg success = False - end_time = datetime.datetime.utcnow() + end_time = datetime.utcnow() delta = end_time - start_time response_time = '%s.%s' % (delta.seconds, delta.microseconds) diff --git a/GeoHealthCheck/init.py b/GeoHealthCheck/init.py index 7be7d8bb..a7017762 100644 --- a/GeoHealthCheck/init.py +++ b/GeoHealthCheck/init.py @@ -29,6 +29,7 @@ import os import sys +import logging from flask import Flask from flask_sqlalchemy import SQLAlchemy from flask_babel import Babel @@ -55,69 +56,71 @@ class App: babel_instance = None plugins_instance = None home_dir = None + count = 0 @staticmethod def init(): # Do init once - if not App.app_instance: - app = Flask(__name__) + app = Flask(__name__) - # Read and override configs - app.config.from_pyfile('config_main.py') - app.config.from_pyfile('../instance/config_site.py') + # Read and override configs + app.config.from_pyfile('config_main.py') + app.config.from_pyfile('../instance/config_site.py') - app.config['GHC_SITE_URL'] = \ - app.config['GHC_SITE_URL'].rstrip('/') + logging.basicConfig(level=logging.INFO) + if app.config['DEBUG'] is True: + logging.basicConfig(level=logging.DEBUG) - app.secret_key = app.config['SECRET_KEY'] + app.config['GHC_SITE_URL'] = \ + app.config['GHC_SITE_URL'].rstrip('/') - App.db_instance = SQLAlchemy(app) - App.babel_instance = Babel(app) + app.secret_key = app.config['SECRET_KEY'] - # Plugins (via Docker ENV) must be list, but may have been - # specified as comma-separated string, or older set notation - app.config['GHC_PLUGINS'] = to_list(app.config['GHC_PLUGINS']) - app.config['GHC_USER_PLUGINS'] = \ - to_list(app.config['GHC_USER_PLUGINS']) + App.db_instance = SQLAlchemy(app) + App.babel_instance = Babel(app) - # Concatenate core- and user-Plugins - App.plugins_instance = \ - app.config['GHC_PLUGINS'] + app.config['GHC_USER_PLUGINS'] + # Plugins (via Docker ENV) must be list, but may have been + # specified as comma-separated string, or older set notation + app.config['GHC_PLUGINS'] = to_list(app.config['GHC_PLUGINS']) + app.config['GHC_USER_PLUGINS'] = \ + to_list(app.config['GHC_USER_PLUGINS']) - # Needed to find Plugins - home_dir = os.path.dirname(os.path.abspath(__file__)) - App.home_dir = sys.path.append('%s/..' % home_dir) + # Concatenate core- and user-Plugins + App.plugins_instance = \ + app.config['GHC_PLUGINS'] + app.config['GHC_USER_PLUGINS'] - # Finally assign app-instance - App.app_instance = app - print("init.py: created GHC App instance") + # Needed to find Plugins + home_dir = os.path.dirname(os.path.abspath(__file__)) + App.home_dir = sys.path.append('%s/..' % home_dir) + + # Finally assign app-instance + App.app_instance = app + App.count += 1 + logging.info("init.py: created GHC App instance #%d" % App.count) @staticmethod def get_app(): - App.init() return App.app_instance @staticmethod def get_babel(): - App.init() return App.babel_instance @staticmethod def get_config(): - App.init() return App.app_instance.config @staticmethod def get_db(): - App.init() return App.db_instance @staticmethod def get_home_dir(): - App.init() return App.home_dir @staticmethod def get_plugins(): - App.init() return App.plugins_instance + + +App.init() diff --git a/GeoHealthCheck/migrations/versions/34531bfd7cab_.py b/GeoHealthCheck/migrations/versions/34531bfd7cab_.py new file mode 100644 index 00000000..71d76b52 --- /dev/null +++ b/GeoHealthCheck/migrations/versions/34531bfd7cab_.py @@ -0,0 +1,47 @@ +"""empty message + +Revision ID: 34531bfd7cab +Revises: bb91fb332c36 +Create Date: 2018-03-19 16:59:42.235474 + +""" +from alembic import op +import sqlalchemy as sa +from GeoHealthCheck.migrations import alembic_helpers + +# revision identifiers, used by Alembic. +revision = '34531bfd7cab' +down_revision = 'bb91fb332c36' +branch_labels = None +depends_on = None + + +def upgrade(): + if not alembic_helpers.table_has_column('resource', 'run_frequency'): + print('Column run_frequency not present in resource table, will create') + op.add_column(u'resource', sa.Column('run_frequency', sa.Integer(), + nullable=False, default=60, server_default=60)) + else: + print('Column run_frequency already present in resource table') + + if not alembic_helpers.tables_exist(['resource_lock']): + print('Table for Resource locking not present, will create') + # ### commands auto generated by Alembic - please adjust! ### + op.create_table('resource_lock', + sa.Column('identifier', sa.Integer(), nullable=False, unique=True), + sa.Column('resource_identifier', sa.Integer(), nullable=False, unique=True), + sa.Column('owner', sa.Text, nullable=False, default='NOT SET', server_default='NOT SET'), + sa.Column('start_time', sa.DateTime, nullable=False), + sa.Column('end_time', sa.DateTime, nullable=False), + sa.ForeignKeyConstraint(['resource_identifier'], ['resource.identifier'], ), + sa.PrimaryKeyConstraint('identifier') + ) + # ### end Alembic commands ### + else: + print('Table for Resource-locking already present, will not create') + + +def downgrade(): + print('Dropping Column run_frequency from resource table') + op.drop_column(u'resource', 'run_frequency') + op.drop_table('resource_lock') diff --git a/GeoHealthCheck/models.py b/GeoHealthCheck/models.py index d26ff7ee..a33c9541 100644 --- a/GeoHealthCheck/models.py +++ b/GeoHealthCheck/models.py @@ -31,7 +31,7 @@ import json import logging from flask_babel import gettext as _ -from datetime import datetime +from datetime import datetime, timedelta from sqlalchemy import func, and_ from sqlalchemy.orm import deferred @@ -48,6 +48,21 @@ LOGGER = logging.getLogger(__name__) +# Complete handle of old runs deletion +def flush_runs(): + APP = App.get_app() + retention_time = timedelta(*APP.config['GHC_RETENTION_DAYS']) + + all_runs = Run.query.all() + for run in all_runs: + how_old = (datetime.utcnow() - run.checked_datetime) + if how_old > retention_time: + DB.session.delete(run) + db_commit() + + DB.session.remove() + + class Run(DB.Model): """measurement of resource state""" @@ -55,7 +70,8 @@ class Run(DB.Model): resource_identifier = DB.Column(DB.Integer, DB.ForeignKey('resource.identifier')) resource = DB.relationship('Resource', - backref=DB.backref('runs', lazy='dynamic')) + backref=DB.backref('runs', lazy='dynamic', + cascade="all,delete")) checked_datetime = DB.Column(DB.DateTime, nullable=False) success = DB.Column(DB.Boolean, nullable=False) response_time = DB.Column(DB.Float, nullable=False) @@ -215,6 +231,7 @@ class dummy_value(object): @staticmethod def gettext(*args, **kwargs): return _(*args, **kwargs) + dummy_form = None v(dummy_form, dummy_value()) @@ -272,11 +289,11 @@ def is_webhook(self): @classmethod def burry_dead(cls): RN = ResourceNotification - q = DB.session.query(cls)\ - .join(RN, - RN.recipient_id == cls.id, - isouter=True)\ - .filter(RN.recipient_id.is_(None)) + q = DB.session.query(cls) \ + .join(RN, + RN.recipient_id == cls.id, + isouter=True) \ + .filter(RN.recipient_id.is_(None)) for item in q: DB.session.delete(item) @@ -288,10 +305,10 @@ def get_or_create(cls, channel, location): raise ValueError("invalid value {}: {}".format(location, err)) try: - r = DB.session.query(cls)\ - .filter(and_(cls.channel == channel, - cls.location == location))\ - .one() + r = DB.session.query(cls) \ + .filter(and_(cls.channel == channel, + cls.location == location)) \ + .one() except (MultipleResultsFound, NoResultFound,): r = cls(channel=channel, location=location) DB.session.add(r) @@ -308,12 +325,12 @@ def get_suggestions(cls, channel, for_user): Res = Resource ResNot = ResourceNotification - q = DB.session.query(Rcp.location)\ - .join(ResNot, ResNot.recipient_id == Rcp.id)\ - .join(Res, Res.identifier == ResNot.resource_id)\ - .group_by(Rcp.location)\ - .filter(and_(Res.owner_identifier == for_user, - Rcp.channel == channel)) + q = DB.session.query(Rcp.location) \ + .join(ResNot, ResNot.recipient_id == Rcp.id) \ + .join(Res, Res.identifier == ResNot.resource_id) \ + .group_by(Rcp.location) \ + .filter(and_(Res.owner_identifier == for_user, + Rcp.channel == channel)) return [item[0] for item in q] @@ -347,6 +364,7 @@ class Resource(DB.Model): owner = DB.relationship('User', backref=DB.backref('username2', lazy='dynamic')) tags = DB.relationship('Tag', secondary=resource_tags, backref='resource') + run_frequency = DB.Column(DB.Integer, default=60) def __init__(self, owner, resource_type, title, url, tags): self.resource_type = resource_type @@ -471,26 +489,26 @@ def clear_recipients(self, channel=None, burry_dead=True): # clear specific channel to_delete = self.get_recipients(channel) if to_delete: - to_del_rcp = DB.session.query(RN)\ - .join(Rcp, - Rcp.id == RN.recipient_id)\ - .filter( - and_(RN.resource_id == - self.identifier, - Rcp.channel == channel, - Rcp.location.in_(to_delete)) - ) + to_del_rcp = DB.session.query(RN) \ + .join(Rcp, + Rcp.id == RN.recipient_id) \ + .filter( + and_(RN.resource_id == + self.identifier, + Rcp.channel == channel, + Rcp.location.in_(to_delete)) + ) else: to_del_rcp = [] else: # remove all m2m connections for Resource<->Recipient - to_del_rcp = DB.session.query(RN)\ - .join(Rcp, - Rcp.id == RN.recipient_id)\ - .filter( - RN.resource_id == - self.identifier, - ) + to_del_rcp = DB.session.query(RN) \ + .join(Rcp, + Rcp.id == RN.recipient_id) \ + .filter( + RN.resource_id == + self.identifier, + ) for rcp_ntf in to_del_rcp: DB.session.delete(rcp_ntf) if burry_dead: @@ -527,6 +545,47 @@ def dump_recipients(self): return out +class ResourceLock(DB.Model): + """lock resource for multiprocessing runs""" + + identifier = DB.Column(DB.Integer, + primary_key=True, autoincrement=False, unique=True) + resource_identifier = DB.Column( + DB.Integer, DB.ForeignKey('resource.identifier'), unique=True) + resource = DB.relationship('Resource', + backref=DB.backref('locks', lazy='dynamic', + cascade="all,delete")) + owner = DB.Column(DB.Text, nullable=False, default='NOT SET') + + start_time = DB.Column(DB.DateTime, nullable=False) + end_time = DB.Column(DB.DateTime, nullable=False) + + def __init__(self, resource, owner, interval_mins): + self.identifier = resource.identifier + self.resource = resource + self.owner = owner + self.init_datetimes(interval_mins) + + def init_datetimes(self, interval_mins): + self.start_time = datetime.utcnow() + self.end_time = self.start_time + timedelta(minutes=interval_mins) + + def has_expired(self): + now = datetime.utcnow() + return now > self.end_time + + def obtain(self, owner, frequency): + if not self.has_expired(): + return False + + self.owner = owner + self.init_datetimes(frequency) + return True + + def __repr__(self): + return '' % self.identifier + + class User(DB.Model): """user accounts""" diff --git a/GeoHealthCheck/scheduler.py b/GeoHealthCheck/scheduler.py new file mode 100644 index 00000000..4b8b2f41 --- /dev/null +++ b/GeoHealthCheck/scheduler.py @@ -0,0 +1,207 @@ +# coding=utf-8 +# ================================================================= +# +# Authors: Tom Kralidis +# Just van den Broecke +# +# Copyright (c) 2014 Tom Kralidis +# +# Permission is hereby granted, free of charge, to any person +# obtaining a copy of this software and associated documentation +# files (the "Software"), to deal in the Software without +# restriction, including without limitation the rights to use, +# copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following +# conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +# OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# +# ================================================================= + +import logging +import os +import random +import string +from models import Resource, ResourceLock, flush_runs +from healthcheck import run_resource +from apscheduler.schedulers.background import BackgroundScheduler +from apscheduler.jobstores.base import JobLookupError +from init import App + +LOGGER = logging.getLogger(__name__) +DB = App.get_db() + +# Create scheduler +scheduler = BackgroundScheduler() + + +# commit or rollback shorthand +def db_commit(): + err = None + try: + DB.session.commit() + except Exception as err: + DB.session.rollback() + # finally: + # DB.session.close() + return err + + +def run_job(resource_id, frequency): + # Generate unique id + # https://stackoverflow.com/questions/2257441/random-string-generation-with-upper-case-letters-and-digits-in-python + uuid = '%d-%s' % (os.getpid(), ''.join(random.choice( + string.ascii_uppercase + string.digits) for _ in range(8))) + + resource = Resource.query.filter_by(identifier=resource_id).first() + + if not resource: + stop_job(resource_id) + return + + lock = ResourceLock.query.filter_by(identifier=resource_id).first() + + if not lock: + # No lock at all on Resource: (hope) we're first + # obtain fresh lock, back-off if failed + LOGGER.info('%d No Lock at all: obtain new' % resource_id) + + lock = ResourceLock(resource, uuid, frequency) + DB.session.add(lock) + lock_err = db_commit() + if lock_err: + # Another process may have been there first! + LOGGER.info('%d Cannot obtain Lock %s' % + (resource_id, str(lock_err))) + return + else: + # Lock is there, look if available + LOGGER.info('%d Lock avail: try obtaining..' % resource_id) + if not lock.obtain(uuid, frequency): + LOGGER.info('%d Cannot obtain' % resource_id) + return + else: + LOGGER.info('%d Lock obtained, delete and renew' % resource_id) + DB.session.delete(lock) + lock_err = db_commit() + if lock_err: + # Another process may have been there first! + LOGGER.info('%d Lock Delete failed' % resource_id) + return + + LOGGER.info('%d Lock deleted, add new' % resource_id) + # (hope) we're first + # obtain fresh lock, back-off if failed + lock = ResourceLock(resource, uuid, frequency) + DB.session.add(lock) + lock_err = db_commit() + if lock_err: + # Another process may have been there first! + LOGGER.info('%d Lock Add failed' % resource_id) + return + + # Check if we really own the lock + LOGGER.info('%d Lock Add OK' % resource_id) + lock = ResourceLock.query.filter_by( + identifier=resource_id).first() + + if lock.owner != uuid: + LOGGER.info('%d Lock Add OK, not owner: back-off' + % resource_id) + return + + if lock: + try: + run_resource(resource_id) + LOGGER.info('%d run_resource OK' % resource_id) + finally: + pass + + +def start_schedule(): + # Cold start every cron of every Resource + for resource in Resource.query.all(): + add_job(resource) + + # change configuration + scheduler.configure(job_defaults={ + 'coalesce': False, + 'max_instances': 100000 + }) + + scheduler.add_job(flush_runs, 'interval', minutes=60) + scheduler.add_job(check_schedule, 'interval', minutes=5) + + scheduler.start() + import atexit + atexit.register(lambda: stop_schedule()) + + +def check_schedule(): + LOGGER.info('Checking Job schedules') + # Check the schedule for changed jobs + for resource in Resource.query.all(): + job = scheduler.get_job(str(resource.identifier)) + if job is None: + add_job(resource) + + current_freq = job.args[1] + + # Run frequency changed? + if current_freq != resource.run_frequency: + # Reschedule Job + update_job(resource) + + +def update_job(resource): + stop_job(resource.identifier) + + # Add job to Scheduler + add_job(resource) + + +def add_job(resource): + LOGGER.info('Starting job for resource=%d' % resource.identifier) + freq = resource.run_frequency + + scheduler.add_job( + run_job, 'interval', args=[resource.identifier, freq], + minutes=freq, + id=str(resource.identifier)) + + +def stop_job(resource_id): + LOGGER.info('Stopping job for resource=%d' % resource_id) + + # Try to remove job from scheduler + try: + scheduler.remove_job(str(resource_id)) + except JobLookupError: + pass + + +def stop_schedule(): + LOGGER.info('Stopping Scheduler') + scheduler.shutdown() + + +if __name__ == '__main__': + import time + + # Start scheduler + start_schedule() + + while True: + print("This prints once a minute.") + time.sleep(60) # Delay for 1 minute (60 seconds). diff --git a/GeoHealthCheck/templates/edit_resource.html b/GeoHealthCheck/templates/edit_resource.html index bb9722fb..bc45d251 100644 --- a/GeoHealthCheck/templates/edit_resource.html +++ b/GeoHealthCheck/templates/edit_resource.html @@ -88,6 +88,12 @@

[{{ _('Edit') }}] {{ resour + + {{ _('Run Every') }} + + + + Probes @@ -258,6 +264,9 @@

[{{ _('Edit') }}] {{ resour // Collect tags var new_tags = $('#resource_tags').val(); + // Collect test_frequency + var new_frequency = $('input[name="resource_frequency_value"]').val(); + // Collect active var new_active = $('#input_resource_active').prop('checked'); @@ -335,6 +344,7 @@

[{{ _('Edit') }}] {{ resour title: new_title, active: new_active, tags: new_tags, + run_frequency: new_frequency, probes: new_probes, notify_emails: new_notify_emails, notify_webhooks: new_notify_webhooks diff --git a/GeoHealthCheck/templates/resource.html b/GeoHealthCheck/templates/resource.html index 723f75fb..dc42185c 100644 --- a/GeoHealthCheck/templates/resource.html +++ b/GeoHealthCheck/templates/resource.html @@ -54,6 +54,12 @@

{{ _('Monitoring Period') }}: {{ resource.first_run.chec + + {{ _('Run Every') }} + + {{ resource.run_frequency }} {{ _('minutes') }} + + Probes diff --git a/GeoHealthCheck/translations/en/LC_MESSAGES/messages.po b/GeoHealthCheck/translations/en/LC_MESSAGES/messages.po index 14061347..946f1bef 100644 --- a/GeoHealthCheck/translations/en/LC_MESSAGES/messages.po +++ b/GeoHealthCheck/translations/en/LC_MESSAGES/messages.po @@ -381,6 +381,16 @@ msgstr "" msgid "Response Times (seconds)" msgstr "" +#: GeoHealthCheck/templates/resource.html:60 +#: GeoHealthCheck/templates/edit_resource.html:94 +msgid "minutes" +msgstr "" + +#: GeoHealthCheck/templates/resource.html:58 +#: GeoHealthCheck/templates/edit_resource.html:92 +msgid "Run Every" +msgstr "" + #: GeoHealthCheck/templates/resource.html:71 msgid "Min" msgstr "" diff --git a/GeoHealthCheck/translations/nl_NL/LC_MESSAGES/messages.po b/GeoHealthCheck/translations/nl_NL/LC_MESSAGES/messages.po index 775833b9..779a19d8 100644 --- a/GeoHealthCheck/translations/nl_NL/LC_MESSAGES/messages.po +++ b/GeoHealthCheck/translations/nl_NL/LC_MESSAGES/messages.po @@ -381,6 +381,16 @@ msgstr "Monitoring Periode" msgid "Response Times (seconds)" msgstr "Response Tijden (seconden)" +#: GeoHealthCheck/templates/resource.html:60 +#: GeoHealthCheck/templates/edit_resource.html:94 +msgid "minutes" +msgstr "minuten" + +#: GeoHealthCheck/templates/resource.html:58 +#: GeoHealthCheck/templates/edit_resource.html:92 +msgid "Run Every" +msgstr "Run Elke" + #: GeoHealthCheck/templates/resource.html:71 msgid "Min" msgstr "Min" diff --git a/pavement.py b/pavement.py index 3a5ceacb..ec4cdd32 100644 --- a/pavement.py +++ b/pavement.py @@ -283,6 +283,12 @@ def update_translations(): options.base.pot, options.base.translations)) +@task +def runner_daemon(): + """Run the HealthCheck runner scheduler""" + sh('python GeoHealthCheck/scheduler.py') + + def sphinx_make(): """return what command Sphinx is using for make""" From 75b4647b1fa83058d4316bc1f0faf451236fa38c Mon Sep 17 00:00:00 2001 From: Just van den Broecke Date: Tue, 20 Mar 2018 19:25:30 +0100 Subject: [PATCH 02/20] #132 first stab at healthcheck scheduler using daemon process, fix tests --- .travis.yml | 2 +- GeoHealthCheck/healthcheck.py | 30 +++++++++++++++------- GeoHealthCheck/models.py | 42 +------------------------------ docker/compose/docker-compose.yml | 35 +++----------------------- docker/run-scheduler.sh | 8 ++++++ 5 files changed, 34 insertions(+), 83 deletions(-) create mode 100644 docker/run-scheduler.sh diff --git a/.travis.yml b/.travis.yml index 2cf45c9f..c17dcbf9 100644 --- a/.travis.yml +++ b/.travis.yml @@ -14,7 +14,7 @@ script: - echo -e "admin\ntest\ntest\nyou@example.com\nyou@example.com" | python GeoHealthCheck/models.py create - flake8 - python GeoHealthCheck/models.py load tests/data/fixtures.json y - - python GeoHealthCheck/models.py run + - python GeoHealthCheck/healthcheck.py - python tests/run_tests.py - cd docs && make html diff --git a/GeoHealthCheck/healthcheck.py b/GeoHealthCheck/healthcheck.py index cb5624a8..ae29671d 100644 --- a/GeoHealthCheck/healthcheck.py +++ b/GeoHealthCheck/healthcheck.py @@ -33,7 +33,6 @@ from urllib2 import urlopen from urlparse import urlparse from functools import partial -from pprint import pprint from flask_babel import gettext from owslib.wms import WebMapService @@ -69,6 +68,15 @@ def db_commit(): return err +def run_resources(): + + for resource in Resource.query.all(): # run all tests + print('Testing %s %s' % + (resource.resource_type, resource.url)) + + run_resource(resource.identifier) + + # complete handle of resource test def run_resource(resourceid): resource = Resource.query.filter_by(identifier=resourceid).first() @@ -290,12 +298,16 @@ def geonode_make_tags(base_url): if __name__ == '__main__': - import sys logging.basicConfig(level=logging.INFO) - from init import App - if len(sys.argv) < 3: - print('Usage: %s ' % sys.argv[0]) - sys.exit(1) - - # TODO: need APP.config here, None for now - pprint(sniff_test_resource(App.get_config(), sys.argv[1], sys.argv[2])) + print('START - Running health check tests on %s' + % datetime.utcnow().isoformat()) + run_resources() + print('END - Running health check tests on %s' + % datetime.utcnow().isoformat()) + # from init import App + # if len(sys.argv) < 3: + # print('Usage: %s ' % sys.argv[0]) + # sys.exit(1) + # + # # TODO: need APP.config here, None for now + # pprint(sniff_test_resource(App.get_config(), sys.argv[1], sys.argv[2])) diff --git a/GeoHealthCheck/models.py b/GeoHealthCheck/models.py index a33c9541..81c9406b 100644 --- a/GeoHealthCheck/models.py +++ b/GeoHealthCheck/models.py @@ -41,7 +41,6 @@ from enums import RESOURCE_TYPES from factory import Factory from init import App -from notifications import notify from wtforms.validators import Email, ValidationError DB = App.get_db() @@ -824,46 +823,7 @@ def db_commit(): elif sys.argv[1] == 'run': print('START - Running health check tests on %s' % datetime.utcnow().isoformat()) - from healthcheck import run_test_resource - - for resource in Resource.query.all(): # run all tests - print('Testing %s %s' % - (resource.resource_type, resource.url)) - - if not resource.active: - print('Resource is not active. Skipping') - continue - - # Get the status of the last run, - # assume success if there is none - last_run_success = True - last_run = resource.last_run - if last_run: - last_run_success = last_run.success - - # Run test - result = run_test_resource(resource) - - run1 = Run(resource, result) - - print('Adding Run: success=%s, response_time=%ss\n' - % (str(run1.success), run1.response_time)) - - DB.session.add(run1) - - # commit or rollback each run to avoid long-lived transactions - # see https://github.com/geopython/GeoHealthCheck/issues/14 - db_commit() - - if APP.config['GHC_NOTIFICATIONS']: - # Attempt notification - try: - notify(APP.config, resource, run1, last_run_success) - except Exception as err: - LOGGER.error("Cannot send notifications: %s", - err, - exc_info=err) - + # run_resources() print('END - Running health check tests on %s' % datetime.utcnow().isoformat()) elif sys.argv[1] == 'flush': diff --git a/docker/compose/docker-compose.yml b/docker/compose/docker-compose.yml index 4ba9e155..2bf32c9d 100644 --- a/docker/compose/docker-compose.yml +++ b/docker/compose/docker-compose.yml @@ -14,7 +14,7 @@ services: # See https://docs.docker.com/compose/compose-file/#volumes # - ./../GeoHealthCheck/plugins:/plugins:ro - geohealthcheck-cron-hourly: + geohealthcheck-runner: image: geopython/geohealthcheck:latest depends_on: - geohealthcheck @@ -38,42 +38,13 @@ services: entrypoint: - bash - - /cron-jobs-hourly.sh + - /run-scheduler.sh + volumes: - ghc_sqlitedb:/GeoHealthCheck/DB # Optional Plugins, Path on the host, relative to the Compose file # See https://docs.docker.com/compose/compose-file/#volumes # - ./../GeoHealthCheck/plugins:/plugins:ro - labels: - io.ghc-cron-hourly: 'true' - - geohealthcheck-cron-daily: - image: geopython/geohealthcheck:latest - depends_on: - - geohealthcheck - entrypoint: - - bash - - /cron-jobs-daily.sh - volumes: - - ghc_sqlitedb:/GeoHealthCheck/DB - labels: - io.ghc-cron-daily: 'true' - - jobber: - image: blacklabelops/jobber:docker.v1.1 - depends_on: - - geohealthcheck-cron-hourly - - geohealthcheck-cron-daily - environment: - # May see warnings, see https://github.com/blacklabelops/rsnapshot/issues/2 but ok. - JOB_NAME1: ghc-cron-hourly - JOB_COMMAND1: docker start $$(docker ps -a -f label=io.ghc-cron-hourly=true --format="{{.ID}}") - JOB_TIME1: 0 0 * - JOB_NAME2: ghc-cron-daily - JOB_COMMAND2: docker start $$(docker ps -a -f label=io.ghc-cron-daily=true --format="{{.ID}}") - JOB_TIME2: 0 45 0 * - volumes: - - /var/run/docker.sock:/var/run/docker.sock # docker-compose v2+ needs separate volumes section volumes: diff --git a/docker/run-scheduler.sh b/docker/run-scheduler.sh new file mode 100644 index 00000000..edb0c612 --- /dev/null +++ b/docker/run-scheduler.sh @@ -0,0 +1,8 @@ +# /bin/bash + +# Run the GHC daemon process that schedules healthcheck jobs +source /venv/bin/activate . +cd /GeoHealthCheck +paver runner_daemon + + From 1588df143c180b8bf4bae898004387da09dbc15f Mon Sep 17 00:00:00 2001 From: Just van den Broecke Date: Thu, 22 Mar 2018 23:43:58 +0100 Subject: [PATCH 03/20] #132 further progress on separate test freq+ runner daemon: Docker support --- Dockerfile | 100 +++++++------- docker/README.md | 161 +++++++++------------- docker/compose/docker-compose.postgis.yml | 78 ++++++++--- docker/compose/docker-compose.yml | 55 ++++---- docker/compose/ghc-postgis.env | 7 + docker/compose/ghc.env | 22 +++ docker/docker-clean.sh | 3 + docker/install-docker-ubuntu.sh | 2 +- docker/run-scheduler.sh | 8 -- docker/{ => scripts}/configure.sh | 2 +- docker/{ => scripts}/cron-jobs-daily.sh | 0 docker/{ => scripts}/cron-jobs-hourly.sh | 0 docker/{ => scripts}/install.sh | 5 +- docker/scripts/run-scheduler.sh | 13 ++ docker/{run.sh => scripts/run-web.sh} | 12 +- docker/scripts/set-timezone.sh | 23 ++++ docs/install.rst | 10 +- requirements.txt | 5 +- 18 files changed, 293 insertions(+), 213 deletions(-) create mode 100644 docker/compose/ghc-postgis.env create mode 100644 docker/compose/ghc.env create mode 100755 docker/docker-clean.sh delete mode 100644 docker/run-scheduler.sh rename docker/{ => scripts}/configure.sh (98%) rename docker/{ => scripts}/cron-jobs-daily.sh (100%) rename docker/{ => scripts}/cron-jobs-hourly.sh (100%) rename docker/{ => scripts}/install.sh (85%) create mode 100755 docker/scripts/run-scheduler.sh rename docker/{run.sh => scripts/run-web.sh} (73%) create mode 100755 docker/scripts/set-timezone.sh diff --git a/Dockerfile b/Dockerfile index d31312bb..633bada5 100644 --- a/Dockerfile +++ b/Dockerfile @@ -4,80 +4,78 @@ FROM python:2.7.13-alpine # FROM debian:jessie # Credits to yjacolin for providing first versions -LABEL original_developer "yjacolin " -LABEL maintainer "Just van den Broecke " +LABEL original_developer="yjacolin " \ + maintainer="Just van den Broecke " # These are default values, # Override when running container via docker(-compose) # General ENV settings -ENV LC_ALL "en_US.UTF-8" -ENV LANG "en_US.UTF-8" -ENV LANGUAGE "en_US.UTF-8" - -# GHC ENV settings -ENV ADMIN_NAME admin -ENV ADMIN_PWD admin -ENV ADMIN_EMAIL admin.istrator@mydomain.com -ENV SQLALCHEMY_DATABASE_URI 'sqlite:////GeoHealthCheck/DB/data.db' -ENV SECRET_KEY 'd544ccc37dc3ad214c09b1b7faaa64c60351d5c8bb48b342' -ENV GHC_RETENTION_DAYS 30 -ENV GHC_RUN_FREQUENCY 'hourly' -ENV GHC_SELF_REGISTER False -ENV GHC_NOTIFICATIONS False -ENV GHC_NOTIFICATIONS_VERBOSITY True -ENV GHC_WWW_LINK_EXCEPTION_CHECK False -ENV GHC_ADMIN_EMAIL 'you@example.com' -ENV GHC_NOTIFICATIONS_EMAIL 'you2@example.com,them@example.com' -ENV GHC_SITE_TITLE 'GeoHealthCheck' -ENV GHC_SITE_URL 'http://localhost' -ENV GHC_SMTP_SERVER None -ENV GHC_SMTP_PORT None -ENV GHC_SMTP_TLS False -ENV GHC_SMTP_SSL False -ENV GHC_SMTP_USERNAME None -ENV GHC_SMTP_PASSWORD None -ENV GHC_METADATA_CACHE_SECS 900 - -# WSGI server settings, assumed is gunicorn -ENV HOST 0.0.0.0 -ENV PORT 80 -ENV WSGI_WORKERS 4 -ENV WSGI_WORKER_TIMEOUT 6000 -ENV WSGI_WORKER_CLASS 'eventlet' - -# GHC Core Plugins modules and/or classes, seldom needed to set: -# if not specified here or in Container environment -# all GHC built-in Plugins will be active. +ENV LC_ALL="en_US.UTF-8" \ + LANG="en_US.UTF-8" \ + LANGUAGE="en_US.UTF-8" \ + \ + \ + # GHC ENV settings\ + ADMIN_NAME=admin \ + ADMIN_PWD=admin \ + ADMIN_EMAIL=admin.istrator@mydomain.com \ + SQLALCHEMY_DATABASE_URI='sqlite:////GeoHealthCheck/DB/data.db' \ + SECRET_KEY='d544ccc37dc3ad214c09b1b7faaa64c60351d5c8bb48b342' \ + GHC_RETENTION_DAYS=30 \ + GHC_RUN_FREQUENCY='hourly' \ + GHC_SELF_REGISTER=False \ + GHC_NOTIFICATIONS=False \ + GHC_NOTIFICATIONS_VERBOSITY=True \ + GHC_WWW_LINK_EXCEPTION_CHECK=False \ + GHC_ADMIN_EMAIL='you@example.com' \ + GHC_NOTIFICATIONS_EMAIL='you2@example.com,them@example.com' \ + GHC_SITE_TITLE='GeoHealthCheck' \ + GHC_SITE_URL='http://localhost' \ + GHC_SMTP_SERVER=None \ + GHC_SMTP_PORT=None \ + GHC_SMTP_TLS=False \ + GHC_SMTP_SSL=False \ + GHC_SMTP_USERNAME=None \ + GHC_SMTP_PASSWORD=None \ + GHC_METADATA_CACHE_SECS=900 \ + \ +# WSGI server settings, assumed is gunicorn \ +HOST=0.0.0.0 \ +PORT=80 \ +WSGI_WORKERS=4 \ +WSGI_WORKER_TIMEOUT=6000 \ +WSGI_WORKER_CLASS='eventlet' \ +\ +# GHC Core Plugins modules and/or classes, seldom needed to set: \ +# if not specified here or in Container environment \ +# all GHC built-in Plugins will be active. \ #ENV GHC_PLUGINS 'GeoHealthCheck.plugins.probe.owsgetcaps,\ # GeoHealthCheck.plugins.probe.wms, ...., ...\ -# GeoHealthCheck.plugins.check.checks' - -# GHC User Plugins, best be overridden via Container environment -ENV GHC_USER_PLUGINS '' +# GeoHealthCheck.plugins.check.checks' \ +\ +# GHC User Plugins, best be overridden via Container environment \ +GHC_USER_PLUGINS='' RUN apk add --no-cache --virtual .build-deps gcc build-base linux-headers postgresql-dev \ - && apk add --no-cache bash vim postgresql-client \ + && apk add --no-cache bash postgresql-client tzdata openntpd \ && pip install virtualenv \ && rm -rf /var/cache/apk/* /tmp/* /var/tmp/* # Add standard files and Add/override Plugins # Alternative Entrypoints to run GHC jobs # Override default Entrypoint with these on Containers -ADD docker/install.sh docker/configure.sh docker/run.sh \ - docker/config_site.py docker/cron-jobs-daily.sh docker/cron-jobs-hourly.sh docker/plugins / -RUN chmod a+x /*.sh +ADD docker/scripts/*.sh docker/config_site.py docker/plugins / # Add Source Code ADD . /GeoHealthCheck # Install and Remove build-related packages for smaller image size -RUN bash install.sh \ - && apk del .build-deps +RUN chmod a+x /*.sh && bash install.sh && apk del .build-deps # For SQLite VOLUME ["/GeoHealthCheck/DB/"] EXPOSE ${PORT} -ENTRYPOINT /configure.sh && /run.sh +ENTRYPOINT /run-web.sh diff --git a/docker/README.md b/docker/README.md index fd784386..f943420e 100644 --- a/docker/README.md +++ b/docker/README.md @@ -7,6 +7,12 @@ In the best case an install/run of GHC is a matter of minutes! If you are reading this from Docker Hub, local links will not work. In that case use the [Docker Readme at GHC GitHub](https://github.com/geopython/GeoHealthCheck/blob/master/docker/README.md). +Since GHC release 0.4.0 GHC can run completely with two Docker containers from the same +GHC Docker image: + +* `ghc_web` the web/Flask app +* `ghc_runner` : the daeomon runner app that schedules and executes GHC's Probes + ## Requirements Docker installed and Docker daemon running. @@ -46,7 +52,7 @@ You may schedule the cron-jobs using the local cron system with the [cron-jobs-hourly](cron-jobs-hourly.sh) and [cron-jobs-daily](cron-jobs-daily.sh) Docker Entrypoints. -But the most optimal way to run GHC with cronjobs and optionally Postgres backend DB, +But the most optimal way to run GHC with scheduled jobs and optionally Postgres as backend DB, is to use [Docker Compose](https://docs.docker.com/compose), see below. ## Using docker-compose @@ -62,8 +68,7 @@ for more info. ### Using sqlite DB (default) Using the default [docker-compose.yml](compose/docker-compose.yml) will run GHC with a SQLite DB. -Cronjobs are scheduled using [Jobber Cron](https://github.com/blacklabelops/jobber-cron/) as -regular Unix `cron` does not play nice with Docker. + To run (`-d` allows running in background): @@ -77,16 +82,15 @@ docker-compose -f docker-compose.yml up [-d] ### Using PostGIS DB -The file [docker-compose.postgis.yml](compose/docker-compose.postgis.yml) -extends/overrides the default [docker-compose.yml](compose/docker-compose.yml) to use Postgres with PostGIS -as database. +The file [docker-compose.postgis.yml](compose/docker-compose.postgis.yml) is +similar but uses Postgres with PostGIS as database. -To run, specify both `.yml` files: +To run: ``` cd docker/compose -docker-compose -f docker-compose.yml -f docker-compose.postgis.yml up [-d] +docker-compose -f docker-compose.postgis.yml up [-d] # go to http://localhost:8083 (port 80 in GHC Container is mapped to 8083 on host) @@ -103,7 +107,7 @@ To access your Postgres DB while running: ``` # Bash into running GHC Container -docker exec -it docker_geohealthcheck_1 bash +docker exec -it ghc_web bash # In GHC Container access DB with psql using DB parms psql -h postgis_ghc -U ghc ghc @@ -127,115 +131,84 @@ the Docker host IP address on your Docker host as follows: ## Cronjobs -Cronjobs via `docker-compose` are scheduled using [Jobber Cron](https://github.com/blacklabelops/jobber-cron/). -See this snippet from the [docker-compose.yml](compose/docker-compose.yml): +Cronjobs via `docker-compose` are since v0.4.0 no longer scheduled via Jobber (cron) +but within GHC itself. The GHC Docker Image can run as a Container in a daemon runner +role that executes all GHC scheduled runs from the DB directly. -``` -jobber: - image: blacklabelops/jobber:docker.v1.1 - depends_on: - - geohealthcheck-cron-hourly - - geohealthcheck-cron-daily - environment: - # May see warnings, see https://github.com/blacklabelops/rsnapshot/issues/2 but ok. - JOB_NAME1: ghc-cron-hourly - JOB_COMMAND1: docker start $$(docker ps -a -f label=io.ghc-cron-hourly=true --format="{{.ID}}") - JOB_TIME1: 0 0 * - JOB_NAME2: ghc-cron-daily - JOB_COMMAND2: docker start $$(docker ps -a -f label=io.ghc-cron-daily=true --format="{{.ID}}") - JOB_TIME2: 0 45 0 * - volumes: - - /var/run/docker.sock:/var/run/docker.sock -``` +## Configuration + +The default GHC configuration is specified within the [Dockerfile](../Dockerfile). +This allows overriding these `ENV` vars during deployment (as opposed to having to build +your own GHC Docker Image). Docker (`-e` options) and Docker Compose (`environment` section) +allow setting Environment variables. -This configures two jobs `geohealthcheck-cron-hourly` (every whole hour) and -`geohealthcheck-cron-daily` (every night at 00:45). Both run the GHC Docker image with different -containers and entrypoints. You may want to setup additional jobs e.g. to backup the database. -For example the daily cron-job entry looks like: +But more flexible is to use `.env` files. + +For example, to enable sending email notifications +in Dutch witin the GHC hourly job, specify the `environment` like: ``` -geohealthcheck-cron-daily: - image: geopython/geohealthcheck:latest - depends_on: - - geohealthcheck - entrypoint: - - bash - - /cron-jobs-daily.sh - volumes: - - ghc_sqlitedb:/GeoHealthCheck/DB - labels: - io.ghc-cron-daily: 'true' -``` + ghc_runner: + image: geopython/geohealthcheck:latest -NB: the Jobber entries are run when docker-compose runs, there may be an error when the DB -is not yet setup in the main GHC Docker Container, you can ignore this. + container_name: ghc_runner -NB you may see these Jobber warnings: + env_file: + - ghc.env -``` -Failed to load jobs for open /dev/null/.jobber: not a directory: sshd -Failed to load jobs for open /dev/null/.jobber: not a directory: guest + depends_on: + - ghc_web + + entrypoint: + - bash + - /run-scheduler.sh ``` -These are not errors, see [this Jobber issue](https://github.com/blacklabelops/rsnapshot/issues/2). -Make sure your jobs are scheduled, via: +This applies variables specified in [ghc.env](compose/ghc.env) to the Docker container. +One may override these with another file as is done in the Postgres +version: ``` -# Enter the container: -$ docker exec -it compose_jobber_1 bash -# List Jobber Jobs: -$ jobber list + ghc_runner: + image: geopython/geohealthcheck:latest -NAME STATUS SEC/MIN/HR/MDAY/MTH/WDAY NEXT RUN TIME NOTIFY ON ERR NOTIFY ON FAIL ERR HANDLER -ghc-cron-hourly Good 0 0 * * * * Jul 9 14:00:00 2017 false false Continue -ghc-cron-daily Good 0 45 0 * * * Jul 10 00:45:00 2017 false false Continue + container_name: ghc_runner -# Test job execution: -$ jobber test ghc-cron-hourly -$ jobber test ghc-cron-daily + env_file: + - ghc.env + - ghc-postgis.env -``` + links: + - postgis_ghc -## Configuration + depends_on: + - postgis_ghc -The default GHC configuration is specified within the [Dockerfile](../Dockerfile). -This allows overriding these `ENV` vars during deployment (as opposed to having to build -your own GHC Docker Image). Docker (`-e` options) and Docker Compose (`environment` section) -allow setting Environment variables. + entrypoint: + - bash + - /run-scheduler.sh + + . + . + postgis_ghc: + image: mdillon/postgis:9.6-alpine -For example, to enable sending email notifications -in Dutch witin the GHC hourly job, specify the `environment` like: + container_name: postgis_ghc -``` - geohealthcheck-cron-hourly: - image: geopython/geohealthcheck:latest - depends_on: - - geohealthcheck - - # Override settings to enable email notifications - environment: - LC_ALL: "nl_NL.UTF-8" - LANG: "nl_NL.UTF-8" - LANGUAGE: "nl_NL.UTF-8" - GHC_NOTIFICATIONS: 'true' - GHC_NOTIFICATIONS_VERBOSITY: 'true' - GHC_ADMIN_EMAIL: 'us@gmail.com' - GHC_NOTIFICATIONS_EMAIL: 'us@gmail.com,them@domain.com' - GHC_SMTP_SERVER: 'smtp.gmail.com' - GHC_SMTP_PORT: 587 - GHC_SMTP_TLS: 'true' - GHC_SMTP_SSL: 'false' - GHC_SMTP_USERNAME: 'us@gmail.com' - GHC_SMTP_PASSWORD: 'the_passw' - GHC_USER_PLUGINS: 'GeoHealthCheck.plugins.user.myplugins' - . - . + environment: + - ghc-postgis.env + + volumes: + - ghc_pgdb:/var/lib/postgresql/data ``` +Here [ghc-postgis.env](compose/ghc-postgis.env) adds extra Postgres-related config settings. + + ## Other tasks You can always `bash` into the GHC container to run maintenance tasks. diff --git a/docker/compose/docker-compose.postgis.yml b/docker/compose/docker-compose.postgis.yml index 79f95818..d310ffe1 100644 --- a/docker/compose/docker-compose.postgis.yml +++ b/docker/compose/docker-compose.postgis.yml @@ -1,42 +1,80 @@ -# Overrides default docker-compose.yml -# To run specify both files: -# sudo docker-compose -f docker-compose.yml -f docker-compose.postgis.yml up [-d] - +# GHC Docker setup with Postgres as backend DB. +# +# To run: +# sudo docker-compose -f docker-compose.postgis.yml up [-d] +# version: "3" services: - geohealthcheck: + ghc_web: + image: geopython/geohealthcheck:latest + + container_name: ghc_web + + env_file: + - ghc.env + - ghc-postgis.env + links: - postgis_ghc + depends_on: - postgis_ghc - environment: - - SQLALCHEMY_DATABASE_URI=postgresql://ghc:ghc@postgis_ghc:5432/ghc - geohealthcheck-cron-daily: + ports: + - 8083:80 + + # volumes: + # Optional Plugins, using Path on the host, relative to this Compose file + # To activate: 2 steps for runner and GHC webapp: + # - configure in ghc.env + # - mount these as docker volume on host + # See https://docs.docker.com/compose/compose-file/#volumes + # - ./../GeoHealthCheck/plugins:/plugins:ro + + ghc_runner: + image: geopython/geohealthcheck:latest + + container_name: ghc_runner + + env_file: + - ghc.env + - ghc-postgis.env + links: - postgis_ghc - environment: - - SQLALCHEMY_DATABASE_URI=postgresql://ghc:ghc@postgis_ghc:5432/ghc - geohealthcheck-cron-hourly: - links: + depends_on: - postgis_ghc - environment: - - SQLALCHEMY_DATABASE_URI=postgresql://ghc:ghc@postgis_ghc:5432/ghc + + entrypoint: + - /run-scheduler.sh + + # volumes: + # Optional Plugins, using Path on the host, relative to this Compose file + # To activate 2 steps: + # - configure in ghc.env + # - mount these as docker volume on host + # See https://docs.docker.com/compose/compose-file/#volumes + # - ./../GeoHealthCheck/plugins:/plugins:ro postgis_ghc: image: mdillon/postgis:9.6-alpine + + container_name: postgis_ghc + + env_file: + - ghc-postgis.env + volumes: - ghc_pgdb:/var/lib/postgresql/data - ports: - - 5432:5432 + +# If you ever need to expose PG ports on host externally +# ports: +# - 5432:5432 + expose: - 5432 - environment: - - POSTGRES_USER=ghc - - POSTGRES_PASSWORD=ghc - - POSTGRES_DB=ghc # docker-compose v2+ needs separate volumes section volumes: diff --git a/docker/compose/docker-compose.yml b/docker/compose/docker-compose.yml index 2bf32c9d..63812f22 100644 --- a/docker/compose/docker-compose.yml +++ b/docker/compose/docker-compose.yml @@ -1,48 +1,49 @@ -# Default Compose config: runs GHC with SQLite backend and cronjobs +# Default Docker Compose config: +# runs GHC Web App (ghc_web) and GHC Daemon Runner (ghc_runner) Docker containers +# with SQLite backend. ghc_runner will run using the schedule +# configured in the GHC Web App. +# +# Plugins (optional) need to be configured for both Docker containers. version: "3" services: - geohealthcheck: + ghc_web: image: geopython/geohealthcheck:latest + + container_name: ghc_web + ports: - 8083:80 -# environment: -# GHC_USER_PLUGINS: 'GeoHealthCheck.plugins.user.mywmsprobe,GeoHealthCheck.plugins.user.mywmsprobe2' + + env_file: + - ghc.env + volumes: - ghc_sqlitedb:/GeoHealthCheck/DB - # Optional Plugins, Path on the host, relative to the Compose file + # Optional Plugins, using Path on the host, relative to this Compose file + # To activate: 2 steps for runner and GHC webapp: + # - configure in ghc.env + # - mount these as docker volume on host # See https://docs.docker.com/compose/compose-file/#volumes # - ./../GeoHealthCheck/plugins:/plugins:ro - geohealthcheck-runner: + ghc_runner: image: geopython/geohealthcheck:latest - depends_on: - - geohealthcheck - - # Override settings to enable email notifications and user Plugins -# environment: -# LC_ALL: "nl_NL.UTF-8" -# LANG: "nl_NL.UTF-8" -# LANGUAGE: "nl_NL.UTF-8" -# GHC_NOTIFICATIONS: 'true' -# GHC_NOTIFICATIONS_VERBOSITY: 'true' -# GHC_ADMIN_EMAIL: 'us@gmail.com' -# GHC_NOTIFICATIONS_EMAIL: 'us@gmail.com,them@domain.com' -# GHC_SMTP_SERVER: 'smtp.gmail.com' -# GHC_SMTP_PORT: 587 -# GHC_SMTP_TLS: 'true' -# GHC_SMTP_SSL: 'false' -# GHC_SMTP_USERNAME: 'us@gmail.com' -# GHC_SMTP_PASSWORD: 'the_passw' -# GHC_USER_PLUGINS: 'GeoHealthCheck.plugins.user.mywmsprobe,GeoHealthCheck.plugins.user.mywmsprobe2' + + container_name: ghc_runner + + env_file: + - ghc.env entrypoint: - - bash - /run-scheduler.sh volumes: - ghc_sqlitedb:/GeoHealthCheck/DB - # Optional Plugins, Path on the host, relative to the Compose file + # Optional Plugins, using Path on the host, relative to this Compose file + # To activate 2 steps: + # - configure in ghc.env + # - mount these as docker volume on host # See https://docs.docker.com/compose/compose-file/#volumes # - ./../GeoHealthCheck/plugins:/plugins:ro diff --git a/docker/compose/ghc-postgis.env b/docker/compose/ghc-postgis.env new file mode 100644 index 00000000..67bd5892 --- /dev/null +++ b/docker/compose/ghc-postgis.env @@ -0,0 +1,7 @@ +# We mainly need to override the default (sqlite) DB URI +SQLALCHEMY_DATABASE_URI=postgresql://ghc:ghc@postgis_ghc:5432/ghc + +# Postgres Docker container settings +POSTGRES_USER=ghc +POSTGRES_PASSWORD=ghc +POSTGRES_DB=ghc diff --git a/docker/compose/ghc.env b/docker/compose/ghc.env new file mode 100644 index 00000000..478f1d3c --- /dev/null +++ b/docker/compose/ghc.env @@ -0,0 +1,22 @@ +SQLALCHEMY_DATABASE_URI=sqlite:////GeoHealthCheck/DB/data.db + +# Core variables settings, change at will. +GHC_NOTIFICATIONS=true +GHC_NOTIFICATIONS_VERBOSITY=true +GHC_ADMIN_EMAIL=us@gmail.com +GHC_NOTIFICATIONS_EMAIL=us@gmail.com,them@domain.com +GHC_SMTP_SERVER=smtp.gmail.com +GHC_SMTP_PORT: 587 +GHC_SMTP_TLS=true +GHC_SMTP_SSL=false +GHC_SMTP_USERNAME=us@gmail.com +GHC_SMTP_PASSWORD=the_passw +# GHC_USER_PLUGINS=GeoHealthCheck.plugins.user.mywmsprobe,GeoHealthCheck.plugins.user.mywmsprobe2 + +# Optionally set container Timezone +# CONTAINER_TIMEZONE=Europe/Amsterdam + +# Optionally: set language +# LC_ALL=nl_NL.UTF-8 +# LANG=nl_NL.UTF-8 +# LANGUAGE=nl_NL.UTF-8 diff --git a/docker/docker-clean.sh b/docker/docker-clean.sh new file mode 100755 index 00000000..87716567 --- /dev/null +++ b/docker/docker-clean.sh @@ -0,0 +1,3 @@ +#!/bin/bash +docker rm $(docker ps -a -q) +docker rmi $(docker images -f dangling=true -q) diff --git a/docker/install-docker-ubuntu.sh b/docker/install-docker-ubuntu.sh index eb1e47cf..a37b3c47 100755 --- a/docker/install-docker-ubuntu.sh +++ b/docker/install-docker-ubuntu.sh @@ -56,6 +56,6 @@ apt-get install docker-ce # service docker start # Docker compose -export dockerComposeVersion="1.13.0" +export dockerComposeVersion="1.20.1" curl -L https://github.com/docker/compose/releases/download/${dockerComposeVersion}/docker-compose-`uname -s`-`uname -m` > /usr/local/bin/docker-compose chmod +x /usr/local/bin/docker-compose diff --git a/docker/run-scheduler.sh b/docker/run-scheduler.sh deleted file mode 100644 index edb0c612..00000000 --- a/docker/run-scheduler.sh +++ /dev/null @@ -1,8 +0,0 @@ -# /bin/bash - -# Run the GHC daemon process that schedules healthcheck jobs -source /venv/bin/activate . -cd /GeoHealthCheck -paver runner_daemon - - diff --git a/docker/configure.sh b/docker/scripts/configure.sh similarity index 98% rename from docker/configure.sh rename to docker/scripts/configure.sh index d2094d4e..b0adb5c2 100755 --- a/docker/configure.sh +++ b/docker/scripts/configure.sh @@ -3,7 +3,7 @@ echo "START /configure.sh" -cd /venv/ && . bin/activate +source /venv/bin/activate . # Determine database type from DB URI DB_TYPE=$(echo ${SQLALCHEMY_DATABASE_URI} | cut -f1 -d:) diff --git a/docker/cron-jobs-daily.sh b/docker/scripts/cron-jobs-daily.sh similarity index 100% rename from docker/cron-jobs-daily.sh rename to docker/scripts/cron-jobs-daily.sh diff --git a/docker/cron-jobs-hourly.sh b/docker/scripts/cron-jobs-hourly.sh similarity index 100% rename from docker/cron-jobs-hourly.sh rename to docker/scripts/cron-jobs-hourly.sh diff --git a/docker/install.sh b/docker/scripts/install.sh similarity index 85% rename from docker/install.sh rename to docker/scripts/install.sh index 36b8c849..c66f34fa 100755 --- a/docker/install.sh +++ b/docker/scripts/install.sh @@ -1,10 +1,10 @@ #!/bin/bash cd / -virtualenv venv && cd $_ +virtualenv venv && cd /venv . bin/activate -# GHC Source was ADDed in Dockerfile +# GHC Source was added in Dockerfile, install cd /GeoHealthCheck pip install Paver pip install sphinx @@ -26,6 +26,7 @@ mv /config_site.py /GeoHealthCheck/instance/config_site.py if [ -d /plugins ] then # Copy possible Plugins into app tree + echo "Installing Plugins..." cp -ar /plugins/* GeoHealthCheck/plugins/ # Remove to allow later Volume mount of /plugins diff --git a/docker/scripts/run-scheduler.sh b/docker/scripts/run-scheduler.sh new file mode 100755 index 00000000..a8feb4b9 --- /dev/null +++ b/docker/scripts/run-scheduler.sh @@ -0,0 +1,13 @@ +#!/bin/bash + +# Set the timezone. +/set-timezone.sh + +# Configure: DB and plugins. +/configure.sh + +# Run the GHC daemon process +# that schedules healthcheck jobs +source /venv/bin/activate . +cd /GeoHealthCheck +paver runner_daemon diff --git a/docker/run.sh b/docker/scripts/run-web.sh similarity index 73% rename from docker/run.sh rename to docker/scripts/run-web.sh index 89e6d418..10a98519 100755 --- a/docker/run.sh +++ b/docker/scripts/run-web.sh @@ -2,9 +2,15 @@ # Runs the GHC app with gunicorn -echo "START /run.sh" +echo "START /run-web.sh" -cd /venv/ && . bin/activate +# Set the timezone. +/set-timezone.sh + +# Configure: DB and plugins. +/configure.sh + +source /venv/bin/activate . echo "Running GHC WSGI on ${HOST}:${PORT} with ${WSGI_WORKERS} workers" cd /GeoHealthCheck @@ -18,4 +24,4 @@ gunicorn --workers ${WSGI_WORKERS} \ # Built-in Flask server, deprecated # python /GeoHealthCheck/GeoHealthCheck/app.py ${HOST}:${PORT} -echo "END /run.sh" +echo "END /run-web.sh" diff --git a/docker/scripts/set-timezone.sh b/docker/scripts/set-timezone.sh new file mode 100755 index 00000000..d1697e77 --- /dev/null +++ b/docker/scripts/set-timezone.sh @@ -0,0 +1,23 @@ +#!/bin/bash + +# Set timezone and time right in container. +# See: https://www.ivankrizsan.se/2015/10/31/time-in-docker-containers/ (Alpine) + +# Set the timezone. Base image does not contain +# the setup-timezone script, so an alternate way is used. +if [ "$CONTAINER_TIMEZONE" = "" ]; +then + CONTAINER_TIMEZONE="Europe/London" +else + echo "Container timezone not modified" +fi + +cp /usr/share/zoneinfo/${CONTAINER_TIMEZONE} /etc/localtime && \ +echo "${CONTAINER_TIMEZONE}" > /etc/timezone && \ +echo "Container timezone set to: $CONTAINER_TIMEZONE" + +# Force immediate synchronisation of the time and start the time-synchronization service. +# In order to be able to use ntpd in the container, it must be run with the SYS_TIME capability. +# In addition you may want to add the SYS_NICE capability, +# in order for ntpd to be able to modify its priority. +# ntpd -s diff --git a/docs/install.rst b/docs/install.rst index 8d58c67a..bdf4415b 100644 --- a/docs/install.rst +++ b/docs/install.rst @@ -19,11 +19,13 @@ for a full guide. Requirements ------------ -GeoHealthCheck is built on the awesome Flask microframework and uses -Flask-SQLAlchemy for database interaction and Flask-Login for authorization. -Flask-Migrate with Alembic and Flask-Script support is used for database upgrades. +GeoHealthCheck is built on the awesome Flask micro-framework and uses +`Flask-SQLAlchemy` for database interaction and Flask-Login for authorization. +`Flask-Migrate` with `Alembic` and `Flask-Script` is used for database upgrades. -OWSLib is used to interact with OGC Web Services. +`OWSLib` is used to interact with OGC Web Services. + +`APScheduler` is used to run scheduled healthchecks. These dependencies are automatically installed (see below). ``Paver`` is used for installation and management. ``Cron`` is used for scheduling the actual diff --git a/requirements.txt b/requirements.txt index 20919a6c..feb96aaa 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,5 +6,6 @@ Flask-Script==2.0.5 Flask-SQLAlchemy==2.1 OWSLib Sphinx -requests -WTForms +requests==2.18.4 +WTForms==2.1 +APScheduler==3.5.1 From 6920ca791039727b098b5c4d8d1474e59b7b8dba Mon Sep 17 00:00:00 2001 From: Just van den Broecke Date: Fri, 23 Mar 2018 18:13:37 +0100 Subject: [PATCH 04/20] #132 further progress on runner daemon: Docker support and docs --- Dockerfile | 4 +- GeoHealthCheck/app.py | 48 +++++---------- GeoHealthCheck/config_main.py | 5 +- GeoHealthCheck/factory.py | 19 +++--- GeoHealthCheck/healthcheck.py | 6 +- GeoHealthCheck/init.py | 10 +-- GeoHealthCheck/models.py | 9 +-- GeoHealthCheck/notifications.py | 10 +-- GeoHealthCheck/plugin.py | 5 +- GeoHealthCheck/scheduler.py | 32 +++++++--- GeoHealthCheck/templates/layout.html | 3 +- GeoHealthCheck/util.py | 4 +- README.md | 1 - docker/README.md | 56 +++++++++-------- docker/compose/docker-compose.postgis.yml | 2 +- docker/compose/docker-compose.yml | 2 +- docker/compose/ghc.env | 4 +- docker/config_site.py | 6 +- docker/scripts/cron-jobs-hourly.sh | 2 +- .../{run-scheduler.sh => run-runner.sh} | 4 ++ docs/_static/ghc-parts.jpg | Bin 0 -> 35594 bytes docs/admin.rst | 58 ++++++++++++++---- docs/architecture.rst | 31 +++++++++- docs/config.rst | 5 +- docs/install.rst | 11 ++-- jobs.cron | 2 +- pavement.py | 8 ++- 27 files changed, 218 insertions(+), 129 deletions(-) rename docker/scripts/{run-scheduler.sh => run-runner.sh} (80%) create mode 100644 docs/_static/ghc-parts.jpg diff --git a/Dockerfile b/Dockerfile index 633bada5..6977724a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -23,12 +23,14 @@ ENV LC_ALL="en_US.UTF-8" \ SQLALCHEMY_DATABASE_URI='sqlite:////GeoHealthCheck/DB/data.db' \ SECRET_KEY='d544ccc37dc3ad214c09b1b7faaa64c60351d5c8bb48b342' \ GHC_RETENTION_DAYS=30 \ - GHC_RUN_FREQUENCY='hourly' \ GHC_SELF_REGISTER=False \ GHC_NOTIFICATIONS=False \ GHC_NOTIFICATIONS_VERBOSITY=True \ GHC_WWW_LINK_EXCEPTION_CHECK=False \ GHC_ADMIN_EMAIL='you@example.com' \ + GHC_RUNNER_IN_WEBAPP=False \ + GHC_LOG_LEVEL=30 \ + GHC_LOG_FORMAT='%(asctime)s - %(name)s - %(levelname)s - %(message)s' \ GHC_NOTIFICATIONS_EMAIL='you2@example.com,them@example.com' \ GHC_SITE_TITLE='GeoHealthCheck' \ GHC_SITE_URL='http://localhost' \ diff --git a/GeoHealthCheck/app.py b/GeoHealthCheck/app.py index aaebe5b6..b4f1e9da 100644 --- a/GeoHealthCheck/app.py +++ b/GeoHealthCheck/app.py @@ -44,7 +44,6 @@ from flask_migrate import Migrate from __init__ import __version__ -from healthcheck import sniff_test_resource, run_test_resource from init import App from enums import RESOURCE_TYPES from models import Resource, Run, ProbeVars, CheckVars, Tag, User, Recipient @@ -73,6 +72,16 @@ ('es_BO', 'Español (Bolivia)') ) +# Should GHC Runner be run within GHC webapp? +if CONFIG['GHC_RUNNER_IN_WEBAPP'] is True: + LOGGER.info('Running GHC Scheduler in WebApp') + from scheduler import start_schedule + + # Start scheduler + start_schedule() +else: + LOGGER.info('NOT Running GHC Scheduler in WebApp') + # commit or rollback shorthand def db_commit(): @@ -121,35 +130,6 @@ def unauthorized_callback(): return redirect(url_for('login', lang=g.current_lang, next=url)) -def next_page_refresh(): - """determines when to refresh webapp based on GHC_RUN_FREQUENCY""" - - now = datetime.now() - - frequency = CONFIG['GHC_RUN_FREQUENCY'] - - if frequency == 'hourly': # get next hour - now2 = now.replace(minute=0, second=0, microsecond=0) - refresh = timedelta(hours=1) - elif frequency == 'daily': # get next day - now2 = now.replace(hour=0, minute=0, second=0, microsecond=0) - refresh = timedelta(days=1) - elif frequency == ['weekly']: # get next day - now2 = now.replace(day=1, hour=0, minute=0, second=0, microsecond=0) - refresh = timedelta(weeks=1) - elif frequency == ['monthly']: - now2 = now.replace(day=1, hour=0, minute=0, second=0, microsecond=0) - refresh = timedelta(weeks=4) - elif frequency == ['yearly']: # get next day - now2 = now.replace(day=1, hour=0, minute=0, second=0, microsecond=0) - refresh = timedelta(weeks=52) - - next_frequency = now2 + refresh - differ = next_frequency - now - - return differ.seconds - - @APP.template_filter('cssize_reliability') def cssize_reliability(value, css_type=None): """returns CSS button class snippet based on score""" @@ -202,7 +182,6 @@ def context_processors(): tags = views.get_tag_counts() return { 'app_version': __version__, - 'next_page_refresh': next_page_refresh(), 'resource_types': RESOURCE_TYPES, 'resource_types_counts': rtc['counts'], 'resources_total': rtc['total'], @@ -489,6 +468,7 @@ def add(): url = request.form['url'].strip() resources_to_add = [] + from healthcheck import sniff_test_resource, run_test_resource sniffed_resources = sniff_test_resource(CONFIG, resource_type, url) if not sniffed_resources: @@ -640,8 +620,8 @@ def update(resource_identifier): # Add ProbeVars anew each with optional CheckVars for probe in value: - print('adding Probe class=%s parms=%s' % - (probe['probe_class'], str(probe))) + LOGGER.info('adding Probe class=%s parms=%s' % + (probe['probe_class'], str(probe))) probe_vars = ProbeVars(resource, probe['probe_class'], probe['parameters']) for check in probe['checks']: @@ -689,6 +669,7 @@ def test(resource_identifier): flash(gettext('Resource not found'), 'danger') return redirect(request.referrer) + from healthcheck import run_test_resource result = run_test_resource( resource) @@ -895,7 +876,6 @@ def api_probes_avail(resource_type=None, resource_id=None): if __name__ == '__main__': # run locally, for fun import sys - logging.basicConfig() HOST = '0.0.0.0' PORT = 8000 if len(sys.argv) > 1: diff --git a/GeoHealthCheck/config_main.py b/GeoHealthCheck/config_main.py index 6ff5d6f5..ba6f93e0 100644 --- a/GeoHealthCheck/config_main.py +++ b/GeoHealthCheck/config_main.py @@ -38,7 +38,6 @@ SECRET_KEY = None GHC_RETENTION_DAYS = 30 -GHC_RUN_FREQUENCY = 'hourly' GHC_SELF_REGISTER = False GHC_NOTIFICATIONS = False GHC_NOTIFICATIONS_VERBOSITY = True @@ -47,6 +46,10 @@ GHC_NOTIFICATIONS_EMAIL = ['you2@example.com'] GHC_SITE_TITLE = 'GeoHealthCheck Demonstration' GHC_SITE_URL = 'http://host' +GHC_RUNNER_IN_WEBAPP = False +# 10=DEBUG 20=INFO 30=WARN(ING) 40=ERROR 50=FATAL/CRITICAL +GHC_LOG_LEVEL = 30 +GHC_LOG_FORMAT = '%(asctime)s - %(name)s - %(levelname)s - %(message)s' # Some GetCaps docs are huge. This allows # caching them for N seconds. Set to -1 to diff --git a/GeoHealthCheck/factory.py b/GeoHealthCheck/factory.py index cf31c27e..ff9ff34a 100644 --- a/GeoHealthCheck/factory.py +++ b/GeoHealthCheck/factory.py @@ -1,3 +1,8 @@ +import logging + +LOGGER = logging.getLogger(__name__) + + class Factory: """ Object, Function class Factory (Pattern). @@ -18,9 +23,9 @@ def create_obj(class_string): # class instance from class object with constructor args return class_obj() - except Exception, e: - print("cannot create object instance from class '%s' e=%s" % - (class_string, str(e))) + except Exception as e: + LOGGER.error("cannot create object instance from class '%s' e=%s" % + (class_string, str(e))) raise e @staticmethod @@ -41,8 +46,8 @@ def create_class(class_string): class_obj = getattr( __import__(module_name, globals(), locals(), [class_name], -1), class_name) - except Exception, e: - print("cannot create class '%s'" % class_string) + except Exception as e: + LOGGER.error("cannot create class '%s'" % class_string) raise e return class_obj @@ -61,8 +66,8 @@ def create_module(module_string): try: module_obj = __import__(module_string, globals(), locals(), fromlist=['']) - except Exception, e: - print("cannot create module from '%s'" % module_string) + except Exception as e: + LOGGER.error("cannot create module from '%s'" % module_string) raise e return module_obj diff --git a/GeoHealthCheck/healthcheck.py b/GeoHealthCheck/healthcheck.py index ae29671d..285f8158 100644 --- a/GeoHealthCheck/healthcheck.py +++ b/GeoHealthCheck/healthcheck.py @@ -69,10 +69,9 @@ def db_commit(): def run_resources(): - for resource in Resource.query.all(): # run all tests - print('Testing %s %s' % - (resource.resource_type, resource.url)) + LOGGER.info('Testing %s %s' % + (resource.resource_type, resource.url)) run_resource(resource.identifier) @@ -298,7 +297,6 @@ def geonode_make_tags(base_url): if __name__ == '__main__': - logging.basicConfig(level=logging.INFO) print('START - Running health check tests on %s' % datetime.utcnow().isoformat()) run_resources() diff --git a/GeoHealthCheck/init.py b/GeoHealthCheck/init.py index a7017762..d9f88f89 100644 --- a/GeoHealthCheck/init.py +++ b/GeoHealthCheck/init.py @@ -34,6 +34,8 @@ from flask_sqlalchemy import SQLAlchemy from flask_babel import Babel +LOGGER = logging.getLogger(__name__) + def to_list(obj): obj_type = type(obj) @@ -67,9 +69,9 @@ def init(): app.config.from_pyfile('config_main.py') app.config.from_pyfile('../instance/config_site.py') - logging.basicConfig(level=logging.INFO) - if app.config['DEBUG'] is True: - logging.basicConfig(level=logging.DEBUG) + # Global Logging config + logging.basicConfig(level=int(app.config['GHC_LOG_LEVEL']), + format=app.config['GHC_LOG_FORMAT']) app.config['GHC_SITE_URL'] = \ app.config['GHC_SITE_URL'].rstrip('/') @@ -96,7 +98,7 @@ def init(): # Finally assign app-instance App.app_instance = app App.count += 1 - logging.info("init.py: created GHC App instance #%d" % App.count) + LOGGER.info("created GHC App instance #%d" % App.count) @staticmethod def get_app(): diff --git a/GeoHealthCheck/models.py b/GeoHealthCheck/models.py index 81c9406b..0b751a3f 100644 --- a/GeoHealthCheck/models.py +++ b/GeoHealthCheck/models.py @@ -761,7 +761,7 @@ def db_commit(): except Exception as err: DB.session.rollback() msg = str(err) - print(msg) + LOGGER.error(msg) if __name__ == '__main__': @@ -821,11 +821,8 @@ def db_commit(): print('Provide path to JSON file, e.g. tests/fixtures.json') elif sys.argv[1] == 'run': - print('START - Running health check tests on %s' - % datetime.utcnow().isoformat()) - # run_resources() - print('END - Running health check tests on %s' - % datetime.utcnow().isoformat()) + print('NOTICE: models.py no longer here.') + print('Use: python healthcheck.py or upcoming cli.py') elif sys.argv[1] == 'flush': retention_days = int(APP.config['GHC_RETENTION_DAYS']) print('Flushing runs older than %d days' % diff --git a/GeoHealthCheck/notifications.py b/GeoHealthCheck/notifications.py index 82547169..5058024f 100644 --- a/GeoHealthCheck/notifications.py +++ b/GeoHealthCheck/notifications.py @@ -90,7 +90,7 @@ def do_email(config, resource, run, status_changed, result): LOGGER.warning("No SMTP configuration. Not sending to %s", notifications_email) - print(msg.as_string()) + LOGGER.warning(msg.as_string()) return server = smtplib.SMTP(config['GHC_SMTP']['server'], @@ -104,7 +104,7 @@ def do_email(config, resource, run, status_changed, result): server.starttls() else: server.connect() - except Exception, err: + except Exception as err: LOGGER.exception("Cannot connect to smtp: %s[:%s]: %s", config['GHC_SMTP']['server'], config['GHC_SMTP']['port'], @@ -114,7 +114,7 @@ def do_email(config, resource, run, status_changed, result): try: server.login(config['GHC_SMTP']['username'], config['GHC_SMTP']['password']) - except Exception, err: + except Exception as err: LOGGER.exception("Cannot log in to smtp: %s", err, exc_info=err) try: @@ -207,7 +207,7 @@ def do_webhook(config, resource, run, status_changed, result): for rcp in recipients: try: url, params = _parse_webhook_location(rcp) - except ValueError, err: + except ValueError as err: LOGGER.warning("Cannot send to {}: {}" .format(rcp, err), exc_info=err) @@ -264,6 +264,6 @@ def notify(config, resource, run, last_run_success): for chann_handler in (do_email, do_webhook,): try: chann_handler(config, resource, run, status_changed, result) - except Exception, err: + except Exception as err: LOGGER.warning("couldn't run notification for %s: %s", chann_handler.func_name, err, exc_info=err) diff --git a/GeoHealthCheck/plugin.py b/GeoHealthCheck/plugin.py index 1f19379e..2b36a7da 100644 --- a/GeoHealthCheck/plugin.py +++ b/GeoHealthCheck/plugin.py @@ -1,10 +1,13 @@ # -*- coding: utf-8 -*- from factory import Factory +import logging import inspect import collections import copy from init import App +LOGGER = logging.getLogger(__name__) + class Plugin(object): """ @@ -176,7 +179,7 @@ def add_result(plugin_name, class_obj): and baseclass != class_obj: add_result(plugin_name, class_obj) except Exception: - print('cannot create obj class=%s' % plugin_name) + LOGGER.warn('cannot create obj class=%s' % plugin_name) return result diff --git a/GeoHealthCheck/scheduler.py b/GeoHealthCheck/scheduler.py index 4b8b2f41..cf2369d9 100644 --- a/GeoHealthCheck/scheduler.py +++ b/GeoHealthCheck/scheduler.py @@ -66,10 +66,12 @@ def run_job(resource_id, frequency): resource = Resource.query.filter_by(identifier=resource_id).first() + # Resource may have been deleted, cancel job if not resource: stop_job(resource_id) return + # Resource exists: try to obtain lock. lock = ResourceLock.query.filter_by(identifier=resource_id).first() if not lock: @@ -121,6 +123,7 @@ def run_job(resource_id, frequency): % resource_id) return + # Run Resource healthchecks only if we have lock. if lock: try: run_resource(resource_id) @@ -130,29 +133,38 @@ def run_job(resource_id, frequency): def start_schedule(): - # Cold start every cron of every Resource - for resource in Resource.query.all(): - add_job(resource) - # change configuration + LOGGER.info('Starting scheduler') + + # Adapt configuration scheduler.configure(job_defaults={ 'coalesce': False, 'max_instances': 100000 }) - scheduler.add_job(flush_runs, 'interval', minutes=60) - scheduler.add_job(check_schedule, 'interval', minutes=5) - + # Start APScheduler scheduler.start() import atexit atexit.register(lambda: stop_schedule()) + # Add GHC jobs, one for each Resource, plus + # maintenance jobs. + + # Cold start every cron of every Resource + for resource in Resource.query.all(): + add_job(resource) + + # Start maintenance jobs + scheduler.add_job(flush_runs, 'interval', minutes=60) + scheduler.add_job(check_schedule, 'interval', minutes=5) + def check_schedule(): LOGGER.info('Checking Job schedules') + # Check the schedule for changed jobs for resource in Resource.query.all(): - job = scheduler.get_job(str(resource.identifier)) + job = get_job(resource) if job is None: add_job(resource) @@ -164,6 +176,10 @@ def check_schedule(): update_job(resource) +def get_job(resource): + return scheduler.get_job(str(resource.identifier)) + + def update_job(resource): stop_job(resource.identifier) diff --git a/GeoHealthCheck/templates/layout.html b/GeoHealthCheck/templates/layout.html index a8acb9ee..d049ad92 100644 --- a/GeoHealthCheck/templates/layout.html +++ b/GeoHealthCheck/templates/layout.html @@ -2,7 +2,6 @@ - @@ -107,7 +106,7 @@ {{ _('Settings') }} diff --git a/GeoHealthCheck/util.py b/GeoHealthCheck/util.py index 10b40f4d..c3811d25 100644 --- a/GeoHealthCheck/util.py +++ b/GeoHealthCheck/util.py @@ -135,8 +135,8 @@ def render_template2(template, template_vars): def send_email(mail_config, fromaddr, toaddr, msg): """convenience function to send an email""" - print(mail_config) - print(msg) + LOGGER.debug(mail_config) + LOGGER.debug(msg) server = smtplib.SMTP('%s:%s' % (mail_config['server'], mail_config['port'])) diff --git a/README.md b/README.md index fb16c6a0..933bc44a 100644 --- a/README.md +++ b/README.md @@ -28,7 +28,6 @@ vi instance/config_site.py # - SQLALCHEMY_DATABASE_URI # - SECRET_KEY # from paver create_secret_key # - GHC_RETENTION_DAYS -# - GHC_RUN_FREQUENCY # - GHC_SELF_REGISTER # - GHC_ADMIN_EMAIL # - GHC_SITE_TITLE diff --git a/docker/README.md b/docker/README.md index f943420e..246be048 100644 --- a/docker/README.md +++ b/docker/README.md @@ -10,8 +10,8 @@ In that case use the [Docker Readme at GHC GitHub](https://github.com/geopython/ Since GHC release 0.4.0 GHC can run completely with two Docker containers from the same GHC Docker image: -* `ghc_web` the web/Flask app -* `ghc_runner` : the daeomon runner app that schedules and executes GHC's Probes +* `GHC Webapp` the web/Flask app +* `GHC Runner` : the daemon runner app that schedules and executes GHC's Probes ## Requirements @@ -21,7 +21,8 @@ Docker installed and Docker daemon running. For installing Docker on Ubuntu there is a [bash helper script](install-docker-ubuntu.sh). -NB: The ``docker`` commands below may need to be prepended with ``sudo``, dependent on your login rights. +NB: The ``docker`` commands below may need to be prepended with +``sudo``, dependent on your login rights. ## Build @@ -42,22 +43,36 @@ For example [run.sh](run.sh) which launches GHC using the robust `gunicorn` WSGI ## Run ``` -docker run -d --name GeoHealthCheck -p 8083:80 -v ghc_sqlitedb:/GeoHealthCheck/DB geopython/geohealthcheck:latest +docker run -d --name ghc_web -p 8083:80 -v ghc_sqlitedb:/GeoHealthCheck/DB geopython/geohealthcheck:latest ``` go to http://localhost:8083 (port 80 in GHC Container is mapped to 8083 on host). -NB this runs GHC standalone with a `SQLite` DB, but without the cron-jobs that perform the healthchecks. -You may schedule the cron-jobs using the local cron system with the -[cron-jobs-hourly](cron-jobs-hourly.sh) and -[cron-jobs-daily](cron-jobs-daily.sh) Docker Entrypoints. +NB this runs GHC standalone with a `SQLite` DB, but without the GHC Runner that performs the +healthchecks. This can be done by setting `GHC_RUNNER_IN_WEBAPP`: -But the most optimal way to run GHC with scheduled jobs and optionally Postgres as backend DB, +``` +docker run --name ghc_web -e GHC_RUNNER_IN_WEBAPP=True -p 8083:80 -v ghc_sqlitedb:/GeoHealthCheck/DB geopython/geohealthcheck:latest + +``` + +This allows both the `GHC Webapp` (Dashboard)and `GHC Runner` within a single Docker container. +But this may in cases not be optimal as the `GHC Webapp` may get overloaded +from processing by the `GHC Runner`. To see logging add `-e GHC_LOG_LEVEL=10 `. + +You can also run `GHC Webapp` (Dashboard) and `GHC Runner` as separate containers by overriding +the default `ENTRYPOINT` with `/run-runner.sh`: + +``` +docker run -d --name ghc_runner --entrypoint "/run-runner.sh" -v ghc_sqlitedb:/GeoHealthCheck/DB geopython/geohealthcheck:latest +``` + +The most optimal way to run GHC with scheduled jobs and optionally Postgres as backend DB, is to use [Docker Compose](https://docs.docker.com/compose), see below. ## Using docker-compose -This allows a complete Docker setup, including cron-jobs and optionally using +This allows a complete Docker setup, including scheduling and optionally using Postgres/PostGIS as database (recommended). See the [Docker Compose Documentation](https://docs.docker.com/compose) for more info. @@ -145,9 +160,6 @@ allow setting Environment variables. But more flexible is to use `.env` files. -For example, to enable sending email notifications -in Dutch witin the GHC hourly job, specify the `environment` like: - ``` ghc_runner: image: geopython/geohealthcheck:latest @@ -157,12 +169,11 @@ in Dutch witin the GHC hourly job, specify the `environment` like: env_file: - ghc.env - depends_on: - - ghc_web - entrypoint: - - bash - - /run-scheduler.sh + - /run-runner.sh + + volumes: + - ghc_sqlitedb:/GeoHealthCheck/DB ``` @@ -188,17 +199,14 @@ version: - postgis_ghc entrypoint: - - bash - - /run-scheduler.sh - - . - . + - /run-runner.sh + postgis_ghc: image: mdillon/postgis:9.6-alpine container_name: postgis_ghc - environment: + env_file: - ghc-postgis.env volumes: diff --git a/docker/compose/docker-compose.postgis.yml b/docker/compose/docker-compose.postgis.yml index d310ffe1..e560558c 100644 --- a/docker/compose/docker-compose.postgis.yml +++ b/docker/compose/docker-compose.postgis.yml @@ -48,7 +48,7 @@ services: - postgis_ghc entrypoint: - - /run-scheduler.sh + - /run-runner.sh # volumes: # Optional Plugins, using Path on the host, relative to this Compose file diff --git a/docker/compose/docker-compose.yml b/docker/compose/docker-compose.yml index 63812f22..a9224587 100644 --- a/docker/compose/docker-compose.yml +++ b/docker/compose/docker-compose.yml @@ -36,7 +36,7 @@ services: - ghc.env entrypoint: - - /run-scheduler.sh + - /run-runner.sh volumes: - ghc_sqlitedb:/GeoHealthCheck/DB diff --git a/docker/compose/ghc.env b/docker/compose/ghc.env index 478f1d3c..985458db 100644 --- a/docker/compose/ghc.env +++ b/docker/compose/ghc.env @@ -6,11 +6,13 @@ GHC_NOTIFICATIONS_VERBOSITY=true GHC_ADMIN_EMAIL=us@gmail.com GHC_NOTIFICATIONS_EMAIL=us@gmail.com,them@domain.com GHC_SMTP_SERVER=smtp.gmail.com -GHC_SMTP_PORT: 587 +GHC_SMTP_PORT=587 GHC_SMTP_TLS=true GHC_SMTP_SSL=false GHC_SMTP_USERNAME=us@gmail.com GHC_SMTP_PASSWORD=the_passw +GHC_LOG_LEVEL=20 + # GHC_USER_PLUGINS=GeoHealthCheck.plugins.user.mywmsprobe,GeoHealthCheck.plugins.user.mywmsprobe2 # Optionally set container Timezone diff --git a/docker/config_site.py b/docker/config_site.py index 36bb64be..b6de1f8b 100644 --- a/docker/config_site.py +++ b/docker/config_site.py @@ -39,8 +39,7 @@ # Replace None with 'your secret key string' in quotes SECRET_KEY = os.environ['SECRET_KEY'] -GHC_RETENTION_DAYS = os.environ['GHC_RETENTION_DAYS'] -GHC_RUN_FREQUENCY = os.environ['GHC_RUN_FREQUENCY'] +GHC_RETENTION_DAYS = int(os.environ['GHC_RETENTION_DAYS']) GHC_SELF_REGISTER = os.environ['GHC_SELF_REGISTER'] GHC_NOTIFICATIONS = os.environ['GHC_NOTIFICATIONS'] GHC_NOTIFICATIONS_VERBOSITY = os.environ['GHC_NOTIFICATIONS_VERBOSITY'] @@ -49,6 +48,9 @@ GHC_ADMIN_EMAIL = os.environ['GHC_ADMIN_EMAIL'] GHC_SITE_TITLE = os.environ['GHC_SITE_TITLE'] GHC_SITE_URL = os.environ['GHC_SITE_URL'] +GHC_RUNNER_IN_WEBAPP = bool(os.environ['GHC_RUNNER_IN_WEBAPP']) +GHC_LOG_LEVEL = int(os.environ['GHC_LOG_LEVEL']) +GHC_LOG_FORMAT = os.environ['GHC_LOG_FORMAT'] # Optional ENV set for GHC_PLUGINS (internal/core Plugins) # if not set default from config_main.py applies diff --git a/docker/scripts/cron-jobs-hourly.sh b/docker/scripts/cron-jobs-hourly.sh index 9166cc41..bd1ab8e4 100755 --- a/docker/scripts/cron-jobs-hourly.sh +++ b/docker/scripts/cron-jobs-hourly.sh @@ -8,4 +8,4 @@ then cp -ar /plugins/* /GeoHealthCheck/GeoHealthCheck/plugins/ fi -python /GeoHealthCheck/GeoHealthCheck/models.py run +python /GeoHealthCheck/GeoHealthCheck/healthcheck.py diff --git a/docker/scripts/run-scheduler.sh b/docker/scripts/run-runner.sh similarity index 80% rename from docker/scripts/run-scheduler.sh rename to docker/scripts/run-runner.sh index a8feb4b9..926f8dbe 100755 --- a/docker/scripts/run-scheduler.sh +++ b/docker/scripts/run-runner.sh @@ -1,5 +1,7 @@ #!/bin/bash +echo "START /run-runner.sh" + # Set the timezone. /set-timezone.sh @@ -11,3 +13,5 @@ source /venv/bin/activate . cd /GeoHealthCheck paver runner_daemon + +echo "END /run-runner.sh" diff --git a/docs/_static/ghc-parts.jpg b/docs/_static/ghc-parts.jpg new file mode 100644 index 0000000000000000000000000000000000000000..865447371b53e07a6fc7e0aa11a4a9405726e92b GIT binary patch literal 35594 zcmeFZ2UL^G)-W83AV^UG1q6bCBE1VpAYh>>L_msErAU+BYbXLr?T zKtcils1knw=ZgT#N8Yxt000dQ06zc#xCkI4p#YE(TOXp$+QbPFwgB=!t^)w# z#J>Ol*=Le}-u+DakGrI=K9l|3X81kue39y=jk~+^JrNN{H{q9-PUcp^7ETT#-Y=a+ z#Dqmf03fip^GgeRD|b$FD{EUvd9K~&b}mj^OL;DRDGgB#=Lc3cwyM6aRyw|#&n$fH zEo3dZzzUoo?|a@3&JI@YFFCy(UOT$o^OooOWAS^$_V3FgT%3OlakrP}GSGO+`M}B5 zic?BhQdpFWINR0o)jjQp%74uz_T;(#+N774m#~+Fu#>B`h?uObtca+%h`6{AafFbY zkE8obZy`rF?tiTC(8|ri)z;bF*2$6c_lhsgojly-xwu4#PZ0U*5x<*%?^S{GKjl9d z_zwpDgMt5G;6E7n4+j44!@xhS9V(e>yPH;QTFIxaey!6>p@iSx3KpE&%Xk$TqI>6eMLsX1t7gbLUx7Zyc+-{ z+HMy}{wRM;BYu#Ol96Aapu9*$eTg`r_A-E!gp7=oob1Ad-;W^)Al?U%U%7DgrkD~1 zgVsw*E*D1e;Dqdp-1n*-S4j;dy)Lm&A*?SKjK2lau zRa4h_rmLrK@Z8YC((0A9jjf%VyN9Qjw~ue=o3QYRx9=holaf_URg!0t#4rV_74t^jSX;Hp(Vc-%EzQ}k#A-k%B zid#Y(&1CNSllmHuOab*--Y(?a!y=?gt!KAG6nIMmWq<KD*!M6kIRX?1Nc9*lAZtfy}$b|ikBWeL_`Kx0Cp9x-{~tGZy{@y*dEMq{7h>c zDJ~$gJ-Dedoi-jMB~HBAUN2G(bTl&rjP~VneId-W)dVyK*pTUPMMn;vxt{(!S~>ho z`{aMaw45((Gp*8rH5l*0R)DS6v+)IK-w$T;cS!B{Nb*0EM&9{{ey$Z&VS-@MR`&} zx)}=$ij39`7wKr-(tLB^NzIIS?M?HfbZo>?3#R@pNCKUCtc@PZbty0tTFEKzsR@@Q zt$F`~nQbAqb}+UFekmf8OMQc{y0_X=v3D4QtlbclffMNwcms}% zi%Anzk@G&*?~VjgtZIr&;7^@!w}C7-3gbL<;!eGj&H+)dL}S})6_tBuNP>0eIiOqR zE0m|5>=;P6>U|DqxpNM9lL_9X#z_+!T}Te-vj3RNCTj2;(4#FK9Vix?2^xFHOLn|> z4!}G+2TVLV2Ppnt>g74$3bzPBEs$~tg5$xQ1H84)0aPI9923@o*of^m0d-eWY_ z_0P$1ST!U84eLt5Kzr}@bI6Zr%JU{4oCAWj>96}+fzAP)UL{DD{!9pg|G#Vn+O0|i zJkc!pEbdQW7@?gI;Bi>s4QQtYCT>t1+UZqlL1^y(jU6B}&T@z{_RwPY4{T`>nnX@1 z)8>Ku^ndTA{SQF?u@?y>B>FkoZ>;dvXYrl`u0x3M4kDR1k>B;vnK}skvC8RM`x=Lt zKlB^;Z=?{P^#_6effVqf1hzj04mYd>9%6Vm_c27gcRFC=c<9+T8qWc+cVKJ@_-`_q zp99_|5K;A;;^6;AaY^FJ2qX~7zVVwX1pZJYI68=o2>5s@!l_1$2;)Ct0^%Mf*#DR4CGO}qulxhO ztk6ztOp*+7Kbv}|_#Dm5_P`Is5%s;?1K3Ty7Z5E_7ETwqnR5>K>=QubHPQb9)gyn+ zPtv+)(95|A`_wodGZudfP9PQ{rIqz3`9U%>G`iA-LN23|C@sv81g;J=GI}(35T!HJ+ zNk4KmK4OU~X71?eoM>&Dug^Ycq`$27-9q+fl=|KIp>x2pCnjTbOmk+3v_ikhZ}Ac1 zQtd0!8X_kT>}DIKHUb{573?7lt zJtkaze-3Cr{rT74?9ufE21qoVVBKnT4j|s)+5*TI;dK24^z@`fBme@G}dI;;p5IOxdkrSwnN)dR_AG<H7B#G7^ahnDOjuAU3V znA-hfZ#jWtoBkzuD*qb1+*HhJnH*7!f2+68-Xlmp77ox@6>&?ysQ|4=_+LaI=nwsy z_7_~ae=Rw&Nc;}SVtM@1->Vep2Fg8Ym1=RoB=}Yr9J%odRe~@wZliVoSU+g}vI|M4rIT{8kP`DN-Q>uN=d1T?8&=vfs*~35-sL zLyviFBDBf<%kZs4={g6GTm{ z;PYWTu0&de+V0SLE?`8F!wdGk$_N8P&`fl?W&G%#80(vVc*We{$2?GlCP+>=P7FH$qqJ8LqV(dPg#3L+L^k{iB^ zlBdj<5m2B2bo5ecX5oF^xIXhVpkpB>Uv zQGcix?T}0{i&Lqugd6*fkgRb#K3I&quS_8iHH=A4yQ8}7?3RIQ9~n!qVDn#;D=0IF0D!gkNp7ZxVH_ zx13-0;l8pG?|eA}IbVMcDET~_$e5M9jG3$T(2&;o39}sWh@vVJ&gn>;TfH6rtRL*d z#s_>`S|7-O(aT!ggj}<`_h_Xprs(nV)chb>e%a82PsKNHJPzD$pu|G*)!$ScOn%oY znpPLlIG7$qP13w<;-KRbt6X?N1HFRSUw=aXDbLyMpvOHCE7zlibroSo6vkh?53-~dJi%TRg1Y2G74611aC)=-A`WKCZFCDj%6*k z9#j^F^qcL0T0h!M*H_4Yg&V%MYB+JeN2e^0uv>3YT7)oSm0j$YzO$YK)Y3I|owoDp z1xYsA%qtcG8Y*0d45D#gNj4FMOyW^dH7Re4HJ_#lj%B@%&WpwbY06n_s=HL%INrr? zxBBgW7me_o(zj8-pSnSAq{Qm;Vk3+XmOYC(-2@X-c*zr@y;kFeS7pigomI{OrWrTL zOK(%qbFOAoRXG%fsAwxPpZ*|Y>{YVqAg6Id>=$1&|C*@8#_;0@Q`~?eH}_6v=zOa{ zNL~bQ_5!SMWjsiB);q4=rlF%ZmtLV=6t>3QwV(P-_F&fx5m-0ibP&zQJbLrgg9%UK zZT0TqQBMGv0t#d}2RH^k!GG)$Irg0^J(Rcmfy=3}7l!Xdj>1~Nn!hFVe^h(?h-N|i ztR%_c2CVLGk2fUBaX?_y0(a#R|LoWbK{2IwQK;$B7v*+HU&J_dE?WB=St!4@5_liFJ;=Td49sSXQA3AnJ`#)0% z8o%=*r9{W=50~+L*zjftY@R-&9Aq~2Ss!nu>;hA_-NJVM@}Q@|TIh}nL1r2w$8f1f z&o5q_4=xRp)!6GWIbr>otyk8__o)9!9d%JGUjJs(N`!jYdx~2Sc?f8=*@*#r32sWK zBoc{}E3(z9>g@Nj7t|61y-*!s%*vKZTb#d$h4o4&8hArgTE+@P!1oNwKHJW6_nCBa z^3cSK+cxQFFS~ec>2!|k$|r@cZd*eeP>iHx9tb3EJzqZ!uyY#ECmkBPnW1+ajuTpf zGSAheENY1F>NUsS{`9%}9V)?GyTrJJ)H7s&>X0i(%wbAEM@A=kIs1a=x9cz3y-8K_ z#qaVYLb>t$wx3gH=lJw36RMQS*}`=~F2cz=Wz&RxW0#oV?5@#f6i(_Xnj=yBpH z`LJe^qH27WknZv(@5HEhL0>jw05Z8^o4`s_XPocG3#ah_(eNUI#wv6$1dqFXGkWStA1pP#l6{3Q%S?t9KRFKhq0hcxQZZEBH$A@`qXC33UZV zh%n1Fr(s(xc zP9k(i$@ z3Ydn}o*qijVEgu@nN3y+r+8Tvr&thP556xfQUjjcT=a}8dT#4E1?!<_zHj-!J+C~& zPd9I9Ml_PWuPk!FDjp@LzjET!ZVQ{@w?}NV;9NY@*y0nWri7Y~$@=#|)v+I5K#F03 z_!jr+PY^S-O7{)2TSE~=wjvJXYZ6xnzN29B={9+Nltm9>!`n=-#`Bl-yBVZ=kVV2# zN3t}$8sy)P$}w9*KLSlY?(asPf;w9b4Z_}J3#*nI841YP2kIy{Z z1zsnZO~x$-^|N3YEn?3B&7+(2$obCsH*?pqYKvy?+G&*Y%rP9r>`hX*Mui6toR_!w zsq5Av0#kM(mNXO4T}O|welYvYtCg4xeZ#oPMEFxngPer#2O-X;RJ|)#Qh;}4+z=ME z)VcxXHcJH+i*2VS?B0UHx6G@aek&JNe*pS=kD5qM|Ra#V8L6-@i9IU*D?vi%~#LhFa*2%IO@WzaH=SLUfGWfk_zkP|DR zllA5grd0*zG0?W=Mb~c$9a3KdkQGeUv9Gvu_Piu1@XsI`vn9_{>@|CsJX zC(@y!{3PR7okE*YSl3Huap&l4mbx0(q`BRIyY&y&;w}e{La%|W(3{WElriaWnF6`>jd_&SGA$Jp9GU&o{)ecjnVKlK$g!09nuDD#j3_)eTK=Bu>8Q zQpzNS%6-43Yz|YLcp?QAVip+Ymcf&E^5Mnw7tMhF0_W*3ZeMY>J6aB$(ursmXT zhD5dP`OE`A_Q%12!($fYR6^d&=WD%+D58#0HBe@{i&myIRq>@n1kEHRO7z|#`!u2B zuo^==s-gF>k$5~BF=N)*4yYa0SdD(JUO&BnDAY1*Z(})&!s?HpoiG^i5o#Int3~`A z;8niSt~35RaIIW@u<9(QR$!NYB;;RtOo`&+K2i)LIt*OzAx^8Y6GL-SYv+KN00yFu zv=kiUzl}6&BK84+5;G}8Gd}_u1Tk^kj6MY0WWtia%6TiKC_CR+0>;W9aGDkE2RXhi zXD$uG2JXa@yJ0(1HhK`JeGVA@fNRYlp1pMs%~J=&dexYh_p{0+2RiPD(YR!ABsXe* zR(P8EmBa2XH(5sTcs;0tWstoqdy6DLCM(cHk$Hi5mfU=H(Ri6AY<)cIYSvO8DmY6W z@V?UG{hof^J=CV6MQm7XfJ>yw8>>28TSNXF383xg1Wc0TaAKX54KOrEO4`#m6y9P;=gYcnmxe4(;?QDdJc~+2BaX&cBgKS}!{?0G!!pSZM z$O_+y;qzIRV`Ncu4#`(a7KJXf>j1JBTD98^qww}u6NWL39^an3*<@=WW;F0wL}&Fa zPc&U2Guy=1ySVOViDPAXv2D{s-~Fzae9EEq;?4G+yS2Eu`d$kGt6!$9nN)jc4_Ekv zZg|zgRwhsNpWw^dB&Tif#fC1~pFtVcn?7ZXCq9BYu(>^~zNztZ(6K79OBTP>fUGw> zgicaD54^d#fpVk0M1BsK?v@($m&EFv-N6|*jMmIkWL=S;OS3&{>=~?~3f+szPs;jO zpYLjhhBT%l;Z0lcnA2-pE^|w)Tb(1_f+kdY>4^@%<_Jxh6-Ez4W35AqqsvMjHM-WE z)ERD`1L*u2@`J{;sxvV2<890MH~iLkNU9d~ zlXB)q{^1E@bILb@$x^+xG|DEQ`#WQVKe`Vj+3BUf@%*MK-On_vt`#sKApKKdX_vv< zNf#X#<C(?c$bz~s5}atZ*b;af5>!37!)Y!qjH;JaPD1Qe)YLVk$BYU$UJB|y zX&3O$;7BT#rZL$!F@AGP4K{lQSSW7C^aCiS$9QhC#-=VYJ=)J)q}XykjUD2=Zql|!(M7E_P!#IYrh1-091q!EK0{^1 ziarty68W|eni({~w|dn|@!}2nug8Z4916!eGnqBZ{6)HfS~GO*pSXIN$r^%j?fC%N zi*$6&0JIX!(40scg(dsp~7j#ZPpjT zYXFe$xC5M=A$Vl-*=yr~$Z2M9lcF~Q9UUOQ@nfIt&e|JyBsYi^H++Si4w|p3x)%BpaUv-S=jL>YmFQ97u>f)7SJO@~(0^Iv)35Wi#Kfd2)ekzNA4d+-9 zz=BsbAJEY6*^h?X3O}O$p;I?uh~I$!9A1)b(#nm@r*3KD<=IKIFZgI$IA#V^knk79 zcy;Yw!f|2sZAmfT2~*lJ^*XS?YqL>NgZc8ntgcI=?8h0go=u4w{~bbke^;~f|naDv=w?dNxdKUmo6pY zD9o2#48Qx~jFQ@;AX5eW-u_nKkky&A`9NJh( zJLWTD;`or(bslx;K}7iGPYz`ME}C z=fLC;jWemM(hC__k|J|g#r?PfHZsJaPvH-huZuVoIDnp86Z>06rE~Jw&l%obUGYVx>tGWl+=bSW4w2|m5q+_=nV+^ zwed}neRk6qHhTCG z*d(C+W$U*=bk=pS@UH^&znv=^L@$dXTWmFT?01*j^KaXv zjV(Xf?E=HP5A!PSXQ0@_!JXB@Z;1xdJESvoP0Hht7!`WA`9N67lKI6wB(TsBXGD)~ z#}Fd}>ehQlZUnskSX5&&u7xAZkj&$iDbAu+R*7%OZF^m<%4nAXzTje%`C?qpA~UzL zs8V09E_6B*ntyv{#n~4{jv_nJuF~q0-42aZRJ~>Wf~Nc_k#!j8H2o@3%no^bmurIA)9?Hcjzi=A8TGd4qu!^X5?5DI@aS z;uRpJ?FMzK&VgoCF`=iO5eDqmZ;FL2(3)Aeql#(RIXLJb!2 zWiyV2VroCl5^>~UN$~p;%ppF19=K*E* zY5Me;p`H86#k_34TXpRc)p*--)=@j{Lgo^Uh8|uAueN%ey00tq7MHT|3$G1uc>v_> zj=vh(taJWt{Tvr&F2{XVL4$Q^?Dl1dv!N|-y@yG>3insw-Hdp5di0B!hiU@a{8{eDIO5Gb; zB+-`O7GE(+ETdyFaHS%Dkzi(PXT}131&yuqoPgdq*d%(TGUudX9gH4F>c={_0AP%6 z%_7d#5B=MM`})C@jAXo=q5a6#yQ0*V1d}+V(c(UJa$`zT)U3SLuV6&!%SPltK2>u= zzp3=@GVbS5evMgWM99pBipd1d7DV~Y1l&hFtOSf9ew85=7V`ne$jB3LZyV<@PIn1! z-R(=4hL0&Qo4sB=GN$#Yg_%044#b6I3tR9ztgt0U@S4LE2KNIMY7oMTE*Ug_=YYeY z{CB-2RZpY2nU>qN-DnogZTThFIx}?In`#5gX9?V2Nh*VOc4q>mMAE7hEDjNd71EZ% z1hCnU(T|k{%6+Kl2`c2(*VJ!PWe}W1BVeK^#r;Tk$O=?t0myyh*Vz)0Qk9 z?8t+RiqQ@PFC`_Wgl7v!wE$+oJ(-y3Qedc$MLo5dTNGMOmys@0yf!ubrI?d#fo=c{L-Eg7w-O#cLDV9x+@#bgUkv>#yMPr(R84`DL%K+ z>iT@IE?SmFD?>1&XKfVH*Vz@`F!MICw8ViV*`jP2W`$IE(ZtLZ;%arsyFx`jF>HPyK^Jws&g!KN8H zQVqe~r6OXNirmKH0aQawt%8)#IChIqrN$~yA1iR0$$&lwEXKts@XnhVcnkMCE{~KG#$31&Y*%_X?yuy1 zNbt@4Fhn9xQbBvSuX&G+)Om2or&r}b>Zm6Vf!04#8ufUKNLqqKdU5rR;(swftgM9K zGS`;$^w??oP1P0;TCzi<{X=lODwtbuP4&3OtXxzF{<1|&v$+!tz5n;z~AVe z%-5Ul=wNdA#gA<=BP1tZIg{h+r{^l^Y68E&5mp81&Bq|FDZYvd^SUr;&=)})dP=`| z3N?_MiCH&23AS)C>n(j4&de^=FfM@|k(dY<-OCrM*@^k!Gvz(>NMW-xkP9sx>(C;# z)T-bhtzFkrKTnRs6(~jMk(cH|FY)pFF_^g#hTE$rdLWVWH-x2jFWSC6wY(>vOtG`- z9&-Bwt9jqY*j3IeiPy=vZ>Iu9J{d}}MxmnA=jvMtOU|;i#+p}K7OC!4ChPN??E00O zJqCX4b!Zf9*rIepyj-(glc*)VD^42L-_N_hX_6Hi$l*P*SWxFLhcvmTac?{6dz247 zV;?k<@97trEoV)7O3KJ1@GqUkc^q%nbwNiDM!jMGd+k%lWr}VLKBa4fjl+tcuNwyPG>!5`&N2TA*-7-g7%Hd;s3yrA z8~d=3d%xtF2~sHQaMjoNLDERal6hs!K*re8*@69-cy#RL3p@$Yq!7chzBjOBANEru zdu4$AhFtsP@lkMy1V7eOAlmJV{QVJ6`B3%T!|&ddZ(AfPymj=m9$6Ch0^*Z%(V0yH^x!-zLyub0Ub`K}(r`vp`yR?8xSU+*kqwQ5% z&alkdb1{3Oa>n8B=l8I+{)fZPRn_r_m!Nhmp)}D>DT<(D7y~9E7-T{e4qaB>=$hu3 z+xs_uXuiK%=*IRgIY9~ODnWC~>f!9Mu9BX|YHH)1QFtRFq6Vg{73Z05<}-+LeG%=} z&!%I6(C(=e+l(G*b8R4bM)Y~f_V2i7fo1I*I)IrQhb2Dov{QvH5JA+{-e*W!zBldn!A}`<+oVrDBEY>lNSD>NWOTJ@| zt^^rGAIQ0kS34;O-o(zg6Wwh@tFQju2Eog@^_1u#yG;z{;Y9vdcbnyVMOD;>*u-3p z!^b^^H5ANH5AuinDXPAL8sPE5q)l-IMsa zp<^omOs2Z{Y~gS_60p&7WdVBCUl~KAa19*;n>v(xznpT@=jvs>TY_-+vJz3sazDD< zvs<`;;d0Xr2u;gR<^1=CbA|{`I7s^y%m}efK;w z5!(fa(`MuwO=CaJZrC}06a$Ss4@huY(N{BnWs}MjCo31IDBi%ydQ|xy1ggZ8>ZkAM z4bPdXOsss+B=x>%?xOCf{V`<4{>$1(8y=A=yuBmnXjR)Bv(ykg#rN{VbDo&d#gv3f z-C$4FZXjU}hql}!GocI*q{69T(!wk-Y2lJyqwec4eMnSa0;`2Azo_{TKG#_gOI@0% z3P?zHKFoJCsP_nH@{+Y(nJh_NF68y_usk#}@iE?0P6!5Bk2TYmbM?2Z=<~QFWD3l* z1f0qGn__K0$1!+ksZbSUYe|le^tf_b%V^z+d8d^@JIR+wu@z4*wn+b`O#!nLTFYcyB&i*;Zs8%Z6Mwao?fE=6+Y55?q=%I$NbswLlV&4rREot-qmBLiTXLr3UiE$) z&e;aI(q>1ubTd(5#?(HrSN*yvehK^wigXI|7ZHi7H6f3ZGl)LsW9pWGkUt==?zvI@ zf=+X`NAXrAiPX|5MOe9a;j`MGAh50cA%6Nz$lL|tJuy^}9Ds^v0YJq|CRws*uag*~ zS~EBC&|G{=m#X)ZoojSvf5FEC3C+bco*nMO&$yCeF}1|N&@4i2fWQ*{uLsEbhyz)v zO$2?^+wvd%l6q;PGsavZIxD?BS?e6gSo?kgyT2?3(xYJqp~c?)I^2U7Q!W-&kHlz8s_+F=b1%4VLfqK5@IgOE3j@@R3QtHg(NUxKyXgCj`ywj!)hv5X?D%VgsE&r6|+E z#>9Psl481}Taa9MEooZgn+r4Bql>+|pO^A$NAwSPlwXH-upAU*?Dm;AZl}BMF!x^E z>dpjHYu_jhQo_ZJSIMe4P*%87vKPmqb{0b8jCHz1v_dW@)v4S{Y4>bZf&BPE*7kbE z4X!mws6Cec+d!`RGkO{xf~MlZQA-q@C+YZFQ}7Rck-u!^(|2c|YqTB#7>N~nKK}39$p40;F3Np4#>xq&GVoxc zkbOhXW9pK%?Q&?}X`si2!fSVzm*e>4Jg4}*ZiMU@M~C z3SIU20uomyaD8mb{e{Hp11K8ud9locO+g}w3CCB+ z;;Yck`^s~^jy}3vikPUU*GBr|ZQOnIg=k;PE4*S+h0i1`2#q;C*XWgFN`DcquU!co z&Z=$!(+WH433+v}_GH&J#N+y3N@QI1T`;P_lpK6hOUE5aOHUl@->WDT&?vIIes#`7 z6E4KFFY9HoGt@m%?@x{GcR}m-=m*|3@@lDV9I0@)T^k)@nP!_fTdB&-z#lH#_j}vE zYp*V6-qk5bLsZsBc77a+`r!FL^r%m^FO0QVHb2@ve$Me7(`@n+1U-Xz%Pl^m&O%3e z-#>ng#s;}vN%eVoU0p7kGnQ=aXZke0aC$Md9us-gMNHzmQrmhhfKEWQ?7&Zh)wcX~ z%7WU%1U0s`ieaZ2<2&4M5Pbp*XMT9gd4*Ot*XCgkwZ*7~+nFLN#@)Sn_g)Pe!t_?r_Y~h~D}SRG<`24I6-=BCskgelpxQ zneYecLqs!|XyEQj z{Ff@*$dcD`N-6Mf&eyYvaYt61Cq6}icaskZv;0jQU1{ z3~y4cz> zGjGHZ2rV7$|MF(T#OvDCk!@Jvt-^*Ne8Jrd2*Kws?{E3MisDT_fW0}@?}EL7R!3L+ zLnc(u3{rP4WGeE0!y>z%&@ z8r3E7IaICs*R7GB_%G^TYg|q`g$7x7I#XXaJKioL{2=CXz7CY|9(mmQL(s8io)!z$ zG!haXz+BVk9j8xNl;P4P2c4DE^!K0IUH|$UJ&%0aU~N4RnT{XjAlJlV z?aR?jySwV+$<5pO$D^mH1!Wj91SW`4itB)0k-Bl>(U&_s1%FzdVfbyhFNf>f>l|JF z8H!ibWOuzm{>@9U2yq-AKEJGmEmFI~1AlM-2`{!x*zrnE$A*6y^ZR2Ud9hWNUZ zYv0w!AN@^2y6=$DI3#K+0;;BCW=`q1S(Ze8rNWWdvgx~FwBD&_-Ph`loFu274IUYF zi}A(Y)o+Z>*8M7XAI0uAvFsEx&*~H?YG-Krq0rW%tL-ZK+ihB)YD}=z$^DZr+nqii z8Tm_=Ir}cW<2gj4SarJn)ZZ1`2hO~-D_s^EPs*L{QjQ2Y%26bMqFT8u%;0xOpC?g=o4H0k32qHFyhetW?%muYA zKl7fJIx>gSyxs3@mmqaz%nU@s;@d=Vu2q=hh?=Z$AiX4uzrcb#t8T;N=NVFCTV1yd z1IoT$c=c+^WKeeQdpux-T}r*yb~N3#)b+46O*3_7ZOHz?rKlcV*mId7wvy`(v-$R= z&yD=>@L;|R$|KbiEIgk5r^B(?@jD>v=}4FRjCbUHlg4v8OSibm52xelPj7Z-F~_$FJ)I9N0_1{XNp*&_n;Bw)D^iw;y%3Dz^C98fLr>4Q{CFiJK6S#H6Nyb z9zix^HFgrq>tmFG&2XqXp72K=K-@gqAulek%? zVl(f+jNjkF|D#Tv6@1cwHrbFRj6=t=dfPlPuvutt+q=JeD!;P`BM*wOQAmewi!;03 zIJ1LyrH}f^0T+=V4oJOvwpS8{z88xm@ahLpKv$N0>*!_33Tp!eBCu8jKIr-rg6R-+ zzT~uuX_O(hy<6Vy>7=QGnPD)%6fmX66K8x5;Lj4ODIp=IbytLbWA^Fef)tAI(OuBalVLSxphY|Qgrs!s`}L#2F`+;a@1(`! zZhNB}BJ@U^>9J9^C6iQKjWq_sJRy9Jp#EDb%vy~pI<|u^f+Mkr1(EJgl#3$WA4OR0 z$0ySC^iq$-Kzf(hlCGuSeKhr@MIbuU6)!LZ(l0miPV3FQg^@3F?XoZ5^o`ARV*h%H z|ACJSPu~~vS5FrW97*0H?)y$OhkaWK?|}b$Jh$p)s;4r&lJQ7Kc;ItowK3smJqPB? zeBVJ_+&nc-3H@>D(Mmy;pIiBZ^6HJdhCS0KNv6th`qjoh31!U*I62b`Mp3eHg0!sD zug41qLD2zC9=vs)#xAF=o;&z2iVqK zNp|pdjO|*}YVG3uq>4e~52w|U3Sp^5oBiqfG~Of^$xB=mY!!#M)X3Sc%1sCboGJPX z)?p=(z&C9)=)mIKp0d{(3SMHy{m$LFpozVjfdRPVnCjTrGa3%>1>n*|gatm`QGq&& zR;l*mCvzbsN2Z&1@AIVc-pu;GPqs)0cpHsOpOT4umTX>7QQU9h)i?%W5ARZDL_U{h zm53Q5+@4l**pr7IsU-T~m@w9z`d2|s4)BQE4K_UHYqx9)m7NPYCBjeMo=vi>@jLyh zHW_ml`BlF}XwNZGvnn!tE?8;Y?ZTsd$;WxarHc+;1R|aX6ew`NJr$kJK$ZuvUuLuQ zZ)?0p${)RPQCk^U{3?Eu*z@KjLZIPH7ZN^y-Ccwn$A~%ZDmNb@Ahj`8MIl3v4RY+g z-@Z$dNK0W?HVEjph#kwMXVDbPSrO9C8aS?s)lL5q2I47scAU#vyjjwE5=e+B4D37& z&vn3rFMY3gl`3SR-efQ!>g;^g+49ij(=3B>n@7!T;*aDYR5Ik;nz#E|6UkOt?)Uds6*;o{Au5pGDnv(*!&6u+DZ{>PA$sU*Y$#Gu@2651=}3Z?)?o zWfI}cD{2(Z-c!#Om32){tV9y+q3t??g)&`QxnY^N^_??z$MWywW%Y0F$=feFMYYlR z-^Fahx@Fn~ypwdB#U>HigtTx+OK}CJP(t;)|3!6wA6vM#y3XBHOV^XRJ30Cq82xmU!FaKY*+Co`rO4 z9JhxYUt)0-Ez$?>A!=R06oGQy_KW9$UUs*og4WM72J620<=?JN=|#qWbMRlkX@k__ zHoDzy(8uF|%9(iRa=B11J((Ch31G`Yn6%R0Cz|&(njBL6Bg?w7trK_0)9Y6J+<$UB zD)oPc4J-9ddyZkXZwmWphA;T(u#|-rcm+mbx*n+6)!+=h8T}a)=-8%Aii*(X+LFCGw)+k;|W2UMXy!uUZR6Z>sc(b z-tx;h48!_C{#Vd-9KD?@SH_j;iF~oN6~RexrqWRv-OYuo?;S559GvPXF|P;Fksx$b z+n%#6`(@UAd>)Goxh~VU@Po9M>p_wQJVY3c)FHin&P5}y9wJ@8HZKJc{@6KRW$ql=G!XVS8+067j%M$TUL z>MzFp0`H&Lp98Wd))ks54qxk|3b#MKGRLaQpTY(6JQT5uwTTLT+QQE@J=Z9SFnH@t#A+#}QLk5}2mEot`QhIvh430m(g`j=O} zqBhf0U}ex)gL;xN6K6N{| zV$2{(@HNDILkznb>-~F5U9-`xlr|Drie2d4ixu}QW|xa zzqi-ReqBZ5o>`rbo#`Xj=_HHx!O(e%mOHc`KU7W27NsAJzSQ?b>kmd^^cXX%Tl@BjF8dU*&4#3)8d^BWv>hE^A!dOZ!hdIhA`~Q|Q;1e!>==XV6;t3+GXtkT z%@4{Y_8&lWmA_*laF4Z_?T;LXQ{CeWbYsiFuY{D375z;0-7pRn!6+8OY^zox<>L<> zn~>|{{l~YXVk5j!hp?VI>U`?E+kd7(5Zta|#sZ#0UjZm%-PWJP2PvE^A8ej_6e-Nx zhJHXI3Kqn$qxbhFcs}GJ@a~ic-yIMfsE-3@*M7cTUepcP*LM0!=Yk!RD0+VK-Zqz; z?PV}(zenC_>#JP;+v4pO>ytkPk^vOeAS?Io_-nr4^y7N&>BxfPI+i7Oj%hgdN7~)c zI0dj-Tkh$fW3M4M(<~Bw4~P-%5u-$<8HqKSqPAHZ$SJl7>Td!3q$lmE$oB2Sb_5Xl zryQCo_v%Mub~UUo-MmnTndSiw6yv{4za+JJckC>2x|Fh0mH`mvMnFH=pyHfY0-_eb zc;BoViKf0a$-V3ehe;`#JP#(+$^YuV@fA1}-dy*R&`KeZ@r9UiOJp53Se0dCXR%JS z@3YccAFYwO;;3n&P8KpZK9xc3Nd|mD1fx{oK!508gI}vY4wr|5o{A;DU9d#OL05DY zW^L$begYs*k#XZ6kEZWWgRn>SK!1AFUs@#S@(!7x%pqc3!+Ww$m5f*B`$G}Iv;-Fx zPM3WccGSVcZXq=j#1ZO?I$8QtA=Iz3F$ZxFvWBu>K+GEsrLHu6IcswdM13=b8osO8 zidxK%sF#C7hW;G=$y1JY{w_qW_-s!UzLN=9v1^)rE?~tQ#oe~T#8`i{5Q@q^>=IUJ zwcUp|xVH7iPv7aX;Qb71fWElEyEybk1y92OU6-o$XRg=FbF_bpJqOs3oxEySgf3)> z?qaRW!G&+Ffy6gLOw;K5K~pL1sb<;7bL_XPr31g=g@#pj#(#S!YWPeC(W>1iIc8dbG7^~9fLB|1mBOW8S+(dxzT~G37#5Yq9(k^ls`Bjf#trra0SoH%4 zQ96MAv!0W5ejkMLStT8D!RK{i2pTVlKs2=yY9x(p-U0-3@M34|kBIEasA#))Z#kzxL z&}-W#8WiST9lWPM7HdpsFq!(s1nmMueShBj|waoxFS?PG^D)Hx3_N2&45o$w-ow)4EI65 z45G>07)js@dGP1nQ*vz)n~achnr(KAeUOUvj=;tX3I`S!LL$O`>NW)5grJAcjsgaN z1BoP&O&1FD6lEELM+)uO?}JPvF?%$~;Xopt7%;37#OU4Tf9+E|g`@j>Wk1Uh4ulW< z)asur`4w$!aY0b(R6O$a)y8NdPh4zL_;*taa)g($(29w=Q2 zqzj|gU;hPFX>zio6A}E|c))&z>oX~rZ_mdYWc4k`FRpL!N1rkuMa!Nz_yBnw<~bji zd@BHXHDOH0XxU7^hV{rw?esTu(LSz(h@FDsuWe#9jKLMD)ZDAKcX|Zu*&NfFEcj9)M zRt+nX*B_Q%V{LVv(Wq$#yYVpeHntKue8midN-)1n4YC+YUE`~fu^Rz?fh0ZI8nXUH z_KbeSSd3D+&1>Jp7irvk0vmSU2LlNV#y;pfqJg1B(IaL@_d&mmZ6Jjih_6zt2aF z?-L5LyYqit;0N+3xGM|)`u?BT0)7dsKfm1{K!BtT$E$bs``b$AuigLM#PIpu?X(d=;{VqEJ@E-_WBY}=y^+#-%%N9#q#GYv zq9BtYdM1m4HIUc659?IQIPcLP+F52)lXQ4IW3VpSlxb;cftM(+pij<-EJFN;%(nkb zW^urZNdEstW`8zlCw?8SpNH9 zUtguH%8D$qFcyfSQ4+!jUM}lUg0QaLval##dNN^n!K!s=mhq{1g)a zq)7rMK2;Fadk(cl!|SSS5e2BLE(0cmIi0F(DlJ8Rv5q6GREA4g8Ux0ESX8`*S@yzz zB4pI8J0(22vic}{IOTdwOWG{V(j*@?1@LEr|TOC!VX@iRtzRIw@y0wE* z?Z@sL(`*3|zHRBiD!E&gb!p47$XaE%-;S>k;&joGvUoFBmIL1 zm=ufxB=p|;GratfG5MFW6Z#IxH>FRpQqczK$Sy6;8w;l#p5%LGB9lr6U`wgWB7r zWosuY@IrNq@qt%&-&XfppR%ds<&tBhKDmXcB7eg8j4#XTmG|Z?Zw{RrQ*O&*wI;up zoBigo$x5{iT|(j)RvXdh%GM%3@Y#e87?^6^jZs%jf2em(_d)Wmy!2i_^&9=m`6vA| zL0~?B`alJT+#)d6vKup}uy|>g=Rav!*IJFy^%EJM`@atU16c^C9KL+^%*6RKoB9Vp z)S|zo_o4Zouj5Bgw%?V3$H#^YaiRT6!pV}|sUsd~FU}>ZYnzoE4D6@#AFb7a8}x6Q zO*pd;!8-33RhBj6W4Q%H=$~-h4kIxV62~i1Ar#qOC2@tQxNi1R>#uW=m-4FtwJIqc zhi2=Yc#?4 zWtk3)+$uldWBQ;)v%30O%crShFn{SiGa~I6;d)N76P*<=x+?wFl&qw=+0>G|P*Rv~ zSDL>U)Ki~kj{Hkt_V?iI|MtbHuu`2yu+yyQFD8~{-#eEkiB)PZX4d$)CFD;dh?ke% zr6FE{_uRrZnuS(L;EdyS@$X!oU>^FZe#Ff*k01p^AQJ;}bUWh9fzYQD1nq}tkrSnD zsM|s2*;r4ifkJYJ*1ab(fzcf%lhh!#t=Xk$d?(3t5g!@+-~d-~mGp5}WT`|t4ZYYw@8xvFffjWv%Apob>nsxub* zx)Qx|V$)vnL(rF6KHg7ra_hkBT_v)Ffk!BxdD;X#JGoaW3D|>_sVO?W18ba1#Knf(A|7m@8NJl!$4v0{1&}DO_x8^o5DuY z>Y$tNc)Qb^^NN> zTH_#Pr!DBVF@Yf&lCQ_=lz*nWjH;Y-P*GSVaOmS|hdxa*=_EOJ%YwUEhHIse*sAzw zgGJWzat2uC6t z@I6&_>dUN$M}29CrldoFP7z!~Xsv&|+mqcBf0?KKxWTKHnyEYt?*0X82j4U(VHT<9zaqxCg)BbEpexh4!dww&%XZy;3`$(@$k zL_=TZ#G1iPv}osT0c~_tVvt&cB#@wCDc5^XwhygQQeIYA7Pj7{5$wrp6L#}kR$6LH zLG)+-@b3trL{lBKG3amm9?7bRcn)V__b$pr3v}JMgGL5z#J~NT_P)Qljb`LtiIJY1i-rOU zH~6irzwa5h(guYhjpbwNa3^k!&!W*?rt-8S>SlqZ(CFC032`Pzib%h>uSVW*^=Rsc zYu&@w$fiRbkB%HV5Ksf6s>#zMh~P8y35gT-&ZIOl3LL1I9PYiuht65vY6tnimP+w5 z2sX!0ghI0r_GqTQxk-TX)S%*9+u1{^YP z8WsX!hjT3;=zUEeq>c)3o80x0M=DNWFS@GOem+Xzp$Ed~^MIIldqkiyyHzE^;UDwLK)C<@^y2B_4X}{C^po={#kNgY-j`qHb z;Q8RgJil;wif=MuPh%TKn3b(_VLvKl`Wl1d>Jv1oFnCx}$a5CRuEKEe)k2w4>;ZqM zqx}cbl@k>KV*IXapHpLBPBD2MH2S9~J8?Uu(3KiwMM!K&fIfZ1y~^UWL+fCbQP-^} zvUvlh))V6idZ-0@=!CDFgf+HXPacmDQ@Y}7$<5)R_I}_+WA!H`J04ERs{D%Q+#%)G z?GnNs5Obb0u17a~AHNbUv5_85D(!cuy^RPHM$!%>ihdr{Xe!y>7{T%u7f8jl)H1W$ z6iU>9NBw21gm8X(O$yT`LFww2rr!?TmSn#0r1={{MP$2v98u?Y0oNbwoHG%0d7A2O zbDP#wjCYLQtu@Gi%lKtWwyy4mTf=I3E1<`Mb}ny`s+eT23H0=hBM0(8`Ug7y*~Fvz zPEP!ra09(p5cz1#dGHSE3JC57?}r{ZN@ASE9<5ERw1qlE9Dn_mCppQ~J95hA;RD#~ zSTdiZw>vt+?o?7;h$-t6#%j0}r5izQM{t`kl^zjek@8c&sClEV_JSKyHLu6o-*YlR z6{1OL!tk||t6;QMCJz!5ZJ^&aK0mY#I;2xBL`dVw$b8T=inyXi6Af#ppgvg6%Wm%_ zCtVn_7`*yIP0G%xSz)yFKbJ8W@YoFQ<738sL z$izo1^~?ThUjzaB-j@q+UXgYWb+dZpP`rIJ+2-5yHwNFb0!}O^?jhX~l475L7OtYGq(svJ0=5p=hI1X7S!^KpxOQJmAMZ%P#-T_D%of_To` z%8(qmXvvEg!kyY_z#h9YbGcckB#P6FN3%Yud~=J0Mio<7@x91ksb`OdgzFCXfCKvc zWzQ|2C@9soXFaP90zK%;`3HUfRThYSz_s~z;qv`=c}j%naUGH7WbVq&Vqs&WWEq3Q zTadQL%4$#-Xvc zy!p+Yb3BWoE#qwc2mY{ythM@WeFA>$sM1v>p@}nVxb{2xvgLIHBQgiRBnLo(;Pt_i zG|>F$&oL>{3^#(+QfU6p1T?uY$;dvJdgCnZYHEnT*A39=pt|&o#`hyxu>hk%|9ka9uXITd}u{5 z0m)h5;=Trb6fw`P#YXRPiOX7HE!~d|`UbSV1@=aIC?bxrcyfRf@g`5D^>{p=45@WZ z-t~Ir6`7`ZObLbG$-UpzHL*7%qNv$c__bEx%2c5&$Yc?4?iG(M=Q;K3Y109+-#bAEtg6;pW3dA0Li@jLVWt7Et;|M zMe^1Byhx}$CeQ>%?`wE!`6<1W^TmsuOe!~R*cswiavXB)`x%}5 ztj}T(e(=(`6yWxp zM}&rCs1f=Xt`t`pe|pFC!AKPZ9?U1nrjjF@W8>=HUdDq7x?ZMNTNRavw4q#rEcFVB`xOU-ZJb8H)Zb>bR6D8 zMby3`&7%-vsm95=@RO17S7YIC*MuczZCMcD*#3ekVQge1sG>QwQ|H&0dVQ~1Ht}ZA z#{vFE%_u*nr!*hz4nE3j^4=$<*;7!Xi=i+xeTkS#L;DZLnU~QhH7=Y%v@JY}Fe!K@2nKX)+3W*KFkR0wXiw{mJ|@j68xi z;+JB)*-2|#GnJTu*=?)!TM!n{O#ThmmRtr_Z{f)expsK?{Gjvu_1p$Q5iPkAIKlT*xAGs@lWb_)D7`eH9!$g}x|AUC$)1clCa z9nfzNOamtpEZ|n|=jI7++@Fc@P`;%uHuiePU!}crZ6`20^q6DGl7{^dK%MK5HBo%9 z7Fhdfwj~bWsSO7OBUW>Y*PU6ZHkVM`zH*p_>{ZA@{)c@~XK)rIl1l?W-!HWA8JgqU ze+A@s7Q-WDy;HQ9x+#pvF@HV@3x<(xi&=_76t>aHVyfWNv{I;Od44AX?ign(k@Jc1opk5T6>#5=~Kj0?^1B0 zZS_YL&(EC%!ii&Qjp(Cc3M5+c>rD*Z)867o#4BHFB|od&RT+7VRF(AGxcSL)>{gVs z*xf)Hxrmm)Ezo#>nmvZy*D(ET{lh!xP*;hw%f3F6)*JjCG`%H#9fOxY)ub}3gLrO) zt^3hT04}qwi=P%wY|bMgRl9Go#L0|af3^iz?9nzS>B{V9^=Dhu4thTkKJaVy=|6(o zUzJ0@r+)e)xcx;r6)ni7AL%(e5sQAhnm(6$UpzjlczG0Oe6H z{1<)yGhGzC|M%@K|6UJxO8mKp8#L0kC<-Minr2@+M|&5$nJ1U$L@2~%C7Bph_Y{pE z9eHEC&QHOU$Itpbo2WY3vL`b4?Y)8K*DrVO^U1DoAjBFT3LlvMD_g%G1TVATu$gmF z=#F*p|~*4rm( z7N<{vy#4;_7Ju?RrKy^4A|i$cxlhdEv8R!!nh)a+PK{)z{AT z&&nK+tN=3IVt?_%(b9%e%vKZQ>?!fr(W($;ooq|{Lh|!QW27~|+_4X$_bIzm8Iv_z zIp6GpC!`Zbuf)l4nyAyi^I@*d`>^E<*6K^OcNy0L(?1lcch3um>ulLc=BSLq|H4+ddrbAX{ZIbb zsHl`eCeVW30e|+eR+L$1PT_VU(LSHN8;y9K{aVQZ*E_)-SAM_spq?0l%>wf}g&{L+ z+NxhVM{~JTN~C;)WEg#fl8BuXN&kdrY;g7r^m@P-Pv>s-XczBivBN_Q*QfSbR&Ulp zwxg3y7A_VYiT+O-S>5YYxBy_Fvpx%8X zO8+cOra)R0q1(lz^+Av4thZPAacI(UFdq@u|8#X@X5h+;G5cKgjd7PJVg>M)Oudxc zmn2PmhTbwa96Fx~@Duq7FK~K$hlln-W4%C_VzTY9d24wMbZHl7MMeRfMe0jh84Dwlf-7|*GeqGl#jKU^i*^ovSGj29-*_X zGDLF4dcgNMUgtISy^iPwk-P#tH2B_wZ6DzP$Wqv6AUjtyPIoW@Lngu2(jm;Gv%1hC zXP_>CW*))QTIMD;>rR~%Q8q*`oZ@|?yow<<&?;+S!8J5{%2Ys^Q&%mAQRY=E#hK4m ze?xe~{=hyc(SkAXt8EV>Sq4U?azbv1J6^aC5?-7kl*7(sqv9zqk;xO7T{h!Q zSc1wKl|!8xD)&h4^}q=y$F2cR9DyqW9HcfBpztd0?neZ#cDAo0Zr%OvAY@D zaPD1Z((G0OrtP71t-^Vw>LvyJB5;gUx89SYdjV)TVhBb#T}e%Slf{cae3IlaZS77w z;7v8$Nk#=zE<;SSNX;Chhdb-o8KJr>)ri%EJDwt152CuQlfxAuJ5Hrm5S>bsJe8u3 z1NcLFVDKLN5yB3KiKgju+y_ZC3k7Vt_09_N_yOA9gkx|HPWg1RUh)k4o2mze^q)kc zyP?8)S9J2$C6nrT0^dXre)3gAE5ah;D#0*%<-53vCXKx{YU8`#7CNR%zZ-F+? z{sZm(-_jw^6I>29O1mTVQ^l$6PJYMae*@tiLq#OUo!pxU#dH+=LDg(Pac z#@@ni*#GMQ`X3O#@ zSEQ?t&ma>#ZWkO2Og=o{=x2oo;~4Ap_&m9rROARR+&`KQdK%G%3U*IjFBz1xG^M@7 zn%!{lGu@YYn6RzpmvM%at-Ml<9d=g+6S2hL_Raz^e{gHv^opDw%gf&0WMn5Q1pl5d z_FnD9BBQicA-j@jXAZhFc)h~&Y-Uevqm1q_?Wy9Z6d~e3_YioUZwu>tgPiytk9=w} zVN&1z5W+#Y61*$6d@(fVWy#1_w$~P{_@(pu3HYZo@_i5C!Z2- zQz17L%E+-8>C`8M3W4Qgp1a55^@6KhCg_gdnQ*gnEPcQhuyW}hZH2a!fwn76LSiIH z@Qo2B|8ZObRuKKgNAFe$fY;q)+q!-jda3Ar(6zCBkh-*=$;@_McL8$e2xJ^Ng`6W5lE20itak-2 zwWk5Fx7#i9s?O?Ej9`?khT?GT#XxM?`!S+$9{H!C5~7BD)%|e5rzacs;7jfAHqM45Uf1%Xza%LK z`N0%E)DiX-1I2yN!yivv zX6FGgXo|1uQC>O5?kLot7h#8rP`I1uEWJEE`uf4q;^GR#ZQ`tkTCkudU=fJ$ zJeBC|ix}7PCb(Ii$ywOLh83IiE170mEOLbhI+Rx@ z%|;2&Wm-h`^1i&ycy%#7v=*1XZ0~l}PuwOxIlg?ss*pZI9M$J~)h#oxwJk;qU1Cyc z!T1SrdmvAYBITqIoOU8%W%VMutSlWK`R2Srcpuh5aM6Xj_I#yac%_&? zth|k3RQY6{zTZj2VBU6261{=vj1~6*S{+%Ph>fK$j)zHe8DaH$j9fQ{U+7e`EN;-N z*v0qNTsj)@vM24#)itfrZD1>KX%0HXgpR;h;ygmCtlfEMBXgqsa&;!O+AD_cm5Gt! zM$SKT5n_?d=bVKkRjgflJ1Wd^^7P`^yxAT)E8A(Wh{C_dm+FzV;32n>yK3u@uIPD# zz9A~>8$C#lBm1cPK#_LsXFhGdBySTS1UT)P+ebEAC%@K@Gqa{|BnYc6x-z#;`Hd7+ zZa`HLwnEMI%EG@sU=4R;*YcJyS<0pL)iNgoUJB69qR_ot*KW0EzoNei*~T=d!((3} z%Jla^2N8mZ%eJVMN0@rZ7vAHES~JEEhtCB*WYYMM;Z;1B)D)6v*^d7_t5^Q>RJLTX z$#Wyz@^J8=p(LQDguy*O?{j|Up8jS|{(GPQ2b&sr781H#+F(s7L(Vw0%~VBF56{ji zzOWc;B<2-*1{|JF;Ai!j>W_%?a9-zavj?mD>3TgB*gd|LJ)zB$FSwU*74S@kVw^68 zlFlRMS778A^Z;Puv_}Syo?ctJbvc=fsVMH4wqUZ`OWc!5$QkU(D!$XH1|_D(u^gpB z&H(S1MM+nnz9%oPH(kq8M?^e9#f)Wk#xNSYdlFUJ^5R95k9A~n><7#txMZ4yQ33ZJ z2~cI8eT3rlNRE>ST9Ml!K;9P^%Fl!wh)Y;w z{D3CBEhn5=i(Efml5{TY>iHTKXpFJDtLMC0qt$FC@TY?xT!S6oq4MpzF@+|i7i2|p zKVUZ;L`Uaw5hQR&^=i>f`CZbtpDeJRRO1{@i4zJ8NM=m;5rt>P8OAE{3hS7DXoQwS zFw^*#=ZjKL-F=HWvDPZ6@#vlr(Kwr{5E*9`O~&2aGK+YF>uWT? z(|MnlNR8ZYX;^%q!Aqs*nRPg3SGI#ubvU(;HV#J2!9g)h|EqKie7C}na?z-@Oz(1XExGQAVAMlFGYpt zIacq+kucJC`F?oD$@IKOYp|L`?a+5UzT%1wJEC8Vh~bDq!zcu^<%!HWJ}LZAf#aUF zc>`vBv^!V16S?kE=YR5T!P~ygFJ`R{PbD9%&mKCBO<_2MIMRENIr^&?=?7cZ*XQyt z25Ti*tM$I}rTRmX?*dPhMZ={eGiqe zv=^jmr;JV&2w}A$w=CdM)?JDadFmZpz#d3*h7QEL)GmYTuZgV~jau@sC%7bq?!D=t zN$`~;IDK&3dEhiE@che@lU#Jxi{zOtm+3uJ=0xa;h&$m{Mz@dLoYZ_Nc>8Q5h;{D| zT!VjeE&m5BqTktV{!e3TRFlrBu8U2OcJ)(t^cDtVxAKDFEd6XjBjGlses8Pt^vOw5 z%58hnJ==mC&unZUQT!%dFZF4fJrD;-4h}Ce77yLvWw-CdnBONmp-?zf1JN?tR5uOX zZ-7`M!DDw$od5xD{wI(+7&DHzCuLwwEQqBzI)YPt)eNyA7!f3;9|rewm^-5cI4dfk zXF&u`U&S?A0v}&UpqVbv?~P(9ewOF#g1P4KL1D`9}A^}e6?Kg856dn z2a$I77#~H1LUy{N_W(}?p02^t`TKk^ zA-umcMG;Kc@)M~|=I!h_75m1aH1&u`=39S`{I~Qc{b|xw04LIqg!T{0!2g4B?SH-Z V-}(Q4!}$0+_5OeL8}R`_. -- interactive - TBF +NB the limitation of cron is that the per `Resource` schedule cannot be applied as +the cron job will run healthchecks on all `Resources`. + +GHC Runner as Daemon +.................... + +In this mode GHC applies internal scheduling for each individual `Resource`. +This is the preferred mode as each `Resource` can have its own schedule (configurable +via Dashboard) and `cron` has dependencies on local environment. +Later versions may phase out cron-scheduling. +The **GHC Runner** can be run via the command `paver runner_daemon` or internally within +the **GHC Webapp** by setting the config variable **GHC_RUNNER_IN_WEBAPP** to True. + Build Documentation ------------------- @@ -129,11 +148,24 @@ Build Documentation Open a command line, (if needed activate your virtualenv) and move into the directory ``GeoHealthCheck/doc/``. In there, type ''make html'' plus ENTER and the documentation should be build locally. -GeoNode Resource type notes +GeoNode Resource Type Notes --------------------------- -*GeoNode* Resource is a virtual resource. It represents one GeoNode instance, but underneath it's trying autodiscovery of OWS endpoints available in that instance. Note, that OWS autodiscovery feature is optional, and you should check if GeoNode instance has this feature enabled. - -When adding *GeoNode instance* Resource, you have to enter url to instance's home page. GeoHealthCheck will construct url to target OWS endpoints listing and create relevant Resources (WMS, WFS, WMTS, OWC Resources). It will check all endpoints provided by GeoNode API, and will reject those which responded with error. - -All resources added in this way will have at least one tag, which is constructed with template: *GeoNode _hostname_*, where *_hostname_* is a host name from url provided. For example, let's assume you add GeoNode instance that is served from `demo.geonode.org`. All resources created in this way will have *GeoNode demo.geonode.org* tag. +*GeoNode* Resource is a virtual resource. +It represents one GeoNode instance, but underneath +auto-discovery is applied of OWS endpoints available +in that instance. Note, that OWS auto-discovery feature is +optional, and you should check if your GeoNode instance has this feature enabled. + +When adding *GeoNode instance* Resource, you have to enter +the url to the GN instance's home page. +GeoHealthCheck will construct the urls to target +OWS endpoints listing and create relevant Resources (WMS, WFS, WMTS, OWC Resources). +It will check all endpoints provided by the GeoNode API, and will reject +those which responded with an error. + +All resources added in this way will have at least one tag, +which is constructed with template: *GeoNode _hostname_*, where *_hostname_* +is a host name from url provided. For example, let's assume you add GeoNode +instance that is served from `demo.geonode.org`. All resources created in this way +will have *GeoNode demo.geonode.org* tag. diff --git a/docs/architecture.rst b/docs/architecture.rst index 7345950b..0da53336 100644 --- a/docs/architecture.rst +++ b/docs/architecture.rst @@ -3,7 +3,36 @@ Architecture ============ -(To be extended) +GeoHealthCheck (GHC) consists of three cooperating parts as depicted in the figure below. + +.. figure:: _static/ghc-parts.jpg + :align: center + :alt: GHC Parts + + *Figure - GHC Parts* + +The **GHC Webapp** provides the Dashboard where users configure web services +(Resources) for (scheduled) healthchecks and view the status of these checks. The **GHC Runner** +performs the actual healthchecks based on what the user configured via the GHC Webapp. +The third part is the **Database** that stores +all information like users, resources, checks, schedules, results etc. + +**GHC Webapp** is run as a standard Python (Flask) webapp. +**GHC Runner** runs as a daemon process using an internal scheduler to invoke the +actual healthchecks. + +**GHC Webapp** and **GHC Runner** can run as separate processes (preferred) or +both within the **GHC Webapp** process. This depends on a configuration option. +If **GHC_RUNNER_IN_WEBAPP** is set to True then the **GHC Runner** is started +within the **GHC Webapp**. + +A third option is to only run the **GHC Webapp** and have the **GHC Runner** scheduled +via `cron`. This was the (only) GHC option before v0.4.0 and will be phased out +as starting with v0.4.0, per Resource scheduling was introduced and `cron` support +is highly platform-dependent (e.g. hard to use with Docker-based technologies). + +Dependent on the database-type (Postgres or SQLite) the **Database** is run +within the above processes (SQLite) or as a separate process (Postgres). Core Concepts ------------- diff --git a/docs/config.rst b/docs/config.rst index c8cc93dc..d3e4935d 100644 --- a/docs/config.rst +++ b/docs/config.rst @@ -11,8 +11,6 @@ You can override these settings in ``instance/config_site.py``: - **SECRET_KEY**: secret key to set when enabling authentication. Use the output of ``paver create_secret_key`` to set this value - **GHC_RETENTION_DAYS**: the number of days to keep run history -- **GHC_RUN_FREQUENCY**: cron keyword used to indicate frequency of runs - (i.e. ``hourly``, ``daily``, ``monthly``) - **GHC_SELF_REGISTER**: allow registrations from users on the website - **GHC_NOTIFICATIONS**: turn on email notifications - **GHC_NOTIFICATIONS_VERBOSITY**: receive additional email notifications than just ``Failing`` and ``Fixed`` (default ``True``) @@ -27,6 +25,8 @@ You can override these settings in ``instance/config_site.py``: - **GHC_USER_PLUGINS**: list of Plugin classes or modules provided by user (you) - **GHC_PROBE_DEFAULTS**: Default `Probe` class to assign on "add" per Resource-type - **GHC_METADATA_CACHE_SECS**: metadata, "Capabilities Docs", cache expiry time, default 900 secs, -1 to disable +- **GHC_RUNNER_IN_WEBAPP**: should the GHC Runner Daemon be run in webapp (default: false) +- **GHC_LOG_LEVEL**: logging level: 10=DEBUG 20=INFO 30=WARN(ING) 40=ERROR 50=FATAL/CRITICAL (default: 30, WARNING) - **GHC_MAP**: default map settings - **url**: URL of TileLayer @@ -35,7 +35,6 @@ You can override these settings in ``instance/config_site.py``: - **maxzoom**: maximum zoom level - **subdomains**: available subdomains to help with parallel requests - Enabling or disabling languages ------------------------------- diff --git a/docs/install.rst b/docs/install.rst index bdf4415b..779f8b64 100644 --- a/docs/install.rst +++ b/docs/install.rst @@ -28,8 +28,8 @@ GeoHealthCheck is built on the awesome Flask micro-framework and uses `APScheduler` is used to run scheduled healthchecks. These dependencies are automatically installed (see below). ``Paver`` is used -for installation and management. ``Cron`` is used for scheduling the actual -healthchecks. +for installation and management. ``Cron`` was used for scheduling the actual +healthchecks before v0.4.0. Install ------- @@ -50,19 +50,22 @@ Install . bin/activate git clone https://github.com/geopython/GeoHealthCheck.git cd GeoHealthCheck + # install paver dependency for admin tool pip install Paver + # setup app paver setup + # create secret key to use for auth paver create_secret_key + # almost there! Customize config vi instance/config_site.py # edit: # - SQLALCHEMY_DATABASE_URI # - SECRET_KEY # from paver create_secret_key # - GHC_RETENTION_DAYS - # - GHC_RUN_FREQUENCY # - GHC_SELF_REGISTER # - GHC_NOTIFICATIONS # - GHC_NOTIFICATIONS_VERBOSITY @@ -76,7 +79,7 @@ Install # init database python GeoHealthCheck/models.py create - # start server + # start web-app python GeoHealthCheck/app.py # http://localhost:8000/ Schedule the cronjobs. diff --git a/jobs.cron b/jobs.cron index 91b7d061..a60c13fc 100644 --- a/jobs.cron +++ b/jobs.cron @@ -1,2 +1,2 @@ -@hourly /virtualenv/bin/python /path/to/GeoHealthCheck/GeoHealthCheck/models.py run +@hourly /virtualenv/bin/python /path/to/GeoHealthCheck/GeoHealthCheck/healthcheck.py run @daily /virtualenv/bin/python /path/to/GeoHealthCheck/GeoHealthCheck/models.py flush diff --git a/pavement.py b/pavement.py index ec4cdd32..3ab2f547 100644 --- a/pavement.py +++ b/pavement.py @@ -285,10 +285,16 @@ def update_translations(): @task def runner_daemon(): - """Run the HealthCheck runner scheduler""" + """Run the HealthCheck runner daemon scheduler""" sh('python GeoHealthCheck/scheduler.py') +@task +def run_healthchecks(): + """Run all HealthChecks directly""" + sh('python GeoHealthCheck/healthcheck.py') + + def sphinx_make(): """return what command Sphinx is using for make""" From 4601ad9c7e269653fefa78325e3b150527191303 Mon Sep 17 00:00:00 2001 From: Just van den Broecke Date: Fri, 23 Mar 2018 18:22:56 +0100 Subject: [PATCH 05/20] #132 further progress on runner daemon: Docker support and docs - flake8 fix --- GeoHealthCheck/app.py | 1 - 1 file changed, 1 deletion(-) diff --git a/GeoHealthCheck/app.py b/GeoHealthCheck/app.py index b4f1e9da..36908e46 100644 --- a/GeoHealthCheck/app.py +++ b/GeoHealthCheck/app.py @@ -32,7 +32,6 @@ import csv import logging import json -from datetime import datetime, timedelta from StringIO import StringIO from itertools import chain From 9053f38e375b2a9a4cf8dee7abdd979591ebb4ce Mon Sep 17 00:00:00 2001 From: Just van den Broecke Date: Sat, 24 Mar 2018 14:55:24 +0100 Subject: [PATCH 06/20] #132 make runner in webapp default, more Docker support and docs --- GeoHealthCheck/config_main.py | 2 +- GeoHealthCheck/scheduler.py | 27 ++++++++- README.md | 48 +++++++-------- docker/README.md | 42 +++++++------ docker/compose/docker-compose.postgis.yml | 6 ++ docker/compose/docker-compose.yml | 4 ++ docker/compose/ghc.env | 1 + docker/scripts/run-web.sh | 3 + docs/architecture.rst | 74 +++++++++++++++++++++-- docs/index.rst | 2 +- docs/install.rst | 2 +- pavement.py | 2 +- 12 files changed, 156 insertions(+), 57 deletions(-) diff --git a/GeoHealthCheck/config_main.py b/GeoHealthCheck/config_main.py index ba6f93e0..f8c9ba93 100644 --- a/GeoHealthCheck/config_main.py +++ b/GeoHealthCheck/config_main.py @@ -46,7 +46,7 @@ GHC_NOTIFICATIONS_EMAIL = ['you2@example.com'] GHC_SITE_TITLE = 'GeoHealthCheck Demonstration' GHC_SITE_URL = 'http://host' -GHC_RUNNER_IN_WEBAPP = False +GHC_RUNNER_IN_WEBAPP = True # 10=DEBUG 20=INFO 30=WARN(ING) 40=ERROR 50=FATAL/CRITICAL GHC_LOG_LEVEL = 30 GHC_LOG_FORMAT = '%(asctime)s - %(name)s - %(levelname)s - %(message)s' diff --git a/GeoHealthCheck/scheduler.py b/GeoHealthCheck/scheduler.py index cf2369d9..87081203 100644 --- a/GeoHealthCheck/scheduler.py +++ b/GeoHealthCheck/scheduler.py @@ -59,6 +59,26 @@ def db_commit(): def run_job(resource_id, frequency): + """ + Runs single job (all Probes) for single Resource. + As multiple instances of the job scheduler may run in different + processes and threads, the database is used to synchronize and assure + only one job will run. This is achieved by having one lock per Resource. + Only the process/thread that acquires its related ResourceLock record runs the job. + As to avoid permanent "lockouts", each ResourceLock has a lifetime, namely + the timespan until the next Run as configured for/per Resource. This gives + all job runners a chance to obtain a lock once "time's up" for the ResourceLock. + An extra check for lock obtainment is made via an unique UUID per job runner. + Once the lock is obtained the UUID-field of the lock record is set and committed + to the DB. If we then try to obtain the lock again (by reading from DB) + but the UUID is different this means another job runner instance did the same but + was just before us. The lock timespan will guard that a particular UUID will keep + the lock forever, e.g. if the application is suddenly shutdown. + + :param resource_id: + :param frequency: + :return: + """ # Generate unique id # https://stackoverflow.com/questions/2257441/random-string-generation-with-upper-case-letters-and-digits-in-python uuid = '%d-%s' % (os.getpid(), ''.join(random.choice( @@ -71,7 +91,7 @@ def run_job(resource_id, frequency): stop_job(resource_id) return - # Resource exists: try to obtain lock. + # Resource exists: try to obtain our Resource Lock record. lock = ResourceLock.query.filter_by(identifier=resource_id).first() if not lock: @@ -88,7 +108,7 @@ def run_job(resource_id, frequency): (resource_id, str(lock_err))) return else: - # Lock is there, look if available + # Lock record found: check if available for our UUID. LOGGER.info('%d Lock avail: try obtaining..' % resource_id) if not lock.obtain(uuid, frequency): LOGGER.info('%d Cannot obtain' % resource_id) @@ -167,7 +187,8 @@ def check_schedule(): job = get_job(resource) if job is None: add_job(resource) - + continue + current_freq = job.args[1] # Run frequency changed? diff --git a/README.md b/README.md index 933bc44a..8891d525 100644 --- a/README.md +++ b/README.md @@ -6,11 +6,11 @@ GeoHealthCheck ============== -GeoHealthCheck (GHC) is a Service Status Checker for OGC Web Services and web APIs in general. +GeoHealthCheck (GHC) is a Service Status and QoS Checker for OGC Web Services and web APIs in general. See also the [full GHC documentation](http://docs.geohealthcheck.org/). Easiest is [to run GHC using Docker](https://github.com/geopython/GeoHealthCheck/blob/master/docker/README.md). -Below a quick overview of a manual install on Unix-based systems. +Below a quick overview of a manual install on Unix-based systems like Apple Mac and Linux. ```bash virtualenv GeoHealthCheck && cd $_ @@ -22,33 +22,39 @@ pip install Paver paver setup # generate secret key paver create_secret_key -# setup local configuration +# setup local configuration (overrides GeoHealthCheck/config_main.py) vi instance/config_site.py -# edit: +# edit at least secret key: +# - SECRET_KEY # copy/paste result string from paver create_secret_key + +# Optional: edit other settings or leave defaults # - SQLALCHEMY_DATABASE_URI -# - SECRET_KEY # from paver create_secret_key # - GHC_RETENTION_DAYS # - GHC_SELF_REGISTER +# - GHC_RUNNER_IN_WEBAPP # - GHC_ADMIN_EMAIL # - GHC_SITE_TITLE # - GHC_MAP (or use default settings) -# setup database and superuser account +# setup database and superuser account interactively paver create -# setup cronjobs -vi jobs.cron -# edit paths to scripts -# enable cron -crontab jobs.cron - -# start server (default is 0.0.0.0:8000) +# start webserver with healthcheck runner daemon inside +# (default is 0.0.0.0:8000) python GeoHealthCheck/app.py -# start server on another port +# or start webserver on another port python GeoHealthCheck/app.py 0.0.0.0:8881 -# start server on another IP +# or start webserver on another IP python GeoHealthCheck/app.py 192.168.0.105:8001 +# OR start webserver and separate runner daemon (scheduler) process +vi instance/config_site.py +# GHC_RUNNER_IN_WEBAPP = False +python GeoHealthCheck/scheduler.py & +python GeoHealthCheck/app.py + +# next: use a real webserver or preferably Docker for production + # other commands # # drop database @@ -60,14 +66,4 @@ python GeoHealthCheck/models.py load <.json data file> [y/n] ``` -users -- view all services -- view service - -admin -- drop db -- create db -- upgrade db -- add service -- delete service -- run health check (cron or interactive) +More in the [full GHC documentation](http://docs.geohealthcheck.org/). \ No newline at end of file diff --git a/docker/README.md b/docker/README.md index 246be048..a1b0977f 100644 --- a/docker/README.md +++ b/docker/README.md @@ -48,26 +48,28 @@ docker run -d --name ghc_web -p 8083:80 -v ghc_sqlitedb:/GeoHealthCheck/DB geopy go to http://localhost:8083 (port 80 in GHC Container is mapped to 8083 on host). -NB this runs GHC standalone with a `SQLite` DB, but without the GHC Runner that performs the -healthchecks. This can be done by setting `GHC_RUNNER_IN_WEBAPP`: +NB this runs GHC standalone with a `SQLite` DB and with the GHC Runner that performs the +healthchecks. But this may in cases not be optimal as the `GHC Webapp` may get overloaded +from processing by the `GHC Runner`. Tip: to see detailed logging add `-e GHC_LOG_LEVEL=10 `. + +This mode can be disabled by passing `GHC_RUNNER_IN_WEBAPP` to as an ENV +var to the Docker container: ``` -docker run --name ghc_web -e GHC_RUNNER_IN_WEBAPP=True -p 8083:80 -v ghc_sqlitedb:/GeoHealthCheck/DB geopython/geohealthcheck:latest +docker run --name ghc_web -e GHC_RUNNER_IN_WEBAPP=False -p 8083:80 -v ghc_sqlitedb:/GeoHealthCheck/DB geopython/geohealthcheck:latest ``` -This allows both the `GHC Webapp` (Dashboard)and `GHC Runner` within a single Docker container. -But this may in cases not be optimal as the `GHC Webapp` may get overloaded -from processing by the `GHC Runner`. To see logging add `-e GHC_LOG_LEVEL=10 `. +This allows both the `GHC Webapp` (Dashboard)and `GHC Runner` within a separate Docker containers. -You can also run `GHC Webapp` (Dashboard) and `GHC Runner` as separate containers by overriding +You can then run `GHC Runner` as a separate container by overriding the default `ENTRYPOINT` with `/run-runner.sh`: ``` docker run -d --name ghc_runner --entrypoint "/run-runner.sh" -v ghc_sqlitedb:/GeoHealthCheck/DB geopython/geohealthcheck:latest ``` -The most optimal way to run GHC with scheduled jobs and optionally Postgres as backend DB, +But the most optimal way to run GHC with scheduled jobs and optionally Postgres as backend DB, is to use [Docker Compose](https://docs.docker.com/compose), see below. ## Using docker-compose @@ -75,7 +77,8 @@ is to use [Docker Compose](https://docs.docker.com/compose), see below. This allows a complete Docker setup, including scheduling and optionally using Postgres/PostGIS as database (recommended). See the [Docker Compose Documentation](https://docs.docker.com/compose) -for more info. +for more info. GHC Webapp and Runner are in this case +deployed as separate processes (Docker containers). *Note that the `docker-compose` YAML files below are meant as examples to be adapted to your* *local deployment situation.* @@ -98,7 +101,7 @@ docker-compose -f docker-compose.yml up [-d] ### Using PostGIS DB The file [docker-compose.postgis.yml](compose/docker-compose.postgis.yml) is -similar but uses Postgres with PostGIS as database. +similar but uses Postgres as the database. To run: @@ -144,21 +147,23 @@ the Docker host IP address on your Docker host as follows: ``` -## Cronjobs +## Cron jobs -Cronjobs via `docker-compose` are since v0.4.0 no longer scheduled via Jobber (cron) +DEPRECATED: Cronjobs via `docker-compose` are since v0.4.0 no longer scheduled via Jobber (cron) but within GHC itself. The GHC Docker Image can run as a Container in a daemon runner role that executes all GHC scheduled runs from the DB directly. +But with the per-Resource scheduling introduced in v0.4.0, cron-jobs +cannot support the detailed scheduling required. ## Configuration The default GHC configuration is specified within the [Dockerfile](../Dockerfile). This allows overriding these `ENV` vars during deployment (as opposed to having to build -your own GHC Docker Image). Docker (`-e` options) and Docker Compose (`environment` section) -allow setting Environment variables. +your own GHC Docker Image). Docker (`-e` options) and Docker Compose (`environment` and/or `env_file` +sections) allow setting Environment variables. -But more flexible is to use `.env` files. +Here we use `.env` files. ``` ghc_runner: @@ -216,11 +221,10 @@ version: Here [ghc-postgis.env](compose/ghc-postgis.env) adds extra Postgres-related config settings. - ## Other tasks -You can always `bash` into the GHC container to run maintenance tasks. -The GHC installation is at `/GeoHealthCheck`. +You can always `bash` into the GHC Container to run maintenance tasks. +The GHC installation is at `/GeoHealthCheck` within the Docker Container. ``` @@ -237,3 +241,5 @@ paver upgrade etc ``` +NB: database upgrades (`paver upgrade`) +are always performed automatically when running GHC via Docker. diff --git a/docker/compose/docker-compose.postgis.yml b/docker/compose/docker-compose.postgis.yml index e560558c..c859b38b 100644 --- a/docker/compose/docker-compose.postgis.yml +++ b/docker/compose/docker-compose.postgis.yml @@ -11,6 +11,8 @@ services: container_name: ghc_web + restart: unless-stopped + env_file: - ghc.env - ghc-postgis.env @@ -37,6 +39,8 @@ services: container_name: ghc_runner + restart: unless-stopped + env_file: - ghc.env - ghc-postgis.env @@ -63,6 +67,8 @@ services: container_name: postgis_ghc + restart: unless-stopped + env_file: - ghc-postgis.env diff --git a/docker/compose/docker-compose.yml b/docker/compose/docker-compose.yml index a9224587..ad7f6e34 100644 --- a/docker/compose/docker-compose.yml +++ b/docker/compose/docker-compose.yml @@ -12,6 +12,8 @@ services: container_name: ghc_web + restart: unless-stopped + ports: - 8083:80 @@ -32,6 +34,8 @@ services: container_name: ghc_runner + restart: unless-stopped + env_file: - ghc.env diff --git a/docker/compose/ghc.env b/docker/compose/ghc.env index 985458db..03bf2554 100644 --- a/docker/compose/ghc.env +++ b/docker/compose/ghc.env @@ -1,6 +1,7 @@ SQLALCHEMY_DATABASE_URI=sqlite:////GeoHealthCheck/DB/data.db # Core variables settings, change at will. +GHC_RUNNER_IN_WEBAPP=false GHC_NOTIFICATIONS=true GHC_NOTIFICATIONS_VERBOSITY=true GHC_ADMIN_EMAIL=us@gmail.com diff --git a/docker/scripts/run-web.sh b/docker/scripts/run-web.sh index 10a98519..74f496fd 100755 --- a/docker/scripts/run-web.sh +++ b/docker/scripts/run-web.sh @@ -14,6 +14,9 @@ source /venv/bin/activate . echo "Running GHC WSGI on ${HOST}:${PORT} with ${WSGI_WORKERS} workers" cd /GeoHealthCheck + +paver upgrade + gunicorn --workers ${WSGI_WORKERS} \ --worker-class=${WSGI_WORKER_CLASS} \ --timeout ${WSGI_WORKER_TIMEOUT} \ diff --git a/docs/architecture.rst b/docs/architecture.rst index 0da53336..28645533 100644 --- a/docs/architecture.rst +++ b/docs/architecture.rst @@ -12,28 +12,34 @@ GeoHealthCheck (GHC) consists of three cooperating parts as depicted in the figu *Figure - GHC Parts* The **GHC Webapp** provides the Dashboard where users configure web services -(Resources) for (scheduled) healthchecks and view the status of these checks. The **GHC Runner** -performs the actual healthchecks based on what the user configured via the GHC Webapp. +(Resources) for (scheduled) health-checks and view the status of these checks. +The **GHC Runner** performs the actual health-checks and notifications, +based on what the user configured via the GHC Webapp. + The third part is the **Database** that stores all information like users, resources, checks, schedules, results etc. -**GHC Webapp** is run as a standard Python (Flask) webapp. -**GHC Runner** runs as a daemon process using an internal scheduler to invoke the +The **GHC Webapp** is run as a standard Python (Flask) webapp. +The **GHC Runner** runs as a daemon process using an internal scheduler to invoke the actual healthchecks. **GHC Webapp** and **GHC Runner** can run as separate processes (preferred) or both within the **GHC Webapp** process. This depends on a configuration option. -If **GHC_RUNNER_IN_WEBAPP** is set to True then the **GHC Runner** is started +If **GHC_RUNNER_IN_WEBAPP** is set to True (the default) then the **GHC Runner** is started within the **GHC Webapp**. A third option is to only run the **GHC Webapp** and have the **GHC Runner** scheduled via `cron`. This was the (only) GHC option before v0.4.0 and will be phased out -as starting with v0.4.0, per Resource scheduling was introduced and `cron` support +as starting with v0.4.0, per-Resource scheduling was introduced and `cron` support is highly platform-dependent (e.g. hard to use with Docker-based technologies). Dependent on the database-type (Postgres or SQLite) the **Database** is run within the above processes (SQLite) or as a separate process (Postgres). +So in the most minimal setup, i.e. **GHC Webapp** and **GHC Runner** running within +a single process and using SQLite as the database, only a single GHC process instance +is required. + Core Concepts ------------- @@ -62,3 +68,59 @@ Data Model :alt: GHC Data Model *Figure - GHC Data Model* + +GHC Webapp Design +----------------- + +The **GHC Webapp** is realized as a standard `Flask` web-application using `SQLAlchemy` for +ORM database support. It is the user-visible part of GHC as it runs via the +browser. Its main two functionalities are to allow users to: + +* manage (create, update, delete) Resources, their attributes and their Probes and Checks, and +* view results and statistics of Resources (Dashboard function) + +Deployment can be realized using the various standard Flask deployment methods: standalone, +within a WSGI server etc. + +As an option (via configuration, see above) the **GHC Runner** may run within the **GHC Webapp**. Note that in case +that when the **GHC Webapp** runs as multiple processes and/or threads "Resource Locking" (see below) +will prevent inconsistencies. + +GHC Runner Design +----------------- + +The **GHC Runner** in its core is a job scheduler based on the Python library +`APScheduler `_. Each job scheduled is a +healthcheck runner for a single `Resource` that runs all the `Probes` for that `Resource`. +The run-frequency follows the per-Resource run frequency (since v0.4.0). + +The GHC Runner is thus responsible for running the `Probes` for each `Resource`, storing +the `Results` and doing notifications when needed. + +The **GHC Runner** can run as a separate (Python) process, or within the **GHC WebApp** (see above). +Separate processes is the preferred mode of running. + +Job Runner Synchronization +.......................... + +As multiple instances of the job scheduler (i.e. `APScheduler `_) +may run in different processes and even threads within processes, +the database is used to synchronize and assure only one job will run for a single `Resource`. + +This is achieved by having one lock per `Resource` via the table `ResourceLock`. +Only the process/thread that acquires its related `ResourceLock` +record runs the job. +As to avoid permanent "lockouts", each `ResourceLock` has a lifetime, namely +the timespan until the next Run as configured for/per `Resource`. This gives +all job runners a chance to obtain a lock once "time's up" for the `ResourceLock`. + +Additional lock-liveliness is realized by using a unique **UUID per job runner**. +Once the lock is obtained, the UUID-field of the lock record is set and committed +to the DB. If we then try to obtain the lock again (by reading from DB) +but the UUID is different this means another job runner instance did the same but +was just before us. The lock-lifetime (see above) guards that a particular UUID keeps +the lock forever, e.g. on sudden application shutdown. + +The above locking mechanism is supported for SQLite, but it is strongly +advised to use PostgreSQL for better robustness and performance in general. + diff --git a/docs/index.rst b/docs/index.rst index f15be868..df1de561 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -21,7 +21,7 @@ Features - database agnostic: any SQLAlchemy supported backend - database upgrades: using Alembic with Flask-Migrate - extensible healthchecks via Plugins -- per-resource notifications +- per-resource scheduling and notifications Links ----- diff --git a/docs/install.rst b/docs/install.rst index 779f8b64..f60c0a36 100644 --- a/docs/install.rst +++ b/docs/install.rst @@ -8,7 +8,7 @@ Below are installation notes for GeoHealthCheck (GHC). Docker ------ -Easiest and quickest install/run +The easiest and quickest install for GHC is with Docker/Docker Compose using the GHC images hosted on `Docker Hub `_. diff --git a/pavement.py b/pavement.py index 3ab2f547..8f14bff2 100644 --- a/pavement.py +++ b/pavement.py @@ -242,7 +242,7 @@ def clean(): @task def extract_translations(): - """extrect translations wrapped in _() or gettext()""" + """extract translations wrapped in _() or gettext()""" pot_dir = path('GeoHealthCheck/translations/en/LC_MESSAGES') if not os.path.exists(pot_dir): From af665fbfe0dde1979b98f9fce6e7a7839ed3179a Mon Sep 17 00:00:00 2001 From: Just van den Broecke Date: Sat, 24 Mar 2018 15:12:15 +0100 Subject: [PATCH 07/20] #132 make runner in webapp default, more Docker support and docs - flake8 --- GeoHealthCheck/scheduler.py | 26 ++++++++++++++------------ 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/GeoHealthCheck/scheduler.py b/GeoHealthCheck/scheduler.py index 87081203..37fb170b 100644 --- a/GeoHealthCheck/scheduler.py +++ b/GeoHealthCheck/scheduler.py @@ -64,16 +64,19 @@ def run_job(resource_id, frequency): As multiple instances of the job scheduler may run in different processes and threads, the database is used to synchronize and assure only one job will run. This is achieved by having one lock per Resource. - Only the process/thread that acquires its related ResourceLock record runs the job. - As to avoid permanent "lockouts", each ResourceLock has a lifetime, namely - the timespan until the next Run as configured for/per Resource. This gives - all job runners a chance to obtain a lock once "time's up" for the ResourceLock. - An extra check for lock obtainment is made via an unique UUID per job runner. - Once the lock is obtained the UUID-field of the lock record is set and committed - to the DB. If we then try to obtain the lock again (by reading from DB) - but the UUID is different this means another job runner instance did the same but - was just before us. The lock timespan will guard that a particular UUID will keep - the lock forever, e.g. if the application is suddenly shutdown. + Only the process/thread that acquires its related ResourceLock record + runs the job. As to avoid permanent "lockouts", each ResourceLock has + a lifetime, namely the timespan until the next Run as configured for/per + Resource. This gives all job runners a chance to obtain a lock once + "time's up" for the ResourceLock. + + An extra check for lock obtainment is made via an unique UUID per job + runner. Once the lock is obtained the UUID-field of the lock record + is set and committed to the DB. If we then try to obtain the lock again + (by reading from DB) but the UUID is different this means another job + runner instance did the same but was just before us. The lock timespan + will guard that a particular UUID will keep the lock forever, e.g. if + the application is suddenly shutdown. :param resource_id: :param frequency: @@ -153,7 +156,6 @@ def run_job(resource_id, frequency): def start_schedule(): - LOGGER.info('Starting scheduler') # Adapt configuration @@ -188,7 +190,7 @@ def check_schedule(): if job is None: add_job(resource) continue - + current_freq = job.args[1] # Run frequency changed? From eca4ef6661b29ffb99750eb7b0a973db9ee893c6 Mon Sep 17 00:00:00 2001 From: Just van den Broecke Date: Tue, 29 May 2018 16:16:21 +0200 Subject: [PATCH 08/20] #132 fix Alembic upgrade error server_default with quotes --- GeoHealthCheck/migrations/versions/34531bfd7cab_.py | 2 +- docker/compose/ghc.env | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/GeoHealthCheck/migrations/versions/34531bfd7cab_.py b/GeoHealthCheck/migrations/versions/34531bfd7cab_.py index 71d76b52..f7896ad8 100644 --- a/GeoHealthCheck/migrations/versions/34531bfd7cab_.py +++ b/GeoHealthCheck/migrations/versions/34531bfd7cab_.py @@ -20,7 +20,7 @@ def upgrade(): if not alembic_helpers.table_has_column('resource', 'run_frequency'): print('Column run_frequency not present in resource table, will create') op.add_column(u'resource', sa.Column('run_frequency', sa.Integer(), - nullable=False, default=60, server_default=60)) + nullable=False, default=60, server_default='60')) else: print('Column run_frequency already present in resource table') diff --git a/docker/compose/ghc.env b/docker/compose/ghc.env index 03bf2554..20c3bb43 100644 --- a/docker/compose/ghc.env +++ b/docker/compose/ghc.env @@ -1,15 +1,15 @@ SQLALCHEMY_DATABASE_URI=sqlite:////GeoHealthCheck/DB/data.db # Core variables settings, change at will. -GHC_RUNNER_IN_WEBAPP=false -GHC_NOTIFICATIONS=true -GHC_NOTIFICATIONS_VERBOSITY=true +GHC_RUNNER_IN_WEBAPP=False +GHC_NOTIFICATIONS=True +GHC_NOTIFICATIONS_VERBOSITY=True GHC_ADMIN_EMAIL=us@gmail.com GHC_NOTIFICATIONS_EMAIL=us@gmail.com,them@domain.com GHC_SMTP_SERVER=smtp.gmail.com GHC_SMTP_PORT=587 -GHC_SMTP_TLS=true -GHC_SMTP_SSL=false +GHC_SMTP_TLS=True +GHC_SMTP_SSL=False GHC_SMTP_USERNAME=us@gmail.com GHC_SMTP_PASSWORD=the_passw GHC_LOG_LEVEL=20 From 60aa6a6f950275f794bb9efda1e3b095cf82b7cc Mon Sep 17 00:00:00 2001 From: Just van den Broecke Date: Tue, 29 May 2018 19:08:39 +0200 Subject: [PATCH 09/20] #132 fix job for flushing runs (retention days format) --- GeoHealthCheck/models.py | 23 ++++++++--------------- 1 file changed, 8 insertions(+), 15 deletions(-) diff --git a/GeoHealthCheck/models.py b/GeoHealthCheck/models.py index 0b751a3f..f413965a 100644 --- a/GeoHealthCheck/models.py +++ b/GeoHealthCheck/models.py @@ -50,18 +50,20 @@ # Complete handle of old runs deletion def flush_runs(): APP = App.get_app() - retention_time = timedelta(*APP.config['GHC_RETENTION_DAYS']) - + retention_days = int(APP.config['GHC_RETENTION_DAYS']) + LOGGER.info('Flushing runs older than %d days' % retention_days) all_runs = Run.query.all() + run_count = 0 for run in all_runs: - how_old = (datetime.utcnow() - run.checked_datetime) - if how_old > retention_time: + days_old = (datetime.utcnow() - run.checked_datetime).days + if days_old > retention_days: + run_count += 1 DB.session.delete(run) db_commit() + LOGGER.info('Deleted %d Runs' % run_count) DB.session.remove() - class Run(DB.Model): """measurement of resource state""" @@ -824,15 +826,6 @@ def db_commit(): print('NOTICE: models.py no longer here.') print('Use: python healthcheck.py or upcoming cli.py') elif sys.argv[1] == 'flush': - retention_days = int(APP.config['GHC_RETENTION_DAYS']) - print('Flushing runs older than %d days' % - retention_days) - all_runs = Run.query.all() - for run in all_runs: - days_old = (datetime.utcnow() - run.checked_datetime).days - if days_old > retention_days: - print('Run older than %d days. Deleting' % days_old) - DB.session.delete(run) - db_commit() + flush_runs() DB.session.remove() From 982fd98d2e87a282349d9b72c5fd238551fb941b Mon Sep 17 00:00:00 2001 From: Just van den Broecke Date: Tue, 29 May 2018 21:53:35 +0200 Subject: [PATCH 10/20] #132 fix conveting bool env vars in config_site.py for Docker --- GeoHealthCheck/models.py | 1 + docker/config_site.py | 15 ++++++++++----- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/GeoHealthCheck/models.py b/GeoHealthCheck/models.py index f413965a..ca224933 100644 --- a/GeoHealthCheck/models.py +++ b/GeoHealthCheck/models.py @@ -64,6 +64,7 @@ def flush_runs(): DB.session.remove() + class Run(DB.Model): """measurement of resource state""" diff --git a/docker/config_site.py b/docker/config_site.py index b6de1f8b..0cfff5b2 100644 --- a/docker/config_site.py +++ b/docker/config_site.py @@ -28,6 +28,11 @@ # ================================================================= import os + +def str2bool(v): + return v.lower() in ("yes", "true", "t", "1") + + DEBUG = False SQLALCHEMY_ECHO = False @@ -40,15 +45,15 @@ SECRET_KEY = os.environ['SECRET_KEY'] GHC_RETENTION_DAYS = int(os.environ['GHC_RETENTION_DAYS']) -GHC_SELF_REGISTER = os.environ['GHC_SELF_REGISTER'] -GHC_NOTIFICATIONS = os.environ['GHC_NOTIFICATIONS'] -GHC_NOTIFICATIONS_VERBOSITY = os.environ['GHC_NOTIFICATIONS_VERBOSITY'] -GHC_WWW_LINK_EXCEPTION_CHECK = os.environ['GHC_WWW_LINK_EXCEPTION_CHECK'] +GHC_SELF_REGISTER = str2bool(os.environ['GHC_SELF_REGISTER']) +GHC_NOTIFICATIONS = str2bool(os.environ['GHC_NOTIFICATIONS']) +GHC_NOTIFICATIONS_VERBOSITY = str2bool(os.environ['GHC_NOTIFICATIONS_VERBOSITY']) +GHC_WWW_LINK_EXCEPTION_CHECK = str2bool(os.environ['GHC_WWW_LINK_EXCEPTION_CHECK']) GHC_NOTIFICATIONS_EMAIL = os.environ['GHC_NOTIFICATIONS_EMAIL'] GHC_ADMIN_EMAIL = os.environ['GHC_ADMIN_EMAIL'] GHC_SITE_TITLE = os.environ['GHC_SITE_TITLE'] GHC_SITE_URL = os.environ['GHC_SITE_URL'] -GHC_RUNNER_IN_WEBAPP = bool(os.environ['GHC_RUNNER_IN_WEBAPP']) +GHC_RUNNER_IN_WEBAPP = str2bool(os.environ['GHC_RUNNER_IN_WEBAPP']) GHC_LOG_LEVEL = int(os.environ['GHC_LOG_LEVEL']) GHC_LOG_FORMAT = os.environ['GHC_LOG_FORMAT'] From 5a3f24d41f030432576cf0f8477b99fc9270aa46 Mon Sep 17 00:00:00 2001 From: Just van den Broecke Date: Sat, 2 Jun 2018 12:42:50 +0200 Subject: [PATCH 11/20] #132 extend end-time for ResourceLock to prevent missing obtainments --- GeoHealthCheck/models.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/GeoHealthCheck/models.py b/GeoHealthCheck/models.py index ca224933..2f88a0ed 100644 --- a/GeoHealthCheck/models.py +++ b/GeoHealthCheck/models.py @@ -570,7 +570,9 @@ def __init__(self, resource, owner, interval_mins): def init_datetimes(self, interval_mins): self.start_time = datetime.utcnow() - self.end_time = self.start_time + timedelta(minutes=interval_mins) + # Add some space to end-time to prevent expiry + minutes = interval_mins + 3 + self.end_time = self.start_time + timedelta(minutes) def has_expired(self): now = datetime.utcnow() From 080547d0b9ff2d194b51bbe881d00975107fc418 Mon Sep 17 00:00:00 2001 From: Just van den Broecke Date: Sat, 2 Jun 2018 14:34:12 +0200 Subject: [PATCH 12/20] #132 less end-time for ResourceLock to prevent missing obtainments --- GeoHealthCheck/models.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/GeoHealthCheck/models.py b/GeoHealthCheck/models.py index 2f88a0ed..327f14b9 100644 --- a/GeoHealthCheck/models.py +++ b/GeoHealthCheck/models.py @@ -570,8 +570,8 @@ def __init__(self, resource, owner, interval_mins): def init_datetimes(self, interval_mins): self.start_time = datetime.utcnow() - # Add some space to end-time to prevent expiry - minutes = interval_mins + 3 + # Subtract some space from end-time to allow obtain at scheduled time + minutes = interval_mins - 1 self.end_time = self.start_time + timedelta(minutes) def has_expired(self): From d629897a84d9bb2f81c21c94e3674e30bc5d6688 Mon Sep 17 00:00:00 2001 From: Just van den Broecke Date: Sun, 3 Jun 2018 07:36:51 +0200 Subject: [PATCH 13/20] #132 less end-time for ResourceLock to prevent missing obtainments --- GeoHealthCheck/models.py | 2 +- GeoHealthCheck/scheduler.py | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/GeoHealthCheck/models.py b/GeoHealthCheck/models.py index 327f14b9..95cbd0bb 100644 --- a/GeoHealthCheck/models.py +++ b/GeoHealthCheck/models.py @@ -572,7 +572,7 @@ def init_datetimes(self, interval_mins): self.start_time = datetime.utcnow() # Subtract some space from end-time to allow obtain at scheduled time minutes = interval_mins - 1 - self.end_time = self.start_time + timedelta(minutes) + self.end_time = self.start_time + timedelta(minutes=minutes) def has_expired(self): now = datetime.utcnow() diff --git a/GeoHealthCheck/scheduler.py b/GeoHealthCheck/scheduler.py index 37fb170b..f5dcd4b4 100644 --- a/GeoHealthCheck/scheduler.py +++ b/GeoHealthCheck/scheduler.py @@ -107,14 +107,14 @@ def run_job(resource_id, frequency): lock_err = db_commit() if lock_err: # Another process may have been there first! - LOGGER.info('%d Cannot obtain Lock %s' % + LOGGER.info('%d Error obtaining Lock %s' % (resource_id, str(lock_err))) return else: - # Lock record found: check if available for our UUID. - LOGGER.info('%d Lock avail: try obtaining..' % resource_id) + # Lock record found for Resource: check if available for our UUID. + LOGGER.info('%d Lock present: try obtaining..' % resource_id) if not lock.obtain(uuid, frequency): - LOGGER.info('%d Cannot obtain' % resource_id) + LOGGER.info('%d Cannot obtain lock' % resource_id) return else: LOGGER.info('%d Lock obtained, delete and renew' % resource_id) From 65e85d34e27ea0f8c15f7b861dbf55b2bb03b71f Mon Sep 17 00:00:00 2001 From: Just van den Broecke Date: Mon, 29 Oct 2018 13:45:03 +0100 Subject: [PATCH 14/20] SCRIPT_NAME must not be empty string or only slash --- docker/scripts/configure.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docker/scripts/configure.sh b/docker/scripts/configure.sh index b0adb5c2..26c2d231 100755 --- a/docker/scripts/configure.sh +++ b/docker/scripts/configure.sh @@ -70,4 +70,8 @@ then cp -ar /plugins/* /GeoHealthCheck/GeoHealthCheck/plugins/ fi +# SCRIPT_NAME if set should not be empty string or have value '/' +[ "${SCRIPT_NAME}" = '/' ] && SCRIPT_NAME="" && echo "make SCRIPT_NAME empty from /" +[ -z "${SCRIPT_NAME}" ] && unset ${SCRIPT_NAME} && echo "unset empty SCRIPT_NAME" + echo "END /configure.sh" From c4789174b87ee2e6287f67e5615df7676a8fb7c9 Mon Sep 17 00:00:00 2001 From: Just van den Broecke Date: Mon, 29 Oct 2018 15:14:56 +0100 Subject: [PATCH 15/20] SCRIPT_NAME may not be slash and minimal run frequency --- Dockerfile | 1 + GeoHealthCheck/config_main.py | 1 + GeoHealthCheck/templates/edit_resource.html | 2 +- GeoHealthCheck/templates/layout.html | 1 + docker/config_site.py | 1 + docker/scripts/configure.sh | 4 ---- docker/scripts/run-web.sh | 5 ++++- docs/config.rst | 1 + 8 files changed, 10 insertions(+), 6 deletions(-) diff --git a/Dockerfile b/Dockerfile index a1fd5332..bd04bef0 100644 --- a/Dockerfile +++ b/Dockerfile @@ -23,6 +23,7 @@ ENV LC_ALL="en_US.UTF-8" \ SQLALCHEMY_DATABASE_URI='sqlite:////GeoHealthCheck/DB/data.db' \ SECRET_KEY='d544ccc37dc3ad214c09b1b7faaa64c60351d5c8bb48b342' \ GHC_PROBE_HTTP_TIMEOUT_SECS=30 \ + GHC_MINIMAL_RUN_FREQUENCY_MINS=10 \ GHC_RETENTION_DAYS=30 \ GHC_SELF_REGISTER=False \ GHC_NOTIFICATIONS=False \ diff --git a/GeoHealthCheck/config_main.py b/GeoHealthCheck/config_main.py index a189bcb5..37b4b092 100644 --- a/GeoHealthCheck/config_main.py +++ b/GeoHealthCheck/config_main.py @@ -39,6 +39,7 @@ GHC_RETENTION_DAYS = 30 GHC_PROBE_HTTP_TIMEOUT_SECS = 30 +GHC_MINIMAL_RUN_FREQUENCY_MINS = 10 GHC_SELF_REGISTER = False GHC_NOTIFICATIONS = False GHC_NOTIFICATIONS_VERBOSITY = True diff --git a/GeoHealthCheck/templates/edit_resource.html b/GeoHealthCheck/templates/edit_resource.html index bb052d4e..1569a0fd 100644 --- a/GeoHealthCheck/templates/edit_resource.html +++ b/GeoHealthCheck/templates/edit_resource.html @@ -91,7 +91,7 @@

[{{ _('Edit') }}] {{ resour {{ _('Run Every') }} - + diff --git a/GeoHealthCheck/templates/layout.html b/GeoHealthCheck/templates/layout.html index 2417b3d9..c3bf0b21 100644 --- a/GeoHealthCheck/templates/layout.html +++ b/GeoHealthCheck/templates/layout.html @@ -108,6 +108,7 @@
  • {{ _('History') }}: {{ config['GHC_RETENTION_DAYS'] }} {{ _('days') }}
  • {{ _('Runner in Webapp') }}: {{ config['GHC_RUNNER_IN_WEBAPP'] }}
  • {{ _('Probe Timeout') }}: {{ config['GHC_PROBE_HTTP_TIMEOUT_SECS'] }} seconds
    • +
  • {{ _('Minimal Run Freq') }}: {{ config['GHC_MINIMAL_RUN_FREQUENCY_MINS'] }} minutes
    • diff --git a/docker/config_site.py b/docker/config_site.py index 96cabdab..67c813cc 100644 --- a/docker/config_site.py +++ b/docker/config_site.py @@ -46,6 +46,7 @@ def str2bool(v): GHC_RETENTION_DAYS = int(os.environ['GHC_RETENTION_DAYS']) GHC_PROBE_HTTP_TIMEOUT_SECS = int(os.environ['GHC_PROBE_HTTP_TIMEOUT_SECS']) +GHC_MINIMAL_RUN_FREQUENCY_MINS = int(os.environ['GHC_MINIMAL_RUN_FREQUENCY_MINS']) GHC_SELF_REGISTER = str2bool(os.environ['GHC_SELF_REGISTER']) GHC_NOTIFICATIONS = str2bool(os.environ['GHC_NOTIFICATIONS']) GHC_NOTIFICATIONS_VERBOSITY = str2bool(os.environ['GHC_NOTIFICATIONS_VERBOSITY']) diff --git a/docker/scripts/configure.sh b/docker/scripts/configure.sh index 26c2d231..b0adb5c2 100755 --- a/docker/scripts/configure.sh +++ b/docker/scripts/configure.sh @@ -70,8 +70,4 @@ then cp -ar /plugins/* /GeoHealthCheck/GeoHealthCheck/plugins/ fi -# SCRIPT_NAME if set should not be empty string or have value '/' -[ "${SCRIPT_NAME}" = '/' ] && SCRIPT_NAME="" && echo "make SCRIPT_NAME empty from /" -[ -z "${SCRIPT_NAME}" ] && unset ${SCRIPT_NAME} && echo "unset empty SCRIPT_NAME" - echo "END /configure.sh" diff --git a/docker/scripts/run-web.sh b/docker/scripts/run-web.sh index 74f496fd..04153b38 100755 --- a/docker/scripts/run-web.sh +++ b/docker/scripts/run-web.sh @@ -12,11 +12,14 @@ echo "START /run-web.sh" source /venv/bin/activate . -echo "Running GHC WSGI on ${HOST}:${PORT} with ${WSGI_WORKERS} workers" cd /GeoHealthCheck paver upgrade +# SCRIPT_NAME should not have value '/' +[ "${SCRIPT_NAME}" = '/' ] && export SCRIPT_NAME="" && echo "make SCRIPT_NAME empty from /" + +echo "Running GHC WSGI on ${HOST}:${PORT} with ${WSGI_WORKERS} workers and SCRIPT_NAME=${SCRIPT_NAME}" gunicorn --workers ${WSGI_WORKERS} \ --worker-class=${WSGI_WORKER_CLASS} \ --timeout ${WSGI_WORKER_TIMEOUT} \ diff --git a/docs/config.rst b/docs/config.rst index 21a56b2c..5c2d7775 100644 --- a/docs/config.rst +++ b/docs/config.rst @@ -10,6 +10,7 @@ You can override these settings in ``instance/config_site.py``: - **SECRET_KEY**: secret key to set when enabling authentication. Use the output of ``paver create_secret_key`` to set this value - **GHC_RETENTION_DAYS**: the number of days to keep run history - **GHC_PROBE_HTTP_TIMEOUT_SECS**: stop waiting for the first byte of a probe response after the given number of seconds +- **GHC_MINIMAL_RUN_FREQUENCY_MINS**: minimal run frequency for Resource that can be set in web UI - **GHC_SELF_REGISTER**: allow registrations from users on the website - **GHC_NOTIFICATIONS**: turn on email notifications - **GHC_NOTIFICATIONS_VERBOSITY**: receive additional email notifications than just ``Failing`` and ``Fixed`` (default ``True``) From 5a7560b624006f34d78872d67506bddf4b7889cf Mon Sep 17 00:00:00 2001 From: Just van den Broecke Date: Mon, 29 Oct 2018 16:16:05 +0100 Subject: [PATCH 16/20] safeguard run_frequency lower than confiured minimum --- GeoHealthCheck/app.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/GeoHealthCheck/app.py b/GeoHealthCheck/app.py index 41c720dc..935537ff 100644 --- a/GeoHealthCheck/app.py +++ b/GeoHealthCheck/app.py @@ -642,6 +642,9 @@ def update(resource_identifier): elif getattr(resource, key) != resource_identifier_dict[key]: # Update other resource attrs, mainly 'name' setattr(resource, key, resource_identifier_dict[key]) + min_run_freq = CONFIG['GHC_MINIMAL_RUN_FREQUENCY_MINS'] + if int(resource.run_frequency) < min_run_freq: + resource.run_frequency = min_run_freq update_counter += 1 # Always update geo-IP: maybe failure on creation or From 0833831606458f6814a0227df20d002eaadc807c Mon Sep 17 00:00:00 2001 From: Just van den Broecke Date: Fri, 2 Nov 2018 10:48:41 +0100 Subject: [PATCH 17/20] some docu changes for testing DockerHub webhook --- docker/compose/README.md | 7 +++++++ docker/compose/docker-compose.postgis.yml | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) create mode 100644 docker/compose/README.md diff --git a/docker/compose/README.md b/docker/compose/README.md new file mode 100644 index 00000000..3deade94 --- /dev/null +++ b/docker/compose/README.md @@ -0,0 +1,7 @@ +# GHC with Docker Compose + +[Docker](https://www.docker.com/) is the fastest/recommended way to get GHC up and running. +[Docker Compose](https://docs.docker.com/compose/) is *a tool for defining and running multi-container Docker applications.* + +Within this directory are *examples* for running GHC using Docker compose. +You should copy and adapt these for your own deployment. diff --git a/docker/compose/docker-compose.postgis.yml b/docker/compose/docker-compose.postgis.yml index c859b38b..71a1d16b 100644 --- a/docker/compose/docker-compose.postgis.yml +++ b/docker/compose/docker-compose.postgis.yml @@ -63,7 +63,7 @@ services: # - ./../GeoHealthCheck/plugins:/plugins:ro postgis_ghc: - image: mdillon/postgis:9.6-alpine + image: mdillon/postgis:10-alpine container_name: postgis_ghc From 185e926817a61373051e6dc94e51629734bc892b Mon Sep 17 00:00:00 2001 From: Just van den Broecke Date: Sun, 4 Nov 2018 11:00:57 +0100 Subject: [PATCH 18/20] schedule first run at random time to prevent all_at_once --- GeoHealthCheck/scheduler.py | 7 ++++--- VERSION | 2 +- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/GeoHealthCheck/scheduler.py b/GeoHealthCheck/scheduler.py index f5dcd4b4..888bd2fe 100644 --- a/GeoHealthCheck/scheduler.py +++ b/GeoHealthCheck/scheduler.py @@ -33,6 +33,7 @@ import os import random import string +from datetime import datetime, timedelta from models import Resource, ResourceLock, flush_runs from healthcheck import run_resource from apscheduler.schedulers.background import BackgroundScheduler @@ -177,7 +178,7 @@ def start_schedule(): add_job(resource) # Start maintenance jobs - scheduler.add_job(flush_runs, 'interval', minutes=60) + scheduler.add_job(flush_runs, 'interval', minutes=150) scheduler.add_job(check_schedule, 'interval', minutes=5) @@ -213,10 +214,10 @@ def update_job(resource): def add_job(resource): LOGGER.info('Starting job for resource=%d' % resource.identifier) freq = resource.run_frequency - + next_run_time = datetime.now() + timedelta(minutes=random.randint(0, freq)) scheduler.add_job( run_job, 'interval', args=[resource.identifier, freq], - minutes=freq, + minutes=freq, next_run_time=next_run_time, id=str(resource.identifier)) diff --git a/VERSION b/VERSION index 9645ebec..7979370a 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.4-dev +0.5-dev From 058a88983c3a5bdc98f500375d89afe6798aa0dc Mon Sep 17 00:00:00 2001 From: Just van den Broecke Date: Sun, 4 Nov 2018 17:23:52 +0100 Subject: [PATCH 19/20] refine scheduler and job parameterization for liveliness --- GeoHealthCheck/scheduler.py | 44 ++++++++++++++++++++++++++++++++----- 1 file changed, 39 insertions(+), 5 deletions(-) diff --git a/GeoHealthCheck/scheduler.py b/GeoHealthCheck/scheduler.py index 888bd2fe..ebfa6310 100644 --- a/GeoHealthCheck/scheduler.py +++ b/GeoHealthCheck/scheduler.py @@ -38,6 +38,9 @@ from healthcheck import run_resource from apscheduler.schedulers.background import BackgroundScheduler from apscheduler.jobstores.base import JobLookupError +from apscheduler.events import \ + EVENT_SCHEDULER_STARTED, EVENT_SCHEDULER_SHUTDOWN, \ + EVENT_JOB_MISSED, EVENT_JOB_ERROR from init import App LOGGER = logging.getLogger(__name__) @@ -161,10 +164,16 @@ def start_schedule(): # Adapt configuration scheduler.configure(job_defaults={ - 'coalesce': False, - 'max_instances': 100000 + 'coalesce': True, + 'max_instances': 1, + 'misfire_grace_time': 300 }) + scheduler.add_listener(lifecycle_listener, + EVENT_SCHEDULER_STARTED | EVENT_SCHEDULER_SHUTDOWN) + scheduler.add_listener(error_listener, + EVENT_JOB_MISSED | EVENT_JOB_ERROR) + # Start APScheduler scheduler.start() import atexit @@ -200,6 +209,28 @@ def check_schedule(): update_job(resource) +def lifecycle_listener(event): + event_code = event.code + event_code_str = '' + if event_code == EVENT_SCHEDULER_STARTED: + event_code_str = 'EVENT_SCHEDULER_STARTED' + elif event_code | EVENT_SCHEDULER_SHUTDOWN: + event_code_str = 'EVENT_SCHEDULER_SHUTDOWN' + + LOGGER.info('lifecycle_listener: %s - %s' % (event_code_str, str(event))) + + +def error_listener(event): + event_code = event.code + event_code_str = '' + if event_code | EVENT_JOB_MISSED: + event_code_str = 'EVENT_JOB_MISSED' + elif event_code | EVENT_JOB_ERROR: + event_code_str = 'EVENT_JOB_ERROR' + + LOGGER.error('error_listener: %s - %s' % (event_code_str, str(event))) + + def get_job(resource): return scheduler.get_job(str(resource.identifier)) @@ -217,7 +248,8 @@ def add_job(resource): next_run_time = datetime.now() + timedelta(minutes=random.randint(0, freq)) scheduler.add_job( run_job, 'interval', args=[resource.identifier, freq], - minutes=freq, next_run_time=next_run_time, + minutes=freq, next_run_time=next_run_time, max_instances=1, + misfire_grace_time=(freq * 60) / 2, coalesce=True, id=str(resource.identifier)) @@ -234,6 +266,8 @@ def stop_job(resource_id): def stop_schedule(): LOGGER.info('Stopping Scheduler') scheduler.shutdown() + scheduler.remove_listener(lifecycle_listener) + scheduler.remove_listener(error_listener) if __name__ == '__main__': @@ -243,5 +277,5 @@ def stop_schedule(): start_schedule() while True: - print("This prints once a minute.") - time.sleep(60) # Delay for 1 minute (60 seconds). + LOGGER.info("This prints once in 5 minutes") + time.sleep(300) # Delay for 5 minute (300 seconds). From 67ca249019fab161ffb89058cd800ec38ee02828 Mon Sep 17 00:00:00 2001 From: Just van den Broecke Date: Mon, 5 Nov 2018 17:32:41 +0100 Subject: [PATCH 20/20] set VERSION to master one --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 7979370a..fc576b2f 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.5-dev +0.5.dev0