diff --git a/.travis.yml b/.travis.yml index 2cf45c9f..c17dcbf9 100644 --- a/.travis.yml +++ b/.travis.yml @@ -14,7 +14,7 @@ script: - echo -e "admin\ntest\ntest\nyou@example.com\nyou@example.com" | python GeoHealthCheck/models.py create - flake8 - python GeoHealthCheck/models.py load tests/data/fixtures.json y - - python GeoHealthCheck/models.py run + - python GeoHealthCheck/healthcheck.py - python tests/run_tests.py - cd docs && make html diff --git a/Dockerfile b/Dockerfile index eef50879..bd04bef0 100644 --- a/Dockerfile +++ b/Dockerfile @@ -4,81 +4,82 @@ FROM python:2.7.13-alpine # FROM debian:jessie # Credits to yjacolin for providing first versions -LABEL original_developer "yjacolin " -LABEL maintainer "Just van den Broecke " +LABEL original_developer="yjacolin " \ + maintainer="Just van den Broecke " # These are default values, # Override when running container via docker(-compose) # General ENV settings -ENV LC_ALL "en_US.UTF-8" -ENV LANG "en_US.UTF-8" -ENV LANGUAGE "en_US.UTF-8" - -# GHC ENV settings -ENV ADMIN_NAME admin -ENV ADMIN_PWD admin -ENV ADMIN_EMAIL admin.istrator@mydomain.com -ENV SQLALCHEMY_DATABASE_URI 'sqlite:////GeoHealthCheck/DB/data.db' -ENV SECRET_KEY 'd544ccc37dc3ad214c09b1b7faaa64c60351d5c8bb48b342' -ENV GHC_RETENTION_DAYS 30 -ENV GHC_RUN_FREQUENCY 'hourly' -ENV GHC_PROBE_HTTP_TIMEOUT_SECS 30 -ENV GHC_SELF_REGISTER False -ENV GHC_NOTIFICATIONS False -ENV GHC_NOTIFICATIONS_VERBOSITY True -ENV GHC_WWW_LINK_EXCEPTION_CHECK False -ENV GHC_ADMIN_EMAIL 'you@example.com' -ENV GHC_NOTIFICATIONS_EMAIL 'you2@example.com,them@example.com' -ENV GHC_SITE_TITLE 'GeoHealthCheck' -ENV GHC_SITE_URL 'http://localhost' -ENV GHC_SMTP_SERVER None -ENV GHC_SMTP_PORT None -ENV GHC_SMTP_TLS False -ENV GHC_SMTP_SSL False -ENV GHC_SMTP_USERNAME None -ENV GHC_SMTP_PASSWORD None -ENV GHC_METADATA_CACHE_SECS 900 - -# WSGI server settings, assumed is gunicorn -ENV HOST 0.0.0.0 -ENV PORT 80 -ENV WSGI_WORKERS 4 -ENV WSGI_WORKER_TIMEOUT 6000 -ENV WSGI_WORKER_CLASS 'eventlet' - -# GHC Core Plugins modules and/or classes, seldom needed to set: -# if not specified here or in Container environment -# all GHC built-in Plugins will be active. +ENV LC_ALL="en_US.UTF-8" \ + LANG="en_US.UTF-8" \ + LANGUAGE="en_US.UTF-8" \ + \ + \ + # GHC ENV settings\ + ADMIN_NAME=admin \ + ADMIN_PWD=admin \ + ADMIN_EMAIL=admin.istrator@mydomain.com \ + SQLALCHEMY_DATABASE_URI='sqlite:////GeoHealthCheck/DB/data.db' \ + SECRET_KEY='d544ccc37dc3ad214c09b1b7faaa64c60351d5c8bb48b342' \ + GHC_PROBE_HTTP_TIMEOUT_SECS=30 \ + GHC_MINIMAL_RUN_FREQUENCY_MINS=10 \ + GHC_RETENTION_DAYS=30 \ + GHC_SELF_REGISTER=False \ + GHC_NOTIFICATIONS=False \ + GHC_NOTIFICATIONS_VERBOSITY=True \ + GHC_WWW_LINK_EXCEPTION_CHECK=False \ + GHC_ADMIN_EMAIL='you@example.com' \ + GHC_RUNNER_IN_WEBAPP=False \ + GHC_LOG_LEVEL=30 \ + GHC_LOG_FORMAT='%(asctime)s - %(name)s - %(levelname)s - %(message)s' \ + GHC_NOTIFICATIONS_EMAIL='you2@example.com,them@example.com' \ + GHC_SITE_TITLE='GeoHealthCheck' \ + GHC_SITE_URL='http://localhost' \ + GHC_SMTP_SERVER=None \ + GHC_SMTP_PORT=None \ + GHC_SMTP_TLS=False \ + GHC_SMTP_SSL=False \ + GHC_SMTP_USERNAME=None \ + GHC_SMTP_PASSWORD=None \ + GHC_METADATA_CACHE_SECS=900 \ + \ +# WSGI server settings, assumed is gunicorn \ +HOST=0.0.0.0 \ +PORT=80 \ +WSGI_WORKERS=4 \ +WSGI_WORKER_TIMEOUT=6000 \ +WSGI_WORKER_CLASS='eventlet' \ +\ +# GHC Core Plugins modules and/or classes, seldom needed to set: \ +# if not specified here or in Container environment \ +# all GHC built-in Plugins will be active. \ #ENV GHC_PLUGINS 'GeoHealthCheck.plugins.probe.owsgetcaps,\ # GeoHealthCheck.plugins.probe.wms, ...., ...\ -# GeoHealthCheck.plugins.check.checks' - -# GHC User Plugins, best be overridden via Container environment -ENV GHC_USER_PLUGINS '' +# GeoHealthCheck.plugins.check.checks' \ +\ +# GHC User Plugins, best be overridden via Container environment \ +GHC_USER_PLUGINS='' RUN apk add --no-cache --virtual .build-deps gcc build-base linux-headers postgresql-dev \ - && apk add --no-cache bash vim postgresql-client \ + && apk add --no-cache bash postgresql-client tzdata openntpd \ && pip install virtualenv \ && rm -rf /var/cache/apk/* /tmp/* /var/tmp/* # Add standard files and Add/override Plugins # Alternative Entrypoints to run GHC jobs # Override default Entrypoint with these on Containers -ADD docker/install.sh docker/configure.sh docker/run.sh \ - docker/config_site.py docker/cron-jobs-daily.sh docker/cron-jobs-hourly.sh docker/plugins / -RUN chmod a+x /*.sh +ADD docker/scripts/*.sh docker/config_site.py docker/plugins / # Add Source Code ADD . /GeoHealthCheck # Install and Remove build-related packages for smaller image size -RUN bash install.sh \ - && apk del .build-deps +RUN chmod a+x /*.sh && bash install.sh && apk del .build-deps # For SQLite VOLUME ["/GeoHealthCheck/DB/"] EXPOSE ${PORT} -ENTRYPOINT /configure.sh && /run.sh +ENTRYPOINT /run-web.sh diff --git a/GeoHealthCheck/app.py b/GeoHealthCheck/app.py index 06b9f779..935537ff 100644 --- a/GeoHealthCheck/app.py +++ b/GeoHealthCheck/app.py @@ -32,7 +32,6 @@ import csv import logging import json -from datetime import datetime, timedelta from StringIO import StringIO from itertools import chain @@ -44,7 +43,6 @@ from flask_migrate import Migrate from __init__ import __version__ -from healthcheck import sniff_test_resource, run_test_resource from init import App from enums import RESOURCE_TYPES from models import Resource, Run, ProbeVars, CheckVars, Tag, User, Recipient @@ -74,6 +72,16 @@ ('hr_HR', 'Croatian (Croatia)') ) +# Should GHC Runner be run within GHC webapp? +if CONFIG['GHC_RUNNER_IN_WEBAPP'] is True: + LOGGER.info('Running GHC Scheduler in WebApp') + from scheduler import start_schedule + + # Start scheduler + start_schedule() +else: + LOGGER.info('NOT Running GHC Scheduler in WebApp') + # commit or rollback shorthand def db_commit(): @@ -122,35 +130,6 @@ def unauthorized_callback(): return redirect(url_for('login', lang=g.current_lang, next=url)) -def next_page_refresh(): - """determines when to refresh webapp based on GHC_RUN_FREQUENCY""" - - now = datetime.now() - - frequency = CONFIG['GHC_RUN_FREQUENCY'] - - if frequency == 'hourly': # get next hour - now2 = now.replace(minute=0, second=0, microsecond=0) - refresh = timedelta(hours=1) - elif frequency == 'daily': # get next day - now2 = now.replace(hour=0, minute=0, second=0, microsecond=0) - refresh = timedelta(days=1) - elif frequency == ['weekly']: # get next day - now2 = now.replace(day=1, hour=0, minute=0, second=0, microsecond=0) - refresh = timedelta(weeks=1) - elif frequency == ['monthly']: - now2 = now.replace(day=1, hour=0, minute=0, second=0, microsecond=0) - refresh = timedelta(weeks=4) - elif frequency == ['yearly']: # get next day - now2 = now.replace(day=1, hour=0, minute=0, second=0, microsecond=0) - refresh = timedelta(weeks=52) - - next_frequency = now2 + refresh - differ = next_frequency - now - - return differ.seconds - - @APP.template_filter('cssize_reliability') def cssize_reliability(value, css_type=None): """returns CSS button class snippet based on score""" @@ -162,11 +141,11 @@ def cssize_reliability(value, css_type=None): score = 'danger' panel = 'red' elif (CONFIG['GHC_RELIABILITY_MATRIX']['orange']['min'] <= number <= - CONFIG['GHC_RELIABILITY_MATRIX']['orange']['max']): + CONFIG['GHC_RELIABILITY_MATRIX']['orange']['max']): score = 'warning' panel = 'yellow' elif (CONFIG['GHC_RELIABILITY_MATRIX']['green']['min'] <= number <= - CONFIG['GHC_RELIABILITY_MATRIX']['green']['max']): + CONFIG['GHC_RELIABILITY_MATRIX']['green']['max']): score = 'success' panel = 'green' else: # should never really get here @@ -203,7 +182,6 @@ def context_processors(): tags = views.get_tag_counts() return { 'app_version': __version__, - 'next_page_refresh': next_page_refresh(), 'resource_types': RESOURCE_TYPES, 'resource_types_counts': rtc['counts'], 'resources_total': rtc['total'], @@ -490,6 +468,7 @@ def add(): url = request.form['url'].strip() resources_to_add = [] + from healthcheck import sniff_test_resource, run_test_resource sniffed_resources = sniff_test_resource(CONFIG, resource_type, url) if not sniffed_resources: @@ -557,7 +536,7 @@ def add(): param_vals = {} for param in param_defs: if param_defs[param]['value']: - param_vals[param] =\ + param_vals[param] = \ param_defs[param]['value'] check_vars = CheckVars( probe_to_add, check_class, param_vals) @@ -641,8 +620,8 @@ def update(resource_identifier): # Add ProbeVars anew each with optional CheckVars for probe in value: - print('adding Probe class=%s parms=%s' % - (probe['probe_class'], str(probe))) + LOGGER.info('adding Probe class=%s parms=%s' % + (probe['probe_class'], str(probe))) probe_vars = ProbeVars(resource, probe['probe_class'], probe['parameters']) for check in probe['checks']: @@ -663,6 +642,9 @@ def update(resource_identifier): elif getattr(resource, key) != resource_identifier_dict[key]: # Update other resource attrs, mainly 'name' setattr(resource, key, resource_identifier_dict[key]) + min_run_freq = CONFIG['GHC_MINIMAL_RUN_FREQUENCY_MINS'] + if int(resource.run_frequency) < min_run_freq: + resource.run_frequency = min_run_freq update_counter += 1 # Always update geo-IP: maybe failure on creation or @@ -699,6 +681,7 @@ def test(resource_identifier): flash(gettext('Resource not found'), 'danger') return redirect(request.referrer) + from healthcheck import run_test_resource result = run_test_resource( resource) @@ -754,10 +737,6 @@ def delete(resource_identifier): flash(gettext('Resource not found'), 'danger') return redirect(url_for('home', lang=g.current_lang)) - runs = Run.query.filter_by(resource_identifier=resource_identifier).all() - - for run in runs: - DB.session.delete(run) resource.clear_recipients() DB.session.delete(resource) @@ -908,7 +887,7 @@ def api_probes_avail(resource_type=None, resource_id=None): if __name__ == '__main__': # run locally, for fun import sys - logging.basicConfig() + HOST = '0.0.0.0' PORT = 8000 if len(sys.argv) > 1: diff --git a/GeoHealthCheck/config_main.py b/GeoHealthCheck/config_main.py index c589ef70..37b4b092 100644 --- a/GeoHealthCheck/config_main.py +++ b/GeoHealthCheck/config_main.py @@ -38,8 +38,8 @@ SECRET_KEY = None GHC_RETENTION_DAYS = 30 -GHC_RUN_FREQUENCY = 'hourly' GHC_PROBE_HTTP_TIMEOUT_SECS = 30 +GHC_MINIMAL_RUN_FREQUENCY_MINS = 10 GHC_SELF_REGISTER = False GHC_NOTIFICATIONS = False GHC_NOTIFICATIONS_VERBOSITY = True @@ -48,6 +48,10 @@ GHC_NOTIFICATIONS_EMAIL = ['you2@example.com'] GHC_SITE_TITLE = 'GeoHealthCheck Demonstration' GHC_SITE_URL = 'http://host' +GHC_RUNNER_IN_WEBAPP = True +# 10=DEBUG 20=INFO 30=WARN(ING) 40=ERROR 50=FATAL/CRITICAL +GHC_LOG_LEVEL = 30 +GHC_LOG_FORMAT = '%(asctime)s - %(name)s - %(levelname)s - %(message)s' # Some GetCaps docs are huge. This allows # caching them for N seconds. Set to -1 to diff --git a/GeoHealthCheck/factory.py b/GeoHealthCheck/factory.py index cf31c27e..ff9ff34a 100644 --- a/GeoHealthCheck/factory.py +++ b/GeoHealthCheck/factory.py @@ -1,3 +1,8 @@ +import logging + +LOGGER = logging.getLogger(__name__) + + class Factory: """ Object, Function class Factory (Pattern). @@ -18,9 +23,9 @@ def create_obj(class_string): # class instance from class object with constructor args return class_obj() - except Exception, e: - print("cannot create object instance from class '%s' e=%s" % - (class_string, str(e))) + except Exception as e: + LOGGER.error("cannot create object instance from class '%s' e=%s" % + (class_string, str(e))) raise e @staticmethod @@ -41,8 +46,8 @@ def create_class(class_string): class_obj = getattr( __import__(module_name, globals(), locals(), [class_name], -1), class_name) - except Exception, e: - print("cannot create class '%s'" % class_string) + except Exception as e: + LOGGER.error("cannot create class '%s'" % class_string) raise e return class_obj @@ -61,8 +66,8 @@ def create_module(module_string): try: module_obj = __import__(module_string, globals(), locals(), fromlist=['']) - except Exception, e: - print("cannot create module from '%s'" % module_string) + except Exception as e: + LOGGER.error("cannot create module from '%s'" % module_string) raise e return module_obj diff --git a/GeoHealthCheck/healthcheck.py b/GeoHealthCheck/healthcheck.py index ae0c1df6..285f8158 100644 --- a/GeoHealthCheck/healthcheck.py +++ b/GeoHealthCheck/healthcheck.py @@ -27,13 +27,13 @@ # # ================================================================= -import datetime +from datetime import datetime import logging import json from urllib2 import urlopen from urlparse import urlparse from functools import partial -from pprint import pprint +from flask_babel import gettext from owslib.wms import WebMapService from owslib.wmts import WebMapTileService @@ -44,12 +44,74 @@ from owslib.csw import CatalogueServiceWeb from owslib.sos import SensorObservationService -from flask_babel import gettext +from init import App from enums import RESOURCE_TYPES +from models import Resource, Run from probe import Probe from result import ResourceResult +from notifications import notify LOGGER = logging.getLogger(__name__) +APP = App.get_app() +DB = App.get_db() + + +# commit or rollback shorthand +def db_commit(): + err = None + try: + DB.session.commit() + except Exception as err: + DB.session.rollback() + # finally: + # DB.session.close() + return err + + +def run_resources(): + for resource in Resource.query.all(): # run all tests + LOGGER.info('Testing %s %s' % + (resource.resource_type, resource.url)) + + run_resource(resource.identifier) + + +# complete handle of resource test +def run_resource(resourceid): + resource = Resource.query.filter_by(identifier=resourceid).first() + + if not resource.active: + # Exit test of resource if it's not active + return + + # Get the status of the last run, + # assume success if there is none + last_run_success = True + last_run = resource.last_run + if last_run: + last_run_success = last_run.success + + # Run test + result = run_test_resource(resource) + + run1 = Run(resource, result, datetime.utcnow()) + + DB.session.add(run1) + + # commit or rollback each run to avoid long-lived transactions + # see https://github.com/geopython/GeoHealthCheck/issues/14 + db_commit() + + if APP.config['GHC_NOTIFICATIONS']: + # Attempt notification + try: + notify(APP.config, resource, run1, last_run_success) + except Exception as err: + # Don't bail out on failure in order to commit the Run + msg = str(err) + logging.warn('error notifying: %s' % msg) + if not __name__ == '__main__': + DB.session.remove() def run_test_resource(resource): @@ -81,7 +143,7 @@ def sniff_test_resource(config, resource_type, url): raise RuntimeError(msg2) title = None - start_time = datetime.datetime.utcnow() + start_time = datetime.utcnow() message = None resource_type_map = {'OGC:WMS': [partial(WebMapService, version='1.3.0'), partial(WebMapService, version='1.1.1')], @@ -153,7 +215,7 @@ def sniff_test_resource(config, resource_type, url): title = urlparse(url).hostname elif resource_type == 'OSGeo:GeoNode': endpoints = ows - end_time = datetime.datetime.utcnow() + end_time = datetime.utcnow() delta = end_time - start_time response_time = '%s.%s' % (delta.seconds, delta.microseconds) base_tags = geonode_make_tags(url) @@ -184,7 +246,7 @@ def sniff_test_resource(config, resource_type, url): message = msg success = False - end_time = datetime.datetime.utcnow() + end_time = datetime.utcnow() delta = end_time - start_time response_time = '%s.%s' % (delta.seconds, delta.microseconds) @@ -235,12 +297,15 @@ def geonode_make_tags(base_url): if __name__ == '__main__': - import sys - logging.basicConfig(level=logging.INFO) - from init import App - if len(sys.argv) < 3: - print('Usage: %s ' % sys.argv[0]) - sys.exit(1) - - # TODO: need APP.config here, None for now - pprint(sniff_test_resource(App.get_config(), sys.argv[1], sys.argv[2])) + print('START - Running health check tests on %s' + % datetime.utcnow().isoformat()) + run_resources() + print('END - Running health check tests on %s' + % datetime.utcnow().isoformat()) + # from init import App + # if len(sys.argv) < 3: + # print('Usage: %s ' % sys.argv[0]) + # sys.exit(1) + # + # # TODO: need APP.config here, None for now + # pprint(sniff_test_resource(App.get_config(), sys.argv[1], sys.argv[2])) diff --git a/GeoHealthCheck/init.py b/GeoHealthCheck/init.py index 7be7d8bb..d9f88f89 100644 --- a/GeoHealthCheck/init.py +++ b/GeoHealthCheck/init.py @@ -29,10 +29,13 @@ import os import sys +import logging from flask import Flask from flask_sqlalchemy import SQLAlchemy from flask_babel import Babel +LOGGER = logging.getLogger(__name__) + def to_list(obj): obj_type = type(obj) @@ -55,69 +58,71 @@ class App: babel_instance = None plugins_instance = None home_dir = None + count = 0 @staticmethod def init(): # Do init once - if not App.app_instance: - app = Flask(__name__) + app = Flask(__name__) + + # Read and override configs + app.config.from_pyfile('config_main.py') + app.config.from_pyfile('../instance/config_site.py') - # Read and override configs - app.config.from_pyfile('config_main.py') - app.config.from_pyfile('../instance/config_site.py') + # Global Logging config + logging.basicConfig(level=int(app.config['GHC_LOG_LEVEL']), + format=app.config['GHC_LOG_FORMAT']) - app.config['GHC_SITE_URL'] = \ - app.config['GHC_SITE_URL'].rstrip('/') + app.config['GHC_SITE_URL'] = \ + app.config['GHC_SITE_URL'].rstrip('/') - app.secret_key = app.config['SECRET_KEY'] + app.secret_key = app.config['SECRET_KEY'] - App.db_instance = SQLAlchemy(app) - App.babel_instance = Babel(app) + App.db_instance = SQLAlchemy(app) + App.babel_instance = Babel(app) - # Plugins (via Docker ENV) must be list, but may have been - # specified as comma-separated string, or older set notation - app.config['GHC_PLUGINS'] = to_list(app.config['GHC_PLUGINS']) - app.config['GHC_USER_PLUGINS'] = \ - to_list(app.config['GHC_USER_PLUGINS']) + # Plugins (via Docker ENV) must be list, but may have been + # specified as comma-separated string, or older set notation + app.config['GHC_PLUGINS'] = to_list(app.config['GHC_PLUGINS']) + app.config['GHC_USER_PLUGINS'] = \ + to_list(app.config['GHC_USER_PLUGINS']) - # Concatenate core- and user-Plugins - App.plugins_instance = \ - app.config['GHC_PLUGINS'] + app.config['GHC_USER_PLUGINS'] + # Concatenate core- and user-Plugins + App.plugins_instance = \ + app.config['GHC_PLUGINS'] + app.config['GHC_USER_PLUGINS'] - # Needed to find Plugins - home_dir = os.path.dirname(os.path.abspath(__file__)) - App.home_dir = sys.path.append('%s/..' % home_dir) + # Needed to find Plugins + home_dir = os.path.dirname(os.path.abspath(__file__)) + App.home_dir = sys.path.append('%s/..' % home_dir) - # Finally assign app-instance - App.app_instance = app - print("init.py: created GHC App instance") + # Finally assign app-instance + App.app_instance = app + App.count += 1 + LOGGER.info("created GHC App instance #%d" % App.count) @staticmethod def get_app(): - App.init() return App.app_instance @staticmethod def get_babel(): - App.init() return App.babel_instance @staticmethod def get_config(): - App.init() return App.app_instance.config @staticmethod def get_db(): - App.init() return App.db_instance @staticmethod def get_home_dir(): - App.init() return App.home_dir @staticmethod def get_plugins(): - App.init() return App.plugins_instance + + +App.init() diff --git a/GeoHealthCheck/migrations/versions/34531bfd7cab_.py b/GeoHealthCheck/migrations/versions/34531bfd7cab_.py new file mode 100644 index 00000000..f7896ad8 --- /dev/null +++ b/GeoHealthCheck/migrations/versions/34531bfd7cab_.py @@ -0,0 +1,47 @@ +"""empty message + +Revision ID: 34531bfd7cab +Revises: bb91fb332c36 +Create Date: 2018-03-19 16:59:42.235474 + +""" +from alembic import op +import sqlalchemy as sa +from GeoHealthCheck.migrations import alembic_helpers + +# revision identifiers, used by Alembic. +revision = '34531bfd7cab' +down_revision = 'bb91fb332c36' +branch_labels = None +depends_on = None + + +def upgrade(): + if not alembic_helpers.table_has_column('resource', 'run_frequency'): + print('Column run_frequency not present in resource table, will create') + op.add_column(u'resource', sa.Column('run_frequency', sa.Integer(), + nullable=False, default=60, server_default='60')) + else: + print('Column run_frequency already present in resource table') + + if not alembic_helpers.tables_exist(['resource_lock']): + print('Table for Resource locking not present, will create') + # ### commands auto generated by Alembic - please adjust! ### + op.create_table('resource_lock', + sa.Column('identifier', sa.Integer(), nullable=False, unique=True), + sa.Column('resource_identifier', sa.Integer(), nullable=False, unique=True), + sa.Column('owner', sa.Text, nullable=False, default='NOT SET', server_default='NOT SET'), + sa.Column('start_time', sa.DateTime, nullable=False), + sa.Column('end_time', sa.DateTime, nullable=False), + sa.ForeignKeyConstraint(['resource_identifier'], ['resource.identifier'], ), + sa.PrimaryKeyConstraint('identifier') + ) + # ### end Alembic commands ### + else: + print('Table for Resource-locking already present, will not create') + + +def downgrade(): + print('Dropping Column run_frequency from resource table') + op.drop_column(u'resource', 'run_frequency') + op.drop_table('resource_lock') diff --git a/GeoHealthCheck/models.py b/GeoHealthCheck/models.py index 6a548e49..693a45b6 100644 --- a/GeoHealthCheck/models.py +++ b/GeoHealthCheck/models.py @@ -31,7 +31,7 @@ import json import logging from flask_babel import gettext as _ -from datetime import datetime +from datetime import datetime, timedelta from sqlalchemy import func, and_ from sqlalchemy.orm import deferred @@ -41,7 +41,6 @@ from enums import RESOURCE_TYPES from factory import Factory from init import App -from notifications import notify from wtforms.validators import Email, ValidationError from owslib.util import bind_url @@ -49,6 +48,24 @@ LOGGER = logging.getLogger(__name__) +# Complete handle of old runs deletion +def flush_runs(): + APP = App.get_app() + retention_days = int(APP.config['GHC_RETENTION_DAYS']) + LOGGER.info('Flushing runs older than %d days' % retention_days) + all_runs = Run.query.all() + run_count = 0 + for run in all_runs: + days_old = (datetime.utcnow() - run.checked_datetime).days + if days_old > retention_days: + run_count += 1 + DB.session.delete(run) + db_commit() + LOGGER.info('Deleted %d Runs' % run_count) + + DB.session.remove() + + class Run(DB.Model): """measurement of resource state""" @@ -56,7 +73,8 @@ class Run(DB.Model): resource_identifier = DB.Column(DB.Integer, DB.ForeignKey('resource.identifier')) resource = DB.relationship('Resource', - backref=DB.backref('runs', lazy='dynamic')) + backref=DB.backref('runs', lazy='dynamic', + cascade="all,delete")) checked_datetime = DB.Column(DB.DateTime, nullable=False) success = DB.Column(DB.Boolean, nullable=False) response_time = DB.Column(DB.Float, nullable=False) @@ -216,6 +234,7 @@ class dummy_value(object): @staticmethod def gettext(*args, **kwargs): return _(*args, **kwargs) + dummy_form = None v(dummy_form, dummy_value()) @@ -273,11 +292,11 @@ def is_webhook(self): @classmethod def burry_dead(cls): RN = ResourceNotification - q = DB.session.query(cls)\ - .join(RN, - RN.recipient_id == cls.id, - isouter=True)\ - .filter(RN.recipient_id.is_(None)) + q = DB.session.query(cls) \ + .join(RN, + RN.recipient_id == cls.id, + isouter=True) \ + .filter(RN.recipient_id.is_(None)) for item in q: DB.session.delete(item) @@ -289,10 +308,10 @@ def get_or_create(cls, channel, location): raise ValueError("invalid value {}: {}".format(location, err)) try: - r = DB.session.query(cls)\ - .filter(and_(cls.channel == channel, - cls.location == location))\ - .one() + r = DB.session.query(cls) \ + .filter(and_(cls.channel == channel, + cls.location == location)) \ + .one() except (MultipleResultsFound, NoResultFound,): r = cls(channel=channel, location=location) DB.session.add(r) @@ -309,12 +328,12 @@ def get_suggestions(cls, channel, for_user): Res = Resource ResNot = ResourceNotification - q = DB.session.query(Rcp.location)\ - .join(ResNot, ResNot.recipient_id == Rcp.id)\ - .join(Res, Res.identifier == ResNot.resource_id)\ - .group_by(Rcp.location)\ - .filter(and_(Res.owner_identifier == for_user, - Rcp.channel == channel)) + q = DB.session.query(Rcp.location) \ + .join(ResNot, ResNot.recipient_id == Rcp.id) \ + .join(Res, Res.identifier == ResNot.resource_id) \ + .group_by(Rcp.location) \ + .filter(and_(Res.owner_identifier == for_user, + Rcp.channel == channel)) return [item[0] for item in q] @@ -348,6 +367,7 @@ class Resource(DB.Model): owner = DB.relationship('User', backref=DB.backref('username2', lazy='dynamic')) tags = DB.relationship('Tag', secondary=resource_tags, backref='resource') + run_frequency = DB.Column(DB.Integer, default=60) def __init__(self, owner, resource_type, title, url, tags): self.resource_type = resource_type @@ -465,26 +485,26 @@ def clear_recipients(self, channel=None, burry_dead=True): # clear specific channel to_delete = self.get_recipients(channel) if to_delete: - to_del_rcp = DB.session.query(RN)\ - .join(Rcp, - Rcp.id == RN.recipient_id)\ - .filter( - and_(RN.resource_id == - self.identifier, - Rcp.channel == channel, - Rcp.location.in_(to_delete)) - ) + to_del_rcp = DB.session.query(RN) \ + .join(Rcp, + Rcp.id == RN.recipient_id) \ + .filter( + and_(RN.resource_id == + self.identifier, + Rcp.channel == channel, + Rcp.location.in_(to_delete)) + ) else: to_del_rcp = [] else: # remove all m2m connections for Resource<->Recipient - to_del_rcp = DB.session.query(RN)\ - .join(Rcp, - Rcp.id == RN.recipient_id)\ - .filter( - RN.resource_id == - self.identifier, - ) + to_del_rcp = DB.session.query(RN) \ + .join(Rcp, + Rcp.id == RN.recipient_id) \ + .filter( + RN.resource_id == + self.identifier, + ) for rcp_ntf in to_del_rcp: DB.session.delete(rcp_ntf) if burry_dead: @@ -521,6 +541,49 @@ def dump_recipients(self): return out +class ResourceLock(DB.Model): + """lock resource for multiprocessing runs""" + + identifier = DB.Column(DB.Integer, + primary_key=True, autoincrement=False, unique=True) + resource_identifier = DB.Column( + DB.Integer, DB.ForeignKey('resource.identifier'), unique=True) + resource = DB.relationship('Resource', + backref=DB.backref('locks', lazy='dynamic', + cascade="all,delete")) + owner = DB.Column(DB.Text, nullable=False, default='NOT SET') + + start_time = DB.Column(DB.DateTime, nullable=False) + end_time = DB.Column(DB.DateTime, nullable=False) + + def __init__(self, resource, owner, interval_mins): + self.identifier = resource.identifier + self.resource = resource + self.owner = owner + self.init_datetimes(interval_mins) + + def init_datetimes(self, interval_mins): + self.start_time = datetime.utcnow() + # Subtract some space from end-time to allow obtain at scheduled time + minutes = interval_mins - 1 + self.end_time = self.start_time + timedelta(minutes=minutes) + + def has_expired(self): + now = datetime.utcnow() + return now > self.end_time + + def obtain(self, owner, frequency): + if not self.has_expired(): + return False + + self.owner = owner + self.init_datetimes(frequency) + return True + + def __repr__(self): + return '' % self.identifier + + class User(DB.Model): """user accounts""" @@ -697,7 +760,7 @@ def db_commit(): except Exception as err: DB.session.rollback() msg = str(err) - print(msg) + LOGGER.error(msg) if __name__ == '__main__': @@ -757,60 +820,9 @@ def db_commit(): print('Provide path to JSON file, e.g. tests/fixtures.json') elif sys.argv[1] == 'run': - print('START - Running health check tests on %s' - % datetime.utcnow().isoformat()) - from healthcheck import run_test_resource - - for resource in Resource.query.all(): # run all tests - print('Testing %s %s' % - (resource.resource_type, resource.url)) - - if not resource.active: - print('Resource is not active. Skipping') - continue - - # Get the status of the last run, - # assume success if there is none - last_run_success = True - last_run = resource.last_run - if last_run: - last_run_success = last_run.success - - # Run test - result = run_test_resource(resource) - - run1 = Run(resource, result) - - print('Adding Run: success=%s, response_time=%ss\n' - % (str(run1.success), run1.response_time)) - - DB.session.add(run1) - - # commit or rollback each run to avoid long-lived transactions - # see https://github.com/geopython/GeoHealthCheck/issues/14 - db_commit() - - if APP.config['GHC_NOTIFICATIONS']: - # Attempt notification - try: - notify(APP.config, resource, run1, last_run_success) - except Exception as err: - LOGGER.error("Cannot send notifications: %s", - err, - exc_info=err) - - print('END - Running health check tests on %s' - % datetime.utcnow().isoformat()) + print('NOTICE: models.py no longer here.') + print('Use: python healthcheck.py or upcoming cli.py') elif sys.argv[1] == 'flush': - retention_days = int(APP.config['GHC_RETENTION_DAYS']) - print('Flushing runs older than %d days' % - retention_days) - all_runs = Run.query.all() - for run in all_runs: - days_old = (datetime.utcnow() - run.checked_datetime).days - if days_old > retention_days: - print('Run older than %d days. Deleting' % days_old) - DB.session.delete(run) - db_commit() + flush_runs() DB.session.remove() diff --git a/GeoHealthCheck/plugin.py b/GeoHealthCheck/plugin.py index 1f19379e..2b36a7da 100644 --- a/GeoHealthCheck/plugin.py +++ b/GeoHealthCheck/plugin.py @@ -1,10 +1,13 @@ # -*- coding: utf-8 -*- from factory import Factory +import logging import inspect import collections import copy from init import App +LOGGER = logging.getLogger(__name__) + class Plugin(object): """ @@ -176,7 +179,7 @@ def add_result(plugin_name, class_obj): and baseclass != class_obj: add_result(plugin_name, class_obj) except Exception: - print('cannot create obj class=%s' % plugin_name) + LOGGER.warn('cannot create obj class=%s' % plugin_name) return result diff --git a/GeoHealthCheck/scheduler.py b/GeoHealthCheck/scheduler.py new file mode 100644 index 00000000..ebfa6310 --- /dev/null +++ b/GeoHealthCheck/scheduler.py @@ -0,0 +1,281 @@ +# coding=utf-8 +# ================================================================= +# +# Authors: Tom Kralidis +# Just van den Broecke +# +# Copyright (c) 2014 Tom Kralidis +# +# Permission is hereby granted, free of charge, to any person +# obtaining a copy of this software and associated documentation +# files (the "Software"), to deal in the Software without +# restriction, including without limitation the rights to use, +# copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following +# conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +# OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# +# ================================================================= + +import logging +import os +import random +import string +from datetime import datetime, timedelta +from models import Resource, ResourceLock, flush_runs +from healthcheck import run_resource +from apscheduler.schedulers.background import BackgroundScheduler +from apscheduler.jobstores.base import JobLookupError +from apscheduler.events import \ + EVENT_SCHEDULER_STARTED, EVENT_SCHEDULER_SHUTDOWN, \ + EVENT_JOB_MISSED, EVENT_JOB_ERROR +from init import App + +LOGGER = logging.getLogger(__name__) +DB = App.get_db() + +# Create scheduler +scheduler = BackgroundScheduler() + + +# commit or rollback shorthand +def db_commit(): + err = None + try: + DB.session.commit() + except Exception as err: + DB.session.rollback() + # finally: + # DB.session.close() + return err + + +def run_job(resource_id, frequency): + """ + Runs single job (all Probes) for single Resource. + As multiple instances of the job scheduler may run in different + processes and threads, the database is used to synchronize and assure + only one job will run. This is achieved by having one lock per Resource. + Only the process/thread that acquires its related ResourceLock record + runs the job. As to avoid permanent "lockouts", each ResourceLock has + a lifetime, namely the timespan until the next Run as configured for/per + Resource. This gives all job runners a chance to obtain a lock once + "time's up" for the ResourceLock. + + An extra check for lock obtainment is made via an unique UUID per job + runner. Once the lock is obtained the UUID-field of the lock record + is set and committed to the DB. If we then try to obtain the lock again + (by reading from DB) but the UUID is different this means another job + runner instance did the same but was just before us. The lock timespan + will guard that a particular UUID will keep the lock forever, e.g. if + the application is suddenly shutdown. + + :param resource_id: + :param frequency: + :return: + """ + # Generate unique id + # https://stackoverflow.com/questions/2257441/random-string-generation-with-upper-case-letters-and-digits-in-python + uuid = '%d-%s' % (os.getpid(), ''.join(random.choice( + string.ascii_uppercase + string.digits) for _ in range(8))) + + resource = Resource.query.filter_by(identifier=resource_id).first() + + # Resource may have been deleted, cancel job + if not resource: + stop_job(resource_id) + return + + # Resource exists: try to obtain our Resource Lock record. + lock = ResourceLock.query.filter_by(identifier=resource_id).first() + + if not lock: + # No lock at all on Resource: (hope) we're first + # obtain fresh lock, back-off if failed + LOGGER.info('%d No Lock at all: obtain new' % resource_id) + + lock = ResourceLock(resource, uuid, frequency) + DB.session.add(lock) + lock_err = db_commit() + if lock_err: + # Another process may have been there first! + LOGGER.info('%d Error obtaining Lock %s' % + (resource_id, str(lock_err))) + return + else: + # Lock record found for Resource: check if available for our UUID. + LOGGER.info('%d Lock present: try obtaining..' % resource_id) + if not lock.obtain(uuid, frequency): + LOGGER.info('%d Cannot obtain lock' % resource_id) + return + else: + LOGGER.info('%d Lock obtained, delete and renew' % resource_id) + DB.session.delete(lock) + lock_err = db_commit() + if lock_err: + # Another process may have been there first! + LOGGER.info('%d Lock Delete failed' % resource_id) + return + + LOGGER.info('%d Lock deleted, add new' % resource_id) + # (hope) we're first + # obtain fresh lock, back-off if failed + lock = ResourceLock(resource, uuid, frequency) + DB.session.add(lock) + lock_err = db_commit() + if lock_err: + # Another process may have been there first! + LOGGER.info('%d Lock Add failed' % resource_id) + return + + # Check if we really own the lock + LOGGER.info('%d Lock Add OK' % resource_id) + lock = ResourceLock.query.filter_by( + identifier=resource_id).first() + + if lock.owner != uuid: + LOGGER.info('%d Lock Add OK, not owner: back-off' + % resource_id) + return + + # Run Resource healthchecks only if we have lock. + if lock: + try: + run_resource(resource_id) + LOGGER.info('%d run_resource OK' % resource_id) + finally: + pass + + +def start_schedule(): + LOGGER.info('Starting scheduler') + + # Adapt configuration + scheduler.configure(job_defaults={ + 'coalesce': True, + 'max_instances': 1, + 'misfire_grace_time': 300 + }) + + scheduler.add_listener(lifecycle_listener, + EVENT_SCHEDULER_STARTED | EVENT_SCHEDULER_SHUTDOWN) + scheduler.add_listener(error_listener, + EVENT_JOB_MISSED | EVENT_JOB_ERROR) + + # Start APScheduler + scheduler.start() + import atexit + atexit.register(lambda: stop_schedule()) + + # Add GHC jobs, one for each Resource, plus + # maintenance jobs. + + # Cold start every cron of every Resource + for resource in Resource.query.all(): + add_job(resource) + + # Start maintenance jobs + scheduler.add_job(flush_runs, 'interval', minutes=150) + scheduler.add_job(check_schedule, 'interval', minutes=5) + + +def check_schedule(): + LOGGER.info('Checking Job schedules') + + # Check the schedule for changed jobs + for resource in Resource.query.all(): + job = get_job(resource) + if job is None: + add_job(resource) + continue + + current_freq = job.args[1] + + # Run frequency changed? + if current_freq != resource.run_frequency: + # Reschedule Job + update_job(resource) + + +def lifecycle_listener(event): + event_code = event.code + event_code_str = '' + if event_code == EVENT_SCHEDULER_STARTED: + event_code_str = 'EVENT_SCHEDULER_STARTED' + elif event_code | EVENT_SCHEDULER_SHUTDOWN: + event_code_str = 'EVENT_SCHEDULER_SHUTDOWN' + + LOGGER.info('lifecycle_listener: %s - %s' % (event_code_str, str(event))) + + +def error_listener(event): + event_code = event.code + event_code_str = '' + if event_code | EVENT_JOB_MISSED: + event_code_str = 'EVENT_JOB_MISSED' + elif event_code | EVENT_JOB_ERROR: + event_code_str = 'EVENT_JOB_ERROR' + + LOGGER.error('error_listener: %s - %s' % (event_code_str, str(event))) + + +def get_job(resource): + return scheduler.get_job(str(resource.identifier)) + + +def update_job(resource): + stop_job(resource.identifier) + + # Add job to Scheduler + add_job(resource) + + +def add_job(resource): + LOGGER.info('Starting job for resource=%d' % resource.identifier) + freq = resource.run_frequency + next_run_time = datetime.now() + timedelta(minutes=random.randint(0, freq)) + scheduler.add_job( + run_job, 'interval', args=[resource.identifier, freq], + minutes=freq, next_run_time=next_run_time, max_instances=1, + misfire_grace_time=(freq * 60) / 2, coalesce=True, + id=str(resource.identifier)) + + +def stop_job(resource_id): + LOGGER.info('Stopping job for resource=%d' % resource_id) + + # Try to remove job from scheduler + try: + scheduler.remove_job(str(resource_id)) + except JobLookupError: + pass + + +def stop_schedule(): + LOGGER.info('Stopping Scheduler') + scheduler.shutdown() + scheduler.remove_listener(lifecycle_listener) + scheduler.remove_listener(error_listener) + + +if __name__ == '__main__': + import time + + # Start scheduler + start_schedule() + + while True: + LOGGER.info("This prints once in 5 minutes") + time.sleep(300) # Delay for 5 minute (300 seconds). diff --git a/GeoHealthCheck/templates/edit_resource.html b/GeoHealthCheck/templates/edit_resource.html index 885aaf4e..1569a0fd 100644 --- a/GeoHealthCheck/templates/edit_resource.html +++ b/GeoHealthCheck/templates/edit_resource.html @@ -88,6 +88,12 @@

[{{ _('Edit') }}] {{ resour + + {{ _('Run Every') }} + + + + Probes @@ -258,6 +264,9 @@

[{{ _('Edit') }}] {{ resour // Collect tags var new_tags = $('#resource_tags').val(); + // Collect test_frequency + var new_frequency = $('input[name="resource_frequency_value"]').val(); + // Collect active var new_active = $('#input_resource_active').prop('checked'); @@ -335,6 +344,7 @@

[{{ _('Edit') }}] {{ resour title: new_title, active: new_active, tags: new_tags, + run_frequency: new_frequency, probes: new_probes, notify_emails: new_notify_emails, notify_webhooks: new_notify_webhooks diff --git a/GeoHealthCheck/templates/layout.html b/GeoHealthCheck/templates/layout.html index f751b948..c3bf0b21 100644 --- a/GeoHealthCheck/templates/layout.html +++ b/GeoHealthCheck/templates/layout.html @@ -2,7 +2,6 @@ - @@ -107,8 +106,9 @@ {{ _('Settings') }} diff --git a/GeoHealthCheck/templates/resource.html b/GeoHealthCheck/templates/resource.html index 723f75fb..dc42185c 100644 --- a/GeoHealthCheck/templates/resource.html +++ b/GeoHealthCheck/templates/resource.html @@ -54,6 +54,12 @@

{{ _('Monitoring Period') }}: {{ resource.first_run.chec + + {{ _('Run Every') }} + + {{ resource.run_frequency }} {{ _('minutes') }} + + Probes diff --git a/GeoHealthCheck/translations/en/LC_MESSAGES/messages.po b/GeoHealthCheck/translations/en/LC_MESSAGES/messages.po index 48b76be3..6389bf66 100644 --- a/GeoHealthCheck/translations/en/LC_MESSAGES/messages.po +++ b/GeoHealthCheck/translations/en/LC_MESSAGES/messages.po @@ -381,6 +381,16 @@ msgstr "" msgid "Response Times (seconds)" msgstr "" +#: GeoHealthCheck/templates/resource.html:60 +#: GeoHealthCheck/templates/edit_resource.html:94 +msgid "minutes" +msgstr "" + +#: GeoHealthCheck/templates/resource.html:58 +#: GeoHealthCheck/templates/edit_resource.html:92 +msgid "Run Every" +msgstr "" + #: GeoHealthCheck/templates/resource.html:71 msgid "Min" msgstr "" diff --git a/GeoHealthCheck/translations/nl_NL/LC_MESSAGES/messages.po b/GeoHealthCheck/translations/nl_NL/LC_MESSAGES/messages.po index 9476288d..d6052529 100644 --- a/GeoHealthCheck/translations/nl_NL/LC_MESSAGES/messages.po +++ b/GeoHealthCheck/translations/nl_NL/LC_MESSAGES/messages.po @@ -381,6 +381,16 @@ msgstr "Monitoring Periode" msgid "Response Times (seconds)" msgstr "Response Tijden (seconden)" +#: GeoHealthCheck/templates/resource.html:60 +#: GeoHealthCheck/templates/edit_resource.html:94 +msgid "minutes" +msgstr "minuten" + +#: GeoHealthCheck/templates/resource.html:58 +#: GeoHealthCheck/templates/edit_resource.html:92 +msgid "Run Every" +msgstr "Run Elke" + #: GeoHealthCheck/templates/resource.html:71 msgid "Min" msgstr "Min" diff --git a/GeoHealthCheck/util.py b/GeoHealthCheck/util.py index e23dde26..d74be46b 100644 --- a/GeoHealthCheck/util.py +++ b/GeoHealthCheck/util.py @@ -136,8 +136,8 @@ def render_template2(template, template_vars): def send_email(mail_config, fromaddr, toaddr, msg): """convenience function to send an email""" - print(mail_config) - print(msg) + LOGGER.debug(mail_config) + LOGGER.debug(msg) server = smtplib.SMTP('%s:%s' % (mail_config['server'], mail_config['port'])) diff --git a/README.md b/README.md index fb16c6a0..8891d525 100644 --- a/README.md +++ b/README.md @@ -6,11 +6,11 @@ GeoHealthCheck ============== -GeoHealthCheck (GHC) is a Service Status Checker for OGC Web Services and web APIs in general. +GeoHealthCheck (GHC) is a Service Status and QoS Checker for OGC Web Services and web APIs in general. See also the [full GHC documentation](http://docs.geohealthcheck.org/). Easiest is [to run GHC using Docker](https://github.com/geopython/GeoHealthCheck/blob/master/docker/README.md). -Below a quick overview of a manual install on Unix-based systems. +Below a quick overview of a manual install on Unix-based systems like Apple Mac and Linux. ```bash virtualenv GeoHealthCheck && cd $_ @@ -22,34 +22,39 @@ pip install Paver paver setup # generate secret key paver create_secret_key -# setup local configuration +# setup local configuration (overrides GeoHealthCheck/config_main.py) vi instance/config_site.py -# edit: +# edit at least secret key: +# - SECRET_KEY # copy/paste result string from paver create_secret_key + +# Optional: edit other settings or leave defaults # - SQLALCHEMY_DATABASE_URI -# - SECRET_KEY # from paver create_secret_key # - GHC_RETENTION_DAYS -# - GHC_RUN_FREQUENCY # - GHC_SELF_REGISTER +# - GHC_RUNNER_IN_WEBAPP # - GHC_ADMIN_EMAIL # - GHC_SITE_TITLE # - GHC_MAP (or use default settings) -# setup database and superuser account +# setup database and superuser account interactively paver create -# setup cronjobs -vi jobs.cron -# edit paths to scripts -# enable cron -crontab jobs.cron - -# start server (default is 0.0.0.0:8000) +# start webserver with healthcheck runner daemon inside +# (default is 0.0.0.0:8000) python GeoHealthCheck/app.py -# start server on another port +# or start webserver on another port python GeoHealthCheck/app.py 0.0.0.0:8881 -# start server on another IP +# or start webserver on another IP python GeoHealthCheck/app.py 192.168.0.105:8001 +# OR start webserver and separate runner daemon (scheduler) process +vi instance/config_site.py +# GHC_RUNNER_IN_WEBAPP = False +python GeoHealthCheck/scheduler.py & +python GeoHealthCheck/app.py + +# next: use a real webserver or preferably Docker for production + # other commands # # drop database @@ -61,14 +66,4 @@ python GeoHealthCheck/models.py load <.json data file> [y/n] ``` -users -- view all services -- view service - -admin -- drop db -- create db -- upgrade db -- add service -- delete service -- run health check (cron or interactive) +More in the [full GHC documentation](http://docs.geohealthcheck.org/). \ No newline at end of file diff --git a/docker/README.md b/docker/README.md index fd784386..a1b0977f 100644 --- a/docker/README.md +++ b/docker/README.md @@ -7,6 +7,12 @@ In the best case an install/run of GHC is a matter of minutes! If you are reading this from Docker Hub, local links will not work. In that case use the [Docker Readme at GHC GitHub](https://github.com/geopython/GeoHealthCheck/blob/master/docker/README.md). +Since GHC release 0.4.0 GHC can run completely with two Docker containers from the same +GHC Docker image: + +* `GHC Webapp` the web/Flask app +* `GHC Runner` : the daemon runner app that schedules and executes GHC's Probes + ## Requirements Docker installed and Docker daemon running. @@ -15,7 +21,8 @@ Docker installed and Docker daemon running. For installing Docker on Ubuntu there is a [bash helper script](install-docker-ubuntu.sh). -NB: The ``docker`` commands below may need to be prepended with ``sudo``, dependent on your login rights. +NB: The ``docker`` commands below may need to be prepended with +``sudo``, dependent on your login rights. ## Build @@ -36,25 +43,42 @@ For example [run.sh](run.sh) which launches GHC using the robust `gunicorn` WSGI ## Run ``` -docker run -d --name GeoHealthCheck -p 8083:80 -v ghc_sqlitedb:/GeoHealthCheck/DB geopython/geohealthcheck:latest +docker run -d --name ghc_web -p 8083:80 -v ghc_sqlitedb:/GeoHealthCheck/DB geopython/geohealthcheck:latest ``` go to http://localhost:8083 (port 80 in GHC Container is mapped to 8083 on host). -NB this runs GHC standalone with a `SQLite` DB, but without the cron-jobs that perform the healthchecks. -You may schedule the cron-jobs using the local cron system with the -[cron-jobs-hourly](cron-jobs-hourly.sh) and -[cron-jobs-daily](cron-jobs-daily.sh) Docker Entrypoints. +NB this runs GHC standalone with a `SQLite` DB and with the GHC Runner that performs the +healthchecks. But this may in cases not be optimal as the `GHC Webapp` may get overloaded +from processing by the `GHC Runner`. Tip: to see detailed logging add `-e GHC_LOG_LEVEL=10 `. + +This mode can be disabled by passing `GHC_RUNNER_IN_WEBAPP` to as an ENV +var to the Docker container: + +``` +docker run --name ghc_web -e GHC_RUNNER_IN_WEBAPP=False -p 8083:80 -v ghc_sqlitedb:/GeoHealthCheck/DB geopython/geohealthcheck:latest + +``` + +This allows both the `GHC Webapp` (Dashboard)and `GHC Runner` within a separate Docker containers. -But the most optimal way to run GHC with cronjobs and optionally Postgres backend DB, +You can then run `GHC Runner` as a separate container by overriding +the default `ENTRYPOINT` with `/run-runner.sh`: + +``` +docker run -d --name ghc_runner --entrypoint "/run-runner.sh" -v ghc_sqlitedb:/GeoHealthCheck/DB geopython/geohealthcheck:latest +``` + +But the most optimal way to run GHC with scheduled jobs and optionally Postgres as backend DB, is to use [Docker Compose](https://docs.docker.com/compose), see below. ## Using docker-compose -This allows a complete Docker setup, including cron-jobs and optionally using +This allows a complete Docker setup, including scheduling and optionally using Postgres/PostGIS as database (recommended). See the [Docker Compose Documentation](https://docs.docker.com/compose) -for more info. +for more info. GHC Webapp and Runner are in this case +deployed as separate processes (Docker containers). *Note that the `docker-compose` YAML files below are meant as examples to be adapted to your* *local deployment situation.* @@ -62,8 +86,7 @@ for more info. ### Using sqlite DB (default) Using the default [docker-compose.yml](compose/docker-compose.yml) will run GHC with a SQLite DB. -Cronjobs are scheduled using [Jobber Cron](https://github.com/blacklabelops/jobber-cron/) as -regular Unix `cron` does not play nice with Docker. + To run (`-d` allows running in background): @@ -77,16 +100,15 @@ docker-compose -f docker-compose.yml up [-d] ### Using PostGIS DB -The file [docker-compose.postgis.yml](compose/docker-compose.postgis.yml) -extends/overrides the default [docker-compose.yml](compose/docker-compose.yml) to use Postgres with PostGIS -as database. +The file [docker-compose.postgis.yml](compose/docker-compose.postgis.yml) is +similar but uses Postgres as the database. -To run, specify both `.yml` files: +To run: ``` cd docker/compose -docker-compose -f docker-compose.yml -f docker-compose.postgis.yml up [-d] +docker-compose -f docker-compose.postgis.yml up [-d] # go to http://localhost:8083 (port 80 in GHC Container is mapped to 8083 on host) @@ -103,7 +125,7 @@ To access your Postgres DB while running: ``` # Bash into running GHC Container -docker exec -it docker_geohealthcheck_1 bash +docker exec -it ghc_web bash # In GHC Container access DB with psql using DB parms psql -h postgis_ghc -U ghc ghc @@ -125,121 +147,84 @@ the Docker host IP address on your Docker host as follows: ``` -## Cronjobs +## Cron jobs -Cronjobs via `docker-compose` are scheduled using [Jobber Cron](https://github.com/blacklabelops/jobber-cron/). -See this snippet from the [docker-compose.yml](compose/docker-compose.yml): +DEPRECATED: Cronjobs via `docker-compose` are since v0.4.0 no longer scheduled via Jobber (cron) +but within GHC itself. The GHC Docker Image can run as a Container in a daemon runner +role that executes all GHC scheduled runs from the DB directly. +But with the per-Resource scheduling introduced in v0.4.0, cron-jobs +cannot support the detailed scheduling required. -``` -jobber: - image: blacklabelops/jobber:docker.v1.1 - depends_on: - - geohealthcheck-cron-hourly - - geohealthcheck-cron-daily - environment: - # May see warnings, see https://github.com/blacklabelops/rsnapshot/issues/2 but ok. - JOB_NAME1: ghc-cron-hourly - JOB_COMMAND1: docker start $$(docker ps -a -f label=io.ghc-cron-hourly=true --format="{{.ID}}") - JOB_TIME1: 0 0 * - JOB_NAME2: ghc-cron-daily - JOB_COMMAND2: docker start $$(docker ps -a -f label=io.ghc-cron-daily=true --format="{{.ID}}") - JOB_TIME2: 0 45 0 * - volumes: - - /var/run/docker.sock:/var/run/docker.sock -``` +## Configuration -This configures two jobs `geohealthcheck-cron-hourly` (every whole hour) and -`geohealthcheck-cron-daily` (every night at 00:45). Both run the GHC Docker image with different -containers and entrypoints. You may want to setup additional jobs e.g. to backup the database. -For example the daily cron-job entry looks like: +The default GHC configuration is specified within the [Dockerfile](../Dockerfile). +This allows overriding these `ENV` vars during deployment (as opposed to having to build +your own GHC Docker Image). Docker (`-e` options) and Docker Compose (`environment` and/or `env_file` +sections) allow setting Environment variables. + +Here we use `.env` files. ``` -geohealthcheck-cron-daily: - image: geopython/geohealthcheck:latest - depends_on: - - geohealthcheck - entrypoint: - - bash - - /cron-jobs-daily.sh - volumes: - - ghc_sqlitedb:/GeoHealthCheck/DB - labels: - io.ghc-cron-daily: 'true' -``` + ghc_runner: + image: geopython/geohealthcheck:latest -NB: the Jobber entries are run when docker-compose runs, there may be an error when the DB -is not yet setup in the main GHC Docker Container, you can ignore this. + container_name: ghc_runner -NB you may see these Jobber warnings: + env_file: + - ghc.env -``` -Failed to load jobs for open /dev/null/.jobber: not a directory: sshd -Failed to load jobs for open /dev/null/.jobber: not a directory: guest + entrypoint: + - /run-runner.sh + + volumes: + - ghc_sqlitedb:/GeoHealthCheck/DB ``` -These are not errors, see [this Jobber issue](https://github.com/blacklabelops/rsnapshot/issues/2). -Make sure your jobs are scheduled, via: +This applies variables specified in [ghc.env](compose/ghc.env) to the Docker container. +One may override these with another file as is done in the Postgres +version: ``` -# Enter the container: -$ docker exec -it compose_jobber_1 bash -# List Jobber Jobs: -$ jobber list + ghc_runner: + image: geopython/geohealthcheck:latest -NAME STATUS SEC/MIN/HR/MDAY/MTH/WDAY NEXT RUN TIME NOTIFY ON ERR NOTIFY ON FAIL ERR HANDLER -ghc-cron-hourly Good 0 0 * * * * Jul 9 14:00:00 2017 false false Continue -ghc-cron-daily Good 0 45 0 * * * Jul 10 00:45:00 2017 false false Continue + container_name: ghc_runner -# Test job execution: -$ jobber test ghc-cron-hourly -$ jobber test ghc-cron-daily + env_file: + - ghc.env + - ghc-postgis.env -``` + links: + - postgis_ghc -## Configuration + depends_on: + - postgis_ghc -The default GHC configuration is specified within the [Dockerfile](../Dockerfile). -This allows overriding these `ENV` vars during deployment (as opposed to having to build -your own GHC Docker Image). Docker (`-e` options) and Docker Compose (`environment` section) -allow setting Environment variables. + entrypoint: + - /run-runner.sh -For example, to enable sending email notifications -in Dutch witin the GHC hourly job, specify the `environment` like: + postgis_ghc: + image: mdillon/postgis:9.6-alpine -``` - geohealthcheck-cron-hourly: - image: geopython/geohealthcheck:latest - depends_on: - - geohealthcheck - - # Override settings to enable email notifications - environment: - LC_ALL: "nl_NL.UTF-8" - LANG: "nl_NL.UTF-8" - LANGUAGE: "nl_NL.UTF-8" - GHC_NOTIFICATIONS: 'true' - GHC_NOTIFICATIONS_VERBOSITY: 'true' - GHC_ADMIN_EMAIL: 'us@gmail.com' - GHC_NOTIFICATIONS_EMAIL: 'us@gmail.com,them@domain.com' - GHC_SMTP_SERVER: 'smtp.gmail.com' - GHC_SMTP_PORT: 587 - GHC_SMTP_TLS: 'true' - GHC_SMTP_SSL: 'false' - GHC_SMTP_USERNAME: 'us@gmail.com' - GHC_SMTP_PASSWORD: 'the_passw' - GHC_USER_PLUGINS: 'GeoHealthCheck.plugins.user.myplugins' - . - . + container_name: postgis_ghc + + env_file: + - ghc-postgis.env + + volumes: + - ghc_pgdb:/var/lib/postgresql/data ``` +Here [ghc-postgis.env](compose/ghc-postgis.env) adds extra Postgres-related config settings. + ## Other tasks -You can always `bash` into the GHC container to run maintenance tasks. -The GHC installation is at `/GeoHealthCheck`. +You can always `bash` into the GHC Container to run maintenance tasks. +The GHC installation is at `/GeoHealthCheck` within the Docker Container. ``` @@ -256,3 +241,5 @@ paver upgrade etc ``` +NB: database upgrades (`paver upgrade`) +are always performed automatically when running GHC via Docker. diff --git a/docker/compose/README.md b/docker/compose/README.md new file mode 100644 index 00000000..3deade94 --- /dev/null +++ b/docker/compose/README.md @@ -0,0 +1,7 @@ +# GHC with Docker Compose + +[Docker](https://www.docker.com/) is the fastest/recommended way to get GHC up and running. +[Docker Compose](https://docs.docker.com/compose/) is *a tool for defining and running multi-container Docker applications.* + +Within this directory are *examples* for running GHC using Docker compose. +You should copy and adapt these for your own deployment. diff --git a/docker/compose/docker-compose.postgis.yml b/docker/compose/docker-compose.postgis.yml index 79f95818..71a1d16b 100644 --- a/docker/compose/docker-compose.postgis.yml +++ b/docker/compose/docker-compose.postgis.yml @@ -1,42 +1,86 @@ -# Overrides default docker-compose.yml -# To run specify both files: -# sudo docker-compose -f docker-compose.yml -f docker-compose.postgis.yml up [-d] - +# GHC Docker setup with Postgres as backend DB. +# +# To run: +# sudo docker-compose -f docker-compose.postgis.yml up [-d] +# version: "3" services: - geohealthcheck: + ghc_web: + image: geopython/geohealthcheck:latest + + container_name: ghc_web + + restart: unless-stopped + + env_file: + - ghc.env + - ghc-postgis.env + links: - postgis_ghc + depends_on: - postgis_ghc - environment: - - SQLALCHEMY_DATABASE_URI=postgresql://ghc:ghc@postgis_ghc:5432/ghc - geohealthcheck-cron-daily: + ports: + - 8083:80 + + # volumes: + # Optional Plugins, using Path on the host, relative to this Compose file + # To activate: 2 steps for runner and GHC webapp: + # - configure in ghc.env + # - mount these as docker volume on host + # See https://docs.docker.com/compose/compose-file/#volumes + # - ./../GeoHealthCheck/plugins:/plugins:ro + + ghc_runner: + image: geopython/geohealthcheck:latest + + container_name: ghc_runner + + restart: unless-stopped + + env_file: + - ghc.env + - ghc-postgis.env + links: - postgis_ghc - environment: - - SQLALCHEMY_DATABASE_URI=postgresql://ghc:ghc@postgis_ghc:5432/ghc - geohealthcheck-cron-hourly: - links: + depends_on: - postgis_ghc - environment: - - SQLALCHEMY_DATABASE_URI=postgresql://ghc:ghc@postgis_ghc:5432/ghc + + entrypoint: + - /run-runner.sh + + # volumes: + # Optional Plugins, using Path on the host, relative to this Compose file + # To activate 2 steps: + # - configure in ghc.env + # - mount these as docker volume on host + # See https://docs.docker.com/compose/compose-file/#volumes + # - ./../GeoHealthCheck/plugins:/plugins:ro postgis_ghc: - image: mdillon/postgis:9.6-alpine + image: mdillon/postgis:10-alpine + + container_name: postgis_ghc + + restart: unless-stopped + + env_file: + - ghc-postgis.env + volumes: - ghc_pgdb:/var/lib/postgresql/data - ports: - - 5432:5432 + +# If you ever need to expose PG ports on host externally +# ports: +# - 5432:5432 + expose: - 5432 - environment: - - POSTGRES_USER=ghc - - POSTGRES_PASSWORD=ghc - - POSTGRES_DB=ghc # docker-compose v2+ needs separate volumes section volumes: diff --git a/docker/compose/docker-compose.yml b/docker/compose/docker-compose.yml index 4ba9e155..ad7f6e34 100644 --- a/docker/compose/docker-compose.yml +++ b/docker/compose/docker-compose.yml @@ -1,79 +1,55 @@ -# Default Compose config: runs GHC with SQLite backend and cronjobs +# Default Docker Compose config: +# runs GHC Web App (ghc_web) and GHC Daemon Runner (ghc_runner) Docker containers +# with SQLite backend. ghc_runner will run using the schedule +# configured in the GHC Web App. +# +# Plugins (optional) need to be configured for both Docker containers. version: "3" services: - geohealthcheck: + ghc_web: image: geopython/geohealthcheck:latest + + container_name: ghc_web + + restart: unless-stopped + ports: - 8083:80 -# environment: -# GHC_USER_PLUGINS: 'GeoHealthCheck.plugins.user.mywmsprobe,GeoHealthCheck.plugins.user.mywmsprobe2' + + env_file: + - ghc.env + volumes: - ghc_sqlitedb:/GeoHealthCheck/DB - # Optional Plugins, Path on the host, relative to the Compose file + # Optional Plugins, using Path on the host, relative to this Compose file + # To activate: 2 steps for runner and GHC webapp: + # - configure in ghc.env + # - mount these as docker volume on host # See https://docs.docker.com/compose/compose-file/#volumes # - ./../GeoHealthCheck/plugins:/plugins:ro - geohealthcheck-cron-hourly: + ghc_runner: image: geopython/geohealthcheck:latest - depends_on: - - geohealthcheck - # Override settings to enable email notifications and user Plugins -# environment: -# LC_ALL: "nl_NL.UTF-8" -# LANG: "nl_NL.UTF-8" -# LANGUAGE: "nl_NL.UTF-8" -# GHC_NOTIFICATIONS: 'true' -# GHC_NOTIFICATIONS_VERBOSITY: 'true' -# GHC_ADMIN_EMAIL: 'us@gmail.com' -# GHC_NOTIFICATIONS_EMAIL: 'us@gmail.com,them@domain.com' -# GHC_SMTP_SERVER: 'smtp.gmail.com' -# GHC_SMTP_PORT: 587 -# GHC_SMTP_TLS: 'true' -# GHC_SMTP_SSL: 'false' -# GHC_SMTP_USERNAME: 'us@gmail.com' -# GHC_SMTP_PASSWORD: 'the_passw' -# GHC_USER_PLUGINS: 'GeoHealthCheck.plugins.user.mywmsprobe,GeoHealthCheck.plugins.user.mywmsprobe2' + container_name: ghc_runner + + restart: unless-stopped + + env_file: + - ghc.env entrypoint: - - bash - - /cron-jobs-hourly.sh + - /run-runner.sh + volumes: - ghc_sqlitedb:/GeoHealthCheck/DB - # Optional Plugins, Path on the host, relative to the Compose file + # Optional Plugins, using Path on the host, relative to this Compose file + # To activate 2 steps: + # - configure in ghc.env + # - mount these as docker volume on host # See https://docs.docker.com/compose/compose-file/#volumes # - ./../GeoHealthCheck/plugins:/plugins:ro - labels: - io.ghc-cron-hourly: 'true' - - geohealthcheck-cron-daily: - image: geopython/geohealthcheck:latest - depends_on: - - geohealthcheck - entrypoint: - - bash - - /cron-jobs-daily.sh - volumes: - - ghc_sqlitedb:/GeoHealthCheck/DB - labels: - io.ghc-cron-daily: 'true' - - jobber: - image: blacklabelops/jobber:docker.v1.1 - depends_on: - - geohealthcheck-cron-hourly - - geohealthcheck-cron-daily - environment: - # May see warnings, see https://github.com/blacklabelops/rsnapshot/issues/2 but ok. - JOB_NAME1: ghc-cron-hourly - JOB_COMMAND1: docker start $$(docker ps -a -f label=io.ghc-cron-hourly=true --format="{{.ID}}") - JOB_TIME1: 0 0 * - JOB_NAME2: ghc-cron-daily - JOB_COMMAND2: docker start $$(docker ps -a -f label=io.ghc-cron-daily=true --format="{{.ID}}") - JOB_TIME2: 0 45 0 * - volumes: - - /var/run/docker.sock:/var/run/docker.sock # docker-compose v2+ needs separate volumes section volumes: diff --git a/docker/compose/ghc-postgis.env b/docker/compose/ghc-postgis.env new file mode 100644 index 00000000..67bd5892 --- /dev/null +++ b/docker/compose/ghc-postgis.env @@ -0,0 +1,7 @@ +# We mainly need to override the default (sqlite) DB URI +SQLALCHEMY_DATABASE_URI=postgresql://ghc:ghc@postgis_ghc:5432/ghc + +# Postgres Docker container settings +POSTGRES_USER=ghc +POSTGRES_PASSWORD=ghc +POSTGRES_DB=ghc diff --git a/docker/compose/ghc.env b/docker/compose/ghc.env new file mode 100644 index 00000000..20c3bb43 --- /dev/null +++ b/docker/compose/ghc.env @@ -0,0 +1,25 @@ +SQLALCHEMY_DATABASE_URI=sqlite:////GeoHealthCheck/DB/data.db + +# Core variables settings, change at will. +GHC_RUNNER_IN_WEBAPP=False +GHC_NOTIFICATIONS=True +GHC_NOTIFICATIONS_VERBOSITY=True +GHC_ADMIN_EMAIL=us@gmail.com +GHC_NOTIFICATIONS_EMAIL=us@gmail.com,them@domain.com +GHC_SMTP_SERVER=smtp.gmail.com +GHC_SMTP_PORT=587 +GHC_SMTP_TLS=True +GHC_SMTP_SSL=False +GHC_SMTP_USERNAME=us@gmail.com +GHC_SMTP_PASSWORD=the_passw +GHC_LOG_LEVEL=20 + +# GHC_USER_PLUGINS=GeoHealthCheck.plugins.user.mywmsprobe,GeoHealthCheck.plugins.user.mywmsprobe2 + +# Optionally set container Timezone +# CONTAINER_TIMEZONE=Europe/Amsterdam + +# Optionally: set language +# LC_ALL=nl_NL.UTF-8 +# LANG=nl_NL.UTF-8 +# LANGUAGE=nl_NL.UTF-8 diff --git a/docker/config_site.py b/docker/config_site.py index 36bb64be..67c813cc 100644 --- a/docker/config_site.py +++ b/docker/config_site.py @@ -28,6 +28,11 @@ # ================================================================= import os + +def str2bool(v): + return v.lower() in ("yes", "true", "t", "1") + + DEBUG = False SQLALCHEMY_ECHO = False @@ -39,16 +44,20 @@ # Replace None with 'your secret key string' in quotes SECRET_KEY = os.environ['SECRET_KEY'] -GHC_RETENTION_DAYS = os.environ['GHC_RETENTION_DAYS'] -GHC_RUN_FREQUENCY = os.environ['GHC_RUN_FREQUENCY'] -GHC_SELF_REGISTER = os.environ['GHC_SELF_REGISTER'] -GHC_NOTIFICATIONS = os.environ['GHC_NOTIFICATIONS'] -GHC_NOTIFICATIONS_VERBOSITY = os.environ['GHC_NOTIFICATIONS_VERBOSITY'] -GHC_WWW_LINK_EXCEPTION_CHECK = os.environ['GHC_WWW_LINK_EXCEPTION_CHECK'] +GHC_RETENTION_DAYS = int(os.environ['GHC_RETENTION_DAYS']) +GHC_PROBE_HTTP_TIMEOUT_SECS = int(os.environ['GHC_PROBE_HTTP_TIMEOUT_SECS']) +GHC_MINIMAL_RUN_FREQUENCY_MINS = int(os.environ['GHC_MINIMAL_RUN_FREQUENCY_MINS']) +GHC_SELF_REGISTER = str2bool(os.environ['GHC_SELF_REGISTER']) +GHC_NOTIFICATIONS = str2bool(os.environ['GHC_NOTIFICATIONS']) +GHC_NOTIFICATIONS_VERBOSITY = str2bool(os.environ['GHC_NOTIFICATIONS_VERBOSITY']) +GHC_WWW_LINK_EXCEPTION_CHECK = str2bool(os.environ['GHC_WWW_LINK_EXCEPTION_CHECK']) GHC_NOTIFICATIONS_EMAIL = os.environ['GHC_NOTIFICATIONS_EMAIL'] GHC_ADMIN_EMAIL = os.environ['GHC_ADMIN_EMAIL'] GHC_SITE_TITLE = os.environ['GHC_SITE_TITLE'] GHC_SITE_URL = os.environ['GHC_SITE_URL'] +GHC_RUNNER_IN_WEBAPP = str2bool(os.environ['GHC_RUNNER_IN_WEBAPP']) +GHC_LOG_LEVEL = int(os.environ['GHC_LOG_LEVEL']) +GHC_LOG_FORMAT = os.environ['GHC_LOG_FORMAT'] # Optional ENV set for GHC_PLUGINS (internal/core Plugins) # if not set default from config_main.py applies diff --git a/docker/docker-clean.sh b/docker/docker-clean.sh new file mode 100755 index 00000000..87716567 --- /dev/null +++ b/docker/docker-clean.sh @@ -0,0 +1,3 @@ +#!/bin/bash +docker rm $(docker ps -a -q) +docker rmi $(docker images -f dangling=true -q) diff --git a/docker/install-docker-ubuntu.sh b/docker/install-docker-ubuntu.sh index eb1e47cf..a37b3c47 100755 --- a/docker/install-docker-ubuntu.sh +++ b/docker/install-docker-ubuntu.sh @@ -56,6 +56,6 @@ apt-get install docker-ce # service docker start # Docker compose -export dockerComposeVersion="1.13.0" +export dockerComposeVersion="1.20.1" curl -L https://github.com/docker/compose/releases/download/${dockerComposeVersion}/docker-compose-`uname -s`-`uname -m` > /usr/local/bin/docker-compose chmod +x /usr/local/bin/docker-compose diff --git a/docker/configure.sh b/docker/scripts/configure.sh similarity index 98% rename from docker/configure.sh rename to docker/scripts/configure.sh index d2094d4e..b0adb5c2 100755 --- a/docker/configure.sh +++ b/docker/scripts/configure.sh @@ -3,7 +3,7 @@ echo "START /configure.sh" -cd /venv/ && . bin/activate +source /venv/bin/activate . # Determine database type from DB URI DB_TYPE=$(echo ${SQLALCHEMY_DATABASE_URI} | cut -f1 -d:) diff --git a/docker/cron-jobs-daily.sh b/docker/scripts/cron-jobs-daily.sh similarity index 100% rename from docker/cron-jobs-daily.sh rename to docker/scripts/cron-jobs-daily.sh diff --git a/docker/cron-jobs-hourly.sh b/docker/scripts/cron-jobs-hourly.sh similarity index 76% rename from docker/cron-jobs-hourly.sh rename to docker/scripts/cron-jobs-hourly.sh index 9166cc41..bd1ab8e4 100755 --- a/docker/cron-jobs-hourly.sh +++ b/docker/scripts/cron-jobs-hourly.sh @@ -8,4 +8,4 @@ then cp -ar /plugins/* /GeoHealthCheck/GeoHealthCheck/plugins/ fi -python /GeoHealthCheck/GeoHealthCheck/models.py run +python /GeoHealthCheck/GeoHealthCheck/healthcheck.py diff --git a/docker/install.sh b/docker/scripts/install.sh similarity index 85% rename from docker/install.sh rename to docker/scripts/install.sh index 36b8c849..c66f34fa 100755 --- a/docker/install.sh +++ b/docker/scripts/install.sh @@ -1,10 +1,10 @@ #!/bin/bash cd / -virtualenv venv && cd $_ +virtualenv venv && cd /venv . bin/activate -# GHC Source was ADDed in Dockerfile +# GHC Source was added in Dockerfile, install cd /GeoHealthCheck pip install Paver pip install sphinx @@ -26,6 +26,7 @@ mv /config_site.py /GeoHealthCheck/instance/config_site.py if [ -d /plugins ] then # Copy possible Plugins into app tree + echo "Installing Plugins..." cp -ar /plugins/* GeoHealthCheck/plugins/ # Remove to allow later Volume mount of /plugins diff --git a/docker/scripts/run-runner.sh b/docker/scripts/run-runner.sh new file mode 100755 index 00000000..926f8dbe --- /dev/null +++ b/docker/scripts/run-runner.sh @@ -0,0 +1,17 @@ +#!/bin/bash + +echo "START /run-runner.sh" + +# Set the timezone. +/set-timezone.sh + +# Configure: DB and plugins. +/configure.sh + +# Run the GHC daemon process +# that schedules healthcheck jobs +source /venv/bin/activate . +cd /GeoHealthCheck +paver runner_daemon + +echo "END /run-runner.sh" diff --git a/docker/run.sh b/docker/scripts/run-web.sh similarity index 55% rename from docker/run.sh rename to docker/scripts/run-web.sh index 89e6d418..04153b38 100755 --- a/docker/run.sh +++ b/docker/scripts/run-web.sh @@ -2,12 +2,24 @@ # Runs the GHC app with gunicorn -echo "START /run.sh" +echo "START /run-web.sh" -cd /venv/ && . bin/activate +# Set the timezone. +/set-timezone.sh + +# Configure: DB and plugins. +/configure.sh + +source /venv/bin/activate . -echo "Running GHC WSGI on ${HOST}:${PORT} with ${WSGI_WORKERS} workers" cd /GeoHealthCheck + +paver upgrade + +# SCRIPT_NAME should not have value '/' +[ "${SCRIPT_NAME}" = '/' ] && export SCRIPT_NAME="" && echo "make SCRIPT_NAME empty from /" + +echo "Running GHC WSGI on ${HOST}:${PORT} with ${WSGI_WORKERS} workers and SCRIPT_NAME=${SCRIPT_NAME}" gunicorn --workers ${WSGI_WORKERS} \ --worker-class=${WSGI_WORKER_CLASS} \ --timeout ${WSGI_WORKER_TIMEOUT} \ @@ -18,4 +30,4 @@ gunicorn --workers ${WSGI_WORKERS} \ # Built-in Flask server, deprecated # python /GeoHealthCheck/GeoHealthCheck/app.py ${HOST}:${PORT} -echo "END /run.sh" +echo "END /run-web.sh" diff --git a/docker/scripts/set-timezone.sh b/docker/scripts/set-timezone.sh new file mode 100755 index 00000000..d1697e77 --- /dev/null +++ b/docker/scripts/set-timezone.sh @@ -0,0 +1,23 @@ +#!/bin/bash + +# Set timezone and time right in container. +# See: https://www.ivankrizsan.se/2015/10/31/time-in-docker-containers/ (Alpine) + +# Set the timezone. Base image does not contain +# the setup-timezone script, so an alternate way is used. +if [ "$CONTAINER_TIMEZONE" = "" ]; +then + CONTAINER_TIMEZONE="Europe/London" +else + echo "Container timezone not modified" +fi + +cp /usr/share/zoneinfo/${CONTAINER_TIMEZONE} /etc/localtime && \ +echo "${CONTAINER_TIMEZONE}" > /etc/timezone && \ +echo "Container timezone set to: $CONTAINER_TIMEZONE" + +# Force immediate synchronisation of the time and start the time-synchronization service. +# In order to be able to use ntpd in the container, it must be run with the SYS_TIME capability. +# In addition you may want to add the SYS_NICE capability, +# in order for ntpd to be able to modify its priority. +# ntpd -s diff --git a/docs/_static/ghc-parts.jpg b/docs/_static/ghc-parts.jpg new file mode 100644 index 00000000..86544737 Binary files /dev/null and b/docs/_static/ghc-parts.jpg differ diff --git a/docs/admin.rst b/docs/admin.rst index 2d9bce10..616a3b11 100644 --- a/docs/admin.rst +++ b/docs/admin.rst @@ -87,7 +87,8 @@ The folowing resource types are available: Deleting Resources ------------------ -Open the resource details by clicking its name in the resources list at the Dashboard page. Under the resource title is a red Delete button. +Open the resource details by clicking its name in the resources list at the Dashboard page. +Under the resource title is a red Delete button. Editing Resources ----------------- @@ -99,7 +100,9 @@ The following aspects of a `Resource` can be edited: - Resource name - Resource Tags +- Resource active/non-active - Notification recipients +- Resource run schedule - Resource Probes, select from "Probes Available" - For each Probe: Probe parameters - For each Probe: Probe Checks, select from "Checks Available" @@ -107,21 +110,37 @@ The following aspects of a `Resource` can be edited: By default, when resource is created, owner's email will be added to notifications, however, resource can have arbitrary number or emails to notify. -Scheduling Runs ---------------- +Running Healthchecks +-------------------- -- Permanent Jobs +Healthchecks (Runs) for each Resource can be scheduled via `cron` or +(since v0.4.0) by running the **GHC Runner** app standalone (as daemon) +or within the **GHC Webapp**. + +Scheduling via Cron +................... Edit the file ``jobs.cron`` so that the paths reflect the path to the virtualenv. Set the first argument to the desired monitoring time step. If finished editing, -copy the command line calls e.g. ``/YOURvirtualenv/bin_or_SCRIPTSonwindows/python /path/to/GeoHealthCheck/GeoHealthCheck/models.py run`` +copy the command line calls e.g. ``/YOURvirtualenv/bin_or_SCRIPTSonwindows/python /path/to/GeoHealthCheck/GeoHealthCheck/healthcheck.py run`` to the commandline to test if they work sucessfully. On Windows - do not forget to include the ''.exe.'' file extension to the python executable. For documentation how to create cron jobs see your operating system: on \*NIX systems e.g. ``crontab -e`` and on windows e.g. the `nssm `_. -- interactive - TBF +NB the limitation of cron is that the per `Resource` schedule cannot be applied as +the cron job will run healthchecks on all `Resources`. + +GHC Runner as Daemon +.................... + +In this mode GHC applies internal scheduling for each individual `Resource`. +This is the preferred mode as each `Resource` can have its own schedule (configurable +via Dashboard) and `cron` has dependencies on local environment. +Later versions may phase out cron-scheduling. +The **GHC Runner** can be run via the command `paver runner_daemon` or internally within +the **GHC Webapp** by setting the config variable **GHC_RUNNER_IN_WEBAPP** to True. + Build Documentation ------------------- @@ -129,11 +148,24 @@ Build Documentation Open a command line, (if needed activate your virtualenv) and move into the directory ``GeoHealthCheck/doc/``. In there, type ''make html'' plus ENTER and the documentation should be built locally. -GeoNode Resource type notes +GeoNode Resource Type Notes --------------------------- -*GeoNode* Resource is a virtual resource. It represents one GeoNode instance, but underneath it's trying autodiscovery of OWS endpoints available in that instance. Note, that OWS autodiscovery feature is optional, and you should check if GeoNode instance has this feature enabled. - -When adding *GeoNode instance* Resource, you have to enter url to instance's home page. GeoHealthCheck will construct url to target OWS endpoints listing and create relevant Resources (WMS, WFS, WMTS, OWC Resources). It will check all endpoints provided by GeoNode API, and will reject those which responded with error. - -All resources added in this way will have at least one tag, which is constructed with template: *GeoNode _hostname_*, where *_hostname_* is a host name from url provided. For example, let's assume you add GeoNode instance that is served from `demo.geonode.org`. All resources created in this way will have *GeoNode demo.geonode.org* tag. +*GeoNode* Resource is a virtual resource. +It represents one GeoNode instance, but underneath +auto-discovery is applied of OWS endpoints available +in that instance. Note, that OWS auto-discovery feature is +optional, and you should check if your GeoNode instance has this feature enabled. + +When adding *GeoNode instance* Resource, you have to enter +the url to the GN instance's home page. +GeoHealthCheck will construct the urls to target +OWS endpoints listing and create relevant Resources (WMS, WFS, WMTS, OWC Resources). +It will check all endpoints provided by the GeoNode API, and will reject +those which responded with an error. + +All resources added in this way will have at least one tag, +which is constructed with template: *GeoNode _hostname_*, where *_hostname_* +is a host name from url provided. For example, let's assume you add GeoNode +instance that is served from `demo.geonode.org`. All resources created in this way +will have *GeoNode demo.geonode.org* tag. diff --git a/docs/architecture.rst b/docs/architecture.rst index 7345950b..28645533 100644 --- a/docs/architecture.rst +++ b/docs/architecture.rst @@ -3,7 +3,42 @@ Architecture ============ -(To be extended) +GeoHealthCheck (GHC) consists of three cooperating parts as depicted in the figure below. + +.. figure:: _static/ghc-parts.jpg + :align: center + :alt: GHC Parts + + *Figure - GHC Parts* + +The **GHC Webapp** provides the Dashboard where users configure web services +(Resources) for (scheduled) health-checks and view the status of these checks. +The **GHC Runner** performs the actual health-checks and notifications, +based on what the user configured via the GHC Webapp. + +The third part is the **Database** that stores +all information like users, resources, checks, schedules, results etc. + +The **GHC Webapp** is run as a standard Python (Flask) webapp. +The **GHC Runner** runs as a daemon process using an internal scheduler to invoke the +actual healthchecks. + +**GHC Webapp** and **GHC Runner** can run as separate processes (preferred) or +both within the **GHC Webapp** process. This depends on a configuration option. +If **GHC_RUNNER_IN_WEBAPP** is set to True (the default) then the **GHC Runner** is started +within the **GHC Webapp**. + +A third option is to only run the **GHC Webapp** and have the **GHC Runner** scheduled +via `cron`. This was the (only) GHC option before v0.4.0 and will be phased out +as starting with v0.4.0, per-Resource scheduling was introduced and `cron` support +is highly platform-dependent (e.g. hard to use with Docker-based technologies). + +Dependent on the database-type (Postgres or SQLite) the **Database** is run +within the above processes (SQLite) or as a separate process (Postgres). + +So in the most minimal setup, i.e. **GHC Webapp** and **GHC Runner** running within +a single process and using SQLite as the database, only a single GHC process instance +is required. Core Concepts ------------- @@ -33,3 +68,59 @@ Data Model :alt: GHC Data Model *Figure - GHC Data Model* + +GHC Webapp Design +----------------- + +The **GHC Webapp** is realized as a standard `Flask` web-application using `SQLAlchemy` for +ORM database support. It is the user-visible part of GHC as it runs via the +browser. Its main two functionalities are to allow users to: + +* manage (create, update, delete) Resources, their attributes and their Probes and Checks, and +* view results and statistics of Resources (Dashboard function) + +Deployment can be realized using the various standard Flask deployment methods: standalone, +within a WSGI server etc. + +As an option (via configuration, see above) the **GHC Runner** may run within the **GHC Webapp**. Note that in case +that when the **GHC Webapp** runs as multiple processes and/or threads "Resource Locking" (see below) +will prevent inconsistencies. + +GHC Runner Design +----------------- + +The **GHC Runner** in its core is a job scheduler based on the Python library +`APScheduler `_. Each job scheduled is a +healthcheck runner for a single `Resource` that runs all the `Probes` for that `Resource`. +The run-frequency follows the per-Resource run frequency (since v0.4.0). + +The GHC Runner is thus responsible for running the `Probes` for each `Resource`, storing +the `Results` and doing notifications when needed. + +The **GHC Runner** can run as a separate (Python) process, or within the **GHC WebApp** (see above). +Separate processes is the preferred mode of running. + +Job Runner Synchronization +.......................... + +As multiple instances of the job scheduler (i.e. `APScheduler `_) +may run in different processes and even threads within processes, +the database is used to synchronize and assure only one job will run for a single `Resource`. + +This is achieved by having one lock per `Resource` via the table `ResourceLock`. +Only the process/thread that acquires its related `ResourceLock` +record runs the job. +As to avoid permanent "lockouts", each `ResourceLock` has a lifetime, namely +the timespan until the next Run as configured for/per `Resource`. This gives +all job runners a chance to obtain a lock once "time's up" for the `ResourceLock`. + +Additional lock-liveliness is realized by using a unique **UUID per job runner**. +Once the lock is obtained, the UUID-field of the lock record is set and committed +to the DB. If we then try to obtain the lock again (by reading from DB) +but the UUID is different this means another job runner instance did the same but +was just before us. The lock-lifetime (see above) guards that a particular UUID keeps +the lock forever, e.g. on sudden application shutdown. + +The above locking mechanism is supported for SQLite, but it is strongly +advised to use PostgreSQL for better robustness and performance in general. + diff --git a/docs/config.rst b/docs/config.rst index 33c49dc4..5c2d7775 100644 --- a/docs/config.rst +++ b/docs/config.rst @@ -6,14 +6,11 @@ Configuration Core configuration is set by GeoHealthCheck in ``GeoHealthCheck/config_main.py``. You can override these settings in ``instance/config_site.py``: -- **SQLALCHEMY_DATABASE_URI**: the database configuration. See the - SQLAlchemy documentation for more info -- **SECRET_KEY**: secret key to set when enabling authentication. Use - the output of ``paver create_secret_key`` to set this value +- **SQLALCHEMY_DATABASE_URI**: the database configuration. See the SQLAlchemy documentation for more info +- **SECRET_KEY**: secret key to set when enabling authentication. Use the output of ``paver create_secret_key`` to set this value - **GHC_RETENTION_DAYS**: the number of days to keep run history -- **GHC_RUN_FREQUENCY**: cron keyword used to indicate frequency of runs - (i.e. ``hourly``, ``daily``, ``monthly``) - **GHC_PROBE_HTTP_TIMEOUT_SECS**: stop waiting for the first byte of a probe response after the given number of seconds +- **GHC_MINIMAL_RUN_FREQUENCY_MINS**: minimal run frequency for Resource that can be set in web UI - **GHC_SELF_REGISTER**: allow registrations from users on the website - **GHC_NOTIFICATIONS**: turn on email notifications - **GHC_NOTIFICATIONS_VERBOSITY**: receive additional email notifications than just ``Failing`` and ``Fixed`` (default ``True``) @@ -28,6 +25,8 @@ You can override these settings in ``instance/config_site.py``: - **GHC_USER_PLUGINS**: list of Plugin classes or modules provided by user (you) - **GHC_PROBE_DEFAULTS**: Default `Probe` class to assign on "add" per Resource-type - **GHC_METADATA_CACHE_SECS**: metadata, "Capabilities Docs", cache expiry time, default 900 secs, -1 to disable +- **GHC_RUNNER_IN_WEBAPP**: should the GHC Runner Daemon be run in webapp (default: false) +- **GHC_LOG_LEVEL**: logging level: 10=DEBUG 20=INFO 30=WARN(ING) 40=ERROR 50=FATAL/CRITICAL (default: 30, WARNING) - **GHC_MAP**: default map settings - **url**: URL of TileLayer @@ -36,7 +35,6 @@ You can override these settings in ``instance/config_site.py``: - **maxzoom**: maximum zoom level - **subdomains**: available subdomains to help with parallel requests - Enabling or disabling languages ------------------------------- diff --git a/docs/index.rst b/docs/index.rst index f15be868..df1de561 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -21,7 +21,7 @@ Features - database agnostic: any SQLAlchemy supported backend - database upgrades: using Alembic with Flask-Migrate - extensible healthchecks via Plugins -- per-resource notifications +- per-resource scheduling and notifications Links ----- diff --git a/docs/install.rst b/docs/install.rst index 8ec3df29..db6fdd32 100644 --- a/docs/install.rst +++ b/docs/install.rst @@ -8,7 +8,7 @@ Below are installation notes for GeoHealthCheck (GHC). Docker ------ -Easiest and quickest install/run +The easiest and quickest install for GHC is with Docker/Docker Compose using the GHC images hosted on `Docker Hub `_. @@ -19,15 +19,17 @@ for a full guide. Requirements ------------ -GeoHealthCheck is built on the awesome Flask microframework and uses -Flask-SQLAlchemy for database interaction and Flask-Login for authorization. -Flask-Migrate with Alembic and Flask-Script support is used for database upgrades. +GeoHealthCheck is built on the awesome Flask micro-framework and uses +`Flask-SQLAlchemy` for database interaction and Flask-Login for authorization. +`Flask-Migrate` with `Alembic` and `Flask-Script` is used for database upgrades. -OWSLib is used to interact with OGC Web Services. +`OWSLib` is used to interact with OGC Web Services. + +`APScheduler` is used to run scheduled healthchecks. These dependencies are automatically installed (see below). ``Paver`` is used -for installation and management. ``Cron`` is used for scheduling the actual -healthchecks. +for installation and management. ``Cron`` was used for scheduling the actual +healthchecks before v0.4.0. Install ------- @@ -49,19 +51,22 @@ Install . bin/activate git clone https://github.com/geopython/GeoHealthCheck.git cd GeoHealthCheck + # install paver dependency for admin tool pip install Paver + # setup app paver setup + # create secret key to use for auth paver create_secret_key + # almost there! Customize config vi instance/config_site.py # edit: # - SQLALCHEMY_DATABASE_URI # - SECRET_KEY # from paver create_secret_key # - GHC_RETENTION_DAYS - # - GHC_RUN_FREQUENCY # - GHC_SELF_REGISTER # - GHC_NOTIFICATIONS # - GHC_NOTIFICATIONS_VERBOSITY @@ -75,7 +80,7 @@ Install # init database python GeoHealthCheck/models.py create - # start server + # start web-app python GeoHealthCheck/app.py # http://localhost:8000/ Schedule the cronjobs. diff --git a/jobs.cron b/jobs.cron index 91b7d061..a60c13fc 100644 --- a/jobs.cron +++ b/jobs.cron @@ -1,2 +1,2 @@ -@hourly /virtualenv/bin/python /path/to/GeoHealthCheck/GeoHealthCheck/models.py run +@hourly /virtualenv/bin/python /path/to/GeoHealthCheck/GeoHealthCheck/healthcheck.py run @daily /virtualenv/bin/python /path/to/GeoHealthCheck/GeoHealthCheck/models.py flush diff --git a/pavement.py b/pavement.py index 3a5ceacb..8f14bff2 100644 --- a/pavement.py +++ b/pavement.py @@ -242,7 +242,7 @@ def clean(): @task def extract_translations(): - """extrect translations wrapped in _() or gettext()""" + """extract translations wrapped in _() or gettext()""" pot_dir = path('GeoHealthCheck/translations/en/LC_MESSAGES') if not os.path.exists(pot_dir): @@ -283,6 +283,18 @@ def update_translations(): options.base.pot, options.base.translations)) +@task +def runner_daemon(): + """Run the HealthCheck runner daemon scheduler""" + sh('python GeoHealthCheck/scheduler.py') + + +@task +def run_healthchecks(): + """Run all HealthChecks directly""" + sh('python GeoHealthCheck/healthcheck.py') + + def sphinx_make(): """return what command Sphinx is using for make""" diff --git a/requirements.txt b/requirements.txt index 463acf4a..f2da7536 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,5 +6,6 @@ Flask-Script==2.0.5 Flask-SQLAlchemy==2.1 OWSLib Sphinx -requests -WTForms +requests==2.18.4 +WTForms==2.1 +APScheduler==3.5.1