Per resource scheduling

.travis.yml

@@ -14,7 +14,7 @@ script:
   - echo -e "admin\ntest\ntest\nyou@example.com\nyou@example.com" | python GeoHealthCheck/models.py create
   - flake8
   - python GeoHealthCheck/models.py load tests/data/fixtures.json y
-  - python GeoHealthCheck/models.py run
+  - python GeoHealthCheck/healthcheck.py
   - python tests/run_tests.py
   - cd docs && make html
 

Dockerfile

@@ -4,81 +4,82 @@ FROM python:2.7.13-alpine
 # FROM debian:jessie
 
 # Credits to yjacolin for providing first versions
-LABEL original_developer "yjacolin <yves.jacolin@camptocamp.com>"
-LABEL maintainer "Just van den Broecke <justb4@gmail.com>"
+LABEL original_developer="yjacolin <yves.jacolin@camptocamp.com>" \
+	maintainer="Just van den Broecke <justb4@gmail.com>"
 
 # These are default values,
 # Override when running container via docker(-compose)
 
 # General ENV settings
-ENV LC_ALL "en_US.UTF-8"
-ENV LANG "en_US.UTF-8"
-ENV LANGUAGE "en_US.UTF-8"
-
-# GHC ENV settings
-ENV ADMIN_NAME admin
-ENV ADMIN_PWD admin
-ENV ADMIN_EMAIL admin.istrator@mydomain.com
-ENV SQLALCHEMY_DATABASE_URI 'sqlite:////GeoHealthCheck/DB/data.db'
-ENV SECRET_KEY 'd544ccc37dc3ad214c09b1b7faaa64c60351d5c8bb48b342'
-ENV GHC_RETENTION_DAYS 30
-ENV GHC_RUN_FREQUENCY 'hourly'
-ENV GHC_PROBE_HTTP_TIMEOUT_SECS 30
-ENV GHC_SELF_REGISTER False
-ENV GHC_NOTIFICATIONS False
-ENV GHC_NOTIFICATIONS_VERBOSITY True
-ENV GHC_WWW_LINK_EXCEPTION_CHECK False
-ENV GHC_ADMIN_EMAIL 'you@example.com'
-ENV GHC_NOTIFICATIONS_EMAIL 'you2@example.com,them@example.com'
-ENV GHC_SITE_TITLE 'GeoHealthCheck'
-ENV GHC_SITE_URL 'http://localhost'
-ENV GHC_SMTP_SERVER None
-ENV GHC_SMTP_PORT None
-ENV GHC_SMTP_TLS False
-ENV GHC_SMTP_SSL False
-ENV GHC_SMTP_USERNAME None
-ENV GHC_SMTP_PASSWORD None
-ENV GHC_METADATA_CACHE_SECS 900
-
-# WSGI server settings, assumed is gunicorn
-ENV HOST 0.0.0.0
-ENV PORT 80
-ENV WSGI_WORKERS 4
-ENV WSGI_WORKER_TIMEOUT 6000
-ENV WSGI_WORKER_CLASS 'eventlet'
-
-# GHC Core Plugins modules and/or classes, seldom needed to set:
-# if not specified here or in Container environment
-# all GHC built-in Plugins will be active.
+ENV LC_ALL="en_US.UTF-8" \
+	LANG="en_US.UTF-8" \
+	LANGUAGE="en_US.UTF-8" \
+    \
+	\
+	# GHC ENV settings\
+	ADMIN_NAME=admin \
+	ADMIN_PWD=admin \
+	ADMIN_EMAIL=admin.istrator@mydomain.com \
+	SQLALCHEMY_DATABASE_URI='sqlite:////GeoHealthCheck/DB/data.db' \
+	SECRET_KEY='d544ccc37dc3ad214c09b1b7faaa64c60351d5c8bb48b342' \
+	GHC_PROBE_HTTP_TIMEOUT_SECS=30 \
+	GHC_MINIMAL_RUN_FREQUENCY_MINS=10 \
+	GHC_RETENTION_DAYS=30 \
+	GHC_SELF_REGISTER=False \
+	GHC_NOTIFICATIONS=False \
+	GHC_NOTIFICATIONS_VERBOSITY=True \
+	GHC_WWW_LINK_EXCEPTION_CHECK=False \
+	GHC_ADMIN_EMAIL='you@example.com' \
+	GHC_RUNNER_IN_WEBAPP=False \
+	GHC_LOG_LEVEL=30 \
+	GHC_LOG_FORMAT='%(asctime)s - %(name)s - %(levelname)s - %(message)s' \
+	GHC_NOTIFICATIONS_EMAIL='you2@example.com,them@example.com' \
+	GHC_SITE_TITLE='GeoHealthCheck' \
+	GHC_SITE_URL='http://localhost' \
+	GHC_SMTP_SERVER=None \
+	GHC_SMTP_PORT=None \
+	GHC_SMTP_TLS=False \
+	GHC_SMTP_SSL=False \
+	GHC_SMTP_USERNAME=None \
+	GHC_SMTP_PASSWORD=None \
+	GHC_METADATA_CACHE_SECS=900 \
+    \
+# WSGI server settings, assumed is gunicorn  \
+HOST=0.0.0.0 \
+PORT=80 \
+WSGI_WORKERS=4 \
+WSGI_WORKER_TIMEOUT=6000 \
+WSGI_WORKER_CLASS='eventlet' \
+\
+# GHC Core Plugins modules and/or classes, seldom needed to set:  \
+# if not specified here or in Container environment  \
+# all GHC built-in Plugins will be active.  \
 #ENV GHC_PLUGINS 'GeoHealthCheck.plugins.probe.owsgetcaps,\
 #        GeoHealthCheck.plugins.probe.wms, ...., ...\
-#        GeoHealthCheck.plugins.check.checks'
-
-# GHC User Plugins, best be overridden via Container environment
-ENV GHC_USER_PLUGINS ''
+#        GeoHealthCheck.plugins.check.checks' \
+\
+# GHC User Plugins, best be overridden via Container environment \
+GHC_USER_PLUGINS=''
 
 RUN apk add --no-cache --virtual .build-deps gcc build-base linux-headers postgresql-dev \
-    && apk add --no-cache bash vim postgresql-client \
+    && apk add --no-cache bash postgresql-client tzdata openntpd \
     && pip install virtualenv \
     && rm -rf /var/cache/apk/* /tmp/* /var/tmp/*
 
 # Add standard files and Add/override Plugins
 # Alternative Entrypoints to run GHC jobs
 # Override default Entrypoint with these on Containers
-ADD docker/install.sh docker/configure.sh docker/run.sh \
-	docker/config_site.py docker/cron-jobs-daily.sh docker/cron-jobs-hourly.sh docker/plugins /
-RUN	chmod a+x /*.sh
+ADD docker/scripts/*.sh docker/config_site.py docker/plugins /
 
 # Add Source Code
 ADD . /GeoHealthCheck
 
 # Install and Remove build-related packages for smaller image size
-RUN bash install.sh  \
-	&& apk del .build-deps
+RUN chmod a+x /*.sh && bash install.sh && apk del .build-deps
 
 # For SQLite
 VOLUME ["/GeoHealthCheck/DB/"]
 
 EXPOSE ${PORT}
 
-ENTRYPOINT /configure.sh && /run.sh
+ENTRYPOINT /run-web.sh

GeoHealthCheck/app.py

@@ -32,7 +32,6 @@
 import csv
 import logging
 import json
-from datetime import datetime, timedelta
 from StringIO import StringIO
 from itertools import chain
 
@@ -44,7 +43,6 @@
 from flask_migrate import Migrate
 
 from __init__ import __version__
-from healthcheck import sniff_test_resource, run_test_resource
 from init import App
 from enums import RESOURCE_TYPES
 from models import Resource, Run, ProbeVars, CheckVars, Tag, User, Recipient
@@ -74,6 +72,16 @@
     ('hr_HR', 'Croatian (Croatia)')
 )
 
+# Should GHC Runner be run within GHC webapp?
+if CONFIG['GHC_RUNNER_IN_WEBAPP'] is True:
+    LOGGER.info('Running GHC Scheduler in WebApp')
+    from scheduler import start_schedule
+
+    # Start scheduler
+    start_schedule()
+else:
+    LOGGER.info('NOT Running GHC Scheduler in WebApp')
+
 
 # commit or rollback shorthand
 def db_commit():
@@ -122,35 +130,6 @@ def unauthorized_callback():
     return redirect(url_for('login', lang=g.current_lang, next=url))
 
 
-def next_page_refresh():
-    """determines when to refresh webapp based on GHC_RUN_FREQUENCY"""
-
-    now = datetime.now()
-
-    frequency = CONFIG['GHC_RUN_FREQUENCY']
-
-    if frequency == 'hourly':  # get next hour
-        now2 = now.replace(minute=0, second=0, microsecond=0)
-        refresh = timedelta(hours=1)
-    elif frequency == 'daily':  # get next day
-        now2 = now.replace(hour=0, minute=0, second=0, microsecond=0)
-        refresh = timedelta(days=1)
-    elif frequency == ['weekly']:  # get next day
-        now2 = now.replace(day=1, hour=0, minute=0, second=0, microsecond=0)
-        refresh = timedelta(weeks=1)
-    elif frequency == ['monthly']:
-        now2 = now.replace(day=1, hour=0, minute=0, second=0, microsecond=0)
-        refresh = timedelta(weeks=4)
-    elif frequency == ['yearly']:  # get next day
-        now2 = now.replace(day=1, hour=0, minute=0, second=0, microsecond=0)
-        refresh = timedelta(weeks=52)
-
-    next_frequency = now2 + refresh
-    differ = next_frequency - now
-
-    return differ.seconds
-
-
 @APP.template_filter('cssize_reliability')
 def cssize_reliability(value, css_type=None):
     """returns CSS button class snippet based on score"""
@@ -162,11 +141,11 @@ def cssize_reliability(value, css_type=None):
         score = 'danger'
         panel = 'red'
     elif (CONFIG['GHC_RELIABILITY_MATRIX']['orange']['min'] <= number <=
-            CONFIG['GHC_RELIABILITY_MATRIX']['orange']['max']):
+          CONFIG['GHC_RELIABILITY_MATRIX']['orange']['max']):
         score = 'warning'
         panel = 'yellow'
     elif (CONFIG['GHC_RELIABILITY_MATRIX']['green']['min'] <= number <=
-            CONFIG['GHC_RELIABILITY_MATRIX']['green']['max']):
+          CONFIG['GHC_RELIABILITY_MATRIX']['green']['max']):
         score = 'success'
         panel = 'green'
     else:  # should never really get here
@@ -203,7 +182,6 @@ def context_processors():
     tags = views.get_tag_counts()
     return {
         'app_version': __version__,
-        'next_page_refresh': next_page_refresh(),
         'resource_types': RESOURCE_TYPES,
         'resource_types_counts': rtc['counts'],
         'resources_total': rtc['total'],
@@ -490,6 +468,7 @@ def add():
     url = request.form['url'].strip()
     resources_to_add = []
 
+    from healthcheck import sniff_test_resource, run_test_resource
     sniffed_resources = sniff_test_resource(CONFIG, resource_type, url)
 
     if not sniffed_resources:
@@ -557,7 +536,7 @@ def add():
                             param_vals = {}
                             for param in param_defs:
                                 if param_defs[param]['value']:
-                                    param_vals[param] =\
+                                    param_vals[param] = \
                                         param_defs[param]['value']
                             check_vars = CheckVars(
                                 probe_to_add, check_class, param_vals)
@@ -641,8 +620,8 @@ def update(resource_identifier):
 
                 # Add ProbeVars anew each with optional CheckVars
                 for probe in value:
-                    print('adding Probe class=%s parms=%s' %
-                          (probe['probe_class'], str(probe)))
+                    LOGGER.info('adding Probe class=%s parms=%s' %
+                                (probe['probe_class'], str(probe)))
                     probe_vars = ProbeVars(resource, probe['probe_class'],
                                            probe['parameters'])
                     for check in probe['checks']:
@@ -663,6 +642,9 @@ def update(resource_identifier):
             elif getattr(resource, key) != resource_identifier_dict[key]:
                 # Update other resource attrs, mainly 'name'
                 setattr(resource, key, resource_identifier_dict[key])
+                min_run_freq = CONFIG['GHC_MINIMAL_RUN_FREQUENCY_MINS']
+                if int(resource.run_frequency) < min_run_freq:
+                    resource.run_frequency = min_run_freq
                 update_counter += 1
 
         # Always update geo-IP: maybe failure on creation or
@@ -699,6 +681,7 @@ def test(resource_identifier):
         flash(gettext('Resource not found'), 'danger')
         return redirect(request.referrer)
 
+    from healthcheck import run_test_resource
     result = run_test_resource(
         resource)
 
@@ -754,10 +737,6 @@ def delete(resource_identifier):
         flash(gettext('Resource not found'), 'danger')
         return redirect(url_for('home', lang=g.current_lang))
 
-    runs = Run.query.filter_by(resource_identifier=resource_identifier).all()
-
-    for run in runs:
-        DB.session.delete(run)
     resource.clear_recipients()
     DB.session.delete(resource)
 
@@ -908,7 +887,7 @@ def api_probes_avail(resource_type=None, resource_id=None):
 
 if __name__ == '__main__':  # run locally, for fun
     import sys
-    logging.basicConfig()
+
     HOST = '0.0.0.0'
     PORT = 8000
     if len(sys.argv) > 1:

GeoHealthCheck/config_main.py

@@ -38,8 +38,8 @@
 SECRET_KEY = None
 
 GHC_RETENTION_DAYS = 30
-GHC_RUN_FREQUENCY = 'hourly'
 GHC_PROBE_HTTP_TIMEOUT_SECS = 30
+GHC_MINIMAL_RUN_FREQUENCY_MINS = 10
 GHC_SELF_REGISTER = False
 GHC_NOTIFICATIONS = False
 GHC_NOTIFICATIONS_VERBOSITY = True
@@ -48,6 +48,10 @@
 GHC_NOTIFICATIONS_EMAIL = ['you2@example.com']
 GHC_SITE_TITLE = 'GeoHealthCheck Demonstration'
 GHC_SITE_URL = 'http://host'
+GHC_RUNNER_IN_WEBAPP = True
+# 10=DEBUG 20=INFO 30=WARN(ING) 40=ERROR 50=FATAL/CRITICAL
+GHC_LOG_LEVEL = 30
+GHC_LOG_FORMAT = '%(asctime)s - %(name)s - %(levelname)s - %(message)s'
 
 # Some GetCaps docs are huge. This allows
 # caching them for N seconds. Set to -1 to

GeoHealthCheck/factory.py

@@ -1,3 +1,8 @@
+import logging
+
+LOGGER = logging.getLogger(__name__)
+
+
 class Factory:
     """
     Object, Function class Factory (Pattern).
@@ -18,9 +23,9 @@ def create_obj(class_string):
 
             # class instance from class object with constructor args
             return class_obj()
-        except Exception, e:
-            print("cannot create object instance from class '%s' e=%s" %
-                  (class_string, str(e)))
+        except Exception as e:
+            LOGGER.error("cannot create object instance from class '%s' e=%s" %
+                         (class_string, str(e)))
             raise e
 
     @staticmethod
@@ -41,8 +46,8 @@ def create_class(class_string):
             class_obj = getattr(
                 __import__(module_name, globals(), locals(),
                            [class_name], -1), class_name)
-        except Exception, e:
-            print("cannot create class '%s'" % class_string)
+        except Exception as e:
+            LOGGER.error("cannot create class '%s'" % class_string)
             raise e
 
         return class_obj
@@ -61,8 +66,8 @@ def create_module(module_string):
         try:
             module_obj = __import__(module_string, globals(),
                                     locals(), fromlist=[''])
-        except Exception, e:
-            print("cannot create module from '%s'" % module_string)
+        except Exception as e:
+            LOGGER.error("cannot create module from '%s'" % module_string)
             raise e
 
         return module_obj