Merge pull request #255 from OpenTreeOfLife/development

passes tests on dev

Author: mtholder

phylesystem_api/phylesystem_api/__init__.py

@@ -1,4 +1,5 @@
 from pyramid.config import Configurator
+from .api_utils import get_conf_object, get_phylesystem
 
 
 def main(global_config, **settings):
@@ -9,4 +10,9 @@ def main(global_config, **settings):
         config.add_cors_preflight_handler()
         config.include(".routes")
         config.scan()
+    # for k, v in settings.items():
+    #     print("{k}: {v}".format(k=k, v=repr(v)))
+    localconfig_filename = settings["config_file_path"]
+    conf_obj = get_conf_object(localconfig_filename=localconfig_filename)
+    _ps = get_phylesystem(request=None, conf_obj=conf_obj)
     return config.make_wsgi_app()

phylesystem_api/phylesystem_api/api_utils.py

@@ -10,13 +10,9 @@
 # see exception subclasses at https://docs.pylonsproject.org/projects/pyramid/en/latest/api/httpexceptions.html
 from pyramid.request import Request
 from pyramid.httpexceptions import (
-    HTTPException,
     HTTPOk,
-    HTTPError,
     HTTPServerError,
-    HTTPNotFound,
     HTTPBadRequest,
-    HTTPInternalServerError,
     HTTPForbidden,
 )
 from beaker.cache import cache_managers
@@ -25,8 +21,6 @@
 import json
 import requests
 import os
-import re
-import copy
 import threading
 
 try:
@@ -51,8 +45,8 @@ def get_private_dir(request):
 def atomic_write_json_if_not_found(obj, dest, request):
     if os.path.exists(dest):
         return False
-    dir = get_private_dir(request)
-    handle, tmpfn = tempfile.mkstemp(suffix=".json", dir=dir, text=True)
+    pdir = get_private_dir(request)
+    handle, tmpfn = tempfile.mkstemp(suffix=".json", dir=pdir, text=True)
     # mkstemp opens the file and returns a file descriptor,
     #   but we are using write_as_json to open with the right encoding
     os.close(handle)
@@ -84,7 +78,7 @@ def compose_push_to_github_url(request, resource_id, doc_type):
 _PHYLESYSTEM = None
 
 
-def get_phylesystem(request):
+def get_phylesystem(request, conf_obj=None):
     global READ_ONLY_MODE
     global _PHYLESYSTEM
     # _LOG.debug('@@@ checking for _PHYLESYSTEM singleton...READ_ONLY_MODE? {}'.format(READ_ONLY_MODE))
@@ -94,6 +88,8 @@ def get_phylesystem(request):
     # _LOG.debug('@@@ NOT FOUND, creating now')
     from phylesystem_api.gitdata import GitData
 
+    if conf_obj is None:
+        conf_obj = get_conf_object(request)
     (
         repo_parent,
         repo_remote,
@@ -103,7 +99,7 @@ def get_phylesystem(request):
         max_filesize,
         max_num_trees,
         READ_ONLY_MODE,
-    ) = read_phylesystem_config(request)
+    ) = read_phylesystem_config(request, conf_obj=conf_obj)
     peyotl_config, cfg_filename = read_peyotl_config()
     if "phylesystem" not in peyotl_config.sections():
         peyotl_config.add_section("phylesystem")
@@ -118,10 +114,9 @@ def get_phylesystem(request):
         },
     }
     mirror_info = {"push": pmi}
-    conf = get_conf_object(request)
     a = {}
     try:
-        new_study_prefix = conf.get("apis", "new_study_prefix")
+        new_study_prefix = conf_obj.get("apis", "new_study_prefix")
         a["new_study_prefix"] = new_study_prefix
     except:
         pass
@@ -135,9 +130,9 @@ def get_phylesystem(request):
     )
     # _LOG.debug('[[[[[[ repo_nexml2json = {}'.format(_PHYLESYSTEM.repo_nexml2json))
     if READ_ONLY_MODE:
-        _LOG.warn("phylesytem-api running in READ_ONLY_MODE")
+        _LOG.warning("phylesytem-api running in READ_ONLY_MODE")
     else:
-        _LOG.warn("phylesytem-api NOT running in READ_ONLY_MODE")
+        _LOG.warning("phylesytem-api NOT running in READ_ONLY_MODE")
     return _PHYLESYSTEM
 
 
@@ -167,9 +162,6 @@ def get_tree_collection_store(request):
         },
     }
     mirror_info = {"push": pmi}
-    conf = get_conf_object(request)
-    import pprint
-
     a = {}
     try:
         # any keyword args to pass along from config?
@@ -216,9 +208,6 @@ def get_taxonomic_amendment_store(request):
         },
     }
     mirror_info = {"push": pmi}
-    conf = get_conf_object(request)
-    import pprint
-
     a = {}
     try:
         # any keyword args to pass along from config?
@@ -255,21 +244,34 @@ def get_failed_push_filepath(request, doc_type=None):
     return os.path.join(get_private_dir(request), failure_filename)
 
 
-def get_conf_object(request):
+def get_conf_object(request=None, localconfig_filename=None):
     # There's apparently no easy way to retrieve the fully parsed
     # configuration from within the app. But we can access the variables
     # from the [app:main] setion, so we'll retrieve the full path to
     # our chosen INI file from there.
+    if localconfig_filename is None:
+        assert request is not None
+        localconfig_filename = request.registry.settings["config_file_path"]
+    _LOG.debug(
+        'get_conf_object(localconfig_filename="{}")'.format(localconfig_filename)
+    )
+    if not os.path.isfile(localconfig_filename):
+        raise RuntimeError(
+            "localconfig_filename={} does not exist".format(localconfig_filename)
+        )
     conf = ConfigParser(allow_no_value=True)
-    localconfig_filename = request.registry.settings["config_file_path"]
-    if os.path.isfile(localconfig_filename):
-        conf.readfp(open(localconfig_filename))
+    conf.read(localconfig_filename)
     return conf
 
 
-def read_phylesystem_config(request):
+def read_phylesystem_config(request, conf_obj=None):
     """Load settings for managing the main Nexson docstore"""
-    conf = get_conf_object(request)
+    if conf_obj is None:
+        conf_obj = get_conf_object(request)
+    return _read_phylesystem_from_conf_obj(conf_obj)
+
+
+def _read_phylesystem_from_conf_obj(conf):
     repo_parent = conf.get("apis", "repo_parent")
     repo_remote = conf.get("apis", "repo_remote")
     try:
@@ -660,14 +662,12 @@ def clear_matching_cache_keys(key_pattern):
     assert len(namespaces) == 1
     active_namespace = list(namespaces.values())[0]
     # NB - again, code may change if we use multiple namespaces here
-    item_count_before = len(list(active_namespace.items()))
-    """
-    print("=== %d RAM cache keys BEFORE clearing: ===" % item_count_before)
-    for k, v in active_namespace.items():
-        print('{k} ===> {v}'.format(k=k,v=v))
-    print("===")
-    """
     # _LOG.debug("> clearing cached items matching [%s]" % key_pattern)
+    # item_count_before = len(list(active_namespace.items()))
+    # print("=== %d RAM cache keys BEFORE clearing: ===" % item_count_before)
+    # for k, v in active_namespace.items():
+    #     print('{k} ===> {v}'.format(k=k,v=v))
+    # print("===")
 
     matching_keys = []
     for k, v in active_namespace.items():
@@ -676,14 +676,12 @@ def clear_matching_cache_keys(key_pattern):
     for matching_key in matching_keys:
         del active_namespace[matching_key]
 
-    """
-    item_count_after = len(list(active_namespace.items()))
-    print("=== %d RAM cache keys AFTER clearing: ===" % item_count_after)
-    for k, v in active_namespace.items():
-        print('{k} ===> {v}'.format(k,v))
-    print("===")
-    print("  %d items removed" % (item_count_before - item_count_after,))
-    """
+    # item_count_after = len(list(active_namespace.items()))
+    # print("=== %d RAM cache keys AFTER clearing: ===" % item_count_after)
+    # for k, v in active_namespace.items():
+    #     print('{k} ===> {v}'.format(k,v))
+    # print("===")
+    # print("  %d items removed" % (item_count_before - item_count_after,))
 
 
 def raise_on_CORS_preflight(request):
@@ -875,7 +873,6 @@ def extract_json_from_http_call(request, data_field_name="data", request_params=
 
     request_params can be the Pyramids request.params multidict or just a dict.
     """
-    json_obj = None
     try:
         # check for kwarg data_field_name, or load the full request body
         if data_field_name in request_params: