From 5cff5562ccada674237bbf9c9ba1dd0026610d40 Mon Sep 17 00:00:00 2001 From: Christophe Haen Date: Wed, 3 Jun 2020 16:23:12 +0200 Subject: [PATCH 01/25] Initial introduction of HTTPS service --- ConfigurationSystem/Client/CSAPI.py | 4 +- .../Client/ConfigurationClient.py | 77 +++ .../Service/TornadoConfigurationHandler.py | 141 +++++ ConfigurationSystem/private/Refresher.py | 322 ++++++++--- .../private/ServiceInterface.py | 361 +------------ .../private/ServiceInterfaceBase.py | 365 +++++++++++++ .../private/ServiceInterfaceTornado.py | 33 ++ Core/Base/Client.py | 10 +- Core/DISET/RequestHandler.py | 12 + Core/DISET/private/Service.py | 2 +- Core/Tornado/Client/RPCClientSelector.py | 59 ++ .../Tornado/Client/SpecificClient/__init__.py | 0 Core/Tornado/Client/TornadoClient.py | 72 +++ Core/Tornado/Client/__init__.py | 3 + .../Client/private/TornadoBaseClient.py | 507 ++++++++++++++++++ Core/Tornado/Client/private/__init__.py | 3 + Core/Tornado/Server/HandlerManager.py | 171 ++++++ Core/Tornado/Server/TornadoServer.py | 214 ++++++++ Core/Tornado/Server/TornadoService.py | 497 +++++++++++++++++ Core/Tornado/Server/__init__.py | 3 + Core/Tornado/Utilities/HTTPErrorCodes.py | 20 + Core/Tornado/Utilities/__init__.py | 3 + Core/Tornado/__init__.py | 3 + Core/Tornado/scripts/__init__.py | 1 + Core/Tornado/scripts/tornado-start-CS.py | 50 ++ Core/Tornado/scripts/tornado-start-all.py | 58 ++ ...tReturnedValuesAndCredentialsExtraction.py | 116 ++++ .../test/TestClientSelectionAndInterface.py | 179 +++++++ Core/Tornado/test/TestServerIntegration.py | 91 ++++ .../test/multi-mechanize/distributed-test.py | 45 ++ .../test/multi-mechanize/ping/config.cfg | 13 + .../ping/test_scripts/v_perf.py | 18 + .../multi-mechanize/plot-distributedTest.py | 173 ++++++ Core/Tornado/test/multi-mechanize/plot.py | 53 ++ .../test/multi-mechanize/service/config.cfg | 13 + .../service/test_scripts/v_perf.py | 25 + .../Client/MonitoringClientIOLoop.py | 30 ++ .../DeveloperGuide/ConfigurationSystem.rst | 26 + .../Internals/Core/Authentification.rst | 55 ++ .../Internals/Core/ClientServer.rst | 3 +- .../TornadoServices/clientservice.uml | 15 + .../DeveloperGuide/TornadoServices/index.rst | 221 ++++++++ .../TornadoServices/installTornado.rst | 214 ++++++++ docs/source/DeveloperGuide/index.rst | 1 + docs/source/DeveloperGuide/monitoring.rst | 0 requirements.txt | 5 + .../TornadoServices/ConfigTemplate.cfg | 9 + .../Integration/TornadoServices/DB/UserDB.py | 74 +++ .../Integration/TornadoServices/DB/UserDB.sql | 3 + .../Services/DummyTornadoHandler.py | 22 + .../Services/UserDiracHandler.py | 68 +++ .../TornadoServices/Services/UserHandler.py | 78 +++ .../Test_TornadoAndDISETmixed.py | 190 +++++++ tests/Integration/TornadoServices/getCPUInfos | 37 ++ .../TornadoServices/perf-test-DB/config.cfg | 19 + .../perf-test-DB/test_scripts/v_perf.py | 37 ++ .../TornadoServices/perf-test-ping/config.cfg | 19 + .../perf-test-ping/test_scripts/v_perf.py | 24 + 58 files changed, 4438 insertions(+), 429 deletions(-) create mode 100644 ConfigurationSystem/Client/ConfigurationClient.py create mode 100644 ConfigurationSystem/Service/TornadoConfigurationHandler.py create mode 100644 ConfigurationSystem/private/ServiceInterfaceBase.py create mode 100644 ConfigurationSystem/private/ServiceInterfaceTornado.py create mode 100644 Core/Tornado/Client/RPCClientSelector.py create mode 100644 Core/Tornado/Client/SpecificClient/__init__.py create mode 100644 Core/Tornado/Client/TornadoClient.py create mode 100644 Core/Tornado/Client/__init__.py create mode 100644 Core/Tornado/Client/private/TornadoBaseClient.py create mode 100644 Core/Tornado/Client/private/__init__.py create mode 100644 Core/Tornado/Server/HandlerManager.py create mode 100644 Core/Tornado/Server/TornadoServer.py create mode 100644 Core/Tornado/Server/TornadoService.py create mode 100644 Core/Tornado/Server/__init__.py create mode 100644 Core/Tornado/Utilities/HTTPErrorCodes.py create mode 100644 Core/Tornado/Utilities/__init__.py create mode 100644 Core/Tornado/__init__.py create mode 100644 Core/Tornado/scripts/__init__.py create mode 100644 Core/Tornado/scripts/tornado-start-CS.py create mode 100644 Core/Tornado/scripts/tornado-start-all.py create mode 100644 Core/Tornado/test/TestClientReturnedValuesAndCredentialsExtraction.py create mode 100644 Core/Tornado/test/TestClientSelectionAndInterface.py create mode 100644 Core/Tornado/test/TestServerIntegration.py create mode 100644 Core/Tornado/test/multi-mechanize/distributed-test.py create mode 100644 Core/Tornado/test/multi-mechanize/ping/config.cfg create mode 100644 Core/Tornado/test/multi-mechanize/ping/test_scripts/v_perf.py create mode 100644 Core/Tornado/test/multi-mechanize/plot-distributedTest.py create mode 100755 Core/Tornado/test/multi-mechanize/plot.py create mode 100644 Core/Tornado/test/multi-mechanize/service/config.cfg create mode 100644 Core/Tornado/test/multi-mechanize/service/test_scripts/v_perf.py create mode 100644 FrameworkSystem/Client/MonitoringClientIOLoop.py create mode 100644 docs/source/DeveloperGuide/ConfigurationSystem.rst create mode 100644 docs/source/DeveloperGuide/Internals/Core/Authentification.rst create mode 100644 docs/source/DeveloperGuide/TornadoServices/clientservice.uml create mode 100644 docs/source/DeveloperGuide/TornadoServices/index.rst create mode 100644 docs/source/DeveloperGuide/TornadoServices/installTornado.rst create mode 100644 docs/source/DeveloperGuide/monitoring.rst create mode 100644 tests/Integration/TornadoServices/ConfigTemplate.cfg create mode 100644 tests/Integration/TornadoServices/DB/UserDB.py create mode 100644 tests/Integration/TornadoServices/DB/UserDB.sql create mode 100644 tests/Integration/TornadoServices/Services/DummyTornadoHandler.py create mode 100644 tests/Integration/TornadoServices/Services/UserDiracHandler.py create mode 100644 tests/Integration/TornadoServices/Services/UserHandler.py create mode 100644 tests/Integration/TornadoServices/Test_TornadoAndDISETmixed.py create mode 100755 tests/Integration/TornadoServices/getCPUInfos create mode 100644 tests/Integration/TornadoServices/perf-test-DB/config.cfg create mode 100644 tests/Integration/TornadoServices/perf-test-DB/test_scripts/v_perf.py create mode 100644 tests/Integration/TornadoServices/perf-test-ping/config.cfg create mode 100644 tests/Integration/TornadoServices/perf-test-ping/test_scripts/v_perf.py diff --git a/ConfigurationSystem/Client/CSAPI.py b/ConfigurationSystem/Client/CSAPI.py index e83e313b40d..566e34dbc87 100644 --- a/ConfigurationSystem/Client/CSAPI.py +++ b/ConfigurationSystem/Client/CSAPI.py @@ -8,7 +8,7 @@ import six from DIRAC import gLogger, gConfig, S_OK, S_ERROR -from DIRAC.Core.DISET.RPCClient import RPCClient +from DIRAC.ConfigurationSystem.Client.ConfigurationClient import ConfigurationClient from DIRAC.Core.Utilities import List, Time from DIRAC.Core.Security.X509Chain import X509Chain # pylint: disable=import-error from DIRAC.Core.Security import Locations @@ -90,7 +90,7 @@ def initialize(self): if not retVal['OK']: self.__initialized = S_ERROR("Master server is not known. Is everything initialized?") return self.__initialized - self.__rpcClient = RPCClient(gConfig.getValue("/DIRAC/Configuration/MasterServer", "")) + self.__rpcClient = ConfigurationClient(url=gConfig.getValue("/DIRAC/Configuration/MasterServer", "")) self.__csMod = Modificator(self.__rpcClient, "%s - %s - %s" % (self.__userGroup, self.__userDN, Time.dateTime().strftime("%Y-%m-%d %H:%M:%S"))) retVal = self.downloadCSData() diff --git a/ConfigurationSystem/Client/ConfigurationClient.py b/ConfigurationSystem/Client/ConfigurationClient.py new file mode 100644 index 00000000000..cb7702adbf6 --- /dev/null +++ b/ConfigurationSystem/Client/ConfigurationClient.py @@ -0,0 +1,77 @@ +""" + Custom TornadoClient for Configuration System + Used like a normal client, should be instanciated if and only if we use the configuration service + + Because of limitation with JSON some datas are encoded in base64 + +""" + +from base64 import b64encode, b64decode + +from DIRAC.Core.Tornado.Client.TornadoClient import TornadoClient +from DIRAC.Core.Base.Client import Client + + +class CSJSONClient(TornadoClient): + """ + The specific client for configuration system. + To avoid JSON limitation the HTTPS handler encode data in base64 + before sending them, this class only decode the base64 + + An exception is made with CommitNewData wich ENCODE in base64 + """ + + def getCompressedData(self): + """ + Transmit request to service and get data in base64, + it decode base64 before returning + + :returns str:Configuration data, compressed + """ + retVal = self.executeRPC('getCompressedData') + if retVal['OK']: + retVal['Value'] = b64decode(retVal['Value']) + return retVal + + def getCompressedDataIfNewer(self, sClientVersion): + """ + Transmit request to service and get data in base64, + it decode base64 before returning. + + :returns str:Configuration data, if changed, compressed + """ + retVal = self.executeRPC('getCompressedDataIfNewer', sClientVersion) + if retVal['OK'] and 'data' in retVal['Value']: + retVal['Value']['data'] = b64decode(retVal['Value']['data']) + return retVal + + def commitNewData(self, sData): + """ + Transmit request to service by encoding data in base64. + + :param: Data to commit, you may call this method with CSAPI and Modificator + """ + return self.executeRPC('commitNewData', b64encode(sData)) + + +class ConfigurationClient(Client): + """ + Specific client to speak with ConfigurationServer. + + This class must contain at least the JSON decoder dedicated to + the Configuration Server. + + You can implement more logic or function to the client here if needed, + RPCCall can be made inside this class with executeRPC method. + """ + + # The JSON decoder for Configuration Server + httpsClient = CSJSONClient + + def __init__(self, **kwargs): + # By default we use Configuration/Server as url, client do the resolution + # In some case url has to be static (when a slave register to the master server for example) + # It's why we can use 'url' as keyword arguments + if 'url' not in kwargs: + kwargs['url'] = 'Configuration/Server' + super(ConfigurationClient, self).__init__(**kwargs) diff --git a/ConfigurationSystem/Service/TornadoConfigurationHandler.py b/ConfigurationSystem/Service/TornadoConfigurationHandler.py new file mode 100644 index 00000000000..a5f3512e4b7 --- /dev/null +++ b/ConfigurationSystem/Service/TornadoConfigurationHandler.py @@ -0,0 +1,141 @@ +""" The CS! (Configuration Service) + + Modified to work with Tornado + Encode data in base64 because of JSON limitations + In client side you must use a specific client + +""" + +__RCSID__ = "$Id$" + +from base64 import b64encode, b64decode + +from DIRAC.Core.Utilities.ReturnValues import S_OK, S_ERROR +from DIRAC.ConfigurationSystem.private.ServiceInterfaceTornado import ServiceInterfaceTornado as ServiceInterface +from DIRAC.Core.Utilities import DErrno +from DIRAC.Core.Tornado.Server.TornadoService import TornadoService + + +class TornadoConfigurationHandler(TornadoService): + """ The CS handler + """ + ServiceInterface = None + PilotSynchronizer = None + + @classmethod + def initializeHandler(cls, serviceInfo): + """ + Initialize the configuration server + Behind it starts thread which refresh configuration + """ + cls.ServiceInterface = ServiceInterface(serviceInfo['URL']) + return S_OK() + + def export_getVersion(self): + """ + Returns the version of the configuration + """ + return S_OK(self.ServiceInterface.getVersion()) + + def export_getCompressedData(self): + """ + Returns the configuration + """ + sData = self.ServiceInterface.getCompressedConfigurationData() + return S_OK(b64encode(sData)) + + def export_getCompressedDataIfNewer(self, sClientVersion): + """ + Returns the configuration if a newer configuration exists, if not just returns the version + + :param sClientVersion: Version used by client + """ + sVersion = self.ServiceInterface.getVersion() + retDict = {'newestVersion': sVersion} + if sClientVersion < sVersion: + retDict['data'] = b64encode(self.ServiceInterface.getCompressedConfigurationData()) + return S_OK(retDict) + + def export_publishSlaveServer(self, sURL): + """ + Used by slave server to register as a slave server. + + :param sURL: The url of the slave server. + """ + self.ServiceInterface.publishSlaveServer(sURL) + return S_OK() + + def export_commitNewData(self, sData): + """ + Write the new configuration + """ + credDict = self.getRemoteCredentials() + if 'DN' not in credDict or 'username' not in credDict: + return S_ERROR("You must be authenticated!") + sData = b64decode(sData) + res = self.ServiceInterface.updateConfiguration(sData, credDict['username']) + if not res['OK']: + return res + + # Check the flag for updating the pilot 3 JSON file + if self.srv_getCSOption('UpdatePilotCStoJSONFile', False) and self.ServiceInterface.isMaster(): + if self.PilotSynchronizer is None: + try: + # This import is only needed for the Master CS service, making it conditional avoids + # dependency on the git client preinstalled on all the servers running CS slaves + from DIRAC.WorkloadManagementSystem.Utilities.PilotCStoJSONSynchronizer import PilotCStoJSONSynchronizer + except ImportError as exc: + self.log.exception("Failed to import PilotCStoJSONSynchronizer", repr(exc)) + return S_ERROR(DErrno.EIMPERR, 'Failed to import PilotCStoJSONSynchronizer') + self.PilotSynchronizer = PilotCStoJSONSynchronizer() + return self.PilotSynchronizer.sync() + + return res + + def export_writeEnabled(self): + """ + Used to know if we can change the configuration on this server + """ + return S_OK(self.ServiceInterface.isMaster()) + + def export_getCommitHistory(self, limit=100): + """ + Get the history of modifications in the configuration + """ + if limit > 100: + limit = 100 + history = self.ServiceInterface.getCommitHistory() + if limit: + history = history[:limit] + return S_OK(history) + + def export_getVersionContents(self, versionList): + """ + Get an old version of the configuration, can also be used to get more than one version. + + :param versionList: List of the version we are trying to get + """ + contentsList = [] + for version in versionList: + retVal = self.ServiceInterface.getVersionContents(version) + if retVal['OK']: + contentsList.append(retVal['Value']) + else: + return S_ERROR("Can't get contents for version %s: %s" % (version, retVal['Message'])) + return S_OK(contentsList) + + def export_rollbackToVersion(self, version): + """ + Rollback to an older version of the configuration + + :param version: The version we want to apply + """ + retVal = self.ServiceInterface.getVersionContents(version) + if not retVal['OK']: + return S_ERROR("Can't get contents for version %s: %s" % (version, retVal['Message'])) + credDict = self.getRemoteCredentials() + if 'DN' not in credDict or 'username' not in credDict: + return S_ERROR("You must be authenticated!") + return self.ServiceInterface.updateConfiguration(retVal['Value'], + credDict['username'], + updateVersionOption=True) diff --git a/ConfigurationSystem/private/Refresher.py b/ConfigurationSystem/private/Refresher.py index 4af11e81469..14eabfdaf32 100755 --- a/ConfigurationSystem/private/Refresher.py +++ b/ConfigurationSystem/private/Refresher.py @@ -1,5 +1,5 @@ """ Refresh local CS (if needed) - In every gConfig +Used each time you call gConfig. It keep your configuration up-to-date with the configuration server """ from __future__ import absolute_import from __future__ import division @@ -11,6 +11,9 @@ import thread import time import random +import os + + from DIRAC.ConfigurationSystem.Client.ConfigurationData import gConfigurationData from DIRAC.ConfigurationSystem.Client.PathFinder import getGatewayURLs from DIRAC.FrameworkSystem.Client.Logger import gLogger @@ -20,6 +23,9 @@ def _updateFromRemoteLocation(serviceClient): + """ + Refresh the configuration + """ gLogger.debug("", "Trying to refresh from %s" % serviceClient.serviceURL) localVersion = gConfigurationData.getVersion() retVal = serviceClient.getCompressedDataIfNewer(localVersion) @@ -34,111 +40,93 @@ def _updateFromRemoteLocation(serviceClient): return retVal -class Refresher(threading.Thread): +class RefresherBase(): + """ + Code factorisation for the refresher + """ def __init__(self): - threading.Thread.__init__(self) - self.__automaticUpdate = False - self.__lastUpdateTime = 0 - self.__url = False - self.__refreshEnabled = True - self.__timeout = 60 - self.__callbacks = {'newVersion': []} - gEventDispatcher.registerEvent("CSNewVersion") + self._automaticUpdate = False + self._lastUpdateTime = 0 + self._url = False + self._refreshEnabled = True + self._timeout = 60 + self._callbacks = {'newVersion': []} random.seed() - self.__triggeredRefreshLock = LockRing.LockRing().getLock() + gEventDispatcher.registerEvent("CSNewVersion") def disable(self): - self.__refreshEnabled = False + """ + Disable the refresher and prevent any request to another server + """ + self._refreshEnabled = False def enable(self): - self.__refreshEnabled = True - if self.__lastRefreshExpired(): + """ + Enable the refresher and authorize request to another server + WARNING: It will not activate automatic updates, use autoRefreshAndPublish() for that + """ + self._refreshEnabled = True + if self._lastRefreshExpired(): return self.forceRefresh() return S_OK() def isEnabled(self): - return self.__refreshEnabled + """ + Returns if you can use refresher or not, use automaticUpdateEnabled() to know + if refresh is automatic. + """ + return self._refreshEnabled def addListenerToNewVersionEvent(self, functor): gEventDispatcher.addListener("CSNewVersion", functor) - def __refreshInThread(self): - retVal = self.__refresh() - if not retVal['OK']: - gLogger.error("Error while updating the configuration", retVal['Message']) - - def __lastRefreshExpired(self): - return time.time() - self.__lastUpdateTime >= gConfigurationData.getRefreshTime() - - def refreshConfigurationIfNeeded(self): - if not self.__refreshEnabled or self.__automaticUpdate or not gConfigurationData.getServers(): - return - self.__triggeredRefreshLock.acquire() - try: - if not self.__lastRefreshExpired(): - return - self.__lastUpdateTime = time.time() - finally: - try: - self.__triggeredRefreshLock.release() - except thread.error: - pass - # Launch the refresh - thd = threading.Thread(target=self.__refreshInThread) - thd.setDaemon(1) - thd.start() + def _lastRefreshExpired(self): + """ + Just returns if last refresh must be considered as expired or not + """ + return time.time() - self._lastUpdateTime >= gConfigurationData.getRefreshTime() def forceRefresh(self, fromMaster=False): - if self.__refreshEnabled: - return self.__refresh(fromMaster=fromMaster) + """ + Force refresh + WARNING: If refresher is disabled, force a refresh will do nothing + """ + if self._refreshEnabled: + return self._refresh(fromMaster=fromMaster) return S_OK() - def autoRefreshAndPublish(self, sURL): - gLogger.debug("Setting configuration refresh as automatic") - if not gConfigurationData.getAutoPublish(): - gLogger.debug("Slave server won't auto publish itself") - if not gConfigurationData.getName(): - import DIRAC - DIRAC.abort(10, "Missing configuration name!") - self.__url = sURL - self.__automaticUpdate = True - self.setDaemon(1) - self.start() - - def run(self): - while self.__automaticUpdate: - iWaitTime = gConfigurationData.getPropagationTime() - time.sleep(iWaitTime) - if self.__refreshEnabled: - if not self.__refreshAndPublish(): - gLogger.error("Can't refresh configuration from any source") - - def __refreshAndPublish(self): - self.__lastUpdateTime = time.time() + def _refreshAndPublish(self): + """ + Refresh configuration and publish local updates + """ + self._lastUpdateTime = time.time() gLogger.info("Refreshing from master server") - from DIRAC.Core.DISET.RPCClient import RPCClient sMasterServer = gConfigurationData.getMasterServer() if sMasterServer: - oClient = RPCClient(sMasterServer, timeout=self.__timeout, - useCertificates=gConfigurationData.useServerCertificate(), - skipCACheck=gConfigurationData.skipCACheck()) + from DIRAC.ConfigurationSystem.Client.ConfigurationClient import ConfigurationClient + oClient = ConfigurationClient(url=sMasterServer, timeout=self._timeout, + useCertificates=gConfigurationData.useServerCertificate(), + skipCACheck=gConfigurationData.skipCACheck()) dRetVal = _updateFromRemoteLocation(oClient) if not dRetVal['OK']: gLogger.error("Can't update from master server", dRetVal['Message']) return False if gConfigurationData.getAutoPublish(): gLogger.info("Publishing to master server...") - dRetVal = oClient.publishSlaveServer(self.__url) + dRetVal = oClient.publishSlaveServer(self._url) if not dRetVal['OK']: gLogger.error("Can't publish to master server", dRetVal['Message']) return True else: gLogger.warn("No master server is specified in the configuration, trying to get data from other slaves") - return self.__refresh()['OK'] + return self._refresh()['OK'] - def __refresh(self, fromMaster=False): - self.__lastUpdateTime = time.time() + def _refresh(self, fromMaster=False): + """ + Refresh configuration + """ + self._lastUpdateTime = time.time() gLogger.debug("Refreshing configuration...") gatewayList = getGatewayURLs("Configuration/Server") updatingErrorsList = [] @@ -161,10 +149,10 @@ def __refresh(self, fromMaster=False): gLogger.debug("Randomized server list is %s" % ", ".join(randomServerList)) for sServer in randomServerList: - from DIRAC.Core.DISET.RPCClient import RPCClient - oClient = RPCClient(sServer, - useCertificates=gConfigurationData.useServerCertificate(), - skipCACheck=gConfigurationData.skipCACheck()) + from DIRAC.ConfigurationSystem.Client.ConfigurationClient import ConfigurationClient + oClient = ConfigurationClient(url=sServer, + useCertificates=gConfigurationData.useServerCertificate(), + skipCACheck=gConfigurationData.skipCACheck()) dRetVal = _updateFromRemoteLocation(oClient) if dRetVal['OK']: return dRetVal @@ -175,12 +163,192 @@ def __refresh(self, fromMaster=False): break return S_ERROR("Reason(s):\n\t%s" % "\n\t".join(List.uniqueElements(updatingErrorsList))) + +class Refresher(RefresherBase, threading.Thread): + """ + The refresher + A long time ago, in a code away, far away... + A guy do the code to autorefresh the configuration + To prepare transition to HTTPS we have done separation + between the logic and the implementation of background + tasks, it's the original version, for diset, using thread. + + """ + + def __init__(self): + threading.Thread.__init__(self) + RefresherBase.__init__(self) + self._triggeredRefreshLock = LockRing.LockRing().getLock() + + def _refreshInThread(self): + """ + Refreshing configration in the background. By default it use a thread but it can be + also runned in the IOLoop + """ + retVal = self._refresh() + if not retVal['OK']: + gLogger.error("Error while updating the configuration", retVal['Message']) + + def refreshConfigurationIfNeeded(self): + """ + Refresh the configuration if automatic update are disabled, refresher is enabled and servers are defined + """ + if not self._refreshEnabled or self._automaticUpdate or not gConfigurationData.getServers(): + return + self._triggeredRefreshLock.acquire() + try: + if not self._lastRefreshExpired(): + return + self._lastUpdateTime = time.time() + finally: + try: + self._triggeredRefreshLock.release() + except thread.error: + pass + # Launch the refreshf + thd = threading.Thread(target=self._refreshInThread) + thd.setDaemon(1) + thd.start() + + def autoRefreshAndPublish(self, sURL): + """ + Start the autorefresh background task + + :param str sURL: URL of the configuration server + """ + gLogger.debug("Setting configuration refresh as automatic") + if not gConfigurationData.getAutoPublish(): + gLogger.debug("Slave server won't auto publish itself") + if not gConfigurationData.getName(): + import DIRAC + DIRAC.abort(10, "Missing configuration name!") + self._url = sURL + self._automaticUpdate = True + self.setDaemon(1) + self.start() + + def run(self): + while self._automaticUpdate: + time.sleep(gConfigurationData.getPropagationTime()) + if self._refreshEnabled: + if not self._refreshAndPublish(): + gLogger.error("Can't refresh configuration from any source") + def daemonize(self): + """ + Daemonize the background tasks + """ + self.setDaemon(1) self.start() -gRefresher = Refresher() +class TornadoRefresher(RefresherBase): + """ + The refresher, modified for Tornado + It's the same refresher, the only thing which change is + that we are using the IOLoop instead of threads for background + tasks, so it work with Tornado (HTTPS server). + """ + + # For pylint... + gen = None + + try: + from tornado import gen + from tornado.ioloop import IOLoop + except ImportError: + gLogger.fatal("You should install tornado to use this refresher, please unset USE_TORNADO_IOLOOP") + + def __init__(self): + RefresherBase.__init__(self) + + def refreshConfigurationIfNeeded(self): + """ + Trigger an automatic refresh, most of the time nothing happens because automaticUpdate is enabled. + This function is called by gConfig.getValue most of the time. + + We disable pylint error because this class must be instanciated by a mixin to define the missing methods + """ + if not self._refreshEnabled or self._automaticUpdate: # pylint: disable=no-member + return + if not gConfigurationData.getServers() or not self._lastRefreshExpired(): # pylint: disable=no-member + return + self._lastUpdateTime = time.time() + self.IOLoop.current().run_in_executor(None, self._refresh) # pylint: disable=no-member + return + + def autoRefreshAndPublish(self, sURL): + """ + Start the autorefresh background task, called by ServiceInterface + (the class behind the Configuration/Server handler) + + :param str sURL: URL of the configuration server + """ + gLogger.debug("Setting configuration refresh as automatic") + if not gConfigurationData.getAutoPublish(): + gLogger.debug("Slave server won't auto publish itself") + if not gConfigurationData.getName(): + import DIRAC + DIRAC.abort(10, "Missing configuration name!") + self._url = sURL + self._automaticUpdate = True + + # Tornado replacement solution to the classic thread + # It start the method self.__refreshLoop on the next IOLoop iteration + self.IOLoop.current().spawn_callback(self.__refreshLoop) + + @gen.coroutine + def __refreshLoop(self): + """ + Trigger the autorefresh when configuration is expired + + This task must use Tornado utilities to avoid blocking the ioloop and + pottentialy deadlock the server. + + See http://www.tornadoweb.org/en/stable/guide/coroutines.html#looping + for official documentation about this type of method. + """ + while self._automaticUpdate: + + # This is the sleep from Tornado, like a sleep it wait some time + # But this version is non-blocking, so IOLoop can continue execution + yield self.gen.sleep(gConfigurationData.getPropagationTime()) + # Publish step is blocking so we have to run it in executor + # If we are not doing it, when master try to ping we block the IOLoop + yield self.IOLoop.current().run_in_executor(None, self.__AutoRefresh) + + @gen.coroutine + def __AutoRefresh(self): + """ + Auto refresh the configuration + We disable pylint error because this class must be instanciated + by a mixin to define the methods. + """ + if self._refreshEnabled: # pylint: disable=no-member + if not self._refreshAndPublish(): # pylint: disable=no-member + gLogger.error("Can't refresh configuration from any source") + + def daemonize(self): + """ daemonize is probably not the best name because there is no daemon behind + but we must keep it to the same interface of the DISET refresher """ + self.IOLoop.current().spawn_callback(self.__refreshLoop) + + +# Here we define the refresher which should be used. +# By default we use the original refresher. + +# Be careful, if you never start the IOLoop (with a TornadoServer for example) +# the TornadoRefresher will not work. IOLoop can be started after refresher +# but background tasks will be delayed until IOLoop start. + + +# USE_TORNADO_IOLOOP is defined by starting scripts +if os.environ.get('USE_TORNADO_IOLOOP', 'false').lower() == 'true': + gRefresher = TornadoRefresher() +else: + gRefresher = Refresher() + if __name__ == "__main__": time.sleep(0.1) diff --git a/ConfigurationSystem/private/ServiceInterface.py b/ConfigurationSystem/private/ServiceInterface.py index 927a701593a..39818dcb6bb 100755 --- a/ConfigurationSystem/private/ServiceInterface.py +++ b/ConfigurationSystem/private/ServiceInterface.py @@ -1,368 +1,41 @@ -""" Threaded implementation of services +""" Threaded implementation of service interface """ from __future__ import absolute_import from __future__ import division from __future__ import print_function -__RCSID__ = "$Id$" - -import os import time -import re import threading -import zipfile -import zlib -import requests +from DIRAC import gLogger + +from DIRAC.ConfigurationSystem.private.ServiceInterfaceBase import ServiceInterfaceBase +from DIRAC.ConfigurationSystem.Client.ConfigurationData import gConfigurationData -import DIRAC -from DIRAC.Core.Utilities.File import mkDir -from DIRAC.ConfigurationSystem.Client.ConfigurationData import gConfigurationData, ConfigurationData -from DIRAC.ConfigurationSystem.private.Refresher import gRefresher -from DIRAC.FrameworkSystem.Client.Logger import gLogger -from DIRAC.Core.Utilities.ReturnValues import S_OK, S_ERROR -from DIRAC.Core.DISET.RPCClient import RPCClient -from DIRAC.Core.Utilities.ThreadPool import ThreadPool -from DIRAC.Core.Security.Locations import getHostCertificateAndKeyLocation +__RCSID__ = "$Id$" -class ServiceInterface(threading.Thread): +class ServiceInterface(ServiceInterfaceBase, threading.Thread): + """ + Service interface, manage Slave/Master server for CS + Thread components + """ def __init__(self, sURL): threading.Thread.__init__(self) - self.sURL = sURL - gLogger.info("Initializing Configuration Service", "URL is %s" % sURL) - self.__modificationsIgnoreMask = ['/DIRAC/Configuration/Servers', '/DIRAC/Configuration/Version'] - gConfigurationData.setAsService() - if not gConfigurationData.isMaster(): - gLogger.info("Starting configuration service as slave") - gRefresher.autoRefreshAndPublish(self.sURL) - else: - gLogger.info("Starting configuration service as master") - gRefresher.disable() - self.__loadConfigurationData() - self.dAliveSlaveServers = {} - self.__launchCheckSlaves() - - self.__updateResultDict = {"Successful": {}, "Failed": {}} - - def isMaster(self): - return gConfigurationData.isMaster() + ServiceInterfaceBase.__init__(self, sURL) + self.__launchCheckSlaves() def __launchCheckSlaves(self): + """ + Start loop which check if slaves are alive + """ gLogger.info("Starting purge slaves thread") self.setDaemon(1) self.start() - def __loadConfigurationData(self): - mkDir(os.path.join(DIRAC.rootPath, "etc", "csbackup")) - gConfigurationData.loadConfigurationData() - if gConfigurationData.isMaster(): - bBuiltNewConfiguration = False - if not gConfigurationData.getName(): - DIRAC.abort(10, "Missing name for the configuration to be exported!") - gConfigurationData.exportName() - sVersion = gConfigurationData.getVersion() - if sVersion == "0": - gLogger.info("There's no version. Generating a new one") - gConfigurationData.generateNewVersion() - bBuiltNewConfiguration = True - - if self.sURL not in gConfigurationData.getServers(): - gConfigurationData.setServers(self.sURL) - bBuiltNewConfiguration = True - - gConfigurationData.setMasterServer(self.sURL) - - if bBuiltNewConfiguration: - gConfigurationData.writeRemoteConfigurationToDisk() - - def __generateNewVersion(self): - if gConfigurationData.isMaster(): - gConfigurationData.generateNewVersion() - gConfigurationData.writeRemoteConfigurationToDisk() - - def publishSlaveServer(self, sSlaveURL): - if not gConfigurationData.isMaster(): - return S_ERROR("Configuration modification is not allowed in this server") - gLogger.info("Pinging slave %s" % sSlaveURL) - rpcClient = RPCClient(sSlaveURL, timeout=10, useCertificates=True) - retVal = rpcClient.ping() - if not retVal['OK']: - gLogger.info("Slave %s didn't reply" % sSlaveURL) - return - if retVal['Value']['name'] != 'Configuration/Server': - gLogger.info("Slave %s is not a CS serveR" % sSlaveURL) - return - bNewSlave = False - if sSlaveURL not in self.dAliveSlaveServers: - bNewSlave = True - gLogger.info("New slave registered", sSlaveURL) - self.dAliveSlaveServers[sSlaveURL] = time.time() - if bNewSlave: - gConfigurationData.setServers("%s, %s" % (self.sURL, - ", ".join(self.dAliveSlaveServers.keys()))) - self.__generateNewVersion() - - def __checkSlavesStatus(self, forceWriteConfiguration=False): - gLogger.info("Checking status of slave servers") - iGraceTime = gConfigurationData.getSlavesGraceTime() - bModifiedSlaveServers = False - for sSlaveURL in self.dAliveSlaveServers.keys(): - if time.time() - self.dAliveSlaveServers[sSlaveURL] > iGraceTime: - gLogger.info("Found dead slave", sSlaveURL) - del self.dAliveSlaveServers[sSlaveURL] - bModifiedSlaveServers = True - if bModifiedSlaveServers or forceWriteConfiguration: - gConfigurationData.setServers("%s, %s" % (self.sURL, - ", ".join(self.dAliveSlaveServers.keys()))) - self.__generateNewVersion() - - @staticmethod - def __forceServiceUpdate(url, fromMaster): - """ - Force updating configuration on a given service - - :param str url: service URL - :param bool fromMaster: flag to force updating from the master CS - :return: S_OK/S_ERROR - """ - gLogger.info('Updating service configuration on', url) - if url.startswith('dip'): - rpc = RPCClient(url) - result = rpc.refreshConfiguration(fromMaster) - elif url.startswith('http'): - hostCertTuple = getHostCertificateAndKeyLocation() - resultRequest = requests.get(url, - headers={'X-RefreshConfiguration': "True"}, - cert=hostCertTuple, - verify=False) - result = S_OK() - if resultRequest.status_code != 200: - result = S_ERROR("Status code returned %d" % resultRequest.status_code) - result['URL'] = url - return result - - def __processResults(self, id_, result): - if result['OK']: - self.__updateResultDict['Successful'][result['URL']] = True - else: - gLogger.warn("Failed to update configuration on", result['URL'] + ':' + result['Message']) - self.__updateResultDict['Failed'][result['URL']] = result['Message'] - - def __updateServiceConfiguration(self, urlSet, fromMaster=False): - """ - Update configuration in a set of service in parallel - - :param set urlSet: a set of service URLs - :param fromMaster: flag to force updating from the master CS - :return: S_OK/S_ERROR, Value Successful/Failed dict with service URLs - """ - pool = ThreadPool(len(urlSet)) - for url in urlSet: - pool.generateJobAndQueueIt(self.__forceServiceUpdate, - args=[url, fromMaster], - kwargs={}, - oCallback=self.__processResults) - pool.processAllResults() - return S_OK(self.__updateResultDict) - - def forceSlavesUpdate(self): - """ - Force updating configuration on all the slave configuration servers - - :return: S_OK/S_ERROR, Value Successful/Failed dict with service URLs - """ - gLogger.info("Updating configuration on slave servers") - iGraceTime = gConfigurationData.getSlavesGraceTime() - self.__updateResultDict = {"Successful": {}, "Failed": {}} - urlSet = set() - for slaveURL in self.dAliveSlaveServers: - if time.time() - self.dAliveSlaveServers[slaveURL] <= iGraceTime: - urlSet.add(slaveURL) - return self.__updateServiceConfiguration(urlSet, fromMaster=True) - - def forceGlobalUpdate(self): - """ - Force updating configuration of all the registered services - - :return: S_OK/S_ERROR, Value Successful/Failed dict with service URLs - """ - gLogger.info("Updating services configuration") - # Get URLs of all the services except for Configuration services - cfg = gConfigurationData.remoteCFG.getAsDict()['Systems'] - urlSet = set() - for system_ in cfg: - for instance in cfg[system_]: - for url in cfg[system_][instance]['URLs']: - urlSet = urlSet.union(set([u.strip() for u in cfg[system_][instance]['URLs'][url].split(',') - if 'Configuration/Server' not in u])) - return self.__updateServiceConfiguration(urlSet) - - def updateConfiguration(self, sBuffer, committer="", updateVersionOption=False): - """ - Update the master configuration with the newly received changes - - :param str sBuffer: newly received configuration data - :param str committer: the user name of the committer - :param bool updateVersionOption: flag to update the current configuration version - :return: S_OK/S_ERROR of the write-to-disk of the new configuration - """ - if not gConfigurationData.isMaster(): - return S_ERROR("Configuration modification is not allowed in this server") - # Load the data in a ConfigurationData object - oRemoteConfData = ConfigurationData(False) - oRemoteConfData.loadRemoteCFGFromCompressedMem(sBuffer) - if updateVersionOption: - oRemoteConfData.setVersion(gConfigurationData.getVersion()) - # Test that remote and new versions are the same - sRemoteVersion = oRemoteConfData.getVersion() - sLocalVersion = gConfigurationData.getVersion() - gLogger.info("Checking versions\nremote: %s\nlocal: %s" % (sRemoteVersion, sLocalVersion)) - if sRemoteVersion != sLocalVersion: - if not gConfigurationData.mergingEnabled(): - return S_ERROR("Local and remote versions differ (%s vs %s). Cannot commit." % (sLocalVersion, sRemoteVersion)) - else: - gLogger.info("AutoMerging new data!") - if updateVersionOption: - return S_ERROR("Cannot AutoMerge! version was overwritten") - result = self.__mergeIndependentUpdates(oRemoteConfData) - if not result['OK']: - gLogger.warn("Could not AutoMerge!", result['Message']) - return S_ERROR("AutoMerge failed: %s" % result['Message']) - requestedRemoteCFG = result['Value'] - gLogger.info("AutoMerge successful!") - oRemoteConfData.setRemoteCFG(requestedRemoteCFG) - # Test that configuration names are the same - sRemoteName = oRemoteConfData.getName() - sLocalName = gConfigurationData.getName() - if sRemoteName != sLocalName: - return S_ERROR("Names differ: Server is %s and remote is %s" % (sLocalName, sRemoteName)) - # Update and generate a new version - gLogger.info("Committing new data...") - gConfigurationData.lock() - gLogger.info("Setting the new CFG") - gConfigurationData.setRemoteCFG(oRemoteConfData.getRemoteCFG()) - gConfigurationData.unlock() - gLogger.info("Generating new version") - gConfigurationData.generateNewVersion() - # self.__checkSlavesStatus( forceWriteConfiguration = True ) - gLogger.info("Writing new version to disk") - retVal = gConfigurationData.writeRemoteConfigurationToDisk("%s@%s" % (committer, gConfigurationData.getVersion())) - gLogger.info("New version", gConfigurationData.getVersion()) - - # Attempt to update the configuration on currently registered slave services - if gConfigurationData.getAutoSlaveSync(): - result = self.forceSlavesUpdate() - if not result['OK']: - gLogger.warn('Failed to update slave servers') - - return retVal - - def getCompressedConfigurationData(self): - return gConfigurationData.getCompressedData() - - def getVersion(self): - return gConfigurationData.getVersion() - - def getCommitHistory(self): - files = self.__getCfgBackups(gConfigurationData.getBackupDir()) - backups = [".".join(fileName.split(".")[1:-1]).split("@") for fileName in files] - return backups - def run(self): while True: iWaitTime = gConfigurationData.getSlavesGraceTime() time.sleep(iWaitTime) - self.__checkSlavesStatus() - - def getVersionContents(self, date): - backupDir = gConfigurationData.getBackupDir() - files = self.__getCfgBackups(backupDir, date) - for fileName in files: - with zipfile.ZipFile("%s/%s" % (backupDir, fileName), "r") as zFile: - cfgName = zFile.namelist()[0] - retVal = S_OK(zlib.compress(zFile.read(cfgName), 9)) - return retVal - return S_ERROR("Version %s does not exist" % date) - - def __getCfgBackups(self, basePath, date="", subPath=""): - rs = re.compile(r"^%s\..*%s.*\.zip$" % (gConfigurationData.getName(), date)) - fsEntries = os.listdir("%s/%s" % (basePath, subPath)) - fsEntries.sort(reverse=True) - backupsList = [] - for entry in fsEntries: - entryPath = "%s/%s/%s" % (basePath, subPath, entry) - if os.path.isdir(entryPath): - backupsList.extend(self.__getCfgBackups(basePath, date, "%s/%s" % (subPath, entry))) - elif os.path.isfile(entryPath): - if rs.search(entry): - backupsList.append("%s/%s" % (subPath, entry)) - return backupsList - - def __getPreviousCFG(self, oRemoteConfData): - backupsList = self.__getCfgBackups(gConfigurationData.getBackupDir(), date=oRemoteConfData.getVersion()) - if not backupsList: - return S_ERROR("Could not AutoMerge. Could not retrieve original committer's version") - prevRemoteConfData = ConfigurationData() - backFile = backupsList[0] - if backFile[0] == "/": - backFile = os.path.join(gConfigurationData.getBackupDir(), backFile[1:]) - try: - prevRemoteConfData.loadConfigurationData(backFile) - except Exception as e: - return S_ERROR("Could not load original committer's version: %s" % str(e)) - gLogger.info("Loaded client original version %s" % prevRemoteConfData.getVersion()) - return S_OK(prevRemoteConfData.getRemoteCFG()) - - def _checkConflictsInModifications(self, realModList, reqModList, parentSection=""): - realModifiedSections = dict([(modAc[1], modAc[3]) - for modAc in realModList if modAc[0].find('Sec') == len(modAc[0]) - 3]) - reqOptionsModificationList = dict([(modAc[1], modAc[3]) - for modAc in reqModList if modAc[0].find('Opt') == len(modAc[0]) - 3]) - for modAc in reqModList: - action = modAc[0] - objectName = modAc[1] - if action == "addSec": - if objectName in realModifiedSections: - return S_ERROR("Section %s/%s already exists" % (parentSection, objectName)) - elif action == "delSec": - if objectName in realModifiedSections: - return S_ERROR("Section %s/%s cannot be deleted. It has been modified." % (parentSection, objectName)) - elif action == "modSec": - if objectName in realModifiedSections: - result = self._checkConflictsInModifications(realModifiedSections[objectName], - modAc[3], "%s/%s" % (parentSection, objectName)) - if not result['OK']: - return result - for modAc in realModList: - action = modAc[0] - objectName = modAc[1] - if action.find("Opt") == len(action) - 3: - return S_ERROR( - "Section %s cannot be merged. Option %s/%s has been modified" % - (parentSection, parentSection, objectName)) - return S_OK() + self._checkSlavesStatus() - def __mergeIndependentUpdates(self, oRemoteConfData): - # return S_ERROR( "AutoMerge is still not finished. - # Meanwhile... why don't you get the newest conf and update from there?" ) - # Get all the CFGs - curSrvCFG = gConfigurationData.getRemoteCFG().clone() - curCliCFG = oRemoteConfData.getRemoteCFG().clone() - result = self.__getPreviousCFG(oRemoteConfData) - if not result['OK']: - return result - prevCliCFG = result['Value'] - # Try to merge curCli with curSrv. To do so we check the updates from - # prevCli -> curSrv VS prevCli -> curCli - prevCliToCurCliModList = prevCliCFG.getModifications(curCliCFG) - prevCliToCurSrvModList = prevCliCFG.getModifications(curSrvCFG) - result = self._checkConflictsInModifications(prevCliToCurSrvModList, - prevCliToCurCliModList) - if not result['OK']: - return S_ERROR("Cannot AutoMerge: %s" % result['Message']) - # Merge! - result = curSrvCFG.applyModifications(prevCliToCurCliModList) - if not result['OK']: - return result - return S_OK(curSrvCFG) diff --git a/ConfigurationSystem/private/ServiceInterfaceBase.py b/ConfigurationSystem/private/ServiceInterfaceBase.py new file mode 100644 index 00000000000..45a85334ac1 --- /dev/null +++ b/ConfigurationSystem/private/ServiceInterfaceBase.py @@ -0,0 +1,365 @@ +""" Threaded implementation of services +""" + +__RCSID__ = "$Id$" + +import os +import time +import re +import threading +import zipfile +import zlib +import requests + +import DIRAC +from DIRAC.Core.Utilities.File import mkDir +from DIRAC.ConfigurationSystem.Client.ConfigurationData import gConfigurationData, ConfigurationData +from DIRAC.ConfigurationSystem.private.Refresher import gRefresher +from DIRAC.FrameworkSystem.Client.Logger import gLogger +from DIRAC.Core.Utilities.ReturnValues import S_OK, S_ERROR +from DIRAC.Core.DISET.RPCClient import RPCClient +from DIRAC.Core.Utilities.ThreadPool import ThreadPool +from DIRAC.Core.Security.Locations import getHostCertificateAndKeyLocation + + +class ServiceInterface(threading.Thread): + + def __init__(self, sURL): + threading.Thread.__init__(self) + self.sURL = sURL + gLogger.info("Initializing Configuration Service", "URL is %s" % sURL) + self.__modificationsIgnoreMask = ['/DIRAC/Configuration/Servers', '/DIRAC/Configuration/Version'] + gConfigurationData.setAsService() + if not gConfigurationData.isMaster(): + gLogger.info("Starting configuration service as slave") + gRefresher.autoRefreshAndPublish(self.sURL) + else: + gLogger.info("Starting configuration service as master") + gRefresher.disable() + self.__loadConfigurationData() + self.dAliveSlaveServers = {} + self.__launchCheckSlaves() + + self.__updateResultDict = {"Successful": {}, "Failed": {}} + + def isMaster(self): + return gConfigurationData.isMaster() + + def __launchCheckSlaves(self): + gLogger.info("Starting purge slaves thread") + self.setDaemon(1) + self.start() + + def __loadConfigurationData(self): + mkDir(os.path.join(DIRAC.rootPath, "etc", "csbackup")) + gConfigurationData.loadConfigurationData() + if gConfigurationData.isMaster(): + bBuiltNewConfiguration = False + if not gConfigurationData.getName(): + DIRAC.abort(10, "Missing name for the configuration to be exported!") + gConfigurationData.exportName() + sVersion = gConfigurationData.getVersion() + if sVersion == "0": + gLogger.info("There's no version. Generating a new one") + gConfigurationData.generateNewVersion() + bBuiltNewConfiguration = True + + if self.sURL not in gConfigurationData.getServers(): + gConfigurationData.setServers(self.sURL) + bBuiltNewConfiguration = True + + gConfigurationData.setMasterServer(self.sURL) + + if bBuiltNewConfiguration: + gConfigurationData.writeRemoteConfigurationToDisk() + + def __generateNewVersion(self): + if gConfigurationData.isMaster(): + gConfigurationData.generateNewVersion() + gConfigurationData.writeRemoteConfigurationToDisk() + + def publishSlaveServer(self, sSlaveURL): + if not gConfigurationData.isMaster(): + return S_ERROR("Configuration modification is not allowed in this server") + gLogger.info("Pinging slave %s" % sSlaveURL) + rpcClient = RPCClient(sSlaveURL, timeout=10, useCertificates=True) + retVal = rpcClient.ping() + if not retVal['OK']: + gLogger.info("Slave %s didn't reply" % sSlaveURL) + return + if retVal['Value']['name'] != 'Configuration/Server': + gLogger.info("Slave %s is not a CS serveR" % sSlaveURL) + return + bNewSlave = False + if sSlaveURL not in self.dAliveSlaveServers: + bNewSlave = True + gLogger.info("New slave registered", sSlaveURL) + self.dAliveSlaveServers[sSlaveURL] = time.time() + if bNewSlave: + gConfigurationData.setServers("%s, %s" % (self.sURL, + ", ".join(self.dAliveSlaveServers.keys()))) + self.__generateNewVersion() + + def __checkSlavesStatus(self, forceWriteConfiguration=False): + gLogger.info("Checking status of slave servers") + iGraceTime = gConfigurationData.getSlavesGraceTime() + bModifiedSlaveServers = False + for sSlaveURL in self.dAliveSlaveServers.keys(): + if time.time() - self.dAliveSlaveServers[sSlaveURL] > iGraceTime: + gLogger.info("Found dead slave", sSlaveURL) + del self.dAliveSlaveServers[sSlaveURL] + bModifiedSlaveServers = True + if bModifiedSlaveServers or forceWriteConfiguration: + gConfigurationData.setServers("%s, %s" % (self.sURL, + ", ".join(self.dAliveSlaveServers.keys()))) + self.__generateNewVersion() + + @staticmethod + def __forceServiceUpdate(url, fromMaster): + """ + Force updating configuration on a given service + + :param str url: service URL + :param bool fromMaster: flag to force updating from the master CS + :return: S_OK/S_ERROR + """ + gLogger.info('Updating service configuration on', url) + if url.startswith('dip'): + rpc = RPCClient(url) + result = rpc.refreshConfiguration(fromMaster) + elif url.startswith('http'): + hostCertTuple = getHostCertificateAndKeyLocation() + resultRequest = requests.get(url, + headers={'X-RefreshConfiguration': "True"}, + cert=hostCertTuple, + verify=False) + result = S_OK() + if resultRequest.status_code != 200: + result = S_ERROR("Status code returned %d" % resultRequest.status_code) + result['URL'] = url + return result + + def __processResults(self, id_, result): + if result['OK']: + self.__updateResultDict['Successful'][result['URL']] = True + else: + gLogger.warn("Failed to update configuration on", result['URL'] + ':' + result['Message']) + self.__updateResultDict['Failed'][result['URL']] = result['Message'] + + def __updateServiceConfiguration(self, urlSet, fromMaster=False): + """ + Update configuration in a set of service in parallel + + :param set urlSet: a set of service URLs + :param fromMaster: flag to force updating from the master CS + :return: S_OK/S_ERROR, Value Successful/Failed dict with service URLs + """ + pool = ThreadPool(len(urlSet)) + for url in urlSet: + pool.generateJobAndQueueIt(self.__forceServiceUpdate, + args=[url, fromMaster], + kwargs={}, + oCallback=self.__processResults) + pool.processAllResults() + return S_OK(self.__updateResultDict) + + def forceSlavesUpdate(self): + """ + Force updating configuration on all the slave configuration servers + + :return: S_OK/S_ERROR, Value Successful/Failed dict with service URLs + """ + gLogger.info("Updating configuration on slave servers") + iGraceTime = gConfigurationData.getSlavesGraceTime() + self.__updateResultDict = {"Successful": {}, "Failed": {}} + urlSet = set() + for slaveURL in self.dAliveSlaveServers: + if time.time() - self.dAliveSlaveServers[slaveURL] <= iGraceTime: + urlSet.add(slaveURL) + return self.__updateServiceConfiguration(urlSet, fromMaster=True) + + def forceGlobalUpdate(self): + """ + Force updating configuration of all the registered services + + :return: S_OK/S_ERROR, Value Successful/Failed dict with service URLs + """ + gLogger.info("Updating services configuration") + # Get URLs of all the services except for Configuration services + cfg = gConfigurationData.remoteCFG.getAsDict()['Systems'] + urlSet = set() + for system_ in cfg: + for instance in cfg[system_]: + for url in cfg[system_][instance]['URLs']: + urlSet = urlSet.union(set([u.strip() for u in cfg[system_][instance]['URLs'][url].split(',') + if 'Configuration/Server' not in u])) + return self.__updateServiceConfiguration(urlSet) + + def updateConfiguration(self, sBuffer, committer="", updateVersionOption=False): + """ + Update the master configuration with the newly received changes + + :param str sBuffer: newly received configuration data + :param str committer: the user name of the committer + :param bool updateVersionOption: flag to update the current configuration version + :return: S_OK/S_ERROR of the write-to-disk of the new configuration + """ + if not gConfigurationData.isMaster(): + return S_ERROR("Configuration modification is not allowed in this server") + # Load the data in a ConfigurationData object + oRemoteConfData = ConfigurationData(False) + oRemoteConfData.loadRemoteCFGFromCompressedMem(sBuffer) + if updateVersionOption: + oRemoteConfData.setVersion(gConfigurationData.getVersion()) + # Test that remote and new versions are the same + sRemoteVersion = oRemoteConfData.getVersion() + sLocalVersion = gConfigurationData.getVersion() + gLogger.info("Checking versions\nremote: %s\nlocal: %s" % (sRemoteVersion, sLocalVersion)) + if sRemoteVersion != sLocalVersion: + if not gConfigurationData.mergingEnabled(): + return S_ERROR("Local and remote versions differ (%s vs %s). Cannot commit." % (sLocalVersion, sRemoteVersion)) + else: + gLogger.info("AutoMerging new data!") + if updateVersionOption: + return S_ERROR("Cannot AutoMerge! version was overwritten") + result = self.__mergeIndependentUpdates(oRemoteConfData) + if not result['OK']: + gLogger.warn("Could not AutoMerge!", result['Message']) + return S_ERROR("AutoMerge failed: %s" % result['Message']) + requestedRemoteCFG = result['Value'] + gLogger.info("AutoMerge successful!") + oRemoteConfData.setRemoteCFG(requestedRemoteCFG) + # Test that configuration names are the same + sRemoteName = oRemoteConfData.getName() + sLocalName = gConfigurationData.getName() + if sRemoteName != sLocalName: + return S_ERROR("Names differ: Server is %s and remote is %s" % (sLocalName, sRemoteName)) + # Update and generate a new version + gLogger.info("Committing new data...") + gConfigurationData.lock() + gLogger.info("Setting the new CFG") + gConfigurationData.setRemoteCFG(oRemoteConfData.getRemoteCFG()) + gConfigurationData.unlock() + gLogger.info("Generating new version") + gConfigurationData.generateNewVersion() + # self.__checkSlavesStatus( forceWriteConfiguration = True ) + gLogger.info("Writing new version to disk") + retVal = gConfigurationData.writeRemoteConfigurationToDisk("%s@%s" % (committer, gConfigurationData.getVersion())) + gLogger.info("New version", gConfigurationData.getVersion()) + + # Attempt to update the configuration on currently registered slave services + if gConfigurationData.getAutoSlaveSync(): + result = self.forceSlavesUpdate() + if not result['OK']: + gLogger.warn('Failed to update slave servers') + + return retVal + + def getCompressedConfigurationData(self): + return gConfigurationData.getCompressedData() + + def getVersion(self): + return gConfigurationData.getVersion() + + def getCommitHistory(self): + files = self.__getCfgBackups(gConfigurationData.getBackupDir()) + backups = [".".join(fileName.split(".")[1:-1]).split("@") for fileName in files] + return backups + + def run(self): + while True: + iWaitTime = gConfigurationData.getSlavesGraceTime() + time.sleep(iWaitTime) + self.__checkSlavesStatus() + + def getVersionContents(self, date): + backupDir = gConfigurationData.getBackupDir() + files = self.__getCfgBackups(backupDir, date) + for fileName in files: + with zipfile.ZipFile("%s/%s" % (backupDir, fileName), "r") as zFile: + cfgName = zFile.namelist()[0] + retVal = S_OK(zlib.compress(zFile.read(cfgName), 9)) + return retVal + return S_ERROR("Version %s does not exist" % date) + + def __getCfgBackups(self, basePath, date="", subPath=""): + rs = re.compile(r"^%s\..*%s.*\.zip$" % (gConfigurationData.getName(), date)) + fsEntries = os.listdir("%s/%s" % (basePath, subPath)) + fsEntries.sort(reverse=True) + backupsList = [] + for entry in fsEntries: + entryPath = "%s/%s/%s" % (basePath, subPath, entry) + if os.path.isdir(entryPath): + backupsList.extend(self.__getCfgBackups(basePath, date, "%s/%s" % (subPath, entry))) + elif os.path.isfile(entryPath): + if rs.search(entry): + backupsList.append("%s/%s" % (subPath, entry)) + return backupsList + + def __getPreviousCFG(self, oRemoteConfData): + backupsList = self.__getCfgBackups(gConfigurationData.getBackupDir(), date=oRemoteConfData.getVersion()) + if not backupsList: + return S_ERROR("Could not AutoMerge. Could not retrieve original committer's version") + prevRemoteConfData = ConfigurationData() + backFile = backupsList[0] + if backFile[0] == "/": + backFile = os.path.join(gConfigurationData.getBackupDir(), backFile[1:]) + try: + prevRemoteConfData.loadConfigurationData(backFile) + except Exception as e: + return S_ERROR("Could not load original committer's version: %s" % str(e)) + gLogger.info("Loaded client original version %s" % prevRemoteConfData.getVersion()) + return S_OK(prevRemoteConfData.getRemoteCFG()) + + def _checkConflictsInModifications(self, realModList, reqModList, parentSection=""): + realModifiedSections = dict([(modAc[1], modAc[3]) + for modAc in realModList if modAc[0].find('Sec') == len(modAc[0]) - 3]) + reqOptionsModificationList = dict([(modAc[1], modAc[3]) + for modAc in reqModList if modAc[0].find('Opt') == len(modAc[0]) - 3]) + for modAc in reqModList: + action = modAc[0] + objectName = modAc[1] + if action == "addSec": + if objectName in realModifiedSections: + return S_ERROR("Section %s/%s already exists" % (parentSection, objectName)) + elif action == "delSec": + if objectName in realModifiedSections: + return S_ERROR("Section %s/%s cannot be deleted. It has been modified." % (parentSection, objectName)) + elif action == "modSec": + if objectName in realModifiedSections: + result = self._checkConflictsInModifications(realModifiedSections[objectName], + modAc[3], "%s/%s" % (parentSection, objectName)) + if not result['OK']: + return result + for modAc in realModList: + action = modAc[0] + objectName = modAc[1] + if action.find("Opt") == len(action) - 3: + return S_ERROR( + "Section %s cannot be merged. Option %s/%s has been modified" % + (parentSection, parentSection, objectName)) + return S_OK() + + def __mergeIndependentUpdates(self, oRemoteConfData): + # return S_ERROR( "AutoMerge is still not finished. + # Meanwhile... why don't you get the newest conf and update from there?" ) + # Get all the CFGs + curSrvCFG = gConfigurationData.getRemoteCFG().clone() + curCliCFG = oRemoteConfData.getRemoteCFG().clone() + result = self.__getPreviousCFG(oRemoteConfData) + if not result['OK']: + return result + prevCliCFG = result['Value'] + # Try to merge curCli with curSrv. To do so we check the updates from + # prevCli -> curSrv VS prevCli -> curCli + prevCliToCurCliModList = prevCliCFG.getModifications(curCliCFG) + prevCliToCurSrvModList = prevCliCFG.getModifications(curSrvCFG) + result = self._checkConflictsInModifications(prevCliToCurSrvModList, + prevCliToCurCliModList) + if not result['OK']: + return S_ERROR("Cannot AutoMerge: %s" % result['Message']) + # Merge! + result = curSrvCFG.applyModifications(prevCliToCurCliModList) + if not result['OK']: + return result + return S_OK(curSrvCFG) diff --git a/ConfigurationSystem/private/ServiceInterfaceTornado.py b/ConfigurationSystem/private/ServiceInterfaceTornado.py new file mode 100644 index 00000000000..fe3650ee938 --- /dev/null +++ b/ConfigurationSystem/private/ServiceInterfaceTornado.py @@ -0,0 +1,33 @@ +""" + Service interface adapted to work with tornado, must be used only by tornado service handlers +""" +from tornado import gen +from tornado.ioloop import IOLoop +from DIRAC.ConfigurationSystem.private.ServiceInterfaceBase import ServiceInterfaceBase +from DIRAC.ConfigurationSystem.Client.ConfigurationData import gConfigurationData +from DIRAC import gLogger + + +class ServiceInterfaceTornado(ServiceInterfaceBase): + """ + Service interface adapted to work with tornado + """ + + def __init__(self, sURL): + ServiceInterfaceBase.__init__(self, sURL) + self.__launchCheckSlaves() + + def __launchCheckSlaves(self): + """ + Start loop to check if slaves are alive + """ + IOLoop.current().spawn_callback(self.run) + gLogger.info("Starting purge slaves thread") + + def run(self): + """ + Check if slaves are alive + """ + while True: + yield gen.sleep(gConfigurationData.getSlavesGraceTime()) + self._checkSlavesStatus() diff --git a/Core/Base/Client.py b/Core/Base/Client.py index 92e3ef8fd97..faff844f2ef 100644 --- a/Core/Base/Client.py +++ b/Core/Base/Client.py @@ -11,7 +11,8 @@ import ast import os -from DIRAC.Core.DISET.RPCClient import RPCClient +from DIRAC.Core.Tornado.Client.RPCClientSelector import RPCClientSelector +from DIRAC.Core.Tornado.Client.TornadoClient import TornadoClient from DIRAC.Core.Utilities.Decorators import deprecated from DIRAC.Core.DISET import DEFAULT_RPC_TIMEOUT @@ -25,13 +26,16 @@ class Client(object): - The self.serverURL member should be set by the inheriting class """ + # Default https (RPC)Client + httpsClient = TornadoClient + def __init__(self, **kwargs): """ C'tor. :param kwargs: just stored as an attribute and passed when creating the RPCClient """ - self.serverURL = None + self.serverURL = kwargs.pop('url', None) self.call = None # I suppose it is initialized here to make pylint happy self.__kwargs = kwargs self.timeout = DEFAULT_RPC_TIMEOUT @@ -103,7 +107,7 @@ def _getRPC(self, rpc=None, url='', timeout=None): timeout = self.timeout self.__kwargs['timeout'] = timeout - rpc = RPCClient(url, **self.__kwargs) + rpc = RPCClientSelector(url, httpsClient=self.httpsClient, **self.__kwargs) return rpc diff --git a/Core/DISET/RequestHandler.py b/Core/DISET/RequestHandler.py index 0d423b92ac1..2b91ee2641a 100755 --- a/Core/DISET/RequestHandler.py +++ b/Core/DISET/RequestHandler.py @@ -499,6 +499,18 @@ def export_ping(self): return S_OK(dInfo) + types_whoami = [] + auth_whoami = ['all'] + + def export_whoami(self): + """ + A simple whoami, returns all credential dictionnary, except certificate chain object. + """ + credDict = self.srv_getRemoteCredentials() + if 'x509Chain' in credDict: + del credDict['x509Chain'] + return S_OK(credDict) + types_echo = [basestring] @staticmethod diff --git a/Core/DISET/private/Service.py b/Core/DISET/private/Service.py index d10a751c91b..e007b49d61c 100644 --- a/Core/DISET/private/Service.py +++ b/Core/DISET/private/Service.py @@ -381,7 +381,7 @@ def _processInThread(self, clientTransport): - Receive arguments/file/something else (depending on action) in the RequestHandler - Executing the action asked by the client - :param clientTransport: Object who describe the opened connection (SSLTransport or PlainTransport) + :param clientTransport: Object which describe the opened connection (SSLTransport or PlainTransport) :return: S_OK with "closeTransport" a boolean to indicate if th connection have to be closed e.g. after RPC, closeTransport=True diff --git a/Core/Tornado/Client/RPCClientSelector.py b/Core/Tornado/Client/RPCClientSelector.py new file mode 100644 index 00000000000..cb15cca9b3c --- /dev/null +++ b/Core/Tornado/Client/RPCClientSelector.py @@ -0,0 +1,59 @@ +""" + RPCClientSelector can replace RPCClient (with import RPCClientSelector as RPCClient) + to migrate from DISET to Tornado. This method chooses and returns the client wich should be + used for a service. If the url of the service uses HTTPS, TornadoClient is returned, else it returns RPCClient + + Example:: + + from DIRAC.Core.Tornado.Client.RPCClientSelector import RPCClientSelector as RPCClient + myService = RPCClient("Framework/MyService") + myService.doSomething() +""" + + +from DIRAC.Core.Tornado.Client.TornadoClient import TornadoClient +from DIRAC.Core.DISET.RPCClient import RPCClient +from DIRAC.ConfigurationSystem.Client.PathFinder import getServiceURL +from DIRAC import gLogger + + +def isURL(url): + """ + Just a test to check if URL is already given or not + """ + return url.startswith('http') or url.startswith('dip') + + +def RPCClientSelector(*args, **kwargs): # We use same interface as RPCClient + """ + Select the correct RPCClient, instanciate it, and return it + :param args[0]: url: URL can be just "system/service" or "dips://domain:port/system/service" + """ + + # We detect if we need to use a specific class for the HTTPS client + if 'httpsClient' in kwargs: + TornadoRPCClient = kwargs.pop('httpsClient') + else: + TornadoRPCClient = TornadoClient + + # We have to make URL resolution BEFORE the RPCClient or TornadoClient to determine wich one we want to use + # URL is defined as first argument (called serviceName) in RPCClient + + try: + serviceName = args[0] + gLogger.verbose("Trying to autodetect client for %s" % serviceName) + if not isURL(serviceName): + completeUrl = getServiceURL(serviceName) + gLogger.verbose("URL resolved: %s" % completeUrl) + else: + completeUrl = serviceName + if completeUrl.startswith("http"): + gLogger.info("Using HTTPS for service %s" % serviceName) + rpc = TornadoRPCClient(*args, **kwargs) + else: + rpc = RPCClient(*args, **kwargs) + except Exception: + # If anything went wrong in the resolution, we return default RPCClient + # So the comportement is exactly the same as before implementation of Tornado + rpc = RPCClient(*args, **kwargs) + return rpc diff --git a/Core/Tornado/Client/SpecificClient/__init__.py b/Core/Tornado/Client/SpecificClient/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/Core/Tornado/Client/TornadoClient.py b/Core/Tornado/Client/TornadoClient.py new file mode 100644 index 00000000000..d8078cdd66a --- /dev/null +++ b/Core/Tornado/Client/TornadoClient.py @@ -0,0 +1,72 @@ +""" + TornadoClient is equivalent of the RPCClient but in HTTPS. + Usage of TornadoClient is the same as RPCClient, you can instanciate TornadoClient with + complete url (https://domain/component/service) or just "component/service". Like RPCClient + you can use all method defined in your service, your call will be automatically transformed + in RPC. + + Main changes: + - KeepAliveLapse is removed, requests library manage it itself. + - nbOfRetry (defined as private attribute) is removed, requests library manage it hitself. + - Underneath it use HTTP POST protocol and JSON + + + Example:: + + from DIRAC.Core.Tornado.Client.TornadoClient import TornadoClient + myService = TornadoClient("Framework/MyService") + myService.doSomething() #Returns S_OK/S_ERROR + +""" + +from DIRAC.Core.Utilities.JEncode import encode +from DIRAC.Core.Tornado.Client.private.TornadoBaseClient import TornadoBaseClient + + +class TornadoClient(TornadoBaseClient): + """ + Client for calling tornado services + Interface is based on RPCClient interface + """ + + def __getattr__(self, attrname): + """ + Return the RPC call procedure + + :param str attrname: Name of the procedure we are trying to call + :return: RPC procedure as function + """ + def call(*args): + """ + Just returns the right function for RPC Call + """ + return self.executeRPC(attrname, *args) + return call + + # Name from RPCClient Interface + def executeRPC(self, method, *args): + """ + This function call a remote service + + :param str procedure: remote procedure name + :param args: list of arguments + :returns: decoded response from server, server may return S_OK or S_ERROR + """ + rpcCall = {'method': method, 'args': encode(args)} + # Start request + retVal = self._request(rpcCall) + retVal['rpcStub'] = (self._getBaseStub(), method, args) + return retVal + + +def executeRPCStub(rpcStub): + """ + Playback a stub + # Copy-paste from DIRAC.Core.DISET.RPCClient with RPCClient changed into TornadoClient + """ + # Generate a client with the same parameters + client = TornadoClient(rpcStub[0][0], **rpcStub[0][1]) + # Get a functor to execute the RPC call + rpcFunc = getattr(client, rpcStub[1]) + # Reproduce the call + return rpcFunc(*rpcStub[2]) diff --git a/Core/Tornado/Client/__init__.py b/Core/Tornado/Client/__init__.py new file mode 100644 index 00000000000..cae5bc3f37c --- /dev/null +++ b/Core/Tornado/Client/__init__.py @@ -0,0 +1,3 @@ +""" +DIRAC.TornadosServices +""" diff --git a/Core/Tornado/Client/private/TornadoBaseClient.py b/Core/Tornado/Client/private/TornadoBaseClient.py new file mode 100644 index 00000000000..ffe6de558b6 --- /dev/null +++ b/Core/Tornado/Client/private/TornadoBaseClient.py @@ -0,0 +1,507 @@ +""" + TornadoBaseClient contain all low-levels functionnalities and initilization methods + It must be instanciated from :py:class:`~DIRAC.Core.Tornado.Client.TornadoClient` + + Requests library manage himself retry when connection failed, so the __nbOfRetry attribute is removed from DIRAC + (For each URL requests manage retries himself, if it still fail, we try next url) + KeepAlive lapse is also removed because managed by request, + see http://docs.python-requests.org/en/master/user/advanced/#keep-alive + + If necessary this class can be modified to define number of retry in requests, documentation does not give + lot of informations but you can see this simple solution from StackOverflow. + After some tests request seems to retry 3 times by default. + https://stackoverflow.com/questions/15431044/can-i-set-max-retries-for-requests-request + + *WARNING*: If you use your own certificates, it's like in dips, please take a look at + :ref:`using_own_CA` + + *WARNING*: Lots of method are copy-paste from :py:class:`~DIRAC.Core.DISET.private.BaseClient`. + And some methods are copy-paste AND modifications, for now it permit to fully separate DISET and HTTPS. + + +""" +import requests +import DIRAC + +from DIRAC.ConfigurationSystem.Client.Config import gConfig +from DIRAC.Core.Utilities import List, Network +from DIRAC import S_OK, S_ERROR, gLogger +from DIRAC.ConfigurationSystem.Client.PathFinder import getServiceURL, getServiceFailoverURL +from DIRAC.Core.Utilities.JEncode import decode, encode +from DIRAC.Core.Security import CS +from DIRAC.Core.Security import Locations +from DIRAC.Core.DISET.ThreadConfig import ThreadConfig + + +class TornadoBaseClient(object): + """ + This class contain initialization method and all utilities method used for RPC + """ + __threadConfig = ThreadConfig() + VAL_EXTRA_CREDENTIALS_HOST = "hosts" + + KW_USE_CERTIFICATES = "useCertificates" + KW_EXTRA_CREDENTIALS = "extraCredentials" + KW_TIMEOUT = "timeout" + KW_SETUP = "setup" + KW_VO = "VO" + KW_DELEGATED_DN = "delegatedDN" + KW_DELEGATED_GROUP = "delegatedGroup" + KW_IGNORE_GATEWAYS = "ignoreGateways" + KW_PROXY_LOCATION = "proxyLocation" + KW_PROXY_STRING = "proxyString" + KW_PROXY_CHAIN = "proxyChain" + KW_SKIP_CA_CHECK = "skipCACheck" + KW_KEEP_ALIVE_LAPSE = "keepAliveLapse" + + def __init__(self, serviceName, **kwargs): + """ + :param serviceName: URL of the service (proper uri or just System/Component) + :param useCertificates: If set to True, use the server certificate + :param extraCredentials: + :param timeout: Timeout of the call (default 600 s) + :param setup: Specify the Setup + :param VO: Specify the VO + :param delegatedDN: Not clear what it can be used for. + :param delegatedGroup: Not clear what it can be used for. + :param ignoreGateways: Ignore the DIRAC Gatways settings + :param proxyLocation: Specify the location of the proxy + :param proxyString: Specify the proxy string + :param proxyChain: Specify the proxy chain + :param skipCACheck: Do not check the CA + :param keepAliveLapse: Duration for keepAliveLapse (heartbeat like) (now managed by requests) + """ + + if not isinstance(serviceName, basestring): + raise TypeError("Service name expected to be a string. Received %s type %s" % + (str(serviceName), type(serviceName))) + + self._destinationSrv = serviceName + self._serviceName = serviceName + self.__ca_location = False + + self.kwargs = kwargs + self.__useCertificates = None + # The CS useServerCertificate option can be overridden by explicit argument + self.__forceUseCertificates = self.kwargs.get(self.KW_USE_CERTIFICATES) + self.__initStatus = S_OK() + self.__idDict = {} + self.__extraCredentials = "" + self.__retry = 0 + self.__retryDelay = 0 + # by default we always have 1 url for example: + # RPCClient('dips://volhcb38.cern.ch:9162/Framework/SystemAdministrator') + self.__nbOfUrls = 1 + # self.__nbOfRetry removed in https, see note at the begining of the class + self.__retryCounter = 1 + self.__bannedUrls = [] + + # For pylint... + self.setup = None + self.vo = None + self.serviceURL = None + self.__proxy_location = None + + for initFunc in ( + self.__discoverTimeout, + self.__discoverSetup, + self.__discoverVO, + self.__discoverCredentialsToUse, + self.__discoverExtraCredentials, + self.__discoverURL): + + result = initFunc() + if not result['OK'] and self.__initStatus['OK']: + self.__initStatus = result + + def __discoverSetup(self): + """ Discover which setup to use and stores it in self.setup + The setup is looked for: + * kwargs of the constructor (see KW_SETUP) + * in the CS /DIRAC/Setup + * default to 'Test' + """ + if self.KW_SETUP in self.kwargs and self.kwargs[self.KW_SETUP]: + self.setup = str(self.kwargs[self.KW_SETUP]) + else: + self.setup = self.__threadConfig.getSetup() + if not self.setup: + self.setup = gConfig.getValue("/DIRAC/Setup", "Test") + return S_OK() + + def __discoverURL(self): + """ Calculate the final URL. It is called at initialization and in connect in case of issue + + It sets: + * self.serviceURL: the url (dips) selected as target using __findServiceURL + * self.__URLTuple: a split of serviceURL obtained by Network.splitURL + * self._serviceName: the last part of URLTuple (typically System/Component) + + WARNING: COPY PASTE FROM BaseClient + """ + # Calculate final URL + try: + result = self.__findServiceURL() + except Exception as e: + return S_ERROR(repr(e)) + if not result['OK']: + return result + self.serviceURL = result['Value'] + retVal = Network.splitURL(self.serviceURL) + if not retVal['OK']: + return retVal + self.__URLTuple = retVal['Value'] + self._serviceName = self.__URLTuple[-1] + res = gConfig.getOptionsDict("/DIRAC/ConnConf/%s:%s" % self.__URLTuple[1:3]) + if res['OK']: + opts = res['Value'] + for k in opts: + if k not in self.kwargs: + self.kwargs[k] = opts[k] + return S_OK() + + def __discoverVO(self): + """ Discover which VO to use and stores it in self.vo + The VO is looked for: + * kwargs of the constructor (see KW_VO) + * in the CS /DIRAC/VirtualOrganization + * default to 'unknown' + + WARNING: COPY/PASTE FROM Core/Diset/private/BaseClient FOR NOW + """ + if self.KW_VO in self.kwargs and self.kwargs[self.KW_VO]: + self.vo = str(self.kwargs[self.KW_VO]) + else: + self.vo = gConfig.getValue("/DIRAC/VirtualOrganization", "unknown") + return S_OK() + + def __discoverCredentialsToUse(self): + """ Discovers which credentials to use for connection. + * Server certificate: + -> If KW_USE_CERTIFICATES in kwargs, sets it in self.__useCertificates + -> If not, check gConfig.useServerCertificate(), and sets it in self.__useCertificates + and kwargs[KW_USE_CERTIFICATES] + * Certification Authorities check: + -> if KW_SKIP_CA_CHECK is not in kwargs and we are using the certificates, + set KW_SKIP_CA_CHECK to false in kwargs + -> if KW_SKIP_CA_CHECK is not in kwargs and we are not using the certificate, check the CS.skipCACheck + * Proxy Chain + + WARNING: MOSTLY COPY/PASTE FROM Core/Diset/private/BaseClient + + """ + # Use certificates? + if self.KW_USE_CERTIFICATES in self.kwargs: + self.__useCertificates = self.kwargs[self.KW_USE_CERTIFICATES] + else: + self.__useCertificates = gConfig.useServerCertificate() + self.kwargs[self.KW_USE_CERTIFICATES] = self.__useCertificates + if self.KW_SKIP_CA_CHECK not in self.kwargs: + if self.__useCertificates: + self.kwargs[self.KW_SKIP_CA_CHECK] = False + else: + self.kwargs[self.KW_SKIP_CA_CHECK] = CS.skipCACheck() + + # Rewrite a little bit from here: don't need the proxy string, we use the file + if self.KW_PROXY_CHAIN in self.kwargs: + try: + self.kwargs[self.KW_PROXY_STRING] = self.kwargs[self.KW_PROXY_CHAIN].dumpAllToString()['Value'] + del self.kwargs[self.KW_PROXY_CHAIN] + except BaseException: + return S_ERROR("Invalid proxy chain specified on instantiation") + + # ==== REWRITED FROM HERE ==== + + # Getting proxy + + proxy = Locations.getProxyLocation() + if not proxy: + gLogger.error("No proxy found") + return S_ERROR("No proxy found") + self.__proxy_location = proxy + + # For certs always check CA's. For clients skipServerIdentityCheck + if self.KW_SKIP_CA_CHECK not in self.kwargs or not self.kwargs[self.KW_SKIP_CA_CHECK]: + cafile = Locations.getCAsLocation() + if not cafile: + gLogger.error("No CAs found!") + return S_ERROR("No CAs found!") + else: + self.__ca_location = cafile + + return S_OK() + + def __discoverExtraCredentials(self): + """ Add extra credentials informations. + * self.__extraCredentials + -> if KW_EXTRA_CREDENTIALS in kwargs, we set it + -> Otherwise, if we use the server certificate, we set it to VAL_EXTRA_CREDENTIALS_HOST + -> If we have a delegation (see bellow), we set it to (delegatedDN, delegatedGroup) + -> otherwise it is an empty string + * delegation: + -> if KW_DELEGATED_DN in kwargs, or delegatedDN in threadConfig, put in in self.kwargs + -> If we have a delegated DN but not group, we find the corresponding group in the CS + + WARNING: COPY/PASTE FROM Core/Diset/private/BaseClient + """ + # Wich extra credentials to use? + if self.__useCertificates: + self.__extraCredentials = self.VAL_EXTRA_CREDENTIALS_HOST + else: + self.__extraCredentials = "" + if self.KW_EXTRA_CREDENTIALS in self.kwargs: + self.__extraCredentials = self.kwargs[self.KW_EXTRA_CREDENTIALS] + # Are we delegating something? + delegatedDN, delegatedGroup = self.__threadConfig.getID() + if self.KW_DELEGATED_DN in self.kwargs and self.kwargs[self.KW_DELEGATED_DN]: + delegatedDN = self.kwargs[self.KW_DELEGATED_DN] + elif delegatedDN: + self.kwargs[self.KW_DELEGATED_DN] = delegatedDN + if self.KW_DELEGATED_GROUP in self.kwargs and self.kwargs[self.KW_DELEGATED_GROUP]: + delegatedGroup = self.kwargs[self.KW_DELEGATED_GROUP] + elif delegatedGroup: + self.kwargs[self.KW_DELEGATED_GROUP] = delegatedGroup + if delegatedDN: + if not delegatedGroup: + result = CS.findDefaultGroupForDN(self.kwargs[self.KW_DELEGATED_DN]) + if not result['OK']: + return result + self.__extraCredentials = (delegatedDN, delegatedGroup) + return S_OK() + + def __discoverTimeout(self): + """ Discover which timeout to use and stores it in self.timeout + The timeout can be specified kwargs of the constructor (see KW_TIMEOUT), + with a minimum of 120 seconds. + If unspecified, the timeout will be 600 seconds. + The value is set in self.timeout, as well as in self.kwargs[KW_TIMEOUT] + + WARNING: COPY/PASTE FROM Core/Diset/private/BaseClient + """ + if self.KW_TIMEOUT in self.kwargs: + self.timeout = self.kwargs[self.KW_TIMEOUT] + else: + self.timeout = False + if self.timeout: + self.timeout = max(120, self.timeout) + else: + self.timeout = 600 + self.kwargs[self.KW_TIMEOUT] = self.timeout + return S_OK() + + def __findServiceURL(self): + """ + Discovers the URL of a service, taking into account gateways, multiple URLs, banned URLs + + + If the site on which we run is configured to use gateways (/DIRAC/Gateways/), + these URLs will be used. To ignore the gateway, it is possible to set KW_IGNORE_GATEWAYS + to False in kwargs. + + If self._destinationSrv (given as constructor attribute) is a properly formed URL, + we just return this one. If we have to use a gateway, we just replace the server name in the url. + + The list of URLs defined in the CS (/URLs/) is randomized + + This method also sets some attributes: + * self.__nbOfUrls = number of URLs + * self.__nbOfRetry removed in HTTPS (Managed by requests) + * self.__bannedUrls is reinitialized if all the URLs are banned + + :return: the selected URL + + WARNING (Mostly) COPY PASTE FROM BaseClient (protocols list is changed to https) + + """ + if not self.__initStatus['OK']: + return self.__initStatus + + # Load the Gateways URLs for the current site Name + gatewayURL = False + if self.KW_IGNORE_GATEWAYS not in self.kwargs or not self.kwargs[self.KW_IGNORE_GATEWAYS]: + dRetVal = gConfig.getOption("/DIRAC/Gateways/%s" % DIRAC.siteName()) + if dRetVal['OK']: + rawGatewayURL = List.randomize(List.fromChar(dRetVal['Value'], ","))[0] + gatewayURL = "/".join(rawGatewayURL.split("/")[:3]) + + # If what was given as constructor attribute is a properly formed URL, + # we just return this one. + # If we have to use a gateway, we just replace the server name in it + if self._destinationSrv.startswith("https://"): + gLogger.debug("Already given a valid url", self._destinationSrv) + if not gatewayURL: + return S_OK(self._destinationSrv) + gLogger.debug("Reconstructing given URL to pass through gateway") + path = "/".join(self._destinationSrv.split("/")[3:]) + finalURL = "%s/%s" % (gatewayURL, path) + gLogger.debug("Gateway URL conversion:\n %s -> %s" % (self._destinationSrv, finalURL)) + return S_OK(finalURL) + + if gatewayURL: + gLogger.debug("Using gateway", gatewayURL) + return S_OK("%s/%s" % (gatewayURL, self._destinationSrv)) + + # If nor url is given as constructor, we extract the list of URLs from the CS (System/URLs/Component) + try: + urls = getServiceURL(self._destinationSrv, setup=self.setup) + except Exception as e: + return S_ERROR("Cannot get URL for %s in setup %s: %s" % (self._destinationSrv, self.setup, repr(e))) + if not urls: + return S_ERROR("URL for service %s not found" % self._destinationSrv) + + failoverUrls = [] + # Try if there are some failover URLs to use as last resort + try: + failoverUrlsStr = getServiceFailoverURL(self._destinationSrv, setup=self.setup) + if failoverUrlsStr: + failoverUrls = failoverUrlsStr.split(',') + except Exception as e: + pass + + # We randomize the list, and add at the end the failover URLs (System/FailoverURLs/Component) + urlsList = List.randomize(List.fromChar(urls, ",")) + failoverUrls + self.__nbOfUrls = len(urlsList) + # __nbOfRetry removed in HTTPS (managed by requests) + if self.__nbOfUrls == len(self.__bannedUrls): + self.__bannedUrls = [] # retry all urls + gLogger.debug("Retrying again all URLs") + + if len(self.__bannedUrls) > 0 and len(urlsList) > 1: + # we have host which is not accessible. We remove that host from the list. + # We only remove if we have more than one instance + for i in self.__bannedUrls: + gLogger.debug("Removing banned URL", "%s" % i) + urlsList.remove(i) + + # Take the first URL from the list + # randUrls = List.randomize( urlsList ) + failoverUrls + + sURL = urlsList[0] + + # If we have banned URLs, and several URLs at disposals, we make sure that the selected sURL + # is not on a host which is banned. If it is, we take the next one in the list using __selectUrl + + if len(self.__bannedUrls) > 0 and self.__nbOfUrls > 2: + retVal = Network.splitURL(sURL) + nexturl = None + if retVal['OK']: + nexturl = retVal['Value'] + + found = False + for i in self.__bannedUrls: + retVal = Network.splitURL(i) + if retVal['OK']: + bannedurl = retVal['Value'] + else: + break + # We found a banned URL on the same host as the one we are running on + if nexturl[1] == bannedurl[1]: + found = True + break + if found: + nexturl = self.__selectUrl(nexturl, urlsList[1:]) + if nexturl: # an url found which is in different host + sURL = nexturl + gLogger.debug("Discovering URL for service", "%s -> %s" % (self._destinationSrv, sURL)) + return S_OK(sURL) + + def __selectUrl(self, notselect, urls): + """In case when multiple services are running in the same host, a new url has to be in a different host + Note: If we do not have different host we will use the selected url... + + :param notselect: URL that should NOT be selected + :param urls: list of potential URLs + + :return: selected URL + + WARNING: COPY/PASTE FROM Core/Diset/private/BaseClient + """ + url = None + for i in urls: + retVal = Network.splitURL(i) + if retVal['OK']: + if retVal['Value'][1] != notselect[1]: # the hots are different + url = i + break + else: + gLogger.error(retVal['Message']) + return url + + def getServiceName(self): + """ + Returns the name of the service, if you had given a url at init, returns the URL. + """ + return self._serviceName + + def getDestinationService(self): + """ + Returns the url the service. + """ + return getServiceURL(self._serviceName) + + def _getBaseStub(self): + """ Returns a tuple with (self._destinationSrv, newKwargs) + self._destinationSrv is what was given as first parameter of the init serviceName + + newKwargs is an updated copy of kwargs: + * if set, we remove the useCertificates (KW_USE_CERTIFICATES) in newKwargs + + This method is just used to return information in case of error in the InnerRPCClient + + WARNING: COPY/PASTE FROM Core/Diset/private/BaseClient + """ + newKwargs = dict(self.kwargs) + # Remove useCertificates as the forwarder of the call will have to + # independently decide whether to use their cert or not anyway. + if 'useCertificates' in newKwargs: + del newKwargs['useCertificates'] + return (self._destinationSrv, newKwargs) + + def _request(self, postArguments, retry=0): + """ + Sends the request to server + + :param postArguments: dictionnary with arguments who needs to be sends, + you should have "method" (str, name of distant method) + and "args" (str in JSON, list of arguments for ce procedure) + """ + + # Adding some informations to send + if self.__extraCredentials: + postArguments[self.KW_EXTRA_CREDENTIALS] = encode(self.__extraCredentials) + postArguments["clientVO"] = self.vo + + # Getting URL + url = self.__findServiceURL() + if not url['OK']: + return url + url = url['Value'] + + # Getting CA file (or skip verification) + verify = (not self.kwargs[self.KW_SKIP_CA_CHECK]) + if verify and self.__ca_location: + verify = self.__ca_location + + # getting certificate + if self.kwargs[self.KW_USE_CERTIFICATES]: + cert = Locations.getHostCertificateAndKeyLocation() + else: + cert = self.__proxy_location + + # Do the request + try: + call = requests.post(url, data=postArguments, timeout=self.timeout, verify=verify, + cert=cert) + return decode(call.text)[0] + except Exception as e: + if url not in self.__bannedUrls: + self.__bannedUrls += [url] + if retry < self.__nbOfUrls - 1: + self._request(postArguments, retry + 1) + return S_ERROR(str(e)) + + +# --- TODO ---- +# Rewrite this method if needed: +# /Core/DISET/private/BaseClient.py +# __delegateCredentials diff --git a/Core/Tornado/Client/private/__init__.py b/Core/Tornado/Client/private/__init__.py new file mode 100644 index 00000000000..cae5bc3f37c --- /dev/null +++ b/Core/Tornado/Client/private/__init__.py @@ -0,0 +1,3 @@ +""" +DIRAC.TornadosServices +""" diff --git a/Core/Tornado/Server/HandlerManager.py b/Core/Tornado/Server/HandlerManager.py new file mode 100644 index 00000000000..a830b6cc15e --- /dev/null +++ b/Core/Tornado/Server/HandlerManager.py @@ -0,0 +1,171 @@ +""" + + HandlerManager for tornado + This class search in the CS all services which use HTTPS and load them. + + Must be used with the TornadoServer + +""" + +from tornado.web import url as TornadoURL, RequestHandler + +from DIRAC.Core.Utilities.ObjectLoader import ObjectLoader +from DIRAC import gLogger, S_ERROR, S_OK, gConfig +from DIRAC.Core.Base.private.ModuleLoader import ModuleLoader +from DIRAC.ConfigurationSystem.Client import PathFinder + + +def urlFinder(module): + """ + Try to guess the url with module name + + :param module: path writed like import (e.g. "DIRAC.something.something") + """ + sections = module.split('.') + for section in sections: + # This condition is a bit long + # We search something which look like <...>.System.<...>.Handler + # If find we return // + if(section.find("System") > 0) and (sections[-1].find('Handler') > 0): + return "/%s/%s" % (section.replace("System", ""), sections[-1].replace("Handler", "")) + return None + + +class HandlerManager(object): + """ + This class is designed to work with Tornado + + It search and loads the handlers of services + """ + + def __init__(self, autoDiscovery=True): + """ + Initialization function, you can set autoDiscovery=False to prevent automatic + discovery of handler. If disabled you can use loadHandlersByServiceName() to + load your handlers or loadHandlerInHandlerManager() + + :param autoDiscovery=False: Disable the automatic discovery, can be used to choose service we want to load. + """ + self.__handlers = {} + self.__objectLoader = ObjectLoader() + self.__autoDiscovery = autoDiscovery + self.loader = ModuleLoader("Service", PathFinder.getServiceSection, RequestHandler, moduleSuffix="Handler") + + def __addHandler(self, handlerTuple, url=None): + """ + Function which add handler to list of known handlers + + + :param handlerTuple: (path, class) + """ + # Check if handler not already loaded + if not url or url not in self.__handlers: + gLogger.debug("Find new handler %s" % (handlerTuple[0])) + + # If url is not given, try to discover it + if url is None: + # FIRST TRY: Url is hardcoded + try: + url = handlerTuple[1].LOCATION + # SECOND TRY: URL can be deduced from path + except AttributeError: + gLogger.debug("No location defined for %s try to get it from path" % handlerTuple[0]) + url = urlFinder(handlerTuple[0]) + + # We add "/" if missing at begin, e.g. we found "Framework/Service" + # URL can't be relative in Tornado + if url and not url.startswith('/'): + url = "/%s" % url + elif not url: + gLogger.warn("URL not found for %s" % (handlerTuple[0])) + return S_ERROR("URL not found for %s" % (handlerTuple[0])) + + # Finally add the URL to handlers + if url not in self.__handlers: + self.__handlers[url] = handlerTuple[1] + gLogger.info("New handler: %s with URL %s" % (handlerTuple[0], url)) + else: + gLogger.debug("Handler already loaded %s" % (handlerTuple[0])) + return S_OK + + def discoverHandlers(self): + """ + Force the discovery of URL, automatic call when we try to get handlers for the first time. + You can disable the automatic call with autoDiscovery=False at initialization + """ + gLogger.debug("Trying to auto-discover the handlers for Tornado") + + # Look in config + diracSystems = gConfig.getSections('/Systems') + serviceList = [] + if diracSystems['OK']: + for system in diracSystems['Value']: + try: + instance = PathFinder.getSystemInstance(system) + services = gConfig.getSections('/Systems/%s/%s/Services' % (system, instance)) + if services['OK']: + for service in services['Value']: + newservice = ("%s/%s" % (system, service)) + + # We search in the CS all handlers which used HTTPS as protocol + isHTTPS = gConfig.getValue('/Systems/%s/%s/Services/%s/Protocol' % (system, instance, service)) + if isHTTPS and isHTTPS.lower() == 'https': + serviceList.append(newservice) + # On systems sometime you have things not related to services... + except RuntimeError as e: + pass + return self.loadHandlersByServiceName(serviceList) + + def loadHandlersByServiceName(self, servicesNames): + """ + Load a list of handler from list of service using DIRAC moduleLoader + Use :py:class:`DIRAC.Core.Base.private.ModuleLoader` + + :param servicesNames: list of service, e.g. ['Framework/Hello', 'Configuration/Server'] + """ + + # Use DIRAC system to load: search in CS if path is given and if not defined + # it search in place it should be (e.g. in DIRAC/FrameworkSystem/Service) + if not isinstance(servicesNames, list): + servicesNames = [servicesNames] + + load = self.loader.loadModules(servicesNames) + if not load['OK']: + return load + for module in self.loader.getModules().values(): + url = module['loadName'] + + # URL can be like https://domain:port/service/name or just service/name + # Here we just want the service name, for tornado + serviceTuple = url.replace('https://', '').split('/')[-2:] + url = "%s/%s" % (serviceTuple[0], serviceTuple[1]) + self.__addHandler((module['loadName'], module['classObj']), url) + return S_OK() + + def getHandlersURLs(self): + """ + Get all handler for usage in Tornado, as a list of tornado.web.url + If there is no handler found before, it try to find them + + :returns: a list of URL (not the string with "https://..." but the tornado object) + see http://www.tornadoweb.org/en/stable/web.html#tornado.web.URLSpec + """ + if not self.__handlers and self.__autoDiscovery: + self.__autoDiscovery = False + self.discoverHandlers() + urls = [] + for key in self.__handlers: + urls.append(TornadoURL(key, self.__handlers[key])) + return urls + + def getHandlersDict(self): + """ + Return all handler dictionnary + + :returns: dictionnary with absolute url as key ("/System/Service") + and tornado.web.url object as value + """ + if not self.__handlers and self.__autoDiscovery: + self.__autoDiscovery = False + self.discoverHandlers() + return self.__handlers diff --git a/Core/Tornado/Server/TornadoServer.py b/Core/Tornado/Server/TornadoServer.py new file mode 100644 index 00000000000..3579e01f947 --- /dev/null +++ b/Core/Tornado/Server/TornadoServer.py @@ -0,0 +1,214 @@ +""" +TornadoServer create a web server and load services. +It may work better with TornadoClient but as it accepts HTTPS you can create your own client +""" + +__RCSID__ = "$Id$" + + +import time +import datetime +import os + +from socket import error as socketerror + +import M2Crypto + +import tornado.iostream +tornado.iostream.SSLIOStream.configure( + 'tornado_m2crypto.m2iostream.M2IOStream') # pylint: disable=wrong-import-position + +from tornado.httpserver import HTTPServer +from tornado.web import Application, url +from tornado.ioloop import IOLoop +import tornado.ioloop + +import DIRAC +from DIRAC.Core.Tornado.Server.HandlerManager import HandlerManager +from DIRAC import gLogger, S_ERROR, S_OK, gConfig +from DIRAC.FrameworkSystem.Client.MonitoringClient import MonitoringClient +from DIRAC.ConfigurationSystem.Client import PathFinder +from DIRAC.Core.Security import Locations +from DIRAC.Core.Utilities import MemStat + + +class TornadoServer(object): + """ + Tornado webserver + + Initialize and run a HTTPS Server for DIRAC services. + By default it load all services from configuration, but you can also give an explicit list. + If you gave explicit list of services, only these ones are loaded + + Example 1: Easy way to start tornado:: + + # Initialize server and load services + serverToLaunch = TornadoServer() + + # Start listening when ready + serverToLaunch.startTornado() + + Example 2:We want to debug service1 and service2 only, and use another port for that :: + + services = ['component/service1', 'component/service2'] + serverToLaunch = TornadoServer(services=services, port=1234, debugSSL=True) + serverToLaunch.startTornado() + + + **WARNING:** debug=True enable SSL debug and Tornado autoreload, + for extra logging use -ddd in your command line + """ + + def __init__(self, services=None, debugSSL=False, port=None): + """ + Basic instanciation, set some variables + + :param list services: List of services you want to start, start all by default + :param str debug: Activate debug mode of Tornado (autoreload server + more errors display) and M2Crypto + :param int port: Used to change port, default is 443 + """ + + # If port not precised, get it from config + if port is None: + port = gConfig.getValue("/Systems/Tornado/%s/Port" % PathFinder.getSystemInstance('Tornado'), 443) + + if services and not isinstance(services, list): + services = [services] + + # URLs for services: 1URL/Service + self.urls = [] + # Other infos + self.debugSSL = debugSSL # Used by tornado and M2Crypto + self.port = port + self.handlerManager = HandlerManager() + self._monitor = MonitoringClient() + self.__monitoringLoopDelay = 60 # In secs + + # If services are defined, load only these ones (useful for debug purpose or specific services) + if services: + retVal = self.handlerManager.loadHandlersByServiceName(services) + if not retVal['OK']: + gLogger.error(retVal['Message']) + raise ImportError("Some services can't be loaded, check the service names and configuration.") + + # if no service list is given, load services from configuration + handlerDict = self.handlerManager.getHandlersDict() + for item in handlerDict.iteritems(): + # handlerDict[key].initializeService(key) + self.urls.append(url(item[0], item[1], dict(debug=self.debugSSL))) + # If there is no services loaded: + if not self.urls: + raise ImportError("There is no services loaded, please check your configuration") + self.__report = None + self.__monitorLastStatsUpdate = None + + def startTornado(self, multiprocess=False): + """ + Start the tornado server when ready. + The script is blocked in the Tornado IOLoop. + Multiprocess option is available but may be used with caution + """ + + gLogger.debug("Starting Tornado") + self._initMonitoring() + + if self.debugSSL: + gLogger.warn("Server is running in debug mode") + + router = Application(self.urls, debug=self.debugSSL) + + certs = Locations.getHostCertificateAndKeyLocation() + if certs is False: + gLogger.fatal("Host certificates not found ! Can't start the Server") + raise ImportError("Unable to load certificates") + ca = Locations.getCAsLocation() + ssl_options = { + 'certfile': certs[0], + 'keyfile': certs[1], + 'cert_reqs': M2Crypto.SSL.verify_peer, + 'ca_certs': ca, + 'sslDebug': self.debugSSL + } + + self.__monitorLastStatsUpdate = time.time() + self.__report = self.__startReportToMonitoringLoop() + + # Starting monitoring, IOLoop waiting time in ms, __monitoringLoopDelay is defined in seconds + tornado.ioloop.PeriodicCallback(self.__reportToMonitoring, self.__monitoringLoopDelay * 1000).start() + + # Start server + server = HTTPServer(router, ssl_options=ssl_options) + try: + if multiprocess: + server.bind(self.port) + else: + server.listen(self.port) + except socketerror as e: + gLogger.fatal(e) + return S_ERROR() + gLogger.always("Listening on port %s" % self.port) + for service in self.urls: + gLogger.debug("Available service: %s" % service) + + if multiprocess: + server.start(0) + IOLoop.current().start() + return True # Never called because we are stuck in the IOLoop, but to make pylint happy + + def _initMonitoring(self): + """ + Init extra bits of monitoring + """ + + self._monitor.setComponentType(MonitoringClient.COMPONENT_TORNADO) + self._monitor.initialize() + self._monitor.setComponentName('Tornado') + + self._monitor.registerActivity('CPU', "CPU Usage", 'Framework', "CPU,%", MonitoringClient.OP_MEAN, 600) + self._monitor.registerActivity('MEM', "Memory Usage", 'Framework', 'Memory,MB', MonitoringClient.OP_MEAN, 600) + + self._monitor.setComponentExtraParam('DIRACVersion', DIRAC.version) + self._monitor.setComponentExtraParam('platform', DIRAC.getPlatform()) + self._monitor.setComponentExtraParam('startTime', datetime.datetime.utcnow()) + return S_OK() + + def __reportToMonitoring(self): + """ + *Called every minute* + Every minutes we determine CPU and Memory usage + """ + + # Calculate CPU usage by comparing realtime and cpu time since last report + self.__endReportToMonitoringLoop(*self.__report) + + # Save memory usage and save realtime/CPU time for next call + self.__report = self.__startReportToMonitoringLoop() + + def __startReportToMonitoringLoop(self): + """ + Get time to prepare CPU usage monitoring and send memory usage to monitor + """ + now = time.time() # Used to calulate a delta + stats = os.times() + cpuTime = stats[0] + stats[2] + if now - self.__monitorLastStatsUpdate < 0: + return (now, cpuTime) + # Send CPU consumption mark + self.__monitorLastStatsUpdate = now + # Send Memory consumption mark + membytes = MemStat.VmB('VmRSS:') + if membytes: + mem = membytes / (1024. * 1024.) + self._monitor.addMark('MEM', mem) + return (now, cpuTime) + + def __endReportToMonitoringLoop(self, initialWallTime, initialCPUTime): + """ + Determine CPU usage by comparing walltime and cputime and send it to monitor + """ + wallTime = time.time() - initialWallTime + stats = os.times() + cpuTime = stats[0] + stats[2] - initialCPUTime + percentage = cpuTime / wallTime * 100. + if percentage > 0: + self._monitor.addMark('CPU', percentage) diff --git a/Core/Tornado/Server/TornadoService.py b/Core/Tornado/Server/TornadoService.py new file mode 100644 index 00000000000..93b451e3564 --- /dev/null +++ b/Core/Tornado/Server/TornadoService.py @@ -0,0 +1,497 @@ +""" + TornadoService represent one service services, your handler must inherith form this class + TornadoService may be used only by TornadoServer. + + To create you must write this "minimal" code:: + + from DIRAC.Core.Tornado.Server.TornadoService import TornadoService + class yourServiceHandler(TornadoService): + + @classmethod + def initializeHandler(cls, infosDict): + ## Called 1 time, at first request. + ## You don't need to use super or to call any parents method, it's managed by the server + + def initializeRequest(self): + ## Called at each request + + auth_someMethod = ['all'] + def export_someMethod(self): + #Insert your method here, don't forgot the return + + + Then you must configure service like any other service + +""" + +import os +import time +from datetime import datetime +from tornado.web import RequestHandler +from tornado import gen +import tornado.ioloop +from tornado.ioloop import IOLoop + + +import DIRAC +from DIRAC.Core.Security.X509Chain import X509Chain +from DIRAC.Core.DISET.AuthManager import AuthManager +from DIRAC.ConfigurationSystem.Client import PathFinder +from DIRAC.Core.Utilities.JEncode import decode, encode +from DIRAC import S_OK, S_ERROR, gLogger +from DIRAC.Core.Utilities.DErrno import ENOAUTH +from DIRAC import gConfig +from DIRAC.FrameworkSystem.Client.MonitoringClient import MonitoringClient +from DIRAC.Core.Tornado.Utilities import HTTPErrorCodes + + +class TornadoService(RequestHandler): # pylint: disable=abstract-method + """ + TornadoService main class, manage all tornado services + Instanciated at each request + """ + + # Because we initialize at first request, we use a flag to know if it's already done + __FLAG_INIT_DONE = False + + # MonitoringClient, we don't use gMonitor which is not thread-safe + # We also need to add specific attributes for each service + _monitor = None + + @classmethod + def _initMonitoring(cls, serviceName, fullUrl): + """ + Init monitoring specific to service + """ + + # Init extra bits of monitoring + + cls._monitor = MonitoringClient() + cls._monitor.setComponentType(MonitoringClient.COMPONENT_WEB) + + cls._monitor.initialize() + + if tornado.process.task_id() is None: # Single process mode + cls._monitor.setComponentName('Tornado/%s' % serviceName) + else: + cls._monitor.setComponentName('Tornado/CPU%d/%s' % (tornado.process.task_id(), serviceName)) + + cls._monitor.setComponentLocation(fullUrl) + + cls._monitor.registerActivity("Queries", "Queries served", "Framework", "queries", MonitoringClient.OP_RATE) + + cls._monitor.setComponentExtraParam('DIRACVersion', DIRAC.version) + cls._monitor.setComponentExtraParam('platform', DIRAC.getPlatform()) + cls._monitor.setComponentExtraParam('startTime', datetime.utcnow()) + + cls._stats = {'requests': 0, 'monitorLastStatsUpdate': time.time()} + + return S_OK() + + @classmethod + def __initializeService(cls, relativeUrl, absoluteUrl, debug): + """ + Initialize a service, called at first request + + :param relativeUrl: the url, something like "/component/service" + :param absoluteUrl: the url, something like "https://dirac.cern.ch:1234/component/service" + :param debug: boolean which indicate if server running in debug mode + """ + # Url starts with a "/", we just remove it + serviceName = relativeUrl[1:] + + cls.debug = debug + cls.log = gLogger + cls._startTime = datetime.utcnow() + cls.log.info("First use of %s, initializing service..." % relativeUrl) + cls._authManager = AuthManager("%s/Authorization" % PathFinder.getServiceSection(serviceName)) + + cls._initMonitoring(serviceName, absoluteUrl) + + cls._serviceName = serviceName + cls._validNames = [serviceName] + serviceInfo = {'serviceName': serviceName, + 'serviceSectionPath': PathFinder.getServiceSection(serviceName), + 'csPaths': [PathFinder.getServiceSection(serviceName)], + 'URL': absoluteUrl + } + cls._serviceInfoDict = serviceInfo + + cls.__monitorLastStatsUpdate = time.time() + + try: + cls.initializeHandler(serviceInfo) + # If anything happen during initialization, we return the error + # broad-except is necessary because we can't really control the exception in the handlers + except Exception as e: # pylint: disable=broad-except + gLogger.error(e) + return S_ERROR('Error while initializing') + + cls.__FLAG_INIT_DONE = True + return S_OK() + + @classmethod + def initializeHandler(cls, serviceInfoDict): + """ + This may be overwrited when you write a DIRAC service handler + And it must be a class method. This method is called only one time, + at the first request + + :param dict ServiceInfoDict: infos about services, it contains + 'serviceName', 'serviceSectionPath', + 'csPaths'and 'URL' + """ + pass + + def initializeRequest(self): + """ + Called at every request, may be overwrited in your handler. + """ + pass + + # This function is designed to be overwrited as we want in Tornado + # It's why we should disable pylint for this one + def initialize(self, debug): # pylint: disable=arguments-differ + """ + initialize, called at every request + ..warning:: DO NOT REWRITE THIS FUNCTION IN YOUR HANDLER + ==> initialize in DISET became initializeRequest in HTTPS ! + """ + self.debug = debug + self.authorized = False + self.method = None + self.requestStartTime = time.time() + self.credDict = None + self.authorized = False + self.method = None + + # On internet you can find "HTTP Error Code" or "HTTP Status Code" for that. + # In fact code>=400 is an error (like "404 Not Found"), code<400 is a status (like "200 OK") + self._httpError = HTTPErrorCodes.HTTP_OK + if not self.__FLAG_INIT_DONE: + init = self.__initializeService(self.srv_getURL(), self.request.full_url(), debug) + if not init['OK']: + self._httpError = HTTPErrorCodes.HTTP_INTERNAL_SERVER_ERROR + gLogger.error("Error during initalization on %s" % self.request.full_url()) + gLogger.debug(init) + return False + + self._stats['requests'] += 1 + self._monitor.setComponentExtraParam('queries', self._stats['requests']) + self._monitor.addMark("Queries") + return True + + def prepare(self): + """ + prepare the request, it read certificates and check authorizations. + """ + self.method = self.get_argument("method") + self.log.notice("Incoming request on /%s: %s" % (self._serviceName, self.method)) + + # Init of service must be checked here, because if it have crashed we are + # not able to end request at initialization (can't write on client) + if not self.__FLAG_INIT_DONE: + error = encode("Service can't be initialized ! Check logs on the server for more informations.") + self.__write_return(error) + self.finish() + + try: + self.credDict = self.gatherPeerCredentials() + except Exception: # pylint: disable=broad-except + # If an error occur when reading certificates we close connection + # It can be strange but the RFC, for HTTP, say's that when error happend + # before authenfication we return 401 UNAUTHORIZED instead of 403 FORBIDDEN + self.reportUnauthorizedAccess(HTTPErrorCodes.HTTP_UNAUTHORIZED) + + try: + hardcodedAuth = getattr(self, 'auth_' + self.method) + except AttributeError: + hardcodedAuth = None + + self.authorized = self._authManager.authQuery(self.method, self.credDict, hardcodedAuth) + if not self.authorized: + self.reportUnauthorizedAccess() + + @gen.coroutine + def post(self): # pylint: disable=arguments-differ + """ + HTTP POST, used for RPC + Call the remote method, client may send his method via "method" argument + and list of arguments in JSON in "args" argument + """ + + # Execute the method + # First argument is "None", it's because we let Tornado manage the executor + retVal = yield IOLoop.current().run_in_executor(None, self.__executeMethod) + + # Tornado recommend to write in main thread + self.__write_return(retVal.result()) + self.finish() + + @gen.coroutine + def __executeMethod(self): + """ + Execute the method called, this method is executed in an executor + We have several try except to catch the different problem who can occurs + + - First, the method does not exist => Attribute error, return an error to client + - second, anything happend during execution => General Exception, send error to client + """ + + # getting method + try: + method = getattr(self, 'export_%s' % self.method) + except AttributeError as e: + self._httpError = HTTPErrorCodes.HTTP_NOT_IMPLEMENTED + return S_ERROR("Unknown method %s" % self.method) + + # Decode args + args_encoded = self.get_body_argument('args', default=encode([])) + + args = decode(args_encoded)[0] + # Execute + try: + self.initializeRequest() + retVal = method(*args) + except Exception as e: # pylint: disable=broad-except + retVal = S_ERROR(repr(e)) + self._httpError = HTTPErrorCodes.HTTP_INTERNAL_SERVER_ERROR + + return retVal + + def __write_return(self, dictionnary): + """ + Write to client what we wan't to return to client + It must be a dictionnary + """ + + # In case of error in server side we hide server CallStack to client + if 'CallStack' in dictionnary: + del dictionnary['CallStack'] + + # Write status code before writing, by default error code is "200 OK" + self.set_status(self._httpError) + self.write(encode(dictionnary)) + + def reportUnauthorizedAccess(self, errorCode=401): + """ + This method stop the current request and return an error to client + + + :param int errorCode: Error code, 403 is "Forbidden" and 401 is "Unauthorized" + """ + error = S_ERROR(ENOAUTH, "Unauthorized query") + gLogger.error( + "Unauthorized access to %s: %s(%s) from %s" % + (self.request.path, + self.credDict['CN'], + self.credDict['DN'], + self.request.remote_ip)) + + self._httpError = errorCode + self.__write_return(error) + self.finish() + + def on_finish(self): + """ + Called after the end of HTTP request + """ + requestDuration = time.time() - self.requestStartTime + gLogger.notice("Ending request to %s after %fs" % (self.srv_getURL(), requestDuration)) + + def gatherPeerCredentials(self): + """ + Load client certchain in DIRAC and extract informations. + + The dictionnary returned is designed to work with the AuthManager, + already written for DISET and re-used for HTTPS. + """ + + # This line get certificates, it must be change when M2Crypto will be fully integrated in tornado + chainAsText = self.request.get_ssl_certificate().as_pem() + peerChain = X509Chain() + + # Here we read all certificate chain + cert_chain = self.request.get_ssl_certificate_chain() + for cert in cert_chain: + chainAsText += cert.as_pem() + + # And we let some utilities do the job... + # Following lines just get the right info, at the right place + peerChain.loadChainFromString(chainAsText) + + isProxyChain = peerChain.isProxy()['Value'] + isLimitedProxyChain = peerChain.isLimitedProxy()['Value'] + if isProxyChain: + if peerChain.isPUSP()['Value']: + identitySubject = peerChain.getCertInChain(-2)['Value'].getSubjectNameObject()['Value'] + else: + identitySubject = peerChain.getIssuerCert()['Value'].getSubjectNameObject()['Value'] + else: + identitySubject = peerChain.getCertInChain(0)['Value'].getSubjectNameObject()['Value'] + credDict = {'DN': identitySubject.one_line(), + 'CN': identitySubject.commonName, + 'x509Chain': peerChain, + 'isProxy': isProxyChain, + 'isLimitedProxy': isLimitedProxyChain} + diracGroup = peerChain.getDIRACGroup() + if diracGroup['OK'] and diracGroup['Value']: + credDict['group'] = diracGroup['Value'] + + # We check if client sends extra credentials... + if "extraCredentials" in self.request.arguments: + extraCred = self.get_argument("extraCredentials") + if extraCred: + credDict['extraCredentials'] = decode(extraCred)[0] + return credDict + + +#### +# +# Default method +# +#### + + auth_ping = ['all'] + + def export_ping(self): + """ + Default ping method, returns some info about server. + + It returns the exact same information as DISET, for transparency purpose. + """ + # COPY FROM DIRAC.Core.DISET.RequestHandler + dInfo = {} + dInfo['version'] = DIRAC.version + dInfo['time'] = datetime.utcnow() + # Uptime + try: + with open("/proc/uptime") as oFD: + iUptime = long(float(oFD.readline().split()[0].strip())) + dInfo['host uptime'] = iUptime + except BaseException: + pass + startTime = self._startTime + dInfo['service start time'] = self._startTime + serviceUptime = datetime.utcnow() - startTime + dInfo['service uptime'] = serviceUptime.days * 3600 + serviceUptime.seconds + # Load average + try: + with open("/proc/loadavg") as oFD: + sLine = oFD.readline() + dInfo['load'] = " ".join(sLine.split()[:3]) + except BaseException: + pass + dInfo['name'] = self._serviceInfoDict['serviceName'] + stTimes = os.times() + dInfo['cpu times'] = {'user time': stTimes[0], + 'system time': stTimes[1], + 'children user time': stTimes[2], + 'children system time': stTimes[3], + 'elapsed real time': stTimes[4] + } + + return S_OK(dInfo) + + auth_echo = ['all'] + + @staticmethod + def export_echo(data): + """ + This method used for testing the performance of a service + """ + return S_OK(data) + + auth_whoami = ['authenticated'] + + def export_whoami(self): + """ + A simple whoami, returns all credential dictionnary, except certificate chain object. + """ + credDict = self.srv_getRemoteCredentials() + if 'x509Chain' in credDict: + # Not serializable + del credDict['x509Chain'] + return S_OK(credDict) + +#### +# +# Utilities methods, some getters. +# From DIRAC.Core.DISET.requestHandler to get same interface in the handlers. +# Adapted for Tornado. +# These method are copied from DISET RequestHandler, they are not all used when i'm writing +# these lines. I rewrite them for Tornado to get them ready when a new HTTPS service need them +# +#### + + @classmethod + def srv_getCSOption(cls, optionName, defaultValue=False): + """ + Get an option from the CS section of the services + + :return: Value for serviceSection/optionName in the CS being defaultValue the default + """ + if optionName[0] == "/": + return gConfig.getValue(optionName, defaultValue) + for csPath in cls._serviceInfoDict['csPaths']: + result = gConfig.getOption("%s/%s" % (csPath, optionName, ), defaultValue) + if result['OK']: + return result['Value'] + return defaultValue + + def getCSOption(self, optionName, defaultValue=False): + """ + Just for keeping same public interface + """ + return self.srv_getCSOption(optionName, defaultValue) + + def srv_getRemoteAddress(self): + """ + Get the address of the remote peer. + + :return: Address of remote peer. + """ + return self.request.remote_ip + + def getRemoteAddress(self): + """ + Just for keeping same public interface + """ + return self.srv_getRemoteAddress() + + def srv_getRemoteCredentials(self): + """ + Get the credentials of the remote peer. + + :return: Credentials dictionary of remote peer. + """ + return self.credDict + + def getRemoteCredentials(self): + """ + Get the credentials of the remote peer. + + :return: Credentials dictionary of remote peer. + """ + return self.credDict + + def srv_getFormattedRemoteCredentials(self): + """ + Return the DN of user + """ + try: + return self.credDict['DN'] + except KeyError: # Called before reading certificate chain + return "unknown" + + def srv_getServiceName(self): + """ + Return the service name + """ + return self._serviceInfoDict['serviceName'] + + def srv_getURL(self): + """ + Return the URL + """ + return self.request.path diff --git a/Core/Tornado/Server/__init__.py b/Core/Tornado/Server/__init__.py new file mode 100644 index 00000000000..cae5bc3f37c --- /dev/null +++ b/Core/Tornado/Server/__init__.py @@ -0,0 +1,3 @@ +""" +DIRAC.TornadosServices +""" diff --git a/Core/Tornado/Utilities/HTTPErrorCodes.py b/Core/Tornado/Utilities/HTTPErrorCodes.py new file mode 100644 index 00000000000..a82959b89e4 --- /dev/null +++ b/Core/Tornado/Utilities/HTTPErrorCodes.py @@ -0,0 +1,20 @@ +""" + List of HTTP codes + + https://tools.ietf.org/html/rfc2616#section-6.1.1 +""" + +# Success code +HTTP_OK = 200 + + +# Client error code +HTTP_UNAUTHORIZED = 401 +HTTP_FORBIDDEN = 403 +HTTP_NOT_FOUND = 404 +HTTP_IM_A_TEAPOT = 418 # see https://tools.ietf.org/html/rfc2324 + + +# Server error code +HTTP_INTERNAL_SERVER_ERROR = 500 +HTTP_NOT_IMPLEMENTED = 501 diff --git a/Core/Tornado/Utilities/__init__.py b/Core/Tornado/Utilities/__init__.py new file mode 100644 index 00000000000..cae5bc3f37c --- /dev/null +++ b/Core/Tornado/Utilities/__init__.py @@ -0,0 +1,3 @@ +""" +DIRAC.TornadosServices +""" diff --git a/Core/Tornado/__init__.py b/Core/Tornado/__init__.py new file mode 100644 index 00000000000..cae5bc3f37c --- /dev/null +++ b/Core/Tornado/__init__.py @@ -0,0 +1,3 @@ +""" +DIRAC.TornadosServices +""" diff --git a/Core/Tornado/scripts/__init__.py b/Core/Tornado/scripts/__init__.py new file mode 100644 index 00000000000..4b853732d24 --- /dev/null +++ b/Core/Tornado/scripts/__init__.py @@ -0,0 +1 @@ +""" The TornadoService scripts package """ diff --git a/Core/Tornado/scripts/tornado-start-CS.py b/Core/Tornado/scripts/tornado-start-CS.py new file mode 100644 index 00000000000..8b444c3f1b7 --- /dev/null +++ b/Core/Tornado/scripts/tornado-start-CS.py @@ -0,0 +1,50 @@ +#!/usr/bin/env python +######################################################################## +# File : tornado-start-CS +# Author : Louis MARTIN +######################################################################## +# Just run this script to start Tornado and CS service +# Use dirac.cfg (or other cfg given in the command line) to change port + +__RCSID__ = "$Id$" + + +# Must be define BEFORE any dirac import +import os +import sys +os.environ['USE_TORNADO_IOLOOP'] = "True" + + +from DIRAC.FrameworkSystem.Client.Logger import gLogger +from DIRAC.Core.Tornado.Server.TornadoServer import TornadoServer +from DIRAC.Core.Base import Script + +from DIRAC.ConfigurationSystem.Client.LocalConfiguration import LocalConfiguration + +from DIRAC.Core.Utilities.DErrno import includeExtensionErrors + +from DIRAC.ConfigurationSystem.Client.ConfigurationData import gConfigurationData +from DIRAC.ConfigurationSystem.private.Refresher import gRefresher + +if gConfigurationData.isMaster(): + gRefresher.disable() + +localCfg = LocalConfiguration() +localCfg.addMandatoryEntry("/DIRAC/Setup") +localCfg.addDefaultEntry("/DIRAC/Security/UseServerCertificate", "yes") +localCfg.addDefaultEntry("LogLevel", "INFO") +localCfg.addDefaultEntry("LogColor", True) +resultDict = localCfg.loadUserData() +if not resultDict['OK']: + gLogger.initialize("Tornado-CS", "/") + gLogger.error("There were errors when loading configuration", resultDict['Message']) + sys.exit(1) + +includeExtensionErrors() + + +gLogger.initialize('Tornado-CS', "/") + + +serverToLaunch = TornadoServer(services='Configuration/Server') +serverToLaunch.startTornado() diff --git a/Core/Tornado/scripts/tornado-start-all.py b/Core/Tornado/scripts/tornado-start-all.py new file mode 100644 index 00000000000..01ed73e7c88 --- /dev/null +++ b/Core/Tornado/scripts/tornado-start-all.py @@ -0,0 +1,58 @@ +#!/usr/bin/env python +######################################################################## +# File : tornado-start-all +# Author : Louis MARTIN +######################################################################## +# Just run this script to start Tornado and all services +# Use CS to change port + +__RCSID__ = "$Id$" + + +# Must be define BEFORE any dirac import +import os +import sys +os.environ['USE_TORNADO_IOLOOP'] = "True" + + +from DIRAC.FrameworkSystem.Client.Logger import gLogger +from DIRAC.Core.Tornado.Server.TornadoServer import TornadoServer +from DIRAC.Core.Base import Script + +from DIRAC.ConfigurationSystem.Client.LocalConfiguration import LocalConfiguration +from DIRAC.ConfigurationSystem.Client.ConfigurationData import gConfigurationData +from DIRAC import gConfig +from DIRAC.ConfigurationSystem.Client import PathFinder + +from DIRAC.Core.Utilities.DErrno import includeExtensionErrors + + +# We check if there is no configuration server started as master +# If you want to start a master CS you should use Configuration_Server.cfg and +# use tornado-start-CS.py +if gConfigurationData.isMaster() and gConfig.getValue( + '/Systems/Configuration/%s/Services/Server/Protocol' % + PathFinder.getSystemInstance('Configuration'), + 'dips').lower() == 'https': + gLogger.fatal("You can't run the CS and services in the same server!") + sys.exit(0) + +localCfg = LocalConfiguration() +localCfg.addMandatoryEntry("/DIRAC/Setup") +localCfg.addDefaultEntry("/DIRAC/Security/UseServerCertificate", "yes") +localCfg.addDefaultEntry("LogLevel", "INFO") +localCfg.addDefaultEntry("LogColor", True) +resultDict = localCfg.loadUserData() +if not resultDict['OK']: + gLogger.initialize("Tornado", "/") + gLogger.error("There were errors when loading configuration", resultDict['Message']) + sys.exit(1) + +includeExtensionErrors() + + +gLogger.initialize('Tornado', "/") + + +serverToLaunch = TornadoServer() +serverToLaunch.startTornado() diff --git a/Core/Tornado/test/TestClientReturnedValuesAndCredentialsExtraction.py b/Core/Tornado/test/TestClientReturnedValuesAndCredentialsExtraction.py new file mode 100644 index 00000000000..0a37af48f2f --- /dev/null +++ b/Core/Tornado/test/TestClientReturnedValuesAndCredentialsExtraction.py @@ -0,0 +1,116 @@ +""" + In this test we want to check if Tornado generate the same credentials dictionnary as DIRAC. + It also test if the correct certificates are sended by client. + + To run this test you must have the handlers who returns credentials dictionnary. + Handlers are diracCredDictHandler and tornadoCredDictHandler just returns these dictionnary + and are stored in DIRAC/FrameworkSystem/Service + + Then you have to start tornado using script tornado-start-all.py in DIRAC/TornadoServices/scripts + and diset with ``dirac-service Framework/diracCredDict`` before running test + + In configuration it have to be set as normal services, it will look like: + + ``` + # In Systems/Service//Framework + Services + { + tornadoCredDict + { + protocol = https + } + diracCredDict + { + Port = 3444 + DisableMonitoring = yes + } + } + ``` + + ``` + URLs + { + tornadoCredDict = https://localhost:443/Framework/tornadoCredDict + diracCredDict = dips://MrBoincHost:3444/Framework/diracCredDict + } + ``` + +""" +from DIRAC.Core.Base.Script import parseCommandLine +parseCommandLine() +import pytest +from DIRAC import gConfig +from DIRAC.ConfigurationSystem.Client.ConfigurationData import gConfigurationData + +from DIRAC.Core.Tornado.Client.TornadoClient import TornadoClient +from DIRAC.Core.DISET.RPCClient import RPCClient +from pytest import mark, fixture +parametrize = mark.parametrize + + +def get_RPC_returnedValue(serviceName, Client): + """ + Get credentials extracted tornado server or Dirac server + """ + service = Client(serviceName) + return service.credDict() + + +def get_all_returnedValues(): + """ + Just code factorisation to who call server and get credential dictionnary + """ + serviceNameTornado = 'Framework/tornadoCredDict' + serviceNameDirac = 'Framework/diracCredDict' + repTornado = TornadoClient(serviceNameTornado).whoami() + repDirac = RPCClient(serviceNameDirac).whoami() + return (repTornado, repDirac) + + +@parametrize('UseServerCertificate', ('true', 'false')) +def test_return_credential_are_equals(UseServerCertificate): + """ + Check if certificates sended AND extraction have same comportement is DISET and HTTPS + """ + gConfigurationData.setOptionInCFG('/DIRAC/Security/UseServerCertificate', UseServerCertificate) + + (repTornado, repDirac) = get_all_returnedValues() + + # Service returns credentials + assert repDirac['Value'] == repTornado['Value'] + + +@parametrize('UseServerCertificate', ('True', 'False')) +def test_rpcStubs_are_equals(UseServerCertificate): + """ + Test if Clients returns the same rpcStubs + + Navigating through array is a bit complicated in this test... + repDirac and repTornado may have the same structure: + + repDirac dict{ + OK: True + rpcStub: tuple{ + ServiceName: str + kwargs: dict{ + *** kwargs used to instanciate client *** + } + methodName: str + arguments: list + } + Value: dict { # NOT USED IN THIS TEST + *** Credentials dictionnary extracted by server *** + } + } + """ + + gConfigurationData.setOptionInCFG('/DIRAC/Security/UseServerCertificate', UseServerCertificate) + (repTornado, repDirac) = get_all_returnedValues() + + # Explicitly removed in Tornado + del repDirac['rpcStub'][0][1]['keepAliveLapse'] + + # rep['rpcStub'] is at form (rpcStub, method, args) where rpcStub is tuple with (serviceName, kwargs) + assert repTornado['rpcStub'][0][0] != repDirac['rpcStub'][0][0] # Services name are different + assert repTornado['rpcStub'][0][1] == repDirac['rpcStub'][0][1] # Check kwargs returned by rpcStub + assert repTornado['rpcStub'][1:] != repDirac['rpcStub'][1:] # Check method/args diff --git a/Core/Tornado/test/TestClientSelectionAndInterface.py b/Core/Tornado/test/TestClientSelectionAndInterface.py new file mode 100644 index 00000000000..0eebf1ec8f7 --- /dev/null +++ b/Core/Tornado/test/TestClientSelectionAndInterface.py @@ -0,0 +1,179 @@ +""" + Unit test on client selection: + - By default: RPCClient should be used + - If we use Tornado service TornadoClient is used + + Should work with + - 'Component/Service' + - URL + - List of URL + + Mock Config: + - Service using HTTPS with Tornado + - Service using Diset + + You don't need to setup anything, just run ``pytest TestClientSelection.py`` ! +""" +import os +import re + + +from pytest import mark, fixture + +from DIRAC.Core.Tornado.Client.RPCClientSelector import RPCClientSelector +from DIRAC.Core.Tornado.Client.TornadoClient import TornadoClient +from DIRAC.Core.DISET.RPCClient import RPCClient +from DIRAC.ConfigurationSystem.private.ConfigurationClient import ConfigurationClient +from DIRAC.ConfigurationSystem.Client.ConfigurationData import gConfigurationData +from DIRAC.Core.Utilities.CFG import CFG +from DIRAC.Core.Base.Client import Client +from DIRAC.Core.DISET.private.InnerRPCClient import InnerRPCClient + +parametrize = mark.parametrize + + +testCfgFileName = 'test.cfg' + + +@fixture(scope='function') +def config(request): + """ + fixture is the pytest way to declare initalization function. + Scope = module significate that this function will be called only time for this file. + If no scope precised it call config for each test. + + This function can have a return value, it will be the value of 'config' argument for the tests + """ + + cfgContent = ''' + DIRAC + { + Setup=TestSetup + Setups + { + TestSetup + { + WorkloadManagement=MyWM + } + } + } + Systems + { + WorkloadManagement + { + MyWM + { + URLs + { + ServiceDips = dips://$MAINSERVERS$:1234/WorkloadManagement/ServiceDips + ServiceHttps = https://$MAINSERVERS$:1234/WorkloadManagement/ServiceHttps + } + } + } + } + Operations{ + Defaults + { + MainServers = server1, server2 + } + } + ''' + with open(testCfgFileName, 'w') as f: + f.write(cfgContent) + gConfig = ConfigurationClient(fileToLoadList=[testCfgFileName]) # we replace the configuration by our own one. + + # def tearDown(): + # Wait for teardown + yield config + """ + This function is called at the end of the test. + """ + try: + os.remove(testCfgFileName) + except OSError: + pass + # SUPER UGLY: one must recreate the CFG objects of gConfigurationData + # not to conflict with other tests that might be using a local dirac.cfg + gConfigurationData.localCFG = CFG() + gConfigurationData.remoteCFG = CFG() + gConfigurationData.mergedCFG = CFG() + gConfigurationData.generateNewVersion() + print "TearDown" + # request is given by @fixture decorator, addfinalizer set the function who need to be called after the tests + # request.addfinalizer(tearDown) + + +# Tuple with (expectedClient, serviceName) +client_imp = ( + (TornadoClient, 'WorkloadManagement/ServiceHttps'), + (TornadoClient, 'https://server1:1234/WorkloadManagement/ServiceHttps'), + (TornadoClient, + 'https://server1:1234/WorkloadManagement/ServiceHttps,https://server2:1234/WorkloadManagement/ServiceHttps'), + (RPCClient, 'WorkloadManagement/ServiceDips'), + (RPCClient, 'dips://server1:1234/WorkloadManagement/ServiceDips'), + (RPCClient, + 'dips://server1:1234/WorkloadManagement/ServiceDips,dips://server2:1234/WorkloadManagement/ServiceDips'), +) + + +@parametrize('client', client_imp) +def test_selection_when_using_RPCClientSelector(client, config): + """ + One way to call service is to use RPCClient or TornadoClient + If service is HTTPS, it must return client who work with tornado (TornadoClient) + else it must return the RPCClient + """ + clientWanted = client[0] + component_service = client[1] + clientSelected = RPCClientSelector(component_service) + assert isinstance(clientSelected, clientWanted) + + +error_component = ( + 'Too/Many/Sections', + 'JustAName', + "InexistantComponent/InexistantService", + "dummyProtocol://dummy/url") + + +@parametrize('component_service', error_component) +def test_error(component_service, config): + """ + In any other cases (including error cases) it must return RPCClient by default + This test is NOT testing if RPCClient handle the errors + It just test that we get RPCClient and not Tornadoclient + """ + clientSelected = RPCClientSelector(component_service) + assert isinstance(clientSelected, RPCClient) + + +def test_interface(): + """ + Interface of TornadoClient MUST contain at least interface of RPCClient. + BUT a __getattr__ method extends this interface with interface of InnerRPCClient. + """ + interfaceTornadoClient = dir(TornadoClient) + interfaceRPCClient = dir(RPCClient) + dir(InnerRPCClient) + for element in interfaceRPCClient: + # We don't need to test private methods / attribute + # Private methods/attribute starts with __ + # dir also return private methods named with something like _ClassName__PrivateMethodName + if not element.startswith('_'): + assert element in interfaceTornadoClient + + +client_imp = ( + (2, 'WorkloadManagement/ServiceHttps'), + (1, 'https://server1:1234/WorkloadManagement/ServiceHttps') +) + + +@parametrize('client', client_imp) +def test_urls_used_by_TornadoClient(config, client): + # We can't directly get url because they are randomized but we can check if we have right number of URL + + nbOfUrl = client[0] + component_service = client[1] + clientSelected = RPCClientSelector(component_service) + # Little hack to get the private attribute + assert nbOfUrl == clientSelected._TornadoBaseClient__nbOfUrls diff --git a/Core/Tornado/test/TestServerIntegration.py b/Core/Tornado/test/TestServerIntegration.py new file mode 100644 index 00000000000..ca95a991cc6 --- /dev/null +++ b/Core/Tornado/test/TestServerIntegration.py @@ -0,0 +1,91 @@ +""" + Test if same service work on DIRAC and TORNADO + Testing if basic operation works on a dummy example + + These handlers provide a method who's access is always forbidden to test authorization system + + It's just normal services, entry in dirac.cfg are the same as usual. + To start tornado use DIRAC/TornadoServices/scripts/tornado-start-all.py + ``` + Services + { + User + { + Protocol = https + } + UserDirac + { + Port = 3424 + } + } + ``` + + ``` + URLs + { + User = https://MrBoincHost:443/Framework/User + UserDirac = dips://localhost:3424/Framework/UserDirac + } + ``` + +""" + +from DIRAC.Core.Base.Script import parseCommandLine +parseCommandLine() +from string import printable +import datetime +import sys + +from hypothesis import given, settings +from hypothesis.strategies import text + +from DIRAC.Core.DISET.RPCClient import RPCClient as RPCClientDIRAC +from DIRAC.Core.Tornado.Client.TornadoClient import TornadoClient as RPCClientTornado +from DIRAC.Core.Utilities.DErrno import ENOAUTH +from DIRAC import S_ERROR + +from pytest import mark +parametrize = mark.parametrize + +rpc_imp = ((RPCClientTornado, 'Framework/User'), (RPCClientDIRAC, 'Framework/UserDirac')) + + +@parametrize('rpc', rpc_imp) +def test_authorization(rpc): + service = rpc[0](rpc[1]) + + authorisation = service.unauthorized() + assert authorisation['OK'] is False + assert authorisation['Message'] == S_ERROR(ENOAUTH, "Unauthorized query")['Message'] + + +@parametrize('rpc', rpc_imp) +def test_unknown_method(rpc): + service = rpc[0](rpc[1]) + + unknownmethod = service.ThisMethodMayNotExist() + assert unknownmethod['OK'] is False + assert unknownmethod['Message'] == "Unknown method ThisMethodMayNotExist" + + +@parametrize('rpc', rpc_imp) +def test_ping(rpc): + service = rpc[0](rpc[1]) + + assert service.ping()['OK'] + + +@parametrize('rpc', rpc_imp) +@settings(deadline=None, max_examples=42) +@given(data=text(printable, max_size=64)) +def test_echo(rpc, data): + service = rpc[0](rpc[1]) + + assert service.echo(data)['Value'] == data + + +def test_whoami(): # Only in tornado + credDict = RPCClientTornado('Framework/User').whoami()['Value'] + assert 'DN' in credDict + assert 'CN' in credDict + assert 'isProxy' in credDict diff --git a/Core/Tornado/test/multi-mechanize/distributed-test.py b/Core/Tornado/test/multi-mechanize/distributed-test.py new file mode 100644 index 00000000000..4839dff7f14 --- /dev/null +++ b/Core/Tornado/test/multi-mechanize/distributed-test.py @@ -0,0 +1,45 @@ +import xmlrpclib +import time +import sys +# == BEGIN OF CONFIGURATION == + +# Add all machine who have multimechanize client +serversList = ['137.138.150.194', 'server2'] +# Each multimechanize client must listen the same ports, add the ports here +portList = ['9000', '9001'] + +# END OF CONFIG + +servers = [] + +print "Starting test servers...." +# We send signal to all servers +for port in portList: + for server in serversList: + servers.append(xmlrpclib.ServerProxy("http://%s:%s" % (server, port))) + servers[-1].run_test() + # If there is multiple ports opened on same machine, we wait a little to avoid confusion in multimechanize + time.sleep(2) + + +print "Waiting for results..." +while servers[-1].get_results() == 'Results Not Available': + time.sleep(1) + +# We get all results and write them into files +# There is one file/multimechanize servers +try: + output = sys.argv[1] +except KeyError: + output = str(time.time()) +fileCount = 0 +for server in servers: + fileCount += 1 + fileName = "%s.%s.txt" % (output, fileCount) + print "Writing output file %s" % fileName + file = open(fileName, 'w') + file.write(server.get_results()) + file.close() + +# We print the command you can copy paste to have the results in a plot +print "python plot-distributedTest.py %s %d" % (output, fileCount) diff --git a/Core/Tornado/test/multi-mechanize/ping/config.cfg b/Core/Tornado/test/multi-mechanize/ping/config.cfg new file mode 100644 index 00000000000..b9ba7f702d8 --- /dev/null +++ b/Core/Tornado/test/multi-mechanize/ping/config.cfg @@ -0,0 +1,13 @@ + +[global] +run_time = 60 +rampup = 60 +results_ts_interval = 1 +progress_bar = on +console_logging = off +xml_report = off + + +[user_group-1] +threads = 60 +script = v_perf.py diff --git a/Core/Tornado/test/multi-mechanize/ping/test_scripts/v_perf.py b/Core/Tornado/test/multi-mechanize/ping/test_scripts/v_perf.py new file mode 100644 index 00000000000..b2702a22f84 --- /dev/null +++ b/Core/Tornado/test/multi-mechanize/ping/test_scripts/v_perf.py @@ -0,0 +1,18 @@ +from DIRAC.Core.Tornado.Client.TornadoClient import TornadoClient +from DIRAC.Core.DISET.RPCClient import RPCClient +from time import time +from random import randint +import sys + + +class Transaction(object): + def __init__(self): + # If we want we can force to use dirac + if len(sys.argv) > 2 and sys.argv[2].lower() == 'dirac': + self.client = RPCClient('Framework/UserDirac') + else: + self.client = TornadoClient('Framework/User') + return + + def run(self): + self.client.ping() diff --git a/Core/Tornado/test/multi-mechanize/plot-distributedTest.py b/Core/Tornado/test/multi-mechanize/plot-distributedTest.py new file mode 100644 index 00000000000..9efd48e7d75 --- /dev/null +++ b/Core/Tornado/test/multi-mechanize/plot-distributedTest.py @@ -0,0 +1,173 @@ + +import csv +import matplotlib.pyplot as plt +import sys + + +# == BEGIN OF CONFIGURATION == + +# FOR DISPLAY - Enter settings relative to test +system = 'Test ping - timeout=30 - Diset Server' +multimech_thread = 60 +multimech_time = 300 +multimech_rampup = 200 +multimech_clients = 8 +server_maxThreads = 20 + +# For this line, use vmstat and copy the free memory +memoryOffset = 0 + +plt.suptitle( + '%s with %d threads\n %d threads/client - %d clients (total %d threads) \n \ + duration: %dsec - rampup %dsec \n latency between client starts: 2s' % + (system, + server_maxThreads, + multimech_thread, + multimech_clients, + multimech_clients * multimech_thread, + multimech_time, + multimech_rampup)) + + +# END OF CONFIG + + +def get_results(): + if len(sys.argv) < 3: + print "Usage: python plot-distributedTest NAME NUMBEROFFILE" + print "Example: python plot-distributedTest 1532506328.38 2" + sys.exit(1) + + file = sys.argv[1] + count = int(sys.argv[2]) + results = [] + for i in range(1, count + 1): + fileName = "%s.%s.txt" % (file, i) + with open(fileName, 'r') as content: + print "reading %s" % fileName + lines = content.read().split('\n')[1:-1] + result = [line.split(',') for line in lines] + results.append(result) + content.close() + return results + + +def get_server_stats(): + print "Please specify location to file with server stats:" + serverStatFile = "/tmp/results.txt" # raw_input() + print "Loading %s" % serverStatFile + + serverStats = dict() + with open(serverStatFile, 'r') as content_file: + lines = content_file.read().split('\n')[1:] + for line in lines: + line = line.split(';') + serverStats[line[0]] = line[1:] + return serverStats + + +def get_test_begin_end(results): + """ + First result file contain the test started in first + Last result file contain the test started in last + Every test have same duration + So we read first and last line to have begin hour and end hour + (the '2' is because time is registered in the third row) + """ + return (int(results[0][0][2]), int(results[-1][-1][2])) + + +def process_data(results, serverStats): + # Begin and end are timestamps + (begin, end) = get_test_begin_end(results) + + # Initializing all data list + (time, requestTime, CPU, RAM, reqPerSec, errorRate, loadAvg) = ([], [], [], [], [], [], []) + global memoryOffset + initialRAM = memoryOffset + + for t in range(begin, end): # We determine datas time with timestamp + # Offset to set starttime = 0 + time.append(t - begin) + + # Getting requesttime (mean), number of request/error at a givent time + (reqTime, reqCount, errorCount) = getRequestTimeAndCount(results, t) + requestTime.append(reqTime) + reqPerSec.append(reqCount) + errorRate.append(errorCount) + + # Getting infos from Server, sometimes no info are getted during more than one second + try: + # Get CPU usage + CPU.append(100 - int(serverStats[str(t)][14])) + + # Get Memory used (delta with memory at the beginning) + usedRam = int(serverStats[str(t)][5]) - initialRAM + RAM.append(usedRam) + + # Get the load + loadAvg.append(100 * float(serverStats[str(t)][17])) # 18 + except KeyError: + # If fail in getting value, take previous values + CPU.append(CPU[-1]) + RAM.append(RAM[-1]) + loadAvg.append(loadAvg[-1]) + + print "ERROR - Some values missing for CPU and Memory usage [try to load for time=%s]" % t + + return (time, requestTime, CPU, RAM, reqPerSec, errorRate, loadAvg) + + +def getRequestTimeAndCount(data, time): + reqCount = 0 + errorCount = 0 + totalRequest = 0.0 + + for result in results: + i = 0 + try: + + # Ignore past + while int(result[i][2]) < time: + i += 1 + + # Get infos for present + while int(result[i][2]) == time: + reqCount += 1 + totalRequest += float(result[i][4]) + if result[i][5] != '': + errorCount += 1 + i += 1 + except IndexError: + pass + return (totalRequest / reqCount if reqCount > 0 else 0, reqCount, errorCount) + + +def displayGraph(results, serverStats): + """ + Display all the graph on the same figure + """ + print "Processing data and plot, it may take some time for huge tests" + (time, requestTime, CPU, RAM, reqPerSec, errorCount, loadAvg) = process_data(results, serverStats) + + plt.subplot(221) + plt.plot(time, requestTime, '-', label="Request time (s)") + plt.legend() + plt.subplot(222) + plt.plot(time, CPU, '*', label="CPU usage (%)") + plt.plot(time, loadAvg, '*', label="Load average * 100") + plt.legend() + plt.subplot(223) + plt.plot(time, RAM, '*', label="Used Memory (bytes)") + plt.legend() + plt.subplot(224) + plt.plot(time, reqPerSec, '*', label="Requests/sec") + plt.plot(time, errorCount, '*', label="Errors/sec") + plt.legend() + + +results = get_results() +# process_data(results) +serverStats = get_server_stats() +displayGraph(results, serverStats) +plt.show() diff --git a/Core/Tornado/test/multi-mechanize/plot.py b/Core/Tornado/test/multi-mechanize/plot.py new file mode 100755 index 00000000000..dfba9c2de2b --- /dev/null +++ b/Core/Tornado/test/multi-mechanize/plot.py @@ -0,0 +1,53 @@ +""" + A little script to analyze multi-mechanize tests by ploting every test in a single figure + + Because it can't be automatized you have to define path of the folder who contains results + + testTornado and testDirac must have the same length +""" +import csv +import matplotlib.pyplot as plt + +testTornado = ['ping/results/results_2018.06.22_14.50.53', 'service/results/results_2018.06.22_15.24.18'] +testDirac = ['ping/results/results_2018.06.22_14.52.09', 'service/results/results_2018.06.22_14.55.03'] + + +def read_data(test, groupSize): + with open(test + '/results.csv', 'rb') as csvfile: + reader = csv.reader(csvfile, delimiter=',') + + sumgrouptime = 0 + sumgrouprequestTime = 0 + count = 0 + + time = [] + requestTime = [] + + for row in reader: + count += 1 + sumgrouptime += float(row[1]) + sumgrouprequestTime += float(row[4]) + # We group some points to make graph readable + if(count == groupSize): + time.append(sumgrouptime / groupSize) + requestTime.append(sumgrouprequestTime / groupSize) + sumgrouptime = 0 + sumgrouprequestTime = 0 + count = 0 + return (time, requestTime) + + +def displayGraph(testTornado, testDirac, subplot, groupSize): + plt.subplot(subplot) + plt.ylabel('red = dirac') + + (timeTornado, requestTimeTornado) = read_data(testTornado, groupSize) + (timeDirac, requestTimeDirac) = read_data(testDirac, groupSize) + + plt.plot(timeTornado, requestTimeTornado, 'b-', timeDirac, requestTimeDirac, 'r-') + + +# The "100*len(testTornado)+11+i" can look strange but it define a sublot dynamically +for i in range(len(testTornado)): + displayGraph(testTornado[i], testDirac[i], 100 * len(testTornado) + 11 + i, 42) +plt.show() diff --git a/Core/Tornado/test/multi-mechanize/service/config.cfg b/Core/Tornado/test/multi-mechanize/service/config.cfg new file mode 100644 index 00000000000..b9ba7f702d8 --- /dev/null +++ b/Core/Tornado/test/multi-mechanize/service/config.cfg @@ -0,0 +1,13 @@ + +[global] +run_time = 60 +rampup = 60 +results_ts_interval = 1 +progress_bar = on +console_logging = off +xml_report = off + + +[user_group-1] +threads = 60 +script = v_perf.py diff --git a/Core/Tornado/test/multi-mechanize/service/test_scripts/v_perf.py b/Core/Tornado/test/multi-mechanize/service/test_scripts/v_perf.py new file mode 100644 index 00000000000..4c6a984ac08 --- /dev/null +++ b/Core/Tornado/test/multi-mechanize/service/test_scripts/v_perf.py @@ -0,0 +1,25 @@ +from DIRAC.Core.Tornado.Client.TornadoClient import TornadoClient +from DIRAC.Core.DISET.RPCClient import RPCClient +from time import time +from random import randint +import sys + + +class Transaction(object): + def __init__(self): + # If we want we can force to use dirac + if len(sys.argv) > 2 and sys.argv[2].lower() == 'dirac': + self.client = RPCClient('Framework/UserDirac') + else: + self.client = TornadoClient('Framework/User') + return + + def run(self): + s = "Chaine 1" + s2 = "Chaine %d" % randint(0, 42) + + newUser = self.client.addUser(s) + userID = int(newUser['Value']) + User = self.client.getUserName(userID) + self.client.editUser(userID, s2) + User = self.client.getUserName(userID) diff --git a/FrameworkSystem/Client/MonitoringClientIOLoop.py b/FrameworkSystem/Client/MonitoringClientIOLoop.py new file mode 100644 index 00000000000..41d18b309e1 --- /dev/null +++ b/FrameworkSystem/Client/MonitoringClientIOLoop.py @@ -0,0 +1,30 @@ +""" + Add-on to make the MonitoringClient works with an IOLoop from a Tornado Server +""" + + +import tornado.ioloop +from DIRAC import gLogger + + +class MonitoringFlusherTornado(object): + """ + This class flushes all monitoring clients registered + Works with the Tornado IOLoop + """ + + def __init__(self): + self.__mcList = [] + gLogger.info("Using MonitoringClient in IOLoop mode") + # Here we don't need to use IOLoop.current(), tornado will attach periodic callback to the current IOLoop himself + # We set callback every 5 minnutes + tornado.ioloop.PeriodicCallback(self.flush, 300000).start() + + def flush(self, allData=False): + gLogger.info('Flushing monitoring') + for mc in self.__mcList: + mc.flush(allData) + + def registerMonitoringClient(self, mc): + if mc not in self.__mcList: + self.__mcList.append(mc) diff --git a/docs/source/DeveloperGuide/ConfigurationSystem.rst b/docs/source/DeveloperGuide/ConfigurationSystem.rst new file mode 100644 index 00000000000..07192a8076d --- /dev/null +++ b/docs/source/DeveloperGuide/ConfigurationSystem.rst @@ -0,0 +1,26 @@ +==================== +Configuration System +==================== + +The configuration system works with master server and slave server. + +Master & slave can receive requests from another master or slave and from any client who need the configuration. + +****** +Master +****** +The master Server is the only server who have a local configuration plus a configuration file with shared configuration. He can distribute the shared configuration to every client or synchronize it with a slave server. + +Master server also check if slaves server are alive, in fact he just verify the date of the last registration request to determine if they are considered dead or alive. + +When a slave want to register in the master server (or renew registration) he call ``publishSlaveServer``, then before answering to slave, the master will ping the slave and if ping work, he register the slave and returns ``S_OK`` to slave. + +Master Server also disable the refresher, because he have the configuration and he manage it, so he did'nt need the refresher in the background. + +****** +Slaves +****** +The slave server distribute the shared configuration to every client who need it. +He force the refresher to run in background and to actualize configuration after a few time (every 5 minutes by default). + +At initialization, slave register hitself to the master and get the configuration, then with every refresh he renew its registration to master. diff --git a/docs/source/DeveloperGuide/Internals/Core/Authentification.rst b/docs/source/DeveloperGuide/Internals/Core/Authentification.rst new file mode 100644 index 00000000000..8790b08e272 --- /dev/null +++ b/docs/source/DeveloperGuide/Internals/Core/Authentification.rst @@ -0,0 +1,55 @@ +================================ +Authentication and authorization +================================ +When a client calls a service, he needs to be identified. If a client opens a connection a :py:class:`~DIRAC.Core.DISET.private.Transports.BaseTransport` object is created then the service use the handshake to read certificates, extract informations and store them in a dictionnary so you can use these informations easily. Here an example of possible dictionnary:: + + { + 'DN': '/C=ch/O=DIRAC/[...]', + 'group': 'devGroup', + 'CN': u'ciuser', + 'x509Chain': , + 'isLimitedProxy': False, + 'isProxy': True + } + + +When connection is opened and handshake is done, the service calls the :py:class:`~DIRAC.Core.DISET.AuthManager` and gave him this dictionnary in argument to check the authorizations. More generally you can get this dictionnary with :py:meth:`BaseTransport.getConnectingCredentials `. + + +All procedure have a list of required properties and user may have at least one propertie to execute the procedure. Be careful, properties are associated with groups, not directly with users! + + +There is two main way to define required properties: + +- "Hardcoded" way: Directly in the code, in your request handler you can write ```auth_yourMethodName = listOfProperties```. It can be useful for development or to provide default values. +- Via the configuration system at ```/DIRAC/Systems/(SystemName)/(InstanceName)/Services/(ServiceName)/Authorization/(methodName)```, if you have also define hardcoded properties, hardcoded properties will be ignored. (you can see the administrator guide for more informations) + +A complete list of properties is available in the administrator guide. +If you don't want to define specific properties you can use "authenticated", "any" and "all". + +- "authenticated" allow all users registered in the configuration system to use the procedure (```/DIRAC/Registry/Users```). +- "any" and "all" have the same effect, everyone can call the procedure. It can be dangerous if you allow non-secured connections. + +You also have to define properties for groups of users in the configuration system at ```/DIRAC/Registry/Groups/(groupName)/Properties```. + + +*********** +AuthManager +*********** +AuthManager.authQuery() returns boolean so it is easy to use, you just have to provide a method you want to call, and credDic. It's easy to use but you have to instantiate correctly the AuthManager. For initialization you need the complete path of your service, to get it you may use the PathFinder:: + + from DIRAC.ConfigurationSystem.Client import PathFinder + from DIRAC.Core.DISET.AuthManager import AuthManager + authManager = AuthManager( "%s/Authorization" % PathFinder.getServiceSection("Framework/someService") ) + authManager.authQuery( csAuthPath, credDict, hardcodedMethodAuth ) #return boolean + # csAuthPath is the name of method for RPC or 'typeOfCall/method' + # credDict came from BaseTransport.getConnectingCredentials() + # hardcodedMethodAuth is optional + +To determine if a query can be authorized or not the AuthManager extract valid properties for a given method. +First AuthManager try to get it from gConfig, then try to get it from hardcoded list (hardcodedMethodAuth) in your service and if nothing was found get default properties from gConfig. + +AuthManager also extract properties from user with credential dictionnary and configuration system to check if properties matches. So you don't have to extract properties by yourself, but if needed you can use :py:class:`DIRAC.Core.Security.CS.getPropertiesForGroup()` + + +You can also read `Components authentication and authorization <./componentsAuthNandAuthZ.html>`_ for informations about client-side authentication. \ No newline at end of file diff --git a/docs/source/DeveloperGuide/Internals/Core/ClientServer.rst b/docs/source/DeveloperGuide/Internals/Core/ClientServer.rst index f6ced21499b..311d227181d 100644 --- a/docs/source/DeveloperGuide/Internals/Core/ClientServer.rst +++ b/docs/source/DeveloperGuide/Internals/Core/ClientServer.rst @@ -167,8 +167,7 @@ Complete path of packages are not on the diagram for readability: You can see that the client sends a proposalTuple, proposalTuple contain (service, setup, ClientVO) then (typeOfCall, method) and finaly extra-credentials. -e.g:: - +e.g.:: (('Framework/serviceName', 'DeveloperSetup', 'unknown'), ('RPC', 'methodName'), '') diff --git a/docs/source/DeveloperGuide/TornadoServices/clientservice.uml b/docs/source/DeveloperGuide/TornadoServices/clientservice.uml new file mode 100644 index 00000000000..7a41fac6508 --- /dev/null +++ b/docs/source/DeveloperGuide/TornadoServices/clientservice.uml @@ -0,0 +1,15 @@ +@startuml +Client -> TornadoServer: connect +Client<-> TornadoServer: handshake +Client -> TornadoServer: POST + args +note over TornadoServer: Routing by Tornado +TornadoServer -> TornadoService: initialize() +note over TornadoService: Decoding user certificate\nand extracting credentials +TornadoServer -> TornadoService: prepare() +TornadoService -> AuthManager: authQuery() +AuthManager -> TornadoService: S_OK/S_ERROR +TornadoServer -> TornadoService: post() +TornadoService-> YourServiceHandler: export_method(args) +YourServiceHandler -> TornadoService: S_OK/S_ERROR +TornadoService -> Client: S_OK/S_ERROR +@enduml \ No newline at end of file diff --git a/docs/source/DeveloperGuide/TornadoServices/index.rst b/docs/source/DeveloperGuide/TornadoServices/index.rst new file mode 100644 index 00000000000..c06e771cb71 --- /dev/null +++ b/docs/source/DeveloperGuide/TornadoServices/index.rst @@ -0,0 +1,221 @@ +=========================== +HTTPS Services with Tornado +=========================== + +************ +Presentation +************ +This page summarize changes between DISET and HTTPS. You can all also see these presentations: + +- `Presentation of HTTPS in DIRAC `_. +- `Presentation of HTTPS migration `_. + + +******* +Service +******* + +.. graphviz:: + + digraph { + TornadoServer -> YourServiceHandler [label=use]; + YourServiceHandler -> TornadoService[label=inherit]; + + + TornadoServer [shape=polygon,sides=4, label = "DIRAC.Core.Tornado.Server.TornadoServer"]; + TornadoService [shape=polygon,sides=4, label = "DIRAC.Core.Tornado.Server.TornadoService"]; + YourServiceHandler [shape=polygon,sides=4]; + + } + +Service returns to Client S_OK/S_ERROR encoded in JSON + +********************************************************* +Important changes between DISET server and Tornado Server +********************************************************* + +Internal structure +****************** + +- :py:class:`~DIRAC.Core.DISET.ServiceReactor` is now :py:class:`~DIRAC.Core.Tornado.Server.TornadoServer` +- :py:class:`~DIRAC.Core.DISET.private.Service` and :py:class:`~DIRAC.Core.DISET.RequestHandler` are now merge into :py:class:`~DIRAC.Core.Tornado.Server.TornadoService` +- CallStack from S_ERROR are deleted when they are returned to client. +- Common config for all services, there is no more specific config/service. But you can still give extra config files in the command line when you start a HTTPS server. +- Server returns HTTP status codes like ``200 OK`` or ``401 Forbidden``. Not used by client for now but open possibility for usage with external services (like a REST API) + +How to write service +******************** +Nothing better than example:: + + from DIRAC.Core.Tornado.Server.TornadoService import TornadoService + class yourServiceHandler(TornadoService): + + @classmethod + def initializeHandler(cls, infosDict): + ## Called 1 time, at first request + + def initializeRequest(self): + ## Called at each request + + auth_someMethod = ['all'] + def export_someMethod(self): + ## Insert your method here, don't forget the return should be serializable + ## Returned value may be an S_OK/S_ERROR + ## You don't need to serialize in JSON, Tornado will do it + +Write a service is similar in tornado and diset. You have to define your method starting with ``export_``, your initialization method is a class method called ``initializeHandler`` +Main changes in tornado are: + +- Service are initialized at first request +- You **should not** write method called ``initialize`` because Tornado already use it, so the ``initialize`` from diset handlers became ``initializeRequest`` +- infosDict, arguments of initializedHandler is not really the same as one from diset, all things relative to transport are removed, to write on the transport you can use self.write() but I recommend to avoid his usage, Tornado will encode and write what you return. +- Variables likes ``types_yourMethod`` are ignored, but you can still define ``auth_yourMethod`` if you want. + +Based on DISET request handler, you still have access to some getters in your handler, getters have the same names as DISET, which includes: +``getCSOption``, ``getRemoteAddress``, ``getRemoteCredentials``, ``srv_getCSOption``, ``srv_getRemoteAddress``, ``srv_getRemoteCredentials``, ``srv_getFormattedRemoteCredentials``, ``srv_getServiceName`` and ``srv_getURL``. + + +How to start server +******************* +The easy way, use ``DIRAC/TornadoService/script/tornado-start-all.py`` it will start all services registered in configuration ! To register a service you just have to add the service in the CS and ``Protocol = https``. It may look like this:: + + Systems { + DevInstance + { + Tornado + { + Port = 443 + } + } + Framework + { + DevInstance + { + Services + { + DummyTornado + { + Protocol = https + } + } + } + } + } + + +But you can also control more settings by launching tornado yourself:: + + from DIRAC.Core.Tornado.Server.TornadoServer import TornadoServer + serverToLaunch = TornadoServer(youroptions) + serverToLaunch.startTornado() + +Options availlable are: + +- services, should be a list, to start only these services +- debugSSL, True or False, activate debug mode of Tornado (includes autoreload) and SSL, for extra logs use -ddd in the command line +- port, int, if you want to override value from config. If it's also not defined in config, it use 443. + +This start method can bu usefull for developing new service or create starting script for a specific service, like the Configuration System (as master). + +****** +Client +****** + +.. graphviz:: + + digraph { + TornadoClient -> TornadoBaseClient [label=inherit] + TornadoBaseClient -> Requests [label=use] + + TornadoClient [shape=polygon,sides=4, label="DIRAC.Core.Tornado.Client.TornadoClient"]; + TornadoBaseClient [shape=polygon,sides=4, label="DIRAC.Core.Tornado.Client.private.TornadoBaseClient"]; + Requests [shape=polygon,sides=4] + } + +This diagram present what is behind TornadoClient, but you should use :py:class:`DIRAC.Core.Base.Client` ! The new client integrate a selection system which select for you between HTTPS and DISET client. + +In your client module when you inherit from :py:class:`DIRAC.Core.Base.Client` you can define `httpsClient` with another client, it can be usefull when you can't serialize some data in JSON. Here the step to create and use a JSON patch: + +- Create a class which inherit from :py:class:`~DIRAC.Core.Tornado.Client.TornadoClient` +- For every method who need a JSON patch create a method with the same name as the service +- Use self.executeRPC to send / receive datas + +You can also see this example:: + + class ConfigurationServerJSON(TornadoClient): + """ + The specific client for configuration system. + To avoid JSON limitation the HTTPS handler encode data in base64 + before sending them, this class only decode the base64 + An exception is made with CommitNewData wich ENCODE in base64 + """ + def getCompressedData(self): + """ + Transmit request to service and get data in base64, + it decode base64 before returning + + :returns str:Configuration data, compressed + """ + retVal = self.executeRPC('getCompressedData') + if retVal['OK']: + retVal['Value'] = b64decode(retVal['Value']) + return retVal + + + + +Behind :py:class:`~DIRAC.Core.Tornado.Client.TornadoClient` the `requests `_ library sends a HTTP POST request with: + +- procedure: str with procedure name +- args: your arguments encoded in JSON +- clientVO: The VO of client +- extraCredentials: (if apply) Extra informations to authenticate client + +Service is determined by server thanks to URL rooting, not with port like in DISET. + +By default server listen on port 443, default port for HTTPS. + + +***************************** +Client / Service interactions +***************************** + +.. image:: clientservice.png + :align: center + :alt: Client/Service interactions + +***************************************************** +Important changes between TornadoClient and RPCClient +***************************************************** + +Internal structure +****************** + +- :py:class:`~DIRAC.Core.DISET.private.innerRPCClient` and :py:class:`~DIRAC.Core.DISET.RPCClient` are now a single class: :py:class:`~DIRAC.Core.Tornado.Client.TornadoClient`. Interface and usage stay the same. +- :py:class:`~DIRAC.Core.Tornado.Client.private.TornadoBaseClient` is the new :py:class:`~DIRAC.Core.DISET.private.BaseClient`. Most of code is copied from :py:class:`~DIRAC.Core.DISET.private.BaseClient` but some method have been rewrited to use `Requests `_ instead of Transports. Code duplication is done to fully separate DISET and HTTPS but later, some parts can be merged by using a new common class between DISET and HTTPS (these parts are explicitly given in the docstrings). +- :py:class:`~DIRAC.Core.DISET.private.Transports.BaseTransport`, :py:class:`~DIRAC.Core.DISET.private.Transports.PlainTransport` and :py:class:`~DIRAC.Core.DISET.private.Transports.SSLTransport` are replaced by `Requests `_ +- keepAliveLapse is removed from rpcStub returned by Client because `Requests `_ manage it himself. +- Due to JSON limitation you can write some specifics clients who inherit from :py:class:`~DIRAC.Core.Tornado.Client.TornadoClient`, there is a simple example with :py:class:`~DIRAC.Core.Tornado.Client.SpecificClient.ConfigurationClient` who transfer data in base64 to overcome JSON limitations + + +Connections and certificates +**************************** +`Requests `_ library check more than DISET when reading certificates and do some stuff for us: + +- Server certificate **must** have subject alternative names. Requests also check the hostname and you can have connection errors when using "localhost" for example. To avoid them add subject alternative name in certificate. (You can also see https://github.com/shazow/urllib3/issues/497 ). +- If server certificates are used by clients, you must add clientAuth in the extendedKeyUsage (requests also check that). +- In server side M2Crypto is used instead of GSI and conflict are possible between GSI and M2Crypto, to avoid them you can comment 4 lasts lines at ``DIRAC/Core/Security/__init__.py`` +- ``_connect()``, ``_disconnect()`` and ``_purposeAction()`` are removed, ``_connect``/``_disconnect`` are now managed by `requests `_ and ``_purposeAction`` is no longer used is in HTTPS protocol. + + + +************ +Launch tests +************ + +pytest +****** +Because for now Tornado does not have "Real" services, you must use some fakes services to compare and test with DISET. +You need tornadoCredDict, diracCredDict, User, UserDirac to run tests. Each test explain how to configure in its docstring. + +The only service available is the Configuration/Server, it will work with HTTPS and DISET services who needs to load configuration with a Configuration/Server. diff --git a/docs/source/DeveloperGuide/TornadoServices/installTornado.rst b/docs/source/DeveloperGuide/TornadoServices/installTornado.rst new file mode 100644 index 00000000000..6149e1cf0cc --- /dev/null +++ b/docs/source/DeveloperGuide/TornadoServices/installTornado.rst @@ -0,0 +1,214 @@ +********************** +How to install Tornado +********************** + + +To install and run service on Tornado you should install DIRAC first. You can install DIRAC in the standard way. But for now you don't need to configure it and generate certificates. You just have to install DIRAC:: + + mkdir -p /opt/dirac + useradd dirac + chown dirac:dirac /opt/dirac + su - dirac + cd /opt/dirac + curl -O -L https://raw.githubusercontent.com/DIRACGrid/DIRAC/integration/Core/scripts/dirac-install.py + chmod +x dirac-install.py + ./dirac-install.py -r v6r20p4 -t server + +Once dirac installed you can add Tornado with the following steps: + +*********************** +Installing requirements +*********************** +To install and compile some elements used by Tornado you may install some packages with ``yum``: ``python-devel``, ``m2crypto``, ``gcc``. If you want to do some performance tests please check if ``nscd`` is running on your machine to avoid too many DNS query (on openstack, it is not enabled with SLC6). + +Then you need to install Tornado and M2Crypto (for python), but not from official repo:: + + pip --trusted-host=files.pythonhosted.org --trusted-host=pypi.org --trusted-host=pypi.python.org install git+https://gitlab.com/chaen/m2crypto.git@tmpUntilSwigUpdated + pip --trusted-host=files.pythonhosted.org --trusted-host=pypi.org --trusted-host=pypi.python.org install git+https://github.com/chaen/tornado.git@iostreamConfigurable + pip --trusted-host=files.pythonhosted.org --trusted-host=pypi.org --trusted-host=pypi.python.org install git+https://github.com/chaen/tornado_m2crypto.git + pip --trusted-host=files.pythonhosted.org --trusted-host=pypi.org --trusted-host=pypi.python.org install -r /opt/dirac/DIRAC/requirements.txt + +*********************** +Adding Tornado to DIRAC +*********************** + +Save the DIRAC folder somewhere then clone my GithHub repo, then switch to branch "stage_toDIRAC_clean". You can run the setup.py if ``DIRAC.Core.Tornado`` is not detected by python:: + + mv DIRAC DIRAC.old + git clone https://github.com/louisjdmartin/DIRAC.git + cd DIRAC + git checkout stage_toDIRAC_clean + python setup.py install + + + +********************* +Generate Certificates +********************* +To use HTTPS your certificates must be generated using TLS standard, you can use following lines to generate them yourself:: + + bash + cd /tmp + git clone https://github.com/chaen/DIRAC.git + cd DIRAC + git checkout rel-v6r20_FEAT_correctCA + + export DEVROOT=/tmp + export SERVERINSTALLDIR=/opt/dirac/ + export CI_CONFIG=/tmp/DIRAC/tests/Jenkins/config/ci/ + + source /tmp/DIRAC/tests/Jenkins/utilities.sh + generateCA # automatic + generateCertificates 365 # Certificates copied to /opt/dirac/etc/grid-security + generateUserCredentials 365 # Certificates generated at /opt/dirac/user -> copy to .globus and rename them userkey.pem and usercert.pem + exit + + +********************** +Configuration (server) +********************** +Like in DISET, check your iptable to open some ports if needed ! + +Configuration is mostly the same as before, you just have to define ``Protocol`` to ``HTTPS`` inside the Services and add a new Section for tornado. You can use this example:: + + LocalSite + { + Site = localhost + } + DIRAC + { + + Setup = DeveloperSetup + Setups + { + DeveloperSetup + { + Tornado = DevInstance + Framework = DevInstance + } + } + Security + { + UseServerCertificate=True + CertFile = /opt/dirac/etc/grid-security/hostcert.pem + KeyFile = /opt/dirac/etc/grid-security/hostkey.pem + } + } + + + LocalInstallation + { + Setup = DeveloperSetup + } + + + Systems + { + + Tornado + { + DevInstance + { + + Port = 4444 + } + } + + Framework + { + DevInstance + { + Databases + { + UserDB + { + Host = 127.0.0.1 #localhost + User = root + Password = + DBName = dirac + } + } + Services + { + User + { + # Use this handler to have a dummyService, can be used for testing without load a database + #HandlerPath = DIRAC/FrameworkSystem/Service/DummyTornadoHandler.py + Protocol = https + } + } + } + } + } + Registry + { + # [Add your registry entry, like in DISET] + } + + + +********************** +Configuration (client) +********************** +Nothing change ! +Define your URL as DIRAC service, but use https instead of dips:: + + DIRAC + { + Setup = DeveloperSetup + Setups + { + DeveloperSetup + { + Framework = DevInstance + } + } + } + Systems + { + Framework + { + DevInstance + { + URLs + { + # DISET + #User = dips://server:9135/Framework/User + + #TORNADO + User = https://server:4444/Framework/User + } + } + } + } + +**************** +Start the server +**************** + +To start the server you must define ``OPENSSL_ALLOW_PROXY_CERTS`` and run ``DIRAC/TornadoServices/Scripts/tornado-start-all.py`` (or ``tornado-start-CS.py`` if you try to run a configuration server):: + + OPENSSL_ALLOW_PROXY_CERTS=1 python /opt/dirac/DIRAC/TornadoServices/scripts/tornado-start-all.py + + +You can now run DIRAC services. You can check the docstring of tests file (``DIRAC/test/Integration/TornadoServices`` and ``DIRAC/TornadoServices/test``) to know how to run tests. + + +********************* +Run performance tests +********************* +For performance test unset ``PYTHONOPTIMIZE`` if it is set in your environement:: + + unset PYTHONOPTIMIZE + + +Then you have to start some clients (adapt the port):: + + cd /opt/dirac/DIRAC/test/Integration/TornadoServices + multimech-run perf-test-ping -p 9000 -b 0.0.0.0 + +Modify first lines of ``DIRAC/TornadoServices/test/multi-mechanize/distributed-test.py`` and ``DIRAC/TornadoServices/test/multi-mechanize/plot-distributed-test.py`` (follow instruction of each files) + +On the server start ``DIRAC/test/Integration/TornadoServices/getCPUInfos`` (redirect output to a file) + +Run ``distributed-test.py [NameOfYourTest]`` at the end of execution, the command to plot is given. Before executing command, copy output of ``getCPUInfos`` on ``/tmp/results.txt`` (on your local machine). \ No newline at end of file diff --git a/docs/source/DeveloperGuide/index.rst b/docs/source/DeveloperGuide/index.rst index b407d2d22b9..166f9d74cc3 100644 --- a/docs/source/DeveloperGuide/index.rst +++ b/docs/source/DeveloperGuide/index.rst @@ -41,3 +41,4 @@ group. Internals/index WorkloadManagementSystem/index Externals/index + TornadoServices/index diff --git a/docs/source/DeveloperGuide/monitoring.rst b/docs/source/DeveloperGuide/monitoring.rst new file mode 100644 index 00000000000..e69de29bb2d diff --git a/requirements.txt b/requirements.txt index 871ad7363be..9accdf9daf4 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,11 @@ # From repo fts3-rest +#Patch for tornado +git+https://gitlab.com/chaen/m2crypto.git@tmpUntilSwigUpdated +git+https://github.com/chaen/tornado.git@iostreamConfigurable +git+https://github.com/chaen/tornado_m2crypto.git + # From pypi boto3 #asn1 diff --git a/tests/Integration/TornadoServices/ConfigTemplate.cfg b/tests/Integration/TornadoServices/ConfigTemplate.cfg new file mode 100644 index 00000000000..9dd8512a155 --- /dev/null +++ b/tests/Integration/TornadoServices/ConfigTemplate.cfg @@ -0,0 +1,9 @@ +# This file should replace DIRAC/FrameworkSystem/ConfigTemplate when running test in jenkins (after server installation) +# This file is read by dirac-install to install a fake service for tests +Services +{ + User + { + Port = 9000 + } +} diff --git a/tests/Integration/TornadoServices/DB/UserDB.py b/tests/Integration/TornadoServices/DB/UserDB.py new file mode 100644 index 00000000000..79844839f55 --- /dev/null +++ b/tests/Integration/TornadoServices/DB/UserDB.py @@ -0,0 +1,74 @@ +""" A test DB in DIRAC, using MySQL as backend +""" + +from DIRAC.Core.Base.DB import DB + +from DIRAC import gLogger, S_OK, S_ERROR + + +class UserDB(DB): + """ Database system for users """ + + def __init__(self): + """ + Initialize the DB + """ + + super(UserDB, self).__init__('UserDB', 'Framework/UserDB') + retVal = self.__initializeDB() + if not retVal['OK']: + raise Exception("Can't create tables: %s" % retVal['Message']) + + def __initializeDB(self): + """ + Create the tables + """ + retVal = self._query("show tables") + if not retVal['OK']: + return retVal + + tablesInDB = [t[0] for t in retVal['Value']] + tablesD = {} + + if 'user_mytable' not in tablesInDB: + tablesD['user_mytable'] = {'Fields': {'Id': 'INTEGER NOT NULL AUTO_INCREMENT', 'Name': 'VARCHAR(64) NOT NULL'}, + 'PrimaryKey': ['Id'] + } + + return self._createTables(tablesD) + + def addUser(self, userName): + """ + Add a user + :param str userName: The name of the user we want to add + :return: S_OK or S_ERROR + """ + gLogger.verbose("Insert %s in DB" % userName) + return self.insertFields('user_mytable', ['Name'], [userName]) + + def editUser(self, uid, value): + """ + Edit a user + :param int uid: The Id of the user in database + :param str value: New user name + :return: S_OK or S_ERROR + """ + return self.updateFields('user_mytable', updateDict={'Name': value}, condDict={'Id': uid}) + + def getUserName(self, uid): + """ + Get a user + :param int uid: The Id of the user in database + :return: S_OK with S_OK['Value'] = TheUserName or S_ERROR if not found + """ + user = self.getFields('user_mytable', condDict={'Id': uid}) + if len(user['Value']) == 1: + return S_OK(user['Value'][0][1]) + return S_ERROR('USER NOT FOUND') + + def listUsers(self): + """ + List all users + :return: S_OK with S_OK['Value'] list of [UserId, UserName] + """ + return self._query('SELECT * FROM user_mytable') diff --git a/tests/Integration/TornadoServices/DB/UserDB.sql b/tests/Integration/TornadoServices/DB/UserDB.sql new file mode 100644 index 00000000000..ebb64fd28a4 --- /dev/null +++ b/tests/Integration/TornadoServices/DB/UserDB.sql @@ -0,0 +1,3 @@ +# Everything is created by the DB object _checkTable method. +USE UserDB; + diff --git a/tests/Integration/TornadoServices/Services/DummyTornadoHandler.py b/tests/Integration/TornadoServices/Services/DummyTornadoHandler.py new file mode 100644 index 00000000000..bf726c5a784 --- /dev/null +++ b/tests/Integration/TornadoServices/Services/DummyTornadoHandler.py @@ -0,0 +1,22 @@ +""" + This handler is only here to test if Tornado server starts + It can be used for some tests (like performance test on ping) + This file must be copied in FrameworkSystem/Service to run tests +""" + + +from DIRAC.Core.Tornado.Server.TornadoService import TornadoService +from DIRAC import S_OK, S_ERROR + + +class DummyTornadoHandler(TornadoService): + + auth_true = ['all'] + + def export_true(self): + return S_OK() + + auth_false = ['all'] + + def export_false(self): + return S_ERROR() diff --git a/tests/Integration/TornadoServices/Services/UserDiracHandler.py b/tests/Integration/TornadoServices/Services/UserDiracHandler.py new file mode 100644 index 00000000000..41e09a7ee7b --- /dev/null +++ b/tests/Integration/TornadoServices/Services/UserDiracHandler.py @@ -0,0 +1,68 @@ +""" Dummy Service is a service for testing new dirac protocol + This file must be copied in FrameworkSystem/Service to run tests +""" + +__RCSID__ = "$Id$" + +from DIRAC import S_OK +from DIRAC.Core.DISET.RequestHandler import RequestHandler +# You need to copy ../DB/UserDB in DIRAC/FrameworkSystem/DB +from DIRAC.FrameworkSystem.DB.UserDB import UserDB # pylint: disable=no-name-in-module, import-error +from DIRAC import gConfig + + +class UserDiracHandler(RequestHandler): + """ + A handler designed for testing Tornado by implementing a basic access to database + Designed to compare Diset and Tornado + """ + + @classmethod + def initializeHandler(cls, serviceInfo): + """ Handler initialization + """ + cls.userDB = UserDB() + return S_OK() + + auth_addUser = ['all'] + types_addUser = [basestring] + + def export_addUser(self, whom): + """ Add a user and return user id + """ + newUser = self.userDB.addUser(whom) + if newUser['OK']: + return S_OK(newUser['lastRowId']) + return newUser + + auth_editUser = ['all'] + types_editUser = [int, basestring] + + def export_editUser(self, uid, value): + """ Edit a user """ + return self.userDB.editUser(uid, value) + + auth_getUserName = ['all'] + types_getUserName = [int] + + def export_getUserName(self, uid): + """ Get a user """ + return self.userDB.getUserName(uid) + + auth_listUsers = ['all'] + types_listUsers = [] + + def export_listUsers(self): + return self.userDB.listUsers() + + auth_unauthorized = ['nobody'] + types_unauthorized = [] + + def export_unauthorized(self): + return S_OK() + + auth_getTestValue = ['all'] + types_getTestValue = [] + + def export_getTestValue(self): + return S_OK(gConfig.getValue('/DIRAC/Configuration/TestUpdateValue')) diff --git a/tests/Integration/TornadoServices/Services/UserHandler.py b/tests/Integration/TornadoServices/Services/UserHandler.py new file mode 100644 index 00000000000..2cec09c7913 --- /dev/null +++ b/tests/Integration/TornadoServices/Services/UserHandler.py @@ -0,0 +1,78 @@ +""" Dummy Service is a service for testing new dirac protocol + This file must be copied in FrameworkSystem/Service to run tests +""" + +from DIRAC.Core.Tornado.Server.TornadoService import TornadoService +from DIRAC import S_OK, gLogger +# You need to copy ../DB/UserDB in DIRAC/FrameworkSystem/DB +from DIRAC.FrameworkSystem.DB.UserDB import UserDB # pylint: disable=no-name-in-module, import-error +from DIRAC import gConfig + + +class UserHandler(TornadoService): + + """ + A handler designed for testing Tornado by implementing a basic access to database + """ + + @classmethod + def initializeHandler(cls, serviceInfoDict): + cls.userDB = UserDB() + return S_OK() + + auth_addUser = ['all'] + + def export_addUser(self, whom): + """ + Add a user + + :param str whom: The name of the user we want to add + :return: S_OK with S_OK['Value'] = The_ID_of_the_user or S_ERROR + """ + newUser = self.userDB.addUser(whom) + if newUser['OK']: + return S_OK(newUser['lastRowId']) + return newUser + + auth_editUser = ['all'] + + def export_editUser(self, uid, value): + """ + Edit a user + + :param int uid: The Id of the user in database + :param str value: New user name + :return: S_OK or S_ERROR + """ + return self.userDB.editUser(uid, value) + + auth_getUserName = ['all'] + + def export_getUserName(self, uid): + """ + Get a user + + :param int uid: The Id of the user in database + :return: S_OK with S_OK['Value'] = TheUserName or S_ERROR if not found + """ + return self.userDB.getUserName(uid) + + auth_listUsers = ['all'] + + def export_listUsers(self): + """ + List all users + + :return: S_OK with S_OK['Value'] list of [UserId, UserName] + """ + return self.userDB.listUsers() + + auth_unauthorized = ['nobody'] + + def export_unauthorized(self): + return S_OK() + + auth_getValue = ['all'] + + def export_getTestValue(self): + return S_OK(gConfig.getValue('/DIRAC/Configuration/TestUpdateValue')) diff --git a/tests/Integration/TornadoServices/Test_TornadoAndDISETmixed.py b/tests/Integration/TornadoServices/Test_TornadoAndDISETmixed.py new file mode 100644 index 00000000000..d19d7552489 --- /dev/null +++ b/tests/Integration/TornadoServices/Test_TornadoAndDISETmixed.py @@ -0,0 +1,190 @@ +""" + (Case 1) This is a integration test with a Service under Tornado(tornado-start-all) + AND a configuration server under DISET (dirac-service) + + (Case 2) We run the same test with Service under DISET and CS under Tornado, + use tornado-start-CS to start the CS. + + [tornado-start-all and tornado-start-CS can be found at DIRAC/TornadoServices/scripts] + + + We test with + - for the CS: Configuration/Server (handlerPath define path to load ConfigurationHandler.py, + or TornadoConfigurationHandler.py in the second case) + - for the service: Framework/User (for the second case handlerPath should define + path to UserDiracHandler.py, see example bellow) + + The CS, service and client (this test) must read different dirac.cfg files and are + not necessary on the same computer + + In the config file distributed by the CS you should see something like:: + ``` + + Systems{ + Framework + { + DevInstance + { + Services + { + URLs + { + #User = https://MrBoincHost/Framework/User # Case 1 + User = dips://MrBoincHost/Framework/User # Case 2 + } + #for case 1 + #User + #{ + # Protocol = https + #} + #for case 2 + User + { + HandlerPath = DIRAC/FrameworkSystem/Service/UserDiracHandler.py + Port = 1234 + Protocol = dips # Not necessary defined, it's the default value, but good for human reading + } + } + + } + } + } + Registry {...} + ``` + For the service you need to define the database (case 1 and 2) and Tornado (case 1), + so in the config file loaded by service you should see:: + ``` + DIRAC + { + Setup = DeveloperSetup + Configuration + { + Servers = dips://localhost:9135/Configuration/Server # Case 1 + #Servers = https://localhost:444/Configuration/Server # Case 2 + } + } + Systems { + # Only in case 1 (Service run under tornado) + # Tornado{ + # DevInstance + # { + # # 443 is already the default, next line is technically useless + # Port = 443 + # } + # } + Framework + { + DevInstance + { + Databases + { + UserDB + { + Host = localhost + User = root + Password = + DBName = dirac + } + } + } + } + } + ``` + + The client should have a very minimal config file, and read only this file:: + ``` + DIRAC + { + Setup = DeveloperSetup + Configuration + { + Servers = dips://localhost:9135/Configuration/Server + #Servers = https://localhost:444/Configuration/Server + } + } + ``` + + + This test can be very long depending on refresh time (5minutes by default) + You can set following values in configuration (before starting service/CS) + They define refresh time in seconds, 300 by default (600 for SlavesGraceTime) + - DIRAC/Configuration/RefreshTime + - DIRAC/Configuration/PropagationTime + - DIRAC/Configuration/SlavesGraceTime +""" + +from DIRAC.Core.Base import Script +Script.parseCommandLine() + +import time + +from string import printable + +from hypothesis import given, settings, unlimited +from hypothesis.strategies import text + +from DIRAC.Core.Tornado.Client.RPCClientSelector import RPCClientSelector as RPCClient +from DIRAC.Core.Utilities.DErrno import ENOAUTH +from DIRAC import S_ERROR +from DIRAC.ConfigurationSystem.Client.CSAPI import CSAPI +from DIRAC.ConfigurationSystem.Client.ConfigurationData import gConfigurationData + + +from pytest import mark +parametrize = mark.parametrize + + +def test_authorization(): + service = RPCClient("Framework/User") + + authorisation = service.unauthorized() # In the handler this method have no allowed properties + assert authorisation['OK'] is False + assert authorisation['Message'] == S_ERROR(ENOAUTH, "Unauthorized query")['Message'] + + +def test_unknown_method(): + service = RPCClient("Framework/User") + + unknownmethod = service.ThisMethodMayNotExist() + assert unknownmethod['OK'] is False + assert unknownmethod['Message'] == "Unknown method ThisMethodMayNotExist" + + +def test_ping(): + service = RPCClient("Framework/User") + + assert service.ping()['OK'] + + +@settings(deadline=None, max_examples=4) +@given(data=text(printable, max_size=64)) +def test_echo(data): + service = RPCClient("Framework/User") + + assert service.echo(data)['Value'] == data + + +@settings(deadline=None, max_examples=1, timeout=unlimited) +@given(value1=text(printable, max_size=64), value2=text(printable, max_size=64)) +def test_configurationAutoUpdate(value1, value2): + """ + Test if service refresh his configuration. It sent a random value to the CS + and check if Service can return it. + + """ + csapi = CSAPI() + + # SETTING FIRST VALUE + csapi.modifyValue("/DIRAC/Configuration/TestUpdateValue", value1) + csapi.commitChanges() + + # Wait for automatic refresh (+1 to be sure that request is done) + time.sleep(gConfigurationData.getPropagationTime() + 1) + RPCClient("Framework/User").getTestValue() + assert RPCClient("Framework/User").getTestValue()['Value'] == value1 + + # SETTING SECOND VALUE + csapi.modifyValue("/DIRAC/Configuration/TestUpdateValue", value2) + csapi.commitChanges() + time.sleep(gConfigurationData.getPropagationTime() + 1) + assert RPCClient("Framework/User").getTestValue()['Value'] == value2 diff --git a/tests/Integration/TornadoServices/getCPUInfos b/tests/Integration/TornadoServices/getCPUInfos new file mode 100755 index 00000000000..177613ee8cb --- /dev/null +++ b/tests/Integration/TornadoServices/getCPUInfos @@ -0,0 +1,37 @@ +#!/bin/bash + +## This script write infos about server, you may redirect the output into a file then read it with plot-DistributedTest.py +## It takes output from system commands, replace space/tab by ; +## (In fact it transform output in CSV) + + +# The output contains: +# epoch; vamstat -a [17 values]; /proc/loadavg [3 values] +# +# Example: +# 1533628489;0;0;0;1549064;1109052;939168;0;0;18;2;28;62;0;0;100;0;0;0.00;0.00;0.00 +# +# $ vmstat -a +# procs -----------memory-------------- ---swap-- -----io---- --system-- -----cpu----- +# r b swpd free inact active si so bi bo in cs us sy id wa st +# 0 0 0 1549064 1109052 939168 0 0 18 2 28 62 0 0 100 0 0 <-- we keep these infos +# +# $ cat /proc/loadavg +# 0.00 0.00 0.00 1/369 11332 + + +# ---returned--- --ignored-- +vmstat -a -n 1 | while read line; +do + # If line is not a header + if [ "$(echo $line | grep -c 0)" -ne "0" ]; + then + # print timestamp + echo -n "$(date +%s)"; + # print vmstat info + echo -n ";$line" | sed -E 's/ +/;/g'; + # add load info + echo ";$(cat /proc/loadavg | awk {'print $1,$2,$3'}|tr ' ' ';')"; + + fi; +done \ No newline at end of file diff --git a/tests/Integration/TornadoServices/perf-test-DB/config.cfg b/tests/Integration/TornadoServices/perf-test-DB/config.cfg new file mode 100644 index 00000000000..768fb51aa11 --- /dev/null +++ b/tests/Integration/TornadoServices/perf-test-DB/config.cfg @@ -0,0 +1,19 @@ + +[global] +run_time = 4 +rampup = 2 +results_ts_interval = 1 +progress_bar = on +console_logging = off +xml_report = off + + +[user_group-1] +threads = 60 +script = v_perf.py + +#[user_group-2] +#threads = 60 +#script = v_perf2.py + + diff --git a/tests/Integration/TornadoServices/perf-test-DB/test_scripts/v_perf.py b/tests/Integration/TornadoServices/perf-test-DB/test_scripts/v_perf.py new file mode 100644 index 00000000000..589d0d72a78 --- /dev/null +++ b/tests/Integration/TornadoServices/perf-test-DB/test_scripts/v_perf.py @@ -0,0 +1,37 @@ +""" + This test is designed to check protocol/server performances with a database + + Configuration and how to run it are explained in the documentation +""" + +from DIRAC.Core.Tornado.Client.RPCClientSelector import RPCClientSelector +from time import time +from random import random +import sys +import os + + +class Transaction(object): + def __init__(self): + self.client = RPCClientSelector('Framework/User', timeout=30) + return + + def run(self): + # Generate random name + s = str(int(random() * 100)) + s2 = str(int(random() * 100)) + service = self.client + + # Create a user + newUser = service.addUser(s) + userID = int(newUser['Value']) + + # Check if user exist and name is correct + User = service.getUserName(userID) + assert (User['OK']), 'Error in getting user' + assert (User['Value'] == s), 'Error on insertion' + + # Check if update work + service.editUser(userID, s2) + User = service.getUserName(userID) + assert (User['Value'] == s2), 'Error on update' diff --git a/tests/Integration/TornadoServices/perf-test-ping/config.cfg b/tests/Integration/TornadoServices/perf-test-ping/config.cfg new file mode 100644 index 00000000000..e6b3b5f1513 --- /dev/null +++ b/tests/Integration/TornadoServices/perf-test-ping/config.cfg @@ -0,0 +1,19 @@ + +[global] +run_time = 300 +rampup = 200 +results_ts_interval = 1 +progress_bar = on +console_logging = off +xml_report = off + + +[user_group-1] +threads = 60 +script = v_perf.py + +#[user_group-2] +#threads = 30 +#script = v_perf.py Framework/User2 + + diff --git a/tests/Integration/TornadoServices/perf-test-ping/test_scripts/v_perf.py b/tests/Integration/TornadoServices/perf-test-ping/test_scripts/v_perf.py new file mode 100644 index 00000000000..ca20640831f --- /dev/null +++ b/tests/Integration/TornadoServices/perf-test-ping/test_scripts/v_perf.py @@ -0,0 +1,24 @@ +""" + This test is designed to check protocol/server performances + + Configuration and how to run it are explained in the documentation +""" + +from DIRAC.Core.Tornado.Client.RPCClientSelector import RPCClientSelector +from time import time +from random import randint +import sys +import os + + +class Transaction(object): + def __init__(self): + # If we want we can force to use another service name (testing multiple services for example) + if len(sys.argv) > 2: + self.client = RPCClientSelector(sys.argv[2]) + else: + self.client = RPCClientSelector('Framework/User') + return + + def run(self): + assert (self.client.ping()['OK']), 'error' From dab73e88c201d8e429aa72c5609639261b7c8877 Mon Sep 17 00:00:00 2001 From: Christophe Haen Date: Thu, 4 Jun 2020 18:03:15 +0200 Subject: [PATCH 02/25] Adapt Louis' changes and 4241 --- .../private/ServiceInterface.py | 26 +++++- .../private/ServiceInterfaceBase.py | 93 ++++++++----------- .../private/ServiceInterfaceTornado.py | 15 ++- Core/Base/Client.py | 1 + .../Client/private/TornadoBaseClient.py | 10 +- Core/Tornado/Server/TornadoService.py | 2 +- FrameworkSystem/Client/MonitoringClient.py | 12 ++- environment.yml | 4 +- 8 files changed, 94 insertions(+), 69 deletions(-) diff --git a/ConfigurationSystem/private/ServiceInterface.py b/ConfigurationSystem/private/ServiceInterface.py index 39818dcb6bb..54b41caa629 100755 --- a/ConfigurationSystem/private/ServiceInterface.py +++ b/ConfigurationSystem/private/ServiceInterface.py @@ -6,10 +6,11 @@ import time import threading -from DIRAC import gLogger +from DIRAC import gLogger, S_OK from DIRAC.ConfigurationSystem.private.ServiceInterfaceBase import ServiceInterfaceBase from DIRAC.ConfigurationSystem.Client.ConfigurationData import gConfigurationData +from DIRAC.Core.Utilities.ThreadPool import ThreadPool __RCSID__ = "$Id$" @@ -23,9 +24,8 @@ class ServiceInterface(ServiceInterfaceBase, threading.Thread): def __init__(self, sURL): threading.Thread.__init__(self) ServiceInterfaceBase.__init__(self, sURL) - self.__launchCheckSlaves() - def __launchCheckSlaves(self): + def _launchCheckSlaves(self): """ Start loop which check if slaves are alive """ @@ -39,3 +39,23 @@ def run(self): time.sleep(iWaitTime) self._checkSlavesStatus() + def _updateServiceConfiguration(self, urlSet, fromMaster=False): + """ + Update configuration in a set of service in parallel + + :param set urlSet: a set of service URLs + :param fromMaster: flag to force updating from the master CS + :return: S_OK/S_ERROR, Value Successful/Failed dict with service URLs + """ + pool = ThreadPool(len(urlSet)) + for url in urlSet: + pool.generateJobAndQueueIt(self._forceServiceUpdate, + args=[url, fromMaster], + kwargs={}, + oCallback=self.__processResults) + pool.processAllResults() + return S_OK() + + def __processResults(self, _id, result): + if not result['OK']: + gLogger.warn("Failed to update configuration on", result['URL'] + ':' + result['Message']) diff --git a/ConfigurationSystem/private/ServiceInterfaceBase.py b/ConfigurationSystem/private/ServiceInterfaceBase.py index 45a85334ac1..beee793f7ce 100644 --- a/ConfigurationSystem/private/ServiceInterfaceBase.py +++ b/ConfigurationSystem/private/ServiceInterfaceBase.py @@ -1,31 +1,27 @@ -""" Threaded implementation of services -""" +"""Service interface is the service which provide config for client and synchronize Master/Slave servers""" __RCSID__ = "$Id$" import os import time import re -import threading import zipfile import zlib -import requests import DIRAC from DIRAC.Core.Utilities.File import mkDir +from DIRAC.ConfigurationSystem.Client.ConfigurationClient import ConfigurationClient from DIRAC.ConfigurationSystem.Client.ConfigurationData import gConfigurationData, ConfigurationData from DIRAC.ConfigurationSystem.private.Refresher import gRefresher from DIRAC.FrameworkSystem.Client.Logger import gLogger from DIRAC.Core.Utilities.ReturnValues import S_OK, S_ERROR -from DIRAC.Core.DISET.RPCClient import RPCClient -from DIRAC.Core.Utilities.ThreadPool import ThreadPool -from DIRAC.Core.Security.Locations import getHostCertificateAndKeyLocation +from DIRAC.Core.Base.Client import Client -class ServiceInterface(threading.Thread): +class ServiceInterfaceBase(object): + """Service interface is the service which provide config for client and synchronize Master/Slave servers""" def __init__(self, sURL): - threading.Thread.__init__(self) self.sURL = sURL gLogger.info("Initializing Configuration Service", "URL is %s" % sURL) self.__modificationsIgnoreMask = ['/DIRAC/Configuration/Servers', '/DIRAC/Configuration/Version'] @@ -38,19 +34,19 @@ def __init__(self, sURL): gRefresher.disable() self.__loadConfigurationData() self.dAliveSlaveServers = {} - self.__launchCheckSlaves() - - self.__updateResultDict = {"Successful": {}, "Failed": {}} + self._launchCheckSlaves() def isMaster(self): return gConfigurationData.isMaster() - def __launchCheckSlaves(self): - gLogger.info("Starting purge slaves thread") - self.setDaemon(1) - self.start() + def _launchCheckSlaves(self): + raise NotImplementedError("Should be implemented by the children class") def __loadConfigurationData(self): + """ + As the name said, it just load the configuration + """ + mkDir(os.path.join(DIRAC.rootPath, "etc", "csbackup")) gConfigurationData.loadConfigurationData() if gConfigurationData.isMaster(): @@ -74,15 +70,24 @@ def __loadConfigurationData(self): gConfigurationData.writeRemoteConfigurationToDisk() def __generateNewVersion(self): + """ + After changing configuration, we use this method to save them + """ if gConfigurationData.isMaster(): gConfigurationData.generateNewVersion() gConfigurationData.writeRemoteConfigurationToDisk() def publishSlaveServer(self, sSlaveURL): + """ + Called by the slave server via service, it register a new slave server + + :param sSlaveURL: url of slave server + """ + if not gConfigurationData.isMaster(): return S_ERROR("Configuration modification is not allowed in this server") gLogger.info("Pinging slave %s" % sSlaveURL) - rpcClient = RPCClient(sSlaveURL, timeout=10, useCertificates=True) + rpcClient = ConfigurationClient(url=sSlaveURL, timeout=10, useCertificates=True) retVal = rpcClient.ping() if not retVal['OK']: gLogger.info("Slave %s didn't reply" % sSlaveURL) @@ -100,7 +105,13 @@ def publishSlaveServer(self, sSlaveURL): ", ".join(self.dAliveSlaveServers.keys()))) self.__generateNewVersion() - def __checkSlavesStatus(self, forceWriteConfiguration=False): + def _checkSlavesStatus(self, forceWriteConfiguration=False): + """ + Check if Slaves server are still availlable + + :param forceWriteConfiguration=False: Force rewriting configuration after checking slaves + """ + gLogger.info("Checking status of slave servers") iGraceTime = gConfigurationData.getSlavesGraceTime() bModifiedSlaveServers = False @@ -115,38 +126,22 @@ def __checkSlavesStatus(self, forceWriteConfiguration=False): self.__generateNewVersion() @staticmethod - def __forceServiceUpdate(url, fromMaster): + def _forceServiceUpdate(url, fromMaster): """ Force updating configuration on a given service + This should be called by _updateServiceConfiguration :param str url: service URL :param bool fromMaster: flag to force updating from the master CS :return: S_OK/S_ERROR """ gLogger.info('Updating service configuration on', url) - if url.startswith('dip'): - rpc = RPCClient(url) - result = rpc.refreshConfiguration(fromMaster) - elif url.startswith('http'): - hostCertTuple = getHostCertificateAndKeyLocation() - resultRequest = requests.get(url, - headers={'X-RefreshConfiguration': "True"}, - cert=hostCertTuple, - verify=False) - result = S_OK() - if resultRequest.status_code != 200: - result = S_ERROR("Status code returned %d" % resultRequest.status_code) + + result = Client(url=url).refreshConfiguration(fromMaster) result['URL'] = url return result - def __processResults(self, id_, result): - if result['OK']: - self.__updateResultDict['Successful'][result['URL']] = True - else: - gLogger.warn("Failed to update configuration on", result['URL'] + ':' + result['Message']) - self.__updateResultDict['Failed'][result['URL']] = result['Message'] - - def __updateServiceConfiguration(self, urlSet, fromMaster=False): + def _updateServiceConfiguration(self, urlSet, fromMaster=False): """ Update configuration in a set of service in parallel @@ -154,14 +149,7 @@ def __updateServiceConfiguration(self, urlSet, fromMaster=False): :param fromMaster: flag to force updating from the master CS :return: S_OK/S_ERROR, Value Successful/Failed dict with service URLs """ - pool = ThreadPool(len(urlSet)) - for url in urlSet: - pool.generateJobAndQueueIt(self.__forceServiceUpdate, - args=[url, fromMaster], - kwargs={}, - oCallback=self.__processResults) - pool.processAllResults() - return S_OK(self.__updateResultDict) + raise NotImplementedError("Should be implemented by the children class") def forceSlavesUpdate(self): """ @@ -171,12 +159,11 @@ def forceSlavesUpdate(self): """ gLogger.info("Updating configuration on slave servers") iGraceTime = gConfigurationData.getSlavesGraceTime() - self.__updateResultDict = {"Successful": {}, "Failed": {}} urlSet = set() for slaveURL in self.dAliveSlaveServers: if time.time() - self.dAliveSlaveServers[slaveURL] <= iGraceTime: urlSet.add(slaveURL) - return self.__updateServiceConfiguration(urlSet, fromMaster=True) + return self._updateServiceConfiguration(urlSet, fromMaster=True) def forceGlobalUpdate(self): """ @@ -193,7 +180,7 @@ def forceGlobalUpdate(self): for url in cfg[system_][instance]['URLs']: urlSet = urlSet.union(set([u.strip() for u in cfg[system_][instance]['URLs'][url].split(',') if 'Configuration/Server' not in u])) - return self.__updateServiceConfiguration(urlSet) + return self._updateServiceConfiguration(urlSet) def updateConfiguration(self, sBuffer, committer="", updateVersionOption=False): """ @@ -266,12 +253,6 @@ def getCommitHistory(self): backups = [".".join(fileName.split(".")[1:-1]).split("@") for fileName in files] return backups - def run(self): - while True: - iWaitTime = gConfigurationData.getSlavesGraceTime() - time.sleep(iWaitTime) - self.__checkSlavesStatus() - def getVersionContents(self, date): backupDir = gConfigurationData.getBackupDir() files = self.__getCfgBackups(backupDir, date) diff --git a/ConfigurationSystem/private/ServiceInterfaceTornado.py b/ConfigurationSystem/private/ServiceInterfaceTornado.py index fe3650ee938..e1b845df0ee 100644 --- a/ConfigurationSystem/private/ServiceInterfaceTornado.py +++ b/ConfigurationSystem/private/ServiceInterfaceTornado.py @@ -15,9 +15,8 @@ class ServiceInterfaceTornado(ServiceInterfaceBase): def __init__(self, sURL): ServiceInterfaceBase.__init__(self, sURL) - self.__launchCheckSlaves() - def __launchCheckSlaves(self): + def _launchCheckSlaves(self): """ Start loop to check if slaves are alive """ @@ -31,3 +30,15 @@ def run(self): while True: yield gen.sleep(gConfigurationData.getSlavesGraceTime()) self._checkSlavesStatus() + + def _updateServiceConfiguration(self, urlSet, fromMaster=False): + """ + Update configuration in a set of service in parallel + + :param set urlSet: a set of service URLs + :param fromMaster: flag to force updating from the master CS + :return: S_OK/S_ERROR, Value Successful/Failed dict with service URLs + """ + + for url in urlSet: + IOLoop.current().spawn_callback(self._forceServiceUpdate, [url, fromMaster]) diff --git a/Core/Base/Client.py b/Core/Base/Client.py index faff844f2ef..237280d63a0 100644 --- a/Core/Base/Client.py +++ b/Core/Base/Client.py @@ -17,6 +17,7 @@ from DIRAC.Core.DISET import DEFAULT_RPC_TIMEOUT + class Client(object): """ Simple class to redirect unknown actions directly to the server. Arguments to the constructor are passed to the RPCClient constructor as they are. diff --git a/Core/Tornado/Client/private/TornadoBaseClient.py b/Core/Tornado/Client/private/TornadoBaseClient.py index ffe6de558b6..2b4bdfbb04a 100644 --- a/Core/Tornado/Client/private/TornadoBaseClient.py +++ b/Core/Tornado/Client/private/TornadoBaseClient.py @@ -26,9 +26,11 @@ from DIRAC.ConfigurationSystem.Client.Config import gConfig from DIRAC.Core.Utilities import List, Network from DIRAC import S_OK, S_ERROR, gLogger +from DIRAC.ConfigurationSystem.Client.Helpers.CSGlobals import skipCACheck +from DIRAC.ConfigurationSystem.Client.Helpers.Registry import findDefaultGroupForDN from DIRAC.ConfigurationSystem.Client.PathFinder import getServiceURL, getServiceFailoverURL from DIRAC.Core.Utilities.JEncode import decode, encode -from DIRAC.Core.Security import CS + from DIRAC.Core.Security import Locations from DIRAC.Core.DISET.ThreadConfig import ThreadConfig @@ -184,7 +186,7 @@ def __discoverCredentialsToUse(self): * Certification Authorities check: -> if KW_SKIP_CA_CHECK is not in kwargs and we are using the certificates, set KW_SKIP_CA_CHECK to false in kwargs - -> if KW_SKIP_CA_CHECK is not in kwargs and we are not using the certificate, check the CS.skipCACheck + -> if KW_SKIP_CA_CHECK is not in kwargs and we are not using the certificate, check the skipCACheck * Proxy Chain WARNING: MOSTLY COPY/PASTE FROM Core/Diset/private/BaseClient @@ -200,7 +202,7 @@ def __discoverCredentialsToUse(self): if self.__useCertificates: self.kwargs[self.KW_SKIP_CA_CHECK] = False else: - self.kwargs[self.KW_SKIP_CA_CHECK] = CS.skipCACheck() + self.kwargs[self.KW_SKIP_CA_CHECK] = skipCACheck() # Rewrite a little bit from here: don't need the proxy string, we use the file if self.KW_PROXY_CHAIN in self.kwargs: @@ -263,7 +265,7 @@ def __discoverExtraCredentials(self): self.kwargs[self.KW_DELEGATED_GROUP] = delegatedGroup if delegatedDN: if not delegatedGroup: - result = CS.findDefaultGroupForDN(self.kwargs[self.KW_DELEGATED_DN]) + result = findDefaultGroupForDN(self.kwargs[self.KW_DELEGATED_DN]) if not result['OK']: return result self.__extraCredentials = (delegatedDN, delegatedGroup) diff --git a/Core/Tornado/Server/TornadoService.py b/Core/Tornado/Server/TornadoService.py index 93b451e3564..cbf56f27bcd 100644 --- a/Core/Tornado/Server/TornadoService.py +++ b/Core/Tornado/Server/TornadoService.py @@ -34,7 +34,7 @@ def export_someMethod(self): import DIRAC -from DIRAC.Core.Security.X509Chain import X509Chain +from DIRAC.Core.Security.X509Chain import X509Chain # pylint: disable=import-error from DIRAC.Core.DISET.AuthManager import AuthManager from DIRAC.ConfigurationSystem.Client import PathFinder from DIRAC.Core.Utilities.JEncode import decode, encode diff --git a/FrameworkSystem/Client/MonitoringClient.py b/FrameworkSystem/Client/MonitoringClient.py index d3e3f5885fc..a16f03d8ff5 100755 --- a/FrameworkSystem/Client/MonitoringClient.py +++ b/FrameworkSystem/Client/MonitoringClient.py @@ -8,7 +8,7 @@ __RCSID__ = "$Id" import time - +import os import six import DIRAC @@ -85,7 +85,12 @@ def registerMonitoringClient(self, mc): self.__mcList.append(mc) -gMonitoringFlusher = MonitoringFlusher() +# USE_TORNADO_IOLOOP is defined by starting scripts +if os.environ.get('USE_TORNADO_IOLOOP', 'false').lower() == 'true': + from DIRAC.FrameworkSystem.Client.MonitoringClientIOLoop import MonitoringFlusherTornado + gMonitoringFlusher = MonitoringFlusherTornado() +else: + gMonitoringFlusher = MonitoringFlusher() class MonitoringClient(object): @@ -104,6 +109,7 @@ class MonitoringClient(object): COMPONENT_AGENT = "agent" COMPONENT_WEB = "web" COMPONENT_SCRIPT = "script" + COMPONENT_TORNADO = "tornado" __validMonitoringValues = six.integer_types + (float,) @@ -177,6 +183,8 @@ def initialize(self): self.setComponentName('WebApp') elif self.sourceDict['componentType'] == self.COMPONENT_SCRIPT: self.cfgSection = "/Script" + elif self.sourceDict['componentType'] == self.COMPONENT_TORNADO: + self.cfgSection = "/Tornado" else: raise Exception("Component type has not been defined") gMonitoringFlusher.registerMonitoringClient(self) diff --git a/environment.yml b/environment.yml index f95611f7df0..73dc63fbe44 100644 --- a/environment.yml +++ b/environment.yml @@ -60,9 +60,11 @@ dependencies: - multi-mechanize >=1.2.0 # - readline >=6.2.4 in the standard library - simplejson >=3.8.1 - - tornado >=5.0.0,<6.0.0 + #- tornado >=5.0.0,<6.0.0 - typing >=3.6.6 # Pin OpenSSL to avoid: https://github.com/DIRACGrid/DIRAC/issues/4489 - openssl <1.1 - pip: - diraccfg + - git+https://github.com/chaen/tornado.git@iostreamConfigurable + - git+https://github.com/chaen/tornado_m2crypto From d49132622c414ebc6183a38d43124cd955d10845 Mon Sep 17 00:00:00 2001 From: Christophe Haen Date: Tue, 23 Jun 2020 18:56:10 +0200 Subject: [PATCH 03/25] some doc fixing --- .../DeveloperGuide/TornadoServices/index.rst | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/docs/source/DeveloperGuide/TornadoServices/index.rst b/docs/source/DeveloperGuide/TornadoServices/index.rst index c06e771cb71..7ec6b27aa16 100644 --- a/docs/source/DeveloperGuide/TornadoServices/index.rst +++ b/docs/source/DeveloperGuide/TornadoServices/index.rst @@ -77,12 +77,20 @@ Based on DISET request handler, you still have access to some getters in your ha How to start server ******************* -The easy way, use ``DIRAC/TornadoService/script/tornado-start-all.py`` it will start all services registered in configuration ! To register a service you just have to add the service in the CS and ``Protocol = https``. It may look like this:: +The easy way, use ``DIRAC/Core/Tornado/script/tornado-start-all.py`` it will start all services registered in configuration ! To register a service you just have to add the service in the CS and ``Protocol = https``. It may look like this:: + + DIRAC + { + Setups + { + Tornado = DevInstance + } + } Systems { - DevInstance + Tornado { - Tornado + DevInstance { Port = 443 } From 17462d8771bb10bca1b26552df5f117022b0e8c8 Mon Sep 17 00:00:00 2001 From: Christophe Haen Date: Tue, 23 Jun 2020 18:57:08 +0200 Subject: [PATCH 04/25] TornadoService: properly load remote credentials --- Core/Tornado/Server/TornadoService.py | 29 +++++++++------------------ 1 file changed, 9 insertions(+), 20 deletions(-) diff --git a/Core/Tornado/Server/TornadoService.py b/Core/Tornado/Server/TornadoService.py index cbf56f27bcd..16fb846708f 100644 --- a/Core/Tornado/Server/TornadoService.py +++ b/Core/Tornado/Server/TornadoService.py @@ -196,7 +196,7 @@ def prepare(self): self.finish() try: - self.credDict = self.gatherPeerCredentials() + self.credDict = self._gatherPeerCredentials() except Exception: # pylint: disable=broad-except # If an error occur when reading certificates we close connection # It can be strange but the RFC, for HTTP, say's that when error happend @@ -254,6 +254,7 @@ def __executeMethod(self): self.initializeRequest() retVal = method(*args) except Exception as e: # pylint: disable=broad-except + gLogger.exception("Exception serving request", "%s:%s" % (str(e), repr(e))) retVal = S_ERROR(repr(e)) self._httpError = HTTPErrorCodes.HTTP_INTERNAL_SERVER_ERROR @@ -284,7 +285,6 @@ def reportUnauthorizedAccess(self, errorCode=401): gLogger.error( "Unauthorized access to %s: %s(%s) from %s" % (self.request.path, - self.credDict['CN'], self.credDict['DN'], self.request.remote_ip)) @@ -299,7 +299,7 @@ def on_finish(self): requestDuration = time.time() - self.requestStartTime gLogger.notice("Ending request to %s after %fs" % (self.srv_getURL(), requestDuration)) - def gatherPeerCredentials(self): + def _gatherPeerCredentials(self): """ Load client certchain in DIRAC and extract informations. @@ -320,23 +320,12 @@ def gatherPeerCredentials(self): # Following lines just get the right info, at the right place peerChain.loadChainFromString(chainAsText) - isProxyChain = peerChain.isProxy()['Value'] - isLimitedProxyChain = peerChain.isLimitedProxy()['Value'] - if isProxyChain: - if peerChain.isPUSP()['Value']: - identitySubject = peerChain.getCertInChain(-2)['Value'].getSubjectNameObject()['Value'] - else: - identitySubject = peerChain.getIssuerCert()['Value'].getSubjectNameObject()['Value'] - else: - identitySubject = peerChain.getCertInChain(0)['Value'].getSubjectNameObject()['Value'] - credDict = {'DN': identitySubject.one_line(), - 'CN': identitySubject.commonName, - 'x509Chain': peerChain, - 'isProxy': isProxyChain, - 'isLimitedProxy': isLimitedProxyChain} - diracGroup = peerChain.getDIRACGroup() - if diracGroup['OK'] and diracGroup['Value']: - credDict['group'] = diracGroup['Value'] + # Retrieve the credentials + res = peerChain.getCredentials(withRegistryInfo=False) + if not res['OK']: + raise Exception(res['Message']) + + credDict = res['Value'] # We check if client sends extra credentials... if "extraCredentials" in self.request.arguments: From 52533d6c7776bae07f353a6faa26298cb78309c0 Mon Sep 17 00:00:00 2001 From: Christophe Haen Date: Tue, 23 Jun 2020 19:01:22 +0200 Subject: [PATCH 05/25] TornadoBaseClient: better handling of error --- .../Client/private/TornadoBaseClient.py | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/Core/Tornado/Client/private/TornadoBaseClient.py b/Core/Tornado/Client/private/TornadoBaseClient.py index 2b4bdfbb04a..4c78c305deb 100644 --- a/Core/Tornado/Client/private/TornadoBaseClient.py +++ b/Core/Tornado/Client/private/TornadoBaseClient.py @@ -492,15 +492,29 @@ def _request(self, postArguments, retry=0): # Do the request try: + call = requests.post(url, data=postArguments, timeout=self.timeout, verify=verify, cert=cert) - return decode(call.text)[0] + # raising the exception for status here + # means essentialy that we are losing here the information of what is returned by the server + # as error message, since it is not passed to the exception + # However, we can store the text and return it raw as an error, + # since there is no guarantee that it is any JEncoded text + # Note that we would get an exception only if there is an exception on the server side which + # is not handled. + # Any standard S_ERROR will be transfered as an S_ERROR with a correct code. + rawText = call.text + call.raise_for_status() + return decode(rawText)[0] except Exception as e: if url not in self.__bannedUrls: self.__bannedUrls += [url] if retry < self.__nbOfUrls - 1: self._request(postArguments, retry + 1) - return S_ERROR(str(e)) + + errStr = "%s: %s" % (str(e), rawText) + return S_ERROR(errStr) + # --- TODO ---- From 483c48ecab05769178b3378548ecac3e192981f9 Mon Sep 17 00:00:00 2001 From: Christophe Haen Date: Fri, 26 Jun 2020 16:08:36 +0200 Subject: [PATCH 06/25] Add possibility to stream, and equivalent of TransferFile --- Core/Tornado/Client/TornadoClient.py | 23 ++++- Core/Tornado/Client/TransferClientSelector.py | 59 +++++++++++++ .../Client/private/TornadoBaseClient.py | 86 ++++++++++++++----- Core/Tornado/Server/HandlerManager.py | 2 +- Core/Tornado/Server/TornadoService.py | 42 ++++++++- 5 files changed, 186 insertions(+), 26 deletions(-) create mode 100644 Core/Tornado/Client/TransferClientSelector.py diff --git a/Core/Tornado/Client/TornadoClient.py b/Core/Tornado/Client/TornadoClient.py index d8078cdd66a..d04d8c1a69d 100644 --- a/Core/Tornado/Client/TornadoClient.py +++ b/Core/Tornado/Client/TornadoClient.py @@ -5,12 +5,13 @@ you can use all method defined in your service, your call will be automatically transformed in RPC. + It also exposes the same interface for receiving file than the TransferClient. + Main changes: - KeepAliveLapse is removed, requests library manage it itself. - nbOfRetry (defined as private attribute) is removed, requests library manage it hitself. - Underneath it use HTTP POST protocol and JSON - Example:: from DIRAC.Core.Tornado.Client.TornadoClient import TornadoClient @@ -19,6 +20,8 @@ """ +# pylint: disable=broad-except + from DIRAC.Core.Utilities.JEncode import encode from DIRAC.Core.Tornado.Client.private.TornadoBaseClient import TornadoBaseClient @@ -54,10 +57,26 @@ def executeRPC(self, method, *args): """ rpcCall = {'method': method, 'args': encode(args)} # Start request - retVal = self._request(rpcCall) + retVal = self._request(**rpcCall) + # Should this line bellow go ? I guess yes retVal['rpcStub'] = (self._getBaseStub(), method, args) return retVal + def receiveFile(self, destFile, *args): + """ + Equivalent of the :py:meth`~DIRAC.Core.DISET.TransferClient.TransferClient.receiveFile + + In practice, it calls the remote method `streamToClient` and stores the raw result in a file + + :param str destFile: path where to store the result + :param args: list of arguments + :returns: S_OK/S_ERROR + """ + rpcCall = {'method': 'streamToClient', 'args': encode(args), 'rawContent': True} + # Start request + retVal = self._request(stream=True, outputFile=destFile, **rpcCall) + return retVal + def executeRPCStub(rpcStub): """ diff --git a/Core/Tornado/Client/TransferClientSelector.py b/Core/Tornado/Client/TransferClientSelector.py new file mode 100644 index 00000000000..3d617d46f27 --- /dev/null +++ b/Core/Tornado/Client/TransferClientSelector.py @@ -0,0 +1,59 @@ +""" + TransferClientSelector can replace TransferClient (with import TransferClientSelector as TransferClient) + to migrate from DISET to Tornado. This method chooses and returns the client wich should be + used for a service. If the url of the service uses HTTPS, TornadoClient is returned, else it returns TransferClient + + Example:: + + from DIRAC.Core.Tornado.Client.TransferClientSelector import TransferClientSelector as TransferClient + myService = TransferClient("Framework/MyService") + myService.doSomething() +""" + + +from DIRAC.Core.Tornado.Client.TornadoClient import TornadoClient +from DIRAC.Core.DISET.TransferClient import TransferClient +from DIRAC.ConfigurationSystem.Client.PathFinder import getServiceURL +from DIRAC import gLogger + + +def isURL(url): + """ + Just a test to check if URL is already given or not + """ + return url.startswith('http') or url.startswith('dip') + + +def TransferClientSelector(*args, **kwargs): # We use same interface as TransferClient + """ + Select the correct TransferClient, instanciate it, and return it + :param args[0]: url: URL can be just "system/service" or "dips://domain:port/system/service" + """ + + # We detect if we need to use a specific class for the HTTPS client + if 'httpsClient' in kwargs: + TornadoTransClient = kwargs.pop('httpsClient') + else: + TornadoTransClient = TornadoClient + + # We have to make URL resolution BEFORE the TransferClient or TornadoClient to determine wich one we want to use + # URL is defined as first argument (called serviceName) in TransferClient + + try: + serviceName = args[0] + gLogger.verbose("Trying to autodetect client for %s" % serviceName) + if not isURL(serviceName): + completeUrl = getServiceURL(serviceName) + gLogger.verbose("URL resolved: %s" % completeUrl) + else: + completeUrl = serviceName + if completeUrl.startswith("http"): + gLogger.info("Using HTTPS for service %s" % serviceName) + transClient = TornadoTransClient(*args, **kwargs) + else: + transClient = TransferClient(*args, **kwargs) + except Exception: + # If anything went wrong in the resolution, we return default TransferClient + # So the comportement is exactly the same as before implementation of Tornado + transClient = TransferClient(*args, **kwargs) + return transClient diff --git a/Core/Tornado/Client/private/TornadoBaseClient.py b/Core/Tornado/Client/private/TornadoBaseClient.py index 4c78c305deb..4cb43318c39 100644 --- a/Core/Tornado/Client/private/TornadoBaseClient.py +++ b/Core/Tornado/Client/private/TornadoBaseClient.py @@ -20,6 +20,10 @@ """ + +# pylint: disable=broad-except + +import cStringIO import requests import DIRAC @@ -459,19 +463,34 @@ def _getBaseStub(self): del newKwargs['useCertificates'] return (self._destinationSrv, newKwargs) - def _request(self, postArguments, retry=0): + def _request(self, retry=0, stream=False, outputFile=None, **kwargs): """ Sends the request to server - :param postArguments: dictionnary with arguments who needs to be sends, - you should have "method" (str, name of distant method) - and "args" (str in JSON, list of arguments for ce procedure) + :param retry: internal parameters for recursive call. TODO: remove ? + :param stream: (default False) if True, we will receive the output in several chunks. + It is usefull when receiving big amount of data. Note that this is not optimzied + on the server side (see TornadoService) + It shows primarily useful to receive files. + :param outputFile: (default None) path to a file where to store the received data. + :param **kwargs: Any argument there is used as a post parameter. They are detailed bellow. + :param method: (mandatory) name of the distant method + :param args: (mandatory) json serialized list of argument for the procedure + :param rawContent: if set to True, the data is transmitted and returned from this method + without going through the JEncode serialization. + + + + :returns: The received data. If outputFile is set, return always S_OK + """ + rawContent = kwargs.get('rawContent') + # Adding some informations to send if self.__extraCredentials: - postArguments[self.KW_EXTRA_CREDENTIALS] = encode(self.__extraCredentials) - postArguments["clientVO"] = self.vo + kwargs[self.KW_EXTRA_CREDENTIALS] = encode(self.__extraCredentials) + kwargs["clientVO"] = self.vo # Getting URL url = self.__findServiceURL() @@ -492,31 +511,56 @@ def _request(self, postArguments, retry=0): # Do the request try: + rawText = None + + if not stream: + call = requests.post(url, data=kwargs, + timeout=self.timeout, verify=verify, + cert=cert) + # raising the exception for status here + # means essentialy that we are losing here the information of what is returned by the server + # as error message, since it is not passed to the exception + # However, we can store the text and return it raw as an error, + # since there is no guarantee that it is any JEncoded text + # Note that we would get an exception only if there is an exception on the server side which + # is not handled. + # Any standard S_ERROR will be transfered as an S_ERROR with a correct code. + rawText = call.text + call.raise_for_status() + return decode(rawText)[0] + else: + rawText = None + # Stream download + # https://requests.readthedocs.io/en/latest/user/advanced/#body-content-workflow + with requests.post(url, data=kwargs, timeout=self.timeout, verify=verify, + cert=cert, stream=True) as r: + r.raise_for_status() + contentFile = outputFile if outputFile else cStringIO.StringIO() + + with open(contentFile, 'wb') as f: + for chunk in r.iter_content(): + if chunk: # filter out keep-alive new chuncks + f.write(chunk) + + if outputFile: + return S_OK() + + if rawContent: + return S_OK(contentFile.getvalue()) + return decode(contentFile.getvalue())[0] - call = requests.post(url, data=postArguments, timeout=self.timeout, verify=verify, - cert=cert) - # raising the exception for status here - # means essentialy that we are losing here the information of what is returned by the server - # as error message, since it is not passed to the exception - # However, we can store the text and return it raw as an error, - # since there is no guarantee that it is any JEncoded text - # Note that we would get an exception only if there is an exception on the server side which - # is not handled. - # Any standard S_ERROR will be transfered as an S_ERROR with a correct code. - rawText = call.text - call.raise_for_status() - return decode(rawText)[0] except Exception as e: + # CHRIS TODO review this part. + # Is this list ever cleaned ? if url not in self.__bannedUrls: self.__bannedUrls += [url] if retry < self.__nbOfUrls - 1: - self._request(postArguments, retry + 1) + self._request(retry=retry + 1, stream=stream, outputFile=outputFile, **kwargs) errStr = "%s: %s" % (str(e), rawText) return S_ERROR(errStr) - # --- TODO ---- # Rewrite this method if needed: # /Core/DISET/private/BaseClient.py diff --git a/Core/Tornado/Server/HandlerManager.py b/Core/Tornado/Server/HandlerManager.py index a830b6cc15e..82373c6ca1a 100644 --- a/Core/Tornado/Server/HandlerManager.py +++ b/Core/Tornado/Server/HandlerManager.py @@ -112,7 +112,7 @@ def discoverHandlers(self): if isHTTPS and isHTTPS.lower() == 'https': serviceList.append(newservice) # On systems sometime you have things not related to services... - except RuntimeError as e: + except RuntimeError: pass return self.loadHandlersByServiceName(serviceList) diff --git a/Core/Tornado/Server/TornadoService.py b/Core/Tornado/Server/TornadoService.py index 16fb846708f..ce71bfa201c 100644 --- a/Core/Tornado/Server/TornadoService.py +++ b/Core/Tornado/Server/TornadoService.py @@ -186,6 +186,7 @@ def prepare(self): prepare the request, it read certificates and check authorizations. """ self.method = self.get_argument("method") + self.rawContent = self.get_argument('rawContent', default=False) self.log.notice("Incoming request on /%s: %s" % (self._serviceName, self.method)) # Init of service must be checked here, because if it have crashed we are @@ -225,9 +226,42 @@ def post(self): # pylint: disable=arguments-differ retVal = yield IOLoop.current().run_in_executor(None, self.__executeMethod) # Tornado recommend to write in main thread + # TODO CHRIS READ THAT https://www.tornadoweb.org/en/branch5.1/web.html#thread-safety-notes self.__write_return(retVal.result()) self.finish() + # This nice idea of streaming to the client cannot work because we are ran in an eecutor + # and we should not write back to the client in a different thread. + # See https://www.tornadoweb.org/en/branch5.1/web.html#thread-safety-notes + # def export_streamToClient(self, filename): + # # https://bhch.github.io/posts/2017/12/serving-large-files-with-tornado-safely-without-blocking/ + # #import ipdb; ipdb.set_trace() + # # chunk size to read + # chunk_size = 1024 * 1024 * 1 # 1 MiB + + # with open(filename, 'rb') as f: + # while True: + # chunk = f.read(chunk_size) + # if not chunk: + # break + # try: + # self.write(chunk) # write the chunk to response + # self.flush() # send the chunk to client + # except StreamClosedError: + # # this means the client has closed the connection + # # so break the loop + # break + # finally: + # # deleting the chunk is very important because + # # if many clients are downloading files at the + # # same time, the chunks in memory will keep + # # increasing and will eat up the RAM + # del chunk + # # pause the coroutine so other handlers can run + # yield gen.sleep(0.000000001) # 1 nanosecond + + # return S_OK() + @gen.coroutine def __executeMethod(self): """ @@ -272,7 +306,11 @@ def __write_return(self, dictionnary): # Write status code before writing, by default error code is "200 OK" self.set_status(self._httpError) - self.write(encode(dictionnary)) + + # See 4.5.1 http://www.rfc-editor.org/rfc/rfc2046.txt + self.set_header("Content-Type", "application/octet-stream") + returnedData = dictionnary if self.rawContent else encode(dictionnary) + self.write(returnedData) def reportUnauthorizedAccess(self, errorCode=401): """ @@ -283,7 +321,7 @@ def reportUnauthorizedAccess(self, errorCode=401): """ error = S_ERROR(ENOAUTH, "Unauthorized query") gLogger.error( - "Unauthorized access to %s: %s(%s) from %s" % + "Unauthorized access to %s: %s from %s" % (self.request.path, self.credDict['DN'], self.request.remote_ip)) From 63ff4eed9fdcf68e2198b57dd42a7b0c8abd601c Mon Sep 17 00:00:00 2001 From: Christophe Haen Date: Mon, 8 Jun 2020 17:40:14 +0200 Subject: [PATCH 07/25] Tests: ignore test directory in finding DBs --- tests/Jenkins/utilities.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/Jenkins/utilities.sh b/tests/Jenkins/utilities.sh index 9ebe6316e76..82c603962e8 100644 --- a/tests/Jenkins/utilities.sh +++ b/tests/Jenkins/utilities.sh @@ -178,11 +178,12 @@ findDatabases() { # # We are avoiding, FileCatalogDB FileCatalogWithFkAndPsDB that is installed in other ways # and InstalledComponentsDB which is installed at the beginning + # We also ignore all the DBs in tests directory # if [[ -n "${DBstoExclude}" ]]; then - find ./*DIRAC/ -name "*DB.sql" | grep -vE '(FileCatalogDB|FileCatalogWithFkAndPsDB|InstalledComponentsDB)' | awk -F "/" '{print $3,$5}' | grep -v "${DBstoExclude}" | grep -v 'DIRAC' | sort -u > databases + find ./*DIRAC/ -path "*/tests/*" -prune -o -name "*DB.sql" -print | grep -vE '(FileCatalogDB|FileCatalogWithFkAndPsDB|InstalledComponentsDB)' | awk -F "/" '{print $3,$5}' | grep -v "${DBstoExclude}" | grep -v 'DIRAC' | sort | uniq > databases else - find ./*DIRAC/ -name "*DB.sql" | grep -vE '(FileCatalogDB|FileCatalogWithFkAndPsDB|InstalledComponentsDB)' | awk -F "/" '{print $3,$5}' | grep "${DBstoSearch}" | grep -v 'DIRAC' | sort -u > databases + find ./*DIRAC/ -path "*/tests/*" -prune -o -name "*DB.sql" -print | grep -vE '(FileCatalogDB|FileCatalogWithFkAndPsDB|InstalledComponentsDB)' | awk -F "/" '{print $3,$5}' | grep "${DBstoSearch}" | grep -v 'DIRAC' | sort | uniq > databases fi echo "found $(wc -l databases)" From 8f96687c03afd5f156d5b728757a883e7f576dca Mon Sep 17 00:00:00 2001 From: Christophe Haen Date: Tue, 23 Jun 2020 18:54:44 +0200 Subject: [PATCH 08/25] X509Chain.getCredentials returns a DN entry --- Core/Security/m2crypto/X509Chain.py | 7 +++++++ Core/Security/test/Test_X509Chain.py | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/Core/Security/m2crypto/X509Chain.py b/Core/Security/m2crypto/X509Chain.py index fa08118dc0e..1140db59178 100644 --- a/Core/Security/m2crypto/X509Chain.py +++ b/Core/Security/m2crypto/X509Chain.py @@ -964,6 +964,7 @@ def getCredentials(self, ignoreDefault=False, withRegistryInfo=True): * isLimitedProxy: boolean (see :py:meth:`.isLimitedProxy`) * validDN: boolean if the DN is known to DIRAC * validGroup: False (see further definition) + * DN: either the DN of the host, or the DN of the user corresponding to the proxy Only for proxy: @@ -994,8 +995,14 @@ def getCredentials(self, ignoreDefault=False, withRegistryInfo=True): 'isLimitedProxy': self.__isProxy and self.__isLimitedProxy, 'validDN': False, 'validGroup': False} + + # Add the DN entry as the subject. + credDict['DN'] = credDict['subject'] if self.__isProxy: credDict['identity'] = str(self._certList[self.__firstProxyStep + 1].getSubjectDN()['Value']) # ['Value'] :( + # if the chain is a proxy, then the DN we want to work with is the real one of the + # user, not the one of his proxy + credDict['DN'] = credDict['identity'] # Check if we have the PUSP case result = self.isPUSP() diff --git a/Core/Security/test/Test_X509Chain.py b/Core/Security/test/Test_X509Chain.py index 4b29e4f0fb3..e887985064b 100644 --- a/Core/Security/test/Test_X509Chain.py +++ b/Core/Security/test/Test_X509Chain.py @@ -360,7 +360,7 @@ def test_getCredentials_on_cert(cert_file, get_X509Chain_class): x509Chain = get_X509Chain_class() x509Chain.loadChainFromFile(cert_file) - credentialInfo = ['isLimitedProxy', 'isProxy', 'issuer', 'secondsLeft', 'subject', 'validDN', 'validGroup'] + credentialInfo = ['DN', 'isLimitedProxy', 'isProxy', 'issuer', 'secondsLeft', 'subject', 'validDN', 'validGroup'] res = x509Chain.getCredentials(ignoreDefault=True) From d021d5d9fbc6b1a13b98a4638779484c49637150 Mon Sep 17 00:00:00 2001 From: Christophe Haen Date: Tue, 23 Jun 2020 18:55:28 +0200 Subject: [PATCH 09/25] Tests: ignore Tornado services when doing installation in CI --- tests/Jenkins/utilities.sh | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/Jenkins/utilities.sh b/tests/Jenkins/utilities.sh index 82c603962e8..432d49bd140 100644 --- a/tests/Jenkins/utilities.sh +++ b/tests/Jenkins/utilities.sh @@ -752,7 +752,8 @@ diracAddSite() { diracServices(){ echo '==> [diracServices]' - local services=$(cut -d '.' -f 1 < services | grep -v PilotsLogging | grep -v StorageElementHandler | grep -v ^ConfigurationSystem | grep -v Plotting | grep -v RAWIntegrity | grep -v RunDBInterface | grep -v ComponentMonitoring | sed 's/System / /g' | sed 's/Handler//g' | sed 's/ /\//g') + # Ignore tornado services + local services=$(cut -d '.' -f 1 < services | grep -v Tornado | grep -v PilotsLogging | grep -v StorageElementHandler | grep -v ^ConfigurationSystem | grep -v Plotting | grep -v RAWIntegrity | grep -v RunDBInterface | grep -v ComponentMonitoring | sed 's/System / /g' | sed 's/Handler//g' | sed 's/ /\//g') # group proxy, will be uploaded explicitly # echo '==> getting/uploading proxy for prod' @@ -802,7 +803,8 @@ diracUninstallServices(){ findServices - local services=$(cut -d '.' -f 1 getting/uploading proxy for prod' From 30fd64953bf0283c575c27f460fd3fe5163dfbe0 Mon Sep 17 00:00:00 2001 From: Christophe Haen Date: Thu, 9 Jul 2020 14:21:28 +0200 Subject: [PATCH 10/25] Component installer can install Tornado services --- FrameworkSystem/Client/ComponentInstaller.py | 88 +++++++++++++++ .../scripts/dirac-install-tornado-service.py | 103 ++++++++++++++++++ 2 files changed, 191 insertions(+) create mode 100755 FrameworkSystem/scripts/dirac-install-tornado-service.py diff --git a/FrameworkSystem/Client/ComponentInstaller.py b/FrameworkSystem/Client/ComponentInstaller.py index f6e1a195354..e861fced0db 100644 --- a/FrameworkSystem/Client/ComponentInstaller.py +++ b/FrameworkSystem/Client/ComponentInstaller.py @@ -2507,5 +2507,93 @@ def execCommand(self, timeout, cmd): return result + def installTornado(self): + """ + Install runit directory for the tornado, and add the configuration of the required service + """ + # Check if the Tornado itself is already installed + runitCompDir = os.path.join(self.runitDir, 'Tornado', 'Tornado') + if os.path.exists(runitCompDir): + msg = "Tornado_Tornado already installed" + gLogger.notice(msg) + return S_OK(runitCompDir) + + # Check the setup for the given system + result = gConfig.getOption('DIRAC/Setups/%s/Tornado' % (CSGlobals.getSetup())) + if not result['OK']: + return result + self.instance = result['Value'] + + # Now do the actual installation + try: + + self._createRunitLog(runitCompDir) + + runFile = os.path.join(runitCompDir, 'run') + with open(runFile, 'w') as fd: + fd.write( + """#!/bin/bash + rcfile=%(bashrc)s + [ -e $rcfile ] && source $rcfile + # + exec 2>&1 + # + # + exec python $DIRAC/DIRAC/Core/Tornado/scripts/tornado-start-all.py -ddd + """ % {'bashrc': os.path.join(self.instancePath, 'bashrc')}) + + os.chmod(runFile, self.gDefaultPerms) + + except Exception: + error = 'Failed to prepare self.setup forTornado' + gLogger.exception(error) + if self.exitOnError: + DIRAC.exit(-1) + return S_ERROR(error) + + result = self.execCommand(5, [runFile]) + + gLogger.notice(result['Value'][1]) + + return S_OK(runitCompDir) + + def addTornadoOptionsToCS(self, gConfig_o): + """ + Add the section with the component options to the CS + """ + + if gConfig_o: + gConfig_o.forceRefresh() + + instanceOption = cfgPath('DIRAC', 'Setups', self.setup, 'Tornado') + + if gConfig_o: + compInstance = gConfig_o.getValue(instanceOption, '') + else: + compInstance = self.localCfg.getOption(instanceOption, '') + if not compInstance: + return S_ERROR('%s not defined in %s' % (instanceOption, self.cfgFile)) + tornadoSection = cfgPath('Systems', 'Tornado', compInstance) + + cfg = self.__getCfg(tornadoSection, 'Port', 8443) + # cfg.setOption(cfgPath(tornadoSection, 'Password'), self.mysqlPassword) + return self._addCfgToCS(cfg) + + + + + + + + port = compCfg.getOption('Port', 0) + if port and self.host: + urlsPath = cfgPath('Systems', system, compInstance, 'URLs') + cfg.createNewSection(urlsPath) + failoverUrlsPath = cfgPath('Systems', system, compInstance, 'FailoverURLs') + cfg.createNewSection(failoverUrlsPath) + cfg.setOption(cfgPath(urlsPath, component), + 'dips://%s:%d/%s/%s' % (self.host, port, system, component)) + + gComponentInstaller = ComponentInstaller() diff --git a/FrameworkSystem/scripts/dirac-install-tornado-service.py b/FrameworkSystem/scripts/dirac-install-tornado-service.py new file mode 100755 index 00000000000..c6d06a761a1 --- /dev/null +++ b/FrameworkSystem/scripts/dirac-install-tornado-service.py @@ -0,0 +1,103 @@ +#!/usr/bin/env python +""" +Do the initial installation and configuration of a DIRAC service based on tornado +""" + +from __future__ import absolute_import + +from DIRAC import gConfig, gLogger, S_OK +from DIRAC.ConfigurationSystem.Client.Helpers import getCSExtensions +from DIRAC.FrameworkSystem.Utilities import MonitoringUtilities +from DIRAC.Core.Base import Script +from DIRAC import exit as DIRACexit +from DIRAC.FrameworkSystem.Client.ComponentInstaller import gComponentInstaller + +__RCSID__ = "$Id$" + +gComponentInstaller.exitOnError = True + +overwrite = False + + +def setOverwrite(opVal): + global overwrite + overwrite = True + return S_OK() + + +module = '' +specialOptions = {} + + +def setModule(optVal): + global specialOptions, module + specialOptions['Module'] = optVal + module = optVal + return S_OK() + + +def setSpecialOption(optVal): + global specialOptions + option, value = optVal.split('=') + specialOptions[option] = value + return S_OK() + + +Script.registerSwitch("w", "overwrite", "Overwrite the configuration in the global CS", setOverwrite) +Script.registerSwitch("m:", "module=", "Python module name for the component code", setModule) +Script.registerSwitch("p:", "parameter=", "Special component option ", setSpecialOption) +Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], + 'Usage:', + ' %s [option|cfgfile] ... System Component|System/Component' % Script.scriptName, + 'Arguments:', + ' System: Name of the DIRAC system (ie: WorkloadManagement)', + ' Service: Name of the DIRAC component (ie: Matcher)'])) + +Script.parseCommandLine() +args = Script.getPositionalArgs() + +if len(args) == 1: + args = args[0].split('/') + +if len(args) != 2: + Script.showHelp() + DIRACexit(1) + +system = args[0] +component = args[1] +compOrMod = module if module else component + + +result = gComponentInstaller.addDefaultOptionsToCS(gConfig, 'service', system, component, + getCSExtensions(), + specialOptions=specialOptions, + overwrite=overwrite) + +if not result['OK']: + gLogger.error(result['Message']) + DIRACexit(1) + + +result = gComponentInstaller.addTornadoOptionsToCS(gConfig) +if not result['OK']: + gLogger.error(result['Message']) + DIRACexit(1) + +result = gComponentInstaller.installTornado() +if not result['OK']: + gLogger.error(result['Message']) + DIRACexit(1) + + +gLogger.notice('Successfully installed component %s in %s system, now setting it up' % (component, system)) +result = gComponentInstaller.setupTornadoService(system, component, getCSExtensions(), module) +if not result['OK']: + gLogger.error(result['Message']) + DIRACexit(1) + +result = MonitoringUtilities.monitorInstallation('service', system, component, module) +if not result['OK']: + gLogger.error(result['Message']) + DIRACexit(1) +gLogger.notice('Successfully completed the installation of %s/%s' % (system, component)) +DIRACexit() From c500558efc510bc8a1be4589d488e556717c49ac Mon Sep 17 00:00:00 2001 From: Christophe Haen Date: Thu, 9 Jul 2020 14:23:45 +0200 Subject: [PATCH 11/25] Add TornadoFileCatalogHandler --- DataManagementSystem/ConfigTemplate.cfg | 19 + .../Service/TornadoFileCatalogHandler.py | 770 ++++++++++++++++++ tests/Jenkins/install.cfg | 1 + 3 files changed, 790 insertions(+) create mode 100644 DataManagementSystem/Service/TornadoFileCatalogHandler.py diff --git a/DataManagementSystem/ConfigTemplate.cfg b/DataManagementSystem/ConfigTemplate.cfg index 52293a503b4..39d9a5d1116 100644 --- a/DataManagementSystem/ConfigTemplate.cfg +++ b/DataManagementSystem/ConfigTemplate.cfg @@ -45,6 +45,25 @@ Services Default = authenticated } } + TornadoFileCatalog + { + Protocol = https + UserGroupManager = UserAndGroupManagerDB + SEManager = SEManagerDB + SecurityManager = NoSecurityManager + DirectoryManager = DirectoryLevelTree + FileManager = FileManager + UniqueGUID = False + GlobalReadAccess = True + LFNPFNConvention = Strong + ResolvePFN = True + DefaultUmask = 509 + VisibleStatus = AprioriGood + Authorization + { + Default = authenticated + } + } ##BEGIN StorageElement StorageElement { diff --git a/DataManagementSystem/Service/TornadoFileCatalogHandler.py b/DataManagementSystem/Service/TornadoFileCatalogHandler.py new file mode 100644 index 00000000000..eddf283a4f6 --- /dev/null +++ b/DataManagementSystem/Service/TornadoFileCatalogHandler.py @@ -0,0 +1,770 @@ +######################################################################## +# File: FileCatalogHandler.py +######################################################################## +""" +:mod: FileCatalogHandler + +.. module: FileCatalogHandler + :synopsis: FileCatalogHandler is a simple Replica and Metadata Catalog service + +""" + +__RCSID__ = "$Id$" + +# imports +import six +import cStringIO +import csv +import os +from types import IntType, LongType, DictType, StringTypes, BooleanType, ListType +# from DIRAC +from DIRAC.Core.DISET.RequestHandler import getServiceOption + +from DIRAC import gLogger, S_OK, S_ERROR +from DIRAC.FrameworkSystem.Client.MonitoringClient import gMonitor +from DIRAC.DataManagementSystem.DB.FileCatalogDB import FileCatalogDB + +from DIRAC.Core.Utilities import DErrno +from DIRAC.Core.Tornado.Server.TornadoService import TornadoService + + +class TornadoFileCatalogHandler(TornadoService): + """ + ..class:: FileCatalogHandler + + A simple Replica and Metadata Catalog service. + """ + + @classmethod + def initializeHandler(cls, serviceInfo): + """ handler initialisation """ + + dbLocation = getServiceOption(serviceInfo, 'Database', 'DataManagement/FileCatalogDB') + cls.gFileCatalogDB = FileCatalogDB(dbLocation) + + databaseConfig = {} + # Obtain the plugins to be used for DB interaction + gLogger.info("Initializing with FileCatalog with following managers:") + defaultManagers = {'UserGroupManager': 'UserAndGroupManagerDB', + 'SEManager': 'SEManagerDB', + 'SecurityManager': 'NoSecurityManager', + 'DirectoryManager': 'DirectoryLevelTree', + 'FileManager': 'FileManager', + 'DirectoryMetadata': 'DirectoryMetadata', + 'FileMetadata': 'FileMetadata', + 'DatasetManager': 'DatasetManager'} + for configKey in sorted(defaultManagers.keys()): + defaultValue = defaultManagers[configKey] + configValue = getServiceOption(serviceInfo, configKey, defaultValue) + gLogger.info("%-20s : %-20s" % (str(configKey), str(configValue))) + databaseConfig[configKey] = configValue + + # Obtain some general configuration of the database + gLogger.info("Initializing the FileCatalog with the following configuration:") + defaultConfig = {'UniqueGUID': False, + 'GlobalReadAccess': True, + 'LFNPFNConvention': 'Strong', + 'ResolvePFN': True, + 'DefaultUmask': 0o775, + 'ValidFileStatus': ['AprioriGood', 'Trash', 'Removing', 'Probing'], + 'ValidReplicaStatus': ['AprioriGood', 'Trash', 'Removing', 'Probing'], + 'VisibleFileStatus': ['AprioriGood'], + 'VisibleReplicaStatus': ['AprioriGood']} + for configKey in sorted(defaultConfig.keys()): + defaultValue = defaultConfig[configKey] + configValue = getServiceOption(serviceInfo, configKey, defaultValue) + gLogger.info("%-20s : %-20s" % (str(configKey), str(configValue))) + databaseConfig[configKey] = configValue + res = cls.gFileCatalogDB.setConfig(databaseConfig) + + gMonitor.registerActivity("AddFile", "Amount of addFile calls", + "FileCatalogHandler", "calls/min", gMonitor.OP_SUM) + gMonitor.registerActivity("AddFileSuccessful", "Files successfully added", + "FileCatalogHandler", "files/min", gMonitor.OP_SUM) + gMonitor.registerActivity("AddFileFailed", "Files failed to add", + "FileCatalogHandler", "files/min", gMonitor.OP_SUM) + + gMonitor.registerActivity("RemoveFile", "Amount of removeFile calls", + "FileCatalogHandler", "calls/min", gMonitor.OP_SUM) + gMonitor.registerActivity("RemoveFileSuccessful", "Files successfully removed", + "FileCatalogHandler", "files/min", gMonitor.OP_SUM) + gMonitor.registerActivity("RemoveFileFailed", "Files failed to remove", + "FileCatalogHandler", "files/min", gMonitor.OP_SUM) + + gMonitor.registerActivity("AddReplica", "Amount of addReplica calls", + "FileCatalogHandler", "calls/min", gMonitor.OP_SUM) + gMonitor.registerActivity("AddReplicaSuccessful", "Replicas successfully added", + "FileCatalogHandler", "replicas/min", gMonitor.OP_SUM) + gMonitor.registerActivity("AddReplicaFailed", "Replicas failed to add", + "FileCatalogHandler", "replicas/min", gMonitor.OP_SUM) + + gMonitor.registerActivity("RemoveReplica", "Amount of removeReplica calls", + "FileCatalogHandler", "calls/min", gMonitor.OP_SUM) + gMonitor.registerActivity("RemoveReplicaSuccessful", "Replicas successfully removed", + "FileCatalogHandler", "replicas/min", gMonitor.OP_SUM) + gMonitor.registerActivity("RemoveReplicaFailed", "Replicas failed to remove", + "FileCatalogHandler", "replicas/min", gMonitor.OP_SUM) + + gMonitor.registerActivity("ListDirectory", "Amount of listDirectory calls", + "FileCatalogHandler", "calls/min", gMonitor.OP_SUM) + + return res + + ######################################################################## + # Path operations (not updated) + # + types_changePathOwner = [[ListType, DictType] + list(StringTypes)] + + def export_changePathOwner(self, lfns, recursive=False): + """ Get replica info for the given list of LFNs + """ + return self.gFileCatalogDB.changePathOwner(lfns, self.getRemoteCredentials(), recursive) + + types_changePathGroup = [[ListType, DictType] + list(StringTypes)] + + def export_changePathGroup(self, lfns, recursive=False): + """ Get replica info for the given list of LFNs + """ + return self.gFileCatalogDB.changePathGroup(lfns, self.getRemoteCredentials(), recursive) + + types_changePathMode = [[ListType, DictType] + list(StringTypes)] + + def export_changePathMode(self, lfns, recursive=False): + """ Get replica info for the given list of LFNs + """ + return self.gFileCatalogDB.changePathMode(lfns, self.getRemoteCredentials(), recursive) + + ######################################################################## + # ACL Operations + # + types_getPathPermissions = [[ListType, DictType] + list(StringTypes)] + + def export_getPathPermissions(self, lfns): + """ Determine the ACL information for a supplied path + """ + return self.gFileCatalogDB.getPathPermissions(lfns, self.getRemoteCredentials()) + + types_hasAccess = [[basestring, dict], [basestring, list, dict]] + + def export_hasAccess(self, paths, opType): + """ Determine if the given op can be performed on the paths + The OpType is all the operations exported + + The reason for the param types is backward compatibility. Between v6r14 and v6r15, + the signature of hasAccess has changed, and the two parameters were swapped. + """ + + # The signature of v6r15 is (dict, str) + # The signature of v6r14 is (str, [dict, str, list]) + # We swap the two params if the first attribute is a string + if isinstance(paths, six.string_types): + paths, opType = opType, paths + + return self.gFileCatalogDB.hasAccess(opType, paths, self.getRemoteCredentials()) + + ################################################################### + # + # isOK + # + + types_isOK = [] + + @classmethod + def export_isOK(cls): + """ returns S_OK if DB is connected + """ + if cls.gFileCatalogDB and cls.gFileCatalogDB._connected: + return S_OK() + return S_ERROR('Server not connected to DB') + + ################################################################### + # + # User/Group write operations + # + + types_addUser = [StringTypes] + + def export_addUser(self, userName): + """ Add a new user to the File Catalog """ + return self.gFileCatalogDB.addUser(userName, self.getRemoteCredentials()) + + types_deleteUser = [StringTypes] + + def export_deleteUser(self, userName): + """ Delete user from the File Catalog """ + return self.gFileCatalogDB.deleteUser(userName, self.getRemoteCredentials()) + + types_addGroup = [StringTypes] + + def export_addGroup(self, groupName): + """ Add a new group to the File Catalog """ + return self.gFileCatalogDB.addGroup(groupName, self.getRemoteCredentials()) + + types_deleteGroup = [StringTypes] + + def export_deleteGroup(self, groupName): + """ Delete group from the File Catalog """ + return self.gFileCatalogDB.deleteGroup(groupName, self.getRemoteCredentials()) + + ################################################################### + # + # User/Group read operations + # + + types_getUsers = [] + + def export_getUsers(self): + """ Get all the users defined in the File Catalog """ + return self.gFileCatalogDB.getUsers(self.getRemoteCredentials()) + + types_getGroups = [] + + def export_getGroups(self): + """ Get all the groups defined in the File Catalog """ + return self.gFileCatalogDB.getGroups(self.getRemoteCredentials()) + + ######################################################################## + # + # Path read operations + # + + types_exists = [[ListType, DictType] + list(StringTypes)] + + def export_exists(self, lfns): + """ Check whether the supplied paths exists """ + return self.gFileCatalogDB.exists(lfns, self.getRemoteCredentials()) + + ######################################################################## + # + # File write operations + # + + types_addFile = [[ListType, DictType] + list(StringTypes)] + + def export_addFile(self, lfns): + """ Register supplied files """ + gMonitor.addMark("AddFile", 1) + res = self.gFileCatalogDB.addFile(lfns, self.getRemoteCredentials()) + if res['OK']: + gMonitor.addMark("AddFileSuccessful", len(res.get('Value', {}).get('Successful', []))) + gMonitor.addMark("AddFileFailed", len(res.get('Value', {}).get('Failed', []))) + + return res + + types_removeFile = [[ListType, DictType] + list(StringTypes)] + + def export_removeFile(self, lfns): + """ Remove the supplied lfns """ + gMonitor.addMark("RemoveFile", 1) + res = self.gFileCatalogDB.removeFile(lfns, self.getRemoteCredentials()) + if res['OK']: + gMonitor.addMark("RemoveFileSuccessful", len(res.get('Value', {}).get('Successful', []))) + gMonitor.addMark("RemoveFileFailed", len(res.get('Value', {}).get('Failed', []))) + + return res + + types_setFileStatus = [DictType] + + def export_setFileStatus(self, lfns): + """ Remove the supplied lfns """ + return self.gFileCatalogDB.setFileStatus(lfns, self.getRemoteCredentials()) + + types_addReplica = [[ListType, DictType] + list(StringTypes)] + + def export_addReplica(self, lfns): + """ Register supplied replicas """ + gMonitor.addMark("AddReplica", 1) + res = self.gFileCatalogDB.addReplica(lfns, self.getRemoteCredentials()) + if res['OK']: + gMonitor.addMark("AddReplicaSuccessful", len(res.get('Value', {}).get('Successful', []))) + gMonitor.addMark("AddReplicaFailed", len(res.get('Value', {}).get('Failed', []))) + + return res + + types_removeReplica = [[ListType, DictType] + list(StringTypes)] + + def export_removeReplica(self, lfns): + """ Remove the supplied replicas """ + gMonitor.addMark("RemoveReplica", 1) + res = self.gFileCatalogDB.removeReplica(lfns, self.getRemoteCredentials()) + if res['OK']: + gMonitor.addMark("RemoveReplicaSuccessful", len(res.get('Value', {}).get('Successful', []))) + gMonitor.addMark("RemoveReplicaFailed", len(res.get('Value', {}).get('Failed', []))) + + return res + + types_setReplicaStatus = [[ListType, DictType] + list(StringTypes)] + + def export_setReplicaStatus(self, lfns): + """ Set the status for the supplied replicas """ + return self.gFileCatalogDB.setReplicaStatus(lfns, self.getRemoteCredentials()) + + types_setReplicaHost = [[ListType, DictType] + list(StringTypes)] + + def export_setReplicaHost(self, lfns): + """ Change the registered SE for the supplied replicas """ + return self.gFileCatalogDB.setReplicaHost(lfns, self.getRemoteCredentials()) + + types_addFileAncestors = [DictType] + + def export_addFileAncestors(self, lfns): + """ Add file ancestor information for the given list of LFNs """ + return self.gFileCatalogDB.addFileAncestors(lfns, self.getRemoteCredentials()) + + ######################################################################## + # + # File read operations + # + + types_isFile = [[ListType, DictType] + list(StringTypes)] + + def export_isFile(self, lfns): + """ Check whether the supplied lfns are files """ + return self.gFileCatalogDB.isFile(lfns, self.getRemoteCredentials()) + + types_getFileSize = [[ListType, DictType] + list(StringTypes)] + + def export_getFileSize(self, lfns): + """ Get the size associated to supplied lfns """ + return self.gFileCatalogDB.getFileSize(lfns, self.getRemoteCredentials()) + + types_getFileMetadata = [[ListType, DictType] + list(StringTypes)] + + def export_getFileMetadata(self, lfns): + """ Get the metadata associated to supplied lfns """ + return self.gFileCatalogDB.getFileMetadata(lfns, self.getRemoteCredentials()) + + types_getReplicas = [[ListType, DictType] + list(StringTypes), BooleanType] + + def export_getReplicas(self, lfns, allStatus=False): + """ Get replicas for supplied lfns """ + return self.gFileCatalogDB.getReplicas(lfns, allStatus, self.getRemoteCredentials()) + + types_getReplicaStatus = [[ListType, DictType] + list(StringTypes)] + + def export_getReplicaStatus(self, lfns): + """ Get the status for the supplied replicas """ + return self.gFileCatalogDB.getReplicaStatus(lfns, self.getRemoteCredentials()) + + types_getFileAncestors = [[ListType, DictType], [ListType, IntType, LongType]] + + def export_getFileAncestors(self, lfns, depths): + """ Get the status for the supplied replicas """ + dList = depths + if not isinstance(dList, ListType): + dList = [depths] + lfnDict = dict.fromkeys(lfns, True) + return self.gFileCatalogDB.getFileAncestors(lfnDict, dList, self.getRemoteCredentials()) + + types_getFileDescendents = [[ListType, DictType], [ListType, IntType, LongType]] + + def export_getFileDescendents(self, lfns, depths): + """ Get the status for the supplied replicas """ + dList = depths + if not isinstance(dList, ListType): + dList = [depths] + lfnDict = dict.fromkeys(lfns, True) + return self.gFileCatalogDB.getFileDescendents(lfnDict, dList, self.getRemoteCredentials()) + + types_getLFNForGUID = [[ListType, DictType] + list(StringTypes)] + + def export_getLFNForGUID(self, guids): + """Get the matching lfns for given guids""" + return self.gFileCatalogDB.getLFNForGUID(guids, self.getRemoteCredentials()) + + ######################################################################## + # + # Directory write operations + # + + types_createDirectory = [[ListType, DictType] + list(StringTypes)] + + def export_createDirectory(self, lfns): + """ Create the supplied directories """ + return self.gFileCatalogDB.createDirectory(lfns, self.getRemoteCredentials()) + + types_removeDirectory = [[ListType, DictType] + list(StringTypes)] + + def export_removeDirectory(self, lfns): + """ Remove the supplied directories """ + return self.gFileCatalogDB.removeDirectory(lfns, self.getRemoteCredentials()) + + ######################################################################## + # + # Directory read operations + # + + types_listDirectory = [[ListType, DictType] + list(StringTypes), BooleanType] + + def export_listDirectory(self, lfns, verbose): + """ List the contents of supplied directories """ + gMonitor.addMark('ListDirectory', 1) + return self.gFileCatalogDB.listDirectory(lfns, self.getRemoteCredentials(), verbose=verbose) + + types_isDirectory = [[ListType, DictType] + list(StringTypes)] + + def export_isDirectory(self, lfns): + """ Determine whether supplied path is a directory """ + return self.gFileCatalogDB.isDirectory(lfns, self.getRemoteCredentials()) + + types_getDirectoryMetadata = [[ListType, DictType] + list(StringTypes)] + + def export_getDirectoryMetadata(self, lfns): + """ Get the size of the supplied directory """ + return self.gFileCatalogDB.getDirectoryMetadata(lfns, self.getRemoteCredentials()) + + types_getDirectorySize = [[ListType, DictType] + list(StringTypes)] + + def export_getDirectorySize(self, lfns, longOut=False, fromFiles=False): + """ Get the size of the supplied directory """ + return self.gFileCatalogDB.getDirectorySize(lfns, longOut, fromFiles, self.getRemoteCredentials()) + + types_getDirectoryReplicas = [[ListType, DictType] + list(StringTypes), BooleanType] + + def export_getDirectoryReplicas(self, lfns, allStatus=False): + """ Get replicas for files in the supplied directory """ + return self.gFileCatalogDB.getDirectoryReplicas(lfns, allStatus, self.getRemoteCredentials()) + + ######################################################################## + # + # Administrative database operations + # + + types_getCatalogCounters = [] + + def export_getCatalogCounters(self): + """ Get the number of registered directories, files and replicas in various tables """ + return self.gFileCatalogDB.getCatalogCounters(self.getRemoteCredentials()) + + types_rebuildDirectoryUsage = [] + + @staticmethod + def export_rebuildDirectoryUsage(self): + """ Rebuild DirectoryUsage table from scratch """ + return self.gFileCatalogDB.rebuildDirectoryUsage() + + types_repairCatalog = [] + + def export_repairCatalog(self): + """ Repair the catalog inconsistencies """ + return self.gFileCatalogDB.repairCatalog(self.getRemoteCredentials()) + + ######################################################################## + # Metadata Catalog Operations + # + + types_addMetadataField = [StringTypes, StringTypes, StringTypes] + + def export_addMetadataField(self, fieldName, fieldType, metaType='-d'): + """ Add a new metadata field of the given type + """ + if metaType.lower() == "-d": + return self.gFileCatalogDB.dmeta.addMetadataField( + fieldName, fieldType, self.getRemoteCredentials()) + elif metaType.lower() == "-f": + return self.gFileCatalogDB.fmeta.addMetadataField( + fieldName, fieldType, self.getRemoteCredentials()) + else: + return S_ERROR('Unknown metadata type %s' % metaType) + + types_deleteMetadataField = [StringTypes] + + def export_deleteMetadataField(self, fieldName): + """ Delete the metadata field + """ + result = self.gFileCatalogDB.dmeta.deleteMetadataField(fieldName, self.getRemoteCredentials()) + error = '' + if not result['OK']: + error = result['Message'] + result = self.gFileCatalogDB.fmeta.deleteMetadataField(fieldName, self.getRemoteCredentials()) + if not result['OK']: + if error: + result["Message"] = error + "; " + result["Message"] + + return result + + types_getMetadataFields = [] + + def export_getMetadataFields(self): + """ Get all the metadata fields + """ + resultDir = self.gFileCatalogDB.dmeta.getMetadataFields(self.getRemoteCredentials()) + if not resultDir['OK']: + return resultDir + resultFile = self.gFileCatalogDB.fmeta.getFileMetadataFields(self.getRemoteCredentials()) + if not resultFile['OK']: + return resultFile + + return S_OK({'DirectoryMetaFields': resultDir['Value'], + 'FileMetaFields': resultFile['Value']}) + + types_setMetadata = [StringTypes, DictType] + + def export_setMetadata(self, path, metadatadict): + """ Set metadata parameter for the given path + """ + return self.gFileCatalogDB.setMetadata(path, metadatadict, self.getRemoteCredentials()) + + types_setMetadataBulk = [DictType] + + def export_setMetadataBulk(self, pathMetadataDict): + """ Set metadata parameter for the given path + """ + return self.gFileCatalogDB.setMetadataBulk(pathMetadataDict, self.getRemoteCredentials()) + + types_removeMetadata = [DictType] + + def export_removeMetadata(self, pathMetadataDict): + """ Remove the specified metadata for the given path + """ + return self.gFileCatalogDB.removeMetadata(pathMetadataDict, self.getRemoteCredentials()) + + types_getDirectoryUserMetadata = [StringTypes] + + def export_getDirectoryUserMetadata(self, path): + """ Get all the metadata valid for the given directory path + """ + return self.gFileCatalogDB.dmeta.getDirectoryMetadata(path, self.getRemoteCredentials()) + + types_getFileUserMetadata = [StringTypes] + + def export_getFileUserMetadata(self, path): + """ Get all the metadata valid for the given file + """ + return self.gFileCatalogDB.fmeta.getFileUserMetadata(path, self.getRemoteCredentials()) + + types_findDirectoriesByMetadata = [DictType] + + def export_findDirectoriesByMetadata(self, metaDict, path='/'): + """ Find all the directories satisfying the given metadata set + """ + return self.gFileCatalogDB.dmeta.findDirectoriesByMetadata( + metaDict, path, self.getRemoteCredentials()) + + types_findFilesByMetadata = [DictType, StringTypes] + + def export_findFilesByMetadata(self, metaDict, path='/'): + """ Find all the files satisfying the given metadata set + """ + result = self.gFileCatalogDB.fmeta.findFilesByMetadata(metaDict, path, self.getRemoteCredentials()) + if not result['OK']: + return result + lfns = result['Value'].values() + return S_OK(lfns) + + types_getReplicasByMetadata = [DictType, StringTypes, BooleanType] + + def export_getReplicasByMetadata(self, metaDict, path='/', allStatus=False): + """ Find all the files satisfying the given metadata set + """ + return self.gFileCatalogDB.fileManager.getReplicasByMetadata(metaDict, + path, + allStatus, + self.getRemoteCredentials()) + + types_findFilesByMetadataDetailed = [DictType, StringTypes] + + def export_findFilesByMetadataDetailed(self, metaDict, path='/'): + """ Find all the files satisfying the given metadata set + """ + result = self.gFileCatalogDB.fmeta.findFilesByMetadata(metaDict, path, self.getRemoteCredentials()) + if not result['OK'] or not result['Value']: + return result + + lfns = result['Value'].values() + return self.gFileCatalogDB.getFileDetails(lfns, self.getRemoteCredentials()) + + types_findFilesByMetadataWeb = [DictType, StringTypes, [IntType, LongType], [IntType, LongType]] + + def export_findFilesByMetadataWeb(self, metaDict, path, startItem, maxItems): + """ Find files satisfying the given metadata set + """ + result = self.gFileCatalogDB.dmeta.findFileIDsByMetadata( + metaDict, path, self.getRemoteCredentials(), startItem, maxItems) + if not result['OK'] or not result['Value']: + return result + + fileIDs = result['Value'] + totalRecords = result['TotalRecords'] + + result = self.gFileCatalogDB.fileManager._getFileLFNs(fileIDs) + if not result['OK']: + return result + + lfnsResultList = result['Value']['Successful'].values() + resultDetails = self.gFileCatalogDB.getFileDetails(lfnsResultList, self.getRemoteCredentials()) + if not resultDetails['OK']: + return resultDetails + + result = S_OK({"TotalRecords": totalRecords, "Records": resultDetails['Value']}) + return result + + def findFilesByMetadataWeb(self, metaDict, path, startItem, maxItems): + """ Find all the files satisfying the given metadata set + """ + result = self.gFileCatalogDB.fmeta.findFilesByMetadata(metaDict, path, self.getRemoteCredentials()) + if not result['OK'] or not result['Value']: + return result + + lfns = [] + for directory in result['Value']: + for fname in result['Value'][directory]: + lfns.append(os.path.join(directory, fname)) + + start = startItem + totalRecords = len(lfns) + if start > totalRecords: + return S_ERROR('Requested files out of existing range') + end = start + maxItems + if end > totalRecords: + end = totalRecords + lfnsResultList = lfns[start:end] + + resultDetails = self.gFileCatalogDB.getFileDetails(lfnsResultList, self.getRemoteCredentials()) + if not resultDetails['OK']: + return resultDetails + + result = S_OK({"TotalRecords": totalRecords, "Records": resultDetails['Value']}) + return result + + types_getCompatibleMetadata = [DictType, StringTypes] + + def export_getCompatibleMetadata(self, metaDict, path='/'): + """ Get metadata values compatible with the given metadata subset + """ + return self.gFileCatalogDB.dmeta.getCompatibleMetadata(metaDict, path, self.getRemoteCredentials()) + + types_addMetadataSet = [StringTypes, DictType] + + def export_addMetadataSet(self, setName, setDict): + """ Add a new metadata set + """ + return self.gFileCatalogDB.dmeta.addMetadataSet(setName, setDict, self.getRemoteCredentials()) + + types_getMetadataSet = [StringTypes, BooleanType] + + def export_getMetadataSet(self, setName, expandFlag): + """ Add a new metadata set + """ + return self.gFileCatalogDB.dmeta.getMetadataSet(setName, expandFlag, self.getRemoteCredentials()) + +######################################################################################### +# +# Dataset manipulation methods +# + types_addDataset = [DictType] + + def export_addDataset(self, datasets): + """ Add a new dynamic dataset defined by its meta query + """ + return self.gFileCatalogDB.datasetManager.addDataset(datasets, self.getRemoteCredentials()) + + types_addDatasetAnnotation = [DictType] + + def export_addDatasetAnnotation(self, datasetDict): + """ Add annotation to an already created dataset + """ + return self.gFileCatalogDB.datasetManager.addDatasetAnnotation( + datasetDict, self.getRemoteCredentials()) + + types_removeDataset = [DictType] + + def export_removeDataset(self, datasets): + """ Check the given dynamic dataset for changes since its definition + """ + return self.gFileCatalogDB.datasetManager.removeDataset(datasets, self.getRemoteCredentials()) + + types_checkDataset = [DictType] + + def export_checkDataset(self, datasets): + """ Check the given dynamic dataset for changes since its definition + """ + return self.gFileCatalogDB.datasetManager.checkDataset(datasets, self.getRemoteCredentials()) + + types_updateDataset = [DictType] + + def export_updateDataset(self, datasets): + """ Update the given dynamic dataset for changes since its definition + """ + return self.gFileCatalogDB.datasetManager.updateDataset(datasets, self.getRemoteCredentials()) + + types_getDatasets = [DictType] + + def export_getDatasets(self, datasets): + """ Get parameters of the given dynamic dataset as they are stored in the database + """ + return self.gFileCatalogDB.datasetManager.getDatasets(datasets, self.getRemoteCredentials()) + + types_getDatasetParameters = [DictType] + + def export_getDatasetParameters(self, datasets): + """ Get parameters of the given dynamic dataset as they are stored in the database + """ + return self.gFileCatalogDB.datasetManager.getDatasetParameters(datasets, self.getRemoteCredentials()) + + types_getDatasetAnnotation = [DictType] + + def export_getDatasetAnnotation(self, datasets): + """ Get annotation of the given datasets + """ + return self.gFileCatalogDB.datasetManager.getDatasetAnnotation(datasets, self.getRemoteCredentials()) + + types_freezeDataset = [DictType] + + def export_freezeDataset(self, datasets): + """ Freeze the contents of the dataset making it effectively static + """ + return self.gFileCatalogDB.datasetManager.freezeDataset(datasets, self.getRemoteCredentials()) + + types_releaseDataset = [DictType] + + def export_releaseDataset(self, datasets): + """ Release the contents of the frozen dataset allowing changes in its contents + """ + return self.gFileCatalogDB.datasetManager.releaseDataset(datasets, self.getRemoteCredentials()) + + types_getDatasetFiles = [DictType] + + def export_getDatasetFiles(self, datasets): + """ Get lfns in the given dataset + """ + return self.gFileCatalogDB.datasetManager.getDatasetFiles(datasets, self.getRemoteCredentials()) + + def getSEDump(self, seName): + """ + Return all the files at a given SE, together with checksum and size + + :param seName: name of the StorageElement + + :returns: S_OK with list of tuples (lfn, checksum, size) + """ + return self.gFileCatalogDB.getSEDump(seName)['Value'] + + def export_streamToClient(self, seName): + """ This method used to transfer the SEDump to the client, + formated as CSV with '|' separation + + :param seName: name of the se to dump + + :returns: the result of the FileHelper + + + """ + + retVal = self.getSEDump(seName) + + try: + csvOutput = cStringIO.StringIO() + writer = csv.writer(csvOutput, delimiter='|') + for lfn in retVal: + writer.writerow(lfn) + + #csvOutput.seek(0) + ret = csvOutput.getvalue() + return ret + + except Exception as e: + gLogger.exception("Exception while sending seDump", repr(e)) + return S_ERROR("Exception while sendind seDump: %s" % repr(e)) + finally: + csvOutput.close() diff --git a/tests/Jenkins/install.cfg b/tests/Jenkins/install.cfg index ef6a1f11122..26fd8a35ddd 100644 --- a/tests/Jenkins/install.cfg +++ b/tests/Jenkins/install.cfg @@ -41,6 +41,7 @@ LocalInstallation Systems += Production Systems += Transformation Systems += WorkloadManagement + Systems += Tornado # List of DataBases to be installed - minimal list for a running base server Databases = InstalledComponentsDB Databases += ResourceStatusDB From adcf718355d682daf1b22c7ea94591fe0bb37eb0 Mon Sep 17 00:00:00 2001 From: Christophe Haen Date: Thu, 9 Jul 2020 14:24:05 +0200 Subject: [PATCH 12/25] Make DFC client compatible with Tornado service --- Resources/Catalog/FileCatalogClient.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Resources/Catalog/FileCatalogClient.py b/Resources/Catalog/FileCatalogClient.py index 6f70723e681..56c3dceef1f 100644 --- a/Resources/Catalog/FileCatalogClient.py +++ b/Resources/Catalog/FileCatalogClient.py @@ -7,7 +7,7 @@ import os from DIRAC import S_OK, S_ERROR -from DIRAC.Core.DISET.TransferClient import TransferClient +from DIRAC.Core.Tornado.Client.TransferClientSelector import TransferClientSelector as TransferClient from DIRAC.Core.Security.ProxyInfo import getVOfromProxyGroup from DIRAC.ConfigurationSystem.Client.Helpers.Registry import getVOMSAttributeForGroup, getDNForUsername From 449b477273fca98ab4aae8411ab95cdb30cd2715 Mon Sep 17 00:00:00 2001 From: Christophe Haen Date: Fri, 3 Jul 2020 18:37:35 +0200 Subject: [PATCH 13/25] Run FileCatalog tests with https --- .github/workflows/integration.yml | 10 + .../Client/ConfigurationClient.py | 6 +- .../Service/TornadoConfigurationHandler.py | 9 +- ConfigurationSystem/private/Refresher.py | 13 +- Core/Tornado/Client/RPCClientSelector.py | 3 +- Core/Tornado/Client/TransferClientSelector.py | 3 +- Core/Tornado/Server/TornadoService.py | 36 +-- .../test/TestClientSelectionAndInterface.py | 5 +- .../test/multi-mechanize/distributed-test.py | 9 +- .../multi-mechanize/plot-distributedTest.py | 15 +- DataManagementSystem/ConfigTemplate.cfg | 12 +- .../Service/TornadoFileCatalogHandler.py | 5 +- FrameworkSystem/Client/ComponentInstaller.py | 63 ++++- .../CommandReference/dirac-admin-add-host.rst | 2 +- .../CommandReference/dirac-admin-ce-info.rst | 4 - .../CommandReference/dirac-proxy-init.rst | 1 - .../CommandReference/index.rst | 1 + .../InstallingDiracServer.rst | 2 + .../Internals/Core/Authentification.rst | 55 ----- .../Internals/Core/ClientServer.rst | 1 + .../{ => Systems}/ConfigurationSystem.rst | 0 docs/source/DeveloperGuide/Systems/index.rst | 1 + .../TornadoServices/clientservice.png | Bin 0 -> 24594 bytes .../DeveloperGuide/TornadoServices/index.rst | 223 ++++++++++++++++++ .../TornadoServices/installTornado.rst | 214 ----------------- docs/source/DeveloperGuide/monitoring.rst | 0 .../dirac-dms-clean-directory.rst | 2 +- .../DataManagement/dirac-dms-find-lfns.rst | 2 +- .../Others/dirac-proxy-init.rst | 1 - tests/CI/install_server.sh | 29 +++ .../all_integration_server_tests.sh | 1 + 31 files changed, 388 insertions(+), 340 deletions(-) delete mode 100644 docs/source/DeveloperGuide/Internals/Core/Authentification.rst rename docs/source/DeveloperGuide/{ => Systems}/ConfigurationSystem.rst (100%) create mode 100644 docs/source/DeveloperGuide/TornadoServices/clientservice.png delete mode 100644 docs/source/DeveloperGuide/TornadoServices/installTornado.rst delete mode 100644 docs/source/DeveloperGuide/monitoring.rst diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index d070f7eda4c..527622694eb 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -12,8 +12,10 @@ jobs: MATRIX_DEFAULT_MYSQL_VER: 5.7 MATRIX_DEFAULT_HOST_OS: cc7 MATRIX_DEFAULT_USE_NEWTHREADPOOL: default + MATRIX_DEFAULT_DIRACOSVER: default MATRIX_DEFAULT_SERVER_USE_M2CRYPTO: Yes MATRIX_DEFAULT_CLIENT_USE_M2CRYPTO: Yes + MATRIX_DEFAULT_TEST_HTTPS: No strategy: fail-fast: False @@ -38,6 +40,12 @@ jobs: CLIENT_USE_M2CRYPTO: Yes - SERVER_USE_M2CRYPTO: No CLIENT_USE_M2CRYPTO: No + ###### HTTPS tests + # IMPLICIT: - DIRACOSVER: default + # TEST_HTTPS: No + - TEST_HTTPS: Yes + DIRACOSVER: https + steps: - uses: actions/checkout@v2 @@ -59,6 +67,8 @@ jobs: echo -n "-e MYSQL_VER=${{ matrix.MYSQL_VER || env.MATRIX_DEFAULT_MYSQL_VER }} " >> run_in_container echo -n "-e ES_VER=${{ matrix.ES_VER || env.MATRIX_DEFAULT_ES_VER }} " >> run_in_container if [[ "${{ matrix.USE_NEWTHREADPOOL || env.MATRIX_DEFAULT_USE_NEWTHREADPOOL }}" != "default" ]]; then echo -n "-e DIRAC_USE_NEWTHREADPOOL=${{ matrix.USE_NEWTHREADPOOL || env.MATRIX_DEFAULT_USE_NEWTHREADPOOL }} " >> run_in_container; fi + if [[ "${{ matrix.DIRACOSVER || env.MATRIX_DEFAULT_DIRACOSVER }}" != "default" ]]; then echo -n "-e DIRACOSVER=${{ matrix.DIRACOSVER || env.MATRIX_DEFAULT_DIRACOSVER }} " >> run_in_container; fi + echo -n "-e TEST_HTTPS=${{ matrix.TEST_HTTPS || env.MATRIX_DEFAULT_TEST_HTTPS }} " >> run_in_container echo -n "-e SERVER_USE_M2CRYPTO=${{ matrix.SERVER_USE_M2CRYPTO || env.MATRIX_DEFAULT_SERVER_USE_M2CRYPTO }} " >> run_in_container echo -n "-e CLIENT_USE_M2CRYPTO=${{ matrix.CLIENT_USE_M2CRYPTO || env.MATRIX_DEFAULT_CLIENT_USE_M2CRYPTO }} " >> run_in_container # Finish wrapper script diff --git a/ConfigurationSystem/Client/ConfigurationClient.py b/ConfigurationSystem/Client/ConfigurationClient.py index cb7702adbf6..dd282f31351 100644 --- a/ConfigurationSystem/Client/ConfigurationClient.py +++ b/ConfigurationSystem/Client/ConfigurationClient.py @@ -26,7 +26,7 @@ def getCompressedData(self): Transmit request to service and get data in base64, it decode base64 before returning - :returns str:Configuration data, compressed + :returns str: Configuration data, compressed """ retVal = self.executeRPC('getCompressedData') if retVal['OK']: @@ -38,7 +38,7 @@ def getCompressedDataIfNewer(self, sClientVersion): Transmit request to service and get data in base64, it decode base64 before returning. - :returns str:Configuration data, if changed, compressed + :returns: Configuration data, if changed, compressed """ retVal = self.executeRPC('getCompressedDataIfNewer', sClientVersion) if retVal['OK'] and 'data' in retVal['Value']: @@ -49,7 +49,7 @@ def commitNewData(self, sData): """ Transmit request to service by encoding data in base64. - :param: Data to commit, you may call this method with CSAPI and Modificator + :param sData: Data to commit, you may call this method with CSAPI and Modificator """ return self.executeRPC('commitNewData', b64encode(sData)) diff --git a/ConfigurationSystem/Service/TornadoConfigurationHandler.py b/ConfigurationSystem/Service/TornadoConfigurationHandler.py index a5f3512e4b7..8ffb465d48e 100644 --- a/ConfigurationSystem/Service/TornadoConfigurationHandler.py +++ b/ConfigurationSystem/Service/TornadoConfigurationHandler.py @@ -1,8 +1,8 @@ """ The CS! (Configuration Service) - Modified to work with Tornado - Encode data in base64 because of JSON limitations - In client side you must use a specific client +Modified to work with Tornado +Encode data in base64 because of JSON limitations +In client side you must use a specific client """ @@ -17,7 +17,8 @@ class TornadoConfigurationHandler(TornadoService): - """ The CS handler + """ + The CS handler """ ServiceInterface = None PilotSynchronizer = None diff --git a/ConfigurationSystem/private/Refresher.py b/ConfigurationSystem/private/Refresher.py index 14eabfdaf32..9f59032c4ea 100755 --- a/ConfigurationSystem/private/Refresher.py +++ b/ConfigurationSystem/private/Refresher.py @@ -256,7 +256,10 @@ class TornadoRefresher(RefresherBase): try: from tornado import gen - from tornado.ioloop import IOLoop + # We change the import name otherwise sphinx tries + # to compile the tornado doc and fails + from tornado.ioloop import IOLoop as _IOLoop + except ImportError: gLogger.fatal("You should install tornado to use this refresher, please unset USE_TORNADO_IOLOOP") @@ -275,7 +278,7 @@ def refreshConfigurationIfNeeded(self): if not gConfigurationData.getServers() or not self._lastRefreshExpired(): # pylint: disable=no-member return self._lastUpdateTime = time.time() - self.IOLoop.current().run_in_executor(None, self._refresh) # pylint: disable=no-member + self._IOLoop.current().run_in_executor(None, self._refresh) # pylint: disable=no-member return def autoRefreshAndPublish(self, sURL): @@ -296,7 +299,7 @@ def autoRefreshAndPublish(self, sURL): # Tornado replacement solution to the classic thread # It start the method self.__refreshLoop on the next IOLoop iteration - self.IOLoop.current().spawn_callback(self.__refreshLoop) + self._IOLoop.current().spawn_callback(self.__refreshLoop) @gen.coroutine def __refreshLoop(self): @@ -316,7 +319,7 @@ def __refreshLoop(self): yield self.gen.sleep(gConfigurationData.getPropagationTime()) # Publish step is blocking so we have to run it in executor # If we are not doing it, when master try to ping we block the IOLoop - yield self.IOLoop.current().run_in_executor(None, self.__AutoRefresh) + yield self._IOLoop.current().run_in_executor(None, self.__AutoRefresh) @gen.coroutine def __AutoRefresh(self): @@ -332,7 +335,7 @@ def __AutoRefresh(self): def daemonize(self): """ daemonize is probably not the best name because there is no daemon behind but we must keep it to the same interface of the DISET refresher """ - self.IOLoop.current().spawn_callback(self.__refreshLoop) + self._IOLoop.current().spawn_callback(self.__refreshLoop) # Here we define the refresher which should be used. diff --git a/Core/Tornado/Client/RPCClientSelector.py b/Core/Tornado/Client/RPCClientSelector.py index cb15cca9b3c..a24888d495d 100644 --- a/Core/Tornado/Client/RPCClientSelector.py +++ b/Core/Tornado/Client/RPCClientSelector.py @@ -27,7 +27,8 @@ def isURL(url): def RPCClientSelector(*args, **kwargs): # We use same interface as RPCClient """ Select the correct RPCClient, instanciate it, and return it - :param args[0]: url: URL can be just "system/service" or "dips://domain:port/system/service" + + :param args: URL can be just "system/service" or "dips://domain:port/system/service" """ # We detect if we need to use a specific class for the HTTPS client diff --git a/Core/Tornado/Client/TransferClientSelector.py b/Core/Tornado/Client/TransferClientSelector.py index 3d617d46f27..9f3b2504216 100644 --- a/Core/Tornado/Client/TransferClientSelector.py +++ b/Core/Tornado/Client/TransferClientSelector.py @@ -27,7 +27,8 @@ def isURL(url): def TransferClientSelector(*args, **kwargs): # We use same interface as TransferClient """ Select the correct TransferClient, instanciate it, and return it - :param args[0]: url: URL can be just "system/service" or "dips://domain:port/system/service" + + :param args: URL can be just "system/service" or "dips://domain:port/system/service" """ # We detect if we need to use a specific class for the HTTPS client diff --git a/Core/Tornado/Server/TornadoService.py b/Core/Tornado/Server/TornadoService.py index ce71bfa201c..07896e57c2c 100644 --- a/Core/Tornado/Server/TornadoService.py +++ b/Core/Tornado/Server/TornadoService.py @@ -1,26 +1,26 @@ """ - TornadoService represent one service services, your handler must inherith form this class - TornadoService may be used only by TornadoServer. +TornadoService represent one service services, your handler must inherith form this class +TornadoService may be used only by TornadoServer. - To create you must write this "minimal" code:: +To create you must write this "minimal" code:: - from DIRAC.Core.Tornado.Server.TornadoService import TornadoService - class yourServiceHandler(TornadoService): + from DIRAC.Core.Tornado.Server.TornadoService import TornadoService + class yourServiceHandler(TornadoService): - @classmethod - def initializeHandler(cls, infosDict): - ## Called 1 time, at first request. - ## You don't need to use super or to call any parents method, it's managed by the server + @classmethod + def initializeHandler(cls, infosDict): + ## Called 1 time, at first request. + ## You don't need to use super or to call any parents method, it's managed by the server - def initializeRequest(self): - ## Called at each request + def initializeRequest(self): + ## Called at each request - auth_someMethod = ['all'] - def export_someMethod(self): - #Insert your method here, don't forgot the return + auth_someMethod = ['all'] + def export_someMethod(self): + #Insert your method here, don't forgot the return - Then you must configure service like any other service +Then you must configure service like any other service """ @@ -154,8 +154,10 @@ def initializeRequest(self): def initialize(self, debug): # pylint: disable=arguments-differ """ initialize, called at every request - ..warning:: DO NOT REWRITE THIS FUNCTION IN YOUR HANDLER - ==> initialize in DISET became initializeRequest in HTTPS ! + + ..warning:: + DO NOT REWRITE THIS FUNCTION IN YOUR HANDLER + ==> initialize in DISET became initializeRequest in HTTPS ! """ self.debug = debug self.authorized = False diff --git a/Core/Tornado/test/TestClientSelectionAndInterface.py b/Core/Tornado/test/TestClientSelectionAndInterface.py index 0eebf1ec8f7..c3e427770f6 100644 --- a/Core/Tornado/test/TestClientSelectionAndInterface.py +++ b/Core/Tornado/test/TestClientSelectionAndInterface.py @@ -14,6 +14,7 @@ You don't need to setup anything, just run ``pytest TestClientSelection.py`` ! """ +from __future__ import print_function import os import re @@ -25,7 +26,7 @@ from DIRAC.Core.DISET.RPCClient import RPCClient from DIRAC.ConfigurationSystem.private.ConfigurationClient import ConfigurationClient from DIRAC.ConfigurationSystem.Client.ConfigurationData import gConfigurationData -from DIRAC.Core.Utilities.CFG import CFG +from diraccfg import CFG from DIRAC.Core.Base.Client import Client from DIRAC.Core.DISET.private.InnerRPCClient import InnerRPCClient @@ -98,7 +99,7 @@ def config(request): gConfigurationData.remoteCFG = CFG() gConfigurationData.mergedCFG = CFG() gConfigurationData.generateNewVersion() - print "TearDown" + print("TearDown") # request is given by @fixture decorator, addfinalizer set the function who need to be called after the tests # request.addfinalizer(tearDown) diff --git a/Core/Tornado/test/multi-mechanize/distributed-test.py b/Core/Tornado/test/multi-mechanize/distributed-test.py index 4839dff7f14..8147b1b9307 100644 --- a/Core/Tornado/test/multi-mechanize/distributed-test.py +++ b/Core/Tornado/test/multi-mechanize/distributed-test.py @@ -1,3 +1,4 @@ +from __future__ import print_function import xmlrpclib import time import sys @@ -12,7 +13,7 @@ servers = [] -print "Starting test servers...." +print("Starting test servers....") # We send signal to all servers for port in portList: for server in serversList: @@ -22,7 +23,7 @@ time.sleep(2) -print "Waiting for results..." +print("Waiting for results...") while servers[-1].get_results() == 'Results Not Available': time.sleep(1) @@ -36,10 +37,10 @@ for server in servers: fileCount += 1 fileName = "%s.%s.txt" % (output, fileCount) - print "Writing output file %s" % fileName + print("Writing output file %s" % fileName) file = open(fileName, 'w') file.write(server.get_results()) file.close() # We print the command you can copy paste to have the results in a plot -print "python plot-distributedTest.py %s %d" % (output, fileCount) +print("python plot-distributedTest.py %s %d" % (output, fileCount)) diff --git a/Core/Tornado/test/multi-mechanize/plot-distributedTest.py b/Core/Tornado/test/multi-mechanize/plot-distributedTest.py index 9efd48e7d75..3eeca13b615 100644 --- a/Core/Tornado/test/multi-mechanize/plot-distributedTest.py +++ b/Core/Tornado/test/multi-mechanize/plot-distributedTest.py @@ -1,3 +1,4 @@ +from __future__ import print_function import csv import matplotlib.pyplot as plt @@ -34,8 +35,8 @@ def get_results(): if len(sys.argv) < 3: - print "Usage: python plot-distributedTest NAME NUMBEROFFILE" - print "Example: python plot-distributedTest 1532506328.38 2" + print("Usage: python plot-distributedTest NAME NUMBEROFFILE") + print("Example: python plot-distributedTest 1532506328.38 2") sys.exit(1) file = sys.argv[1] @@ -44,7 +45,7 @@ def get_results(): for i in range(1, count + 1): fileName = "%s.%s.txt" % (file, i) with open(fileName, 'r') as content: - print "reading %s" % fileName + print("reading %s" % fileName) lines = content.read().split('\n')[1:-1] result = [line.split(',') for line in lines] results.append(result) @@ -53,9 +54,9 @@ def get_results(): def get_server_stats(): - print "Please specify location to file with server stats:" + print("Please specify location to file with server stats:") serverStatFile = "/tmp/results.txt" # raw_input() - print "Loading %s" % serverStatFile + print ("Loading %s" % serverStatFile) serverStats = dict() with open(serverStatFile, 'r') as content_file: @@ -113,7 +114,7 @@ def process_data(results, serverStats): RAM.append(RAM[-1]) loadAvg.append(loadAvg[-1]) - print "ERROR - Some values missing for CPU and Memory usage [try to load for time=%s]" % t + print("ERROR - Some values missing for CPU and Memory usage [try to load for time=%s]" % t) return (time, requestTime, CPU, RAM, reqPerSec, errorRate, loadAvg) @@ -147,7 +148,7 @@ def displayGraph(results, serverStats): """ Display all the graph on the same figure """ - print "Processing data and plot, it may take some time for huge tests" + print("Processing data and plot, it may take some time for huge tests") (time, requestTime, CPU, RAM, reqPerSec, errorCount, loadAvg) = process_data(results, serverStats) plt.subplot(221) diff --git a/DataManagementSystem/ConfigTemplate.cfg b/DataManagementSystem/ConfigTemplate.cfg index 39d9a5d1116..8df2f37b4d7 100644 --- a/DataManagementSystem/ConfigTemplate.cfg +++ b/DataManagementSystem/ConfigTemplate.cfg @@ -45,15 +45,17 @@ Services Default = authenticated } } + + # Caution: tech preview, with LHCb specific managers TornadoFileCatalog { Protocol = https UserGroupManager = UserAndGroupManagerDB SEManager = SEManagerDB - SecurityManager = NoSecurityManager - DirectoryManager = DirectoryLevelTree - FileManager = FileManager - UniqueGUID = False + SecurityManager = VOMSSecurityManager + DirectoryManager = DirectoryClosure + FileManager = FileManagerPs + UniqueGUID = True GlobalReadAccess = True LFNPFNConvention = Strong ResolvePFN = True @@ -64,6 +66,8 @@ Services Default = authenticated } } + ###################### + ##BEGIN StorageElement StorageElement { diff --git a/DataManagementSystem/Service/TornadoFileCatalogHandler.py b/DataManagementSystem/Service/TornadoFileCatalogHandler.py index eddf283a4f6..b4210afe606 100644 --- a/DataManagementSystem/Service/TornadoFileCatalogHandler.py +++ b/DataManagementSystem/Service/TornadoFileCatalogHandler.py @@ -5,7 +5,8 @@ :mod: FileCatalogHandler .. module: FileCatalogHandler - :synopsis: FileCatalogHandler is a simple Replica and Metadata Catalog service + +:synopsis: FileCatalogHandler is a simple Replica and Metadata Catalog service """ @@ -759,7 +760,7 @@ def export_streamToClient(self, seName): for lfn in retVal: writer.writerow(lfn) - #csvOutput.seek(0) + # csvOutput.seek(0) ret = csvOutput.getvalue() return ret diff --git a/FrameworkSystem/Client/ComponentInstaller.py b/FrameworkSystem/Client/ComponentInstaller.py index e861fced0db..c08552e2509 100644 --- a/FrameworkSystem/Client/ComponentInstaller.py +++ b/FrameworkSystem/Client/ComponentInstaller.py @@ -464,7 +464,7 @@ def _getCentralCfg(self, installCfg): centralCfg['Registry']['Groups'][group].addKey( # pylint: disable=unsubscriptable-object 'Properties', '', '') - properties = centralCfg['Registry']['Groups'][adminGroupName].getOption( # pylint: disable=unsubscriptable-object + properties = centralCfg['Registry']['Groups'][adminGroupName].getOption( # noqa # pylint: disable=unsubscriptable-object 'Properties', []) for prop in adminGroupProperties: if prop not in properties: @@ -472,7 +472,7 @@ def _getCentralCfg(self, installCfg): centralCfg['Registry']['Groups'][adminGroupName].appendToOption( # pylint: disable=unsubscriptable-object 'Properties', ', %s' % prop) - properties = centralCfg['Registry']['Groups'][defaultGroupName].getOption( # pylint: disable=unsubscriptable-object + properties = centralCfg['Registry']['Groups'][defaultGroupName].getOption( # noqa # pylint: disable=unsubscriptable-object 'Properties', []) for prop in defaultGroupProperties: if prop not in properties: @@ -2579,21 +2579,60 @@ def addTornadoOptionsToCS(self, gConfig_o): # cfg.setOption(cfgPath(tornadoSection, 'Password'), self.mysqlPassword) return self._addCfgToCS(cfg) - + def setupTornadoService(self, system, component, extensions, + componentModule='', checkModule=True): + """ + Install and create link in startup + """ + # Create the startup entry now + # Force the system and component to be 'Tornado' but preserve the interface and the code + # just to allow for easier refactoring maybe later + system = 'Tornado' + component = 'Tornado' + componentType = 'Tornado' + runitCompDir = os.path.join(self.runitDir, 'Tornado', 'Tornado') + startCompDir = os.path.join(self.startDir, '%s_%s' % (system, component)) + mkDir(self.startDir) + if not os.path.lexists(startCompDir): + gLogger.notice('Creating startup link at', startCompDir) + mkLink(runitCompDir, startCompDir) + # Wait for the service to be recognised (can't use isfile as supervise/ok is a device) + start = time.time() + while (time.time() - 10) < start: + time.sleep(1) + if os.path.exists(os.path.join(startCompDir, 'supervise', 'ok')): + break + else: + return S_ERROR('Failed to find supervise/ok for component %s_%s' % (system, component)) + + # Check the runsv status + start = time.time() + while (time.time() - 20) < start: + result = self.getStartupComponentStatus([(system, component)]) + if not result['OK']: + continue + if result['Value'] and result['Value']['%s_%s' % (system, component)]['RunitStatus'] == "Run": + break + time.sleep(1) + else: + return S_ERROR('Failed to start the component %s_%s' % (system, component)) + resDict = {} + resDict['ComponentType'] = componentType + resDict['RunitStatus'] = result['Value']['%s_%s' % (system, component)]['RunitStatus'] - - port = compCfg.getOption('Port', 0) - if port and self.host: - urlsPath = cfgPath('Systems', system, compInstance, 'URLs') - cfg.createNewSection(urlsPath) - failoverUrlsPath = cfgPath('Systems', system, compInstance, 'FailoverURLs') - cfg.createNewSection(failoverUrlsPath) - cfg.setOption(cfgPath(urlsPath, component), - 'dips://%s:%d/%s/%s' % (self.host, port, system, component)) + return S_OK(resDict) + # port = compCfg.getOption('Port', 0) + # if port and self.host: + # urlsPath = cfgPath('Systems', system, compInstance, 'URLs') + # cfg.createNewSection(urlsPath) + # failoverUrlsPath = cfgPath('Systems', system, compInstance, 'FailoverURLs') + # cfg.createNewSection(failoverUrlsPath) + # cfg.setOption(cfgPath(urlsPath, component), + # 'dips://%s:%d/%s/%s' % (self.host, port, system, component)) gComponentInstaller = ComponentInstaller() diff --git a/docs/source/AdministratorGuide/CommandReference/dirac-admin-add-host.rst b/docs/source/AdministratorGuide/CommandReference/dirac-admin-add-host.rst index 9a87ca8a58d..4d9dff48f5b 100644 --- a/docs/source/AdministratorGuide/CommandReference/dirac-admin-add-host.rst +++ b/docs/source/AdministratorGuide/CommandReference/dirac-admin-add-host.rst @@ -12,7 +12,7 @@ Usage:: Arguments:: - Property=: Other properties to be added to the User like (Responsible=XXXX) + Property=: Other properties to be added to the User like (Responsible=XXX) Options:: diff --git a/docs/source/AdministratorGuide/CommandReference/dirac-admin-ce-info.rst b/docs/source/AdministratorGuide/CommandReference/dirac-admin-ce-info.rst index 764ca3944f6..9e946a8927a 100644 --- a/docs/source/AdministratorGuide/CommandReference/dirac-admin-ce-info.rst +++ b/docs/source/AdministratorGuide/CommandReference/dirac-admin-ce-info.rst @@ -14,10 +14,6 @@ Arguments:: CE: Name of the CE -Options:: - - -G --Grid : Define the Grid where to look (Default: LCG) - Example:: diff --git a/docs/source/AdministratorGuide/CommandReference/dirac-proxy-init.rst b/docs/source/AdministratorGuide/CommandReference/dirac-proxy-init.rst index 4fca413987c..8f321187ee6 100644 --- a/docs/source/AdministratorGuide/CommandReference/dirac-proxy-init.rst +++ b/docs/source/AdministratorGuide/CommandReference/dirac-proxy-init.rst @@ -26,7 +26,6 @@ Options:: -r --rfc : Create an RFC proxy, true by default, deprecated flag -L --legacy : Create a legacy non-RFC proxy -U --upload : Upload a long lived proxy to the ProxyManager - -P --uploadPilot : Upload a long lived pilot proxy to the ProxyManager -M --VOMS : Add voms extension Example:: diff --git a/docs/source/AdministratorGuide/CommandReference/index.rst b/docs/source/AdministratorGuide/CommandReference/index.rst index 527159c8566..2c9375946d2 100644 --- a/docs/source/AdministratorGuide/CommandReference/index.rst +++ b/docs/source/AdministratorGuide/CommandReference/index.rst @@ -99,6 +99,7 @@ Managing DIRAC installation: dirac-install-component dirac-install-db dirac-install-web-portal + dirac-install-tornado-service dirac-install dirac-install-extension dirac-uninstall-component diff --git a/docs/source/AdministratorGuide/ServerInstallations/InstallingDiracServer.rst b/docs/source/AdministratorGuide/ServerInstallations/InstallingDiracServer.rst index a7981bfcb84..3a10289c1ed 100644 --- a/docs/source/AdministratorGuide/ServerInstallations/InstallingDiracServer.rst +++ b/docs/source/AdministratorGuide/ServerInstallations/InstallingDiracServer.rst @@ -197,6 +197,8 @@ Couple notes: * SAN in your certificates: if you are contacting a machine using its aliases, make sure that all the aliases are in the SubjectAlternativeName (SAN) field of the certificates * FQDN in the configuration: SAN normally contains only FQDN, so make sure you use the FQDN in the CS as well (e.g. ``mymachine.cern.ch`` and not ``mymachine``) +.. _using_own_CA: + ----------------- Using your own CA ----------------- diff --git a/docs/source/DeveloperGuide/Internals/Core/Authentification.rst b/docs/source/DeveloperGuide/Internals/Core/Authentification.rst deleted file mode 100644 index 8790b08e272..00000000000 --- a/docs/source/DeveloperGuide/Internals/Core/Authentification.rst +++ /dev/null @@ -1,55 +0,0 @@ -================================ -Authentication and authorization -================================ -When a client calls a service, he needs to be identified. If a client opens a connection a :py:class:`~DIRAC.Core.DISET.private.Transports.BaseTransport` object is created then the service use the handshake to read certificates, extract informations and store them in a dictionnary so you can use these informations easily. Here an example of possible dictionnary:: - - { - 'DN': '/C=ch/O=DIRAC/[...]', - 'group': 'devGroup', - 'CN': u'ciuser', - 'x509Chain': , - 'isLimitedProxy': False, - 'isProxy': True - } - - -When connection is opened and handshake is done, the service calls the :py:class:`~DIRAC.Core.DISET.AuthManager` and gave him this dictionnary in argument to check the authorizations. More generally you can get this dictionnary with :py:meth:`BaseTransport.getConnectingCredentials `. - - -All procedure have a list of required properties and user may have at least one propertie to execute the procedure. Be careful, properties are associated with groups, not directly with users! - - -There is two main way to define required properties: - -- "Hardcoded" way: Directly in the code, in your request handler you can write ```auth_yourMethodName = listOfProperties```. It can be useful for development or to provide default values. -- Via the configuration system at ```/DIRAC/Systems/(SystemName)/(InstanceName)/Services/(ServiceName)/Authorization/(methodName)```, if you have also define hardcoded properties, hardcoded properties will be ignored. (you can see the administrator guide for more informations) - -A complete list of properties is available in the administrator guide. -If you don't want to define specific properties you can use "authenticated", "any" and "all". - -- "authenticated" allow all users registered in the configuration system to use the procedure (```/DIRAC/Registry/Users```). -- "any" and "all" have the same effect, everyone can call the procedure. It can be dangerous if you allow non-secured connections. - -You also have to define properties for groups of users in the configuration system at ```/DIRAC/Registry/Groups/(groupName)/Properties```. - - -*********** -AuthManager -*********** -AuthManager.authQuery() returns boolean so it is easy to use, you just have to provide a method you want to call, and credDic. It's easy to use but you have to instantiate correctly the AuthManager. For initialization you need the complete path of your service, to get it you may use the PathFinder:: - - from DIRAC.ConfigurationSystem.Client import PathFinder - from DIRAC.Core.DISET.AuthManager import AuthManager - authManager = AuthManager( "%s/Authorization" % PathFinder.getServiceSection("Framework/someService") ) - authManager.authQuery( csAuthPath, credDict, hardcodedMethodAuth ) #return boolean - # csAuthPath is the name of method for RPC or 'typeOfCall/method' - # credDict came from BaseTransport.getConnectingCredentials() - # hardcodedMethodAuth is optional - -To determine if a query can be authorized or not the AuthManager extract valid properties for a given method. -First AuthManager try to get it from gConfig, then try to get it from hardcoded list (hardcodedMethodAuth) in your service and if nothing was found get default properties from gConfig. - -AuthManager also extract properties from user with credential dictionnary and configuration system to check if properties matches. So you don't have to extract properties by yourself, but if needed you can use :py:class:`DIRAC.Core.Security.CS.getPropertiesForGroup()` - - -You can also read `Components authentication and authorization <./componentsAuthNandAuthZ.html>`_ for informations about client-side authentication. \ No newline at end of file diff --git a/docs/source/DeveloperGuide/Internals/Core/ClientServer.rst b/docs/source/DeveloperGuide/Internals/Core/ClientServer.rst index 311d227181d..3b5b5580e5b 100644 --- a/docs/source/DeveloperGuide/Internals/Core/ClientServer.rst +++ b/docs/source/DeveloperGuide/Internals/Core/ClientServer.rst @@ -168,6 +168,7 @@ Complete path of packages are not on the diagram for readability: You can see that the client sends a proposalTuple, proposalTuple contain (service, setup, ClientVO) then (typeOfCall, method) and finaly extra-credentials. e.g.:: + (('Framework/serviceName', 'DeveloperSetup', 'unknown'), ('RPC', 'methodName'), '') diff --git a/docs/source/DeveloperGuide/ConfigurationSystem.rst b/docs/source/DeveloperGuide/Systems/ConfigurationSystem.rst similarity index 100% rename from docs/source/DeveloperGuide/ConfigurationSystem.rst rename to docs/source/DeveloperGuide/Systems/ConfigurationSystem.rst diff --git a/docs/source/DeveloperGuide/Systems/index.rst b/docs/source/DeveloperGuide/Systems/index.rst index 22f7778ba7b..6dbc1ef892c 100644 --- a/docs/source/DeveloperGuide/Systems/index.rst +++ b/docs/source/DeveloperGuide/Systems/index.rst @@ -9,6 +9,7 @@ Here the reader can find technical documentation for developing DIRAC systems .. toctree:: :maxdepth: 2 + ConfigurationSystem Framework/index Transformation/index Monitoring/index diff --git a/docs/source/DeveloperGuide/TornadoServices/clientservice.png b/docs/source/DeveloperGuide/TornadoServices/clientservice.png new file mode 100644 index 0000000000000000000000000000000000000000..0b8e8e20ca6b8c667d4eb0f6cfbd132b8da6c56a GIT binary patch literal 24594 zcmbrm2Ut^C8!e17iVaW^5J6B85b3>xih>mB0#bssAidWBK~Yg@(nYFN0cnOJH7e43 zH$aH=7C=e@gp#`hO#A-t-v5?)o?(V0oSdAq_j=b_?|SoELrszL)TL8oWMq^|_wQbg*09zX z&)@3W2WysivTNLEnsi80>3oDriutLIkx0MZU6#SOR6n{lY?X|PU$`>M#@q6KE;?2} z)%zID48!TT2$Kl)E1qr9tzloJuTMAKdpDs@2x|3;e(+0>uzG6r2;tF329Gc^Ti}$wAufJ6lny*Z5r(><=U4+Q&Ym@-dR+>pj*yLW{Fu_gOo;Q>Z`epEB?`rHY+bvceFP?sqbe)4GvR7z!-Oc-Es_DE*1Nn(dTBm76UzKlE za)&Wyg>yX@p8rbs-l^pJlBc7GTxuiN!eu_;*f|Vk%G0&pX%4O;zuu}3b3Nn@TI&tH zY1f^kwnnP?$tNH6NzBz1&xzx3w*7)@jFrJmQ04GuN-{FzRi(SPb-aeKNvDmXAMO!v z3qRGGWPkhoXyg;Nxyond%EIRa_`Ff7!~U@`S^4>f_ES7U_T{v!VpSC{#@TLND8*)( zWmyM5djF9hq9CM4c~77qYl)bkPRzD4oi@E#(I!-lDO?+OpKf!N=-hDa*pjxIJ`>&w z2TS8?j`P(&NGq@#%a>f`<0p-$gPwD{u0c&+HAT3N{{om=$dsxCes>;+*Gy16QP z6zX#RwXIH2$5&r3NuqjgY8)|sCDc6kqTnTJrdD!!eOiY?wPJ0CVF-I!lyrphgCis+ zL1t~<)i7+XjVUAB}Gb!=g#ycW+2P_1W z#GTWMiq0aksec~6k+X--honYqugQI*atCzQgM`lr#MHC3wKV}4A%G|UC~emZaT=>^ z`XTQ&|2bDzy-)fP@$sDW++yM`o`I-d3Onn6y&Gp|$D?YmVxyvwUv`*Uw4JCqe*K7{ z!LggEr=8h5)c8u)-uVr+dApRV#BwXCtH)I_Mm0bGkHfr^fp$I)IyHMI@wE^XPMQC$ zzCQELafFzeSl+{}i`X+C?mS3Be{d5pt82g}$&b7jY?Kt)j*KjXw!iQCApfpv+x_-! zfAZt&Z7n|8&_YGHXSOhmenA{s&dIZD(~+pub4FSQ9T-Udl6twyeF3Jxg4Ou;tzA}N zh3`QEWMe*09pyMrdm4%LfS7jv`b=ic!Q zjMs|fRjsr^GUi8RY_LCXOGq$SS`uYFuoEJ-kj4Hy?RJIiBkybE(P=|kPbrx1Z`QF8 z7Kh7~9rnVQww&gA;w<#tvf?Rkil_2wYKrk02v6XLhx=cm6cx%))!q?h8d(BbKb_k4 zCYveu_C`(oV$x!Lx2DYGkaSuQZJb*hT`lKIv)ez>=U5xLgk zR-x$O?%w}8H)CW(wZe5{WhJuwCDr2YMEIlm+oOIP$21kGn9rU2?7L-Tf9zAzU|qWP z?Qe9pajC4828Jdkh2!Iw=XoRJDwAYunj6py^;pKRQ?!-8m;DDv(Yuu%i$mOoqS-AI zgOR)#-!=bm=5nrB;$nG@q%tR4jNbJajx{3r z4-y!8T8zUN;mwh#%TfwAtho6)YjFc*oSb~9o;~;KTZUGRcEtp!W6+W$x}xKGtu6w& zSXSmzXjO%BL=%#QFK-#@l%OpPh6_!~;~?)pI`n>!|Je}gX!50KF^EOBhPbz>nNAQw zlzVOE>9W1~eLnB|_xx48)t;ulP}s)aZg)D%LxzXv%zCrRGaVD9HtI8!;1|UlK5Ff) zG@iEXijWA!Uek_$`}S7pQOOxfO8ZiK!_W1Eh2Vk#k6&LQL;g25HUp2Nt+$ul9r2XP z$8cB{-`@rrZ!wkbg-LQq_s(QD23)SRKhNB1wp?iBZEeBA8hl%n`lw@iug;A@23KBV zg25=dM(P1)#P(}Lhm&!4a90LO5b_SMb8nXK*rO$z8UzqjL&Sh1GL~8#48<(X!$0@j zuXhcg1PnjbqJ}?Isd*L%eJJe5PC@3ds7`ya?OMNd1R^en(U-W}Xg@rf;efCme1j<% z>r<$(ti$p^{5O^~^Yl1uU`c}YnxeWv9J?QFqTI;m@|5?&q$Kl_bR!e*=Ld9nXO46h z*1tOE$-ySWvjLYQ?vz%>*UsA5_x74pY%g70gmCHY)#1?#qgcl1Q}Zax@oU361=SrZ zw(|7xmTz#eF}sGVf`6dhv;(4ln&JuO9ox&q2bT+%tN3_DaLWTPS@S(Sn$X}(oLO-^ z5u^K`6PYM*)Mror3gU8K9;3dlG`%`yT=v8&Xe2FF zq~h|2T_kQmSfUgkNVqwS4Euw29vi@;3;$~|7c7LYOilm#@w(HFNmT$gY_D)BDpcid zUu4zN{3;e<9?n9X9IsDeV>7X?`QeeE*=cx{$3#|-9nw$hx3f9cLh+KXM%elL^K%|0 z@dObVhwg1DyGKlt?qMd3m1i!wS-p4^SLyZf`_vOTuX1C0k*HYwMk5Ogr?ZAt&_f%~ zMHiLPS^-4t5OQS7NM(C4;_H8|D7G*C2C0MTjf;-eE57dSwa(^Zi;Yt6{J{E|msZ%) zXd0q;BB@!k=M#RKzKnCLQ;ryCP(VELxpYnU{?M@R@Vz+3Mt~0Y74~w)Hv4Ua+;;^0 z>ojjm^>DI8>E4%YEOa(jOBtPT`=xts!S;dFOQvr0i-X#X3us%SCGT#uB;X5XIs8+uRs%`eva?;m1 zIMS=7x3`z!TXRW9?@E&qrgercDKUxCTyk*4`cK>f5mmLZoEM^_ub&q#2hks+i9B9= zi|_h%en3^+D}qWh1tc{W`#1$b5xIA7*mKEfwASyV0dLwYHzmSjbavKITMHuTHXGQ} zQ@FZ15OS%X_>E$@Sbjah(SfHl5^1AU>QOjx%!g9(m^-6{d7dth+>X;zuSHKxW}aNd zfP{bO!ZmkBA?U3a-?x{;_57Xq^mx4EukWt+#wCc>ph49ujrdWkPLtQyK~+(YmX|lQ zN39n0J=#+S_pSH#5NSn)O}yjL#b6h^jI*T%w{15007mOKqVWhbbotoqiVVK%J9YV-G%!aW8z^bGhF*9 z723TL;Ws_&MC2X(MwH+u^2Wg#LQl<(2h6qd(YLO@7LQ_Dz)YFp-^pa%xf5rrYTj^W zbE(QLesJk$L{G8#4-8Q@UOb8))|UL^aS*RsOk|W5+Q-&VUEMRjmxFz;e&{^gxaxPT z0PK9=<;u6OgzlJwL-~;%5T`a~L?e~~dmoS)CBSRKQ9?f>0HY!uju~N9#QFiyB2(~s zbjUX>0{sC zug~z&QbM41OytLYJrxoD2g`ht(6%@1h>T$jYHL%KSUOK3pL?A)YF>#=JyG108{s+m z_0oHr(!WS4c=@P>prw^n>$xeaYv^xpZz_?F-Mq8oYaU<2N-@x-llL(7XysX}aqRk? zVGF!V>y^exSP)=Ej+LTx;c$QL&}$DP-zecSB450C(IB{e>5QGd_Qd!8oBcMl$F;tt zx`?OnQZd7`9f=+22R5C1xuP(MHzOiVV{K5x+>lqklGWcwu(Y)N-#CKNe9x$sapfNU znG@fyQk;cb2nxTGxx(Z^wxY=1mOR^Qf0jxjV3U;w>c}d=$9-*F;g7Mm^2N-xi-F8xL1~C)1rhOy+VHe(T`4WD>9B zuFycK-576CjZ-&Scpu=w_{rdIm zi_`QsUx1Gp#i1S>84y6$+@#VUuC1j7(B{|N90Lsv!gCq2yV@GZ*O@F+^&mlXbad3b zF^uQt&CecM^pwI>`^OCJc=I|A;xkuJwFLgKt2v5;L-k$ig*P-Ci^Ch64}CV4WWc`Y z?+ZG^U?!SSUuM<*zCT@&dUy55j~}vr1SScWjMUV}ZSMrMvsAk!I7Ll(#tuIlQEqmJ zF3o&8x3sg>N*&M!Ga2vv zN}rQYUfBy!eOHs;?dj>s$HUXl868qxGTS(>VmmcX80ws16tT&0)oM$22xPjo6LAtz z7&EJQ_pXjD)Px7&I9QN`vMXE3C@3fpK3z$9a5rnG)8(u>604Rf7k-b`4Hb4~t0m?7 zTd^=QO4$zyZ@0;Eaw>%(;KaT0@o_8`dpq#>Tw)&vV^Wr3*=2ut)A6YYv+?ozU~nIN zot&IBG`bQTYjZBd#>UF7{Gd6a7ZE~;fVsEttxmO`rsIAU60*L&j_t{89V)eTS9|~N z-Osdpr)As9HO2RD`g2s^gqH8l`bx+rmCUKNoJmI)+%zmdVk1g(>|a*c;!muWQ%D{S**YJu*Ii0!}T>>K^}o z$|q{c+Cg9_NBX3|S#IS!~sqFE;G%m2n+57lq*5^o* z);d`5|JY6)Q2eKN{5MVn9o*bs@_4PT?0Ma?1L~sxe_s?_GLVr)D?tBjX z=X`QW>`kTeBm3{9APB3{Q+a@DZz5ETosCV2v zixM|(++bx*FDY67`jk9fA@uWh#_C2OwJfNQCJ1P#);qAuvei>q6~iuExByfiv?ZFe zM1qXW35r0*9`oT)Pu5wjudf$8R#^SO)YR0;sb~)`_yJSym=YNmUulCEL%a1pJ9fI3 zxVLR9J-1MZf@Yf0{@5R=?6A}_?ZE2xmk(IXlS#-1ApRhjX?CdOO zlKvqIsfYLaCLqCpCn+2Erira-n`jTSCt4U|C%P>#Qh8Uz1S1>o|-Z;0Ayxc3ACHf|I6Fz_=hGqIgTko;}wAr>% zozn9eEgq^07jJJ_SpGV<(7mizxp9G3(2r1HGmx+kJ|;riTOm--Op`(RGsm+%&mud2 z&*Ji!m_03w;2bo5pk$3QmxU+Z)ZQw|W4RP{=B-isUvzzYDT3|=(DKX<*Luto$Cz(IibH{K~97cc64t7eq>~3B`O7*I?R87M77Yg zLbdf?qjbH}*+9%-aBsHQV#L4=1F4wozh3e(ow#sUj_^T*XM1A9T{b&BU;6K7N7ZtT z1g_HTR%ht!1wzmozB?szxEQ+Z)qYwkrd*aID-;&F>ZC#q>NWVm4+B5Gk{~A4zb%=T z=J>HcnM2uk$E#;f9ncK`sj}$f``B!XznkJfzJG1_Vd)B8^)LSXZQI1Tg_ef$8}`xs zpZkNMRp-Qs6Ppt(M6c;KM`E4(@@UP<%8HoLJwc4$MiOBFph$`z9`)+g32N%_U%!5h z)de{Ympy<9^HAC1DaSm(dGT*!&MXJ9_+Jz-`?0%$DXf2K(~c zQ0=@*Gq*v>4RP_G{rwLOYn>+)B%Z~^#W6hydjx?%w4o_~0kgy=F#Ld5$un@bz;b7E zT-_*Cm%UEd5o{G z?@m($Yq|Zfu9lXTG0X0JULiAl$GIZ!RcNSjwbzQe+-?T4Z~#j|&Fnk=<799`vvXur z6l`OOM=^GSxQ!v!dLRyPJJu+D1f}|3*m~rPGo7s<;1egCUgxbZ464i6X=uD1@r{m+ zWfpgO-1kva*as|-TDtB>0v{x!F82HBX+kP1DqgEtmtW|#9r;om$;Zu|DC_5=qZ3Vn zPpcH4`J7WUewSHd%J1B{Q@gXO!YHF}i&qS1&R@M?;=A&qts=i>!R+$x#TK?!Q*b@5 zUGXTG1mrE7c|I$_!Jz2zw+#&3gz@`rbl`rW|IiMZDLNX9F{A`w!_pFaz-@p;~#H}zZ`60vAHo1=dB?hi#4 zKOMwh=kgWmSnLZfN(>SaS9K(qF-6L=093^ORZZu{^>w$3dr!L4>RE`Jjexyc*C4)p z(Qi(X_SUk+)2SE#V|C;ZmP!-+yu7}QgeA(r@I9F-$BrEvurafT7G5bo2CAp&)b{mC z%0m*ffWo@Ey6*1mOn0Uf7Bs}gd3$@eRCRZEtHklj_-;CRdv8UsDl)^c#haU(Hd`NC za>594qxD;g8CQIKQbAE{Qg_8V}j(KY%(jwRF3{?Prnc zfQ_L0ybexk`^y(`9Qs=z1$?btOmL7ML4E2}+Mj?kTHFTtiLg>9?(G&L-+298LRa;d zgInAR-oMPIUl}^~;ll?S4z=jHLO-ZUJgEN-KYn=m_<#Z;%f^{1^3JJTJ#DbmuYFI#>TqAhxzo0+kJa{B0Jp5 z!uTf7-|*M)xc+X^RnPloTMr%Bp-A>r;!-w*OZu^bc=|ryYW~p9Y?Hu|;=4N890eBO zVsCb53USBE&_kjU8`Y=K?LuFjGsD`bC5Ny~1FGnqOdq3?lG3+t-*^nmd?zCm`}$v$ zZTb=1WqKGr4F_$0&C~hiCe8v81*L9~-|@HXWfnxnMbEL6=XyJ`6Tks#WNrYudRN(Y zgr`(<{eA4Ny5s0Qxg!8%>|fB*P^s(=AJ1IDmYl(Hw4k%=hjnuC4MzVE*ajCZVbMp)22V%{W zIlJ=-b5-&7ms9_G`1qUf6w#X6o^iltFV-JOLymuP)&Eiez81Ite+$5yb1$k^*Vy*O zVE}6IcV+|=Pm(?|RPzjwmq(x`QE~XSPB@!{8ez&=Hq<<^XPKdXUowWHqWnc*y$KV4 zJ`^RNP(xGr?ko0k7k0{VX@NSkq8{tz`T}Lw(b>tz#pU{}p`nEph|I|@=<zC~i~LTxc=;{IsyJ@%hQ5;9z);d}s2v z)>uldatU9rI~rY-RvprhGMDF#x`!P%sHq&L=jYS3$9A__DmF$vv`qO5v*cfnfGv;` z?UK{Kxq>vhudg3GXhK}P^C_vop#12l#8U)8zYWX%vDxlHk4Pw?65>i&CMOGu*M z+f^;-mLH)+Ab};$ZP55?Zx*gg0JgjOEs&M!@+J7*PISBw%ww7PQ*>(uVS`%A!!9kO zYMH7Oiq{HCS7eF19?u};qdZ2yv9hxIwYH{cmN7_`O^j4{4hc37RHWSwyq0l^KwG;g zN~B~|F#htJzM|M`XQhK$=0aw)ddjGLPKJvt+={0aC%Ebh#>S9imW>8Y%W*2 zom5t;n296abN$t=oBzJq)LsCV#3kwW3$hI=tyEpthLqD!c>>68^AITOoA5r6E9!%^1J;l#hpFN z%wS&C0Qz-)sfB*DETJY;XuAdmA6Cc@^jfno-y%-&b>|zx=6vZLw%Z_9e7H)kZeU2l3CV3&t ze~MdZXeMR3z^E;`hTDH<5P^v=50f8;($I20&SY)dmY2oOk;n?H7@++=>GE!46B38-9 zZ9OzO`N1_-zo;mR+$!P(OEHguaOsyznDM#EfQbh5(&FN1R_I{0cQMehpUWhLn75=HHgt*QltZl%cWt#u1@VqyzeQf+Ie{=jC%@b>;ayWB0w%?C1oe=b+JFm3$>r?wU-kQcaqHC>ldvc5-$;tT8 zUi)Epy@0tlWv_2)b44kL>cOi}dl~Q5W&4r5>YaEoOi0xf9UM34`L4!XR8)a?A6?Ax zaL|iWcXC3+^2o6Gi?;xiiQ8WsQ&~EwXCm|SZ1U?M6#0QG$T*AzmM$Ht9j`&sL^UFp66+}PraAptisrDtQ|ai_`N`L%bQ)m8P~UMv@U(AL&=f&S4rH@6pb z`n=U$;q#ey`83;aQGYj`ItPS@%M*m-3FHg@cvT=%$5{kTR6XgSZeOI(4Cg}ca(1ie z2z9YKY&{^o^gDKeA^yr6*CthGU|S?4zCg(ui%GDiMciQXd2IuH*ATOPLydzVdNui~ z>s^|sXpx`dwh{8<$EMCsbw8)Yr{oknySpmN%7^6` zys+$fUV)X#L>nMUmR+g0ht9{3Ac3FJbmRw1Jpc9(pbp0~6`tprBUpg+0c^39t+}}w zcqwfT%Fd7+BU8$W&e z6d4)0Hq)topd<)Vrj_h~GDO#y{ug>ssRtPMtm-7#N8294RuD2X-f~(U;WfkdTnJ_V$OX;CCBz zWoaZZSPLKHS9^MUX_`q6WDy37Yt6SWa-He;`BUG~(Q&HITTDzW=`zq1^yki*iYD}* zugL52FT()SCR18icpVrhLqhLWvxR}43?Cx_N9tcl&_tG!7y}UO6W#uGQ zJqPNs+;-4mxXgZN;{NM_?S~2>7mK}C%gWN@d5$@v^$3Pm6hj*%L|z*0O^*JGTg058oBim zv%m#b6Sqn2NW~P{{#KXwd+!IXXD}Vl&A37nBm_DGaO&jAlaGoXKMg|}8iw-0?kOnv z`1p9t^;j+rl`2-er-Pr3z35t|{5$JWSQDAkt(J?J2+hQx)ki zx$m|x+mJb5*4oW-zZ&>sZ}5+Gxf*YJ%JU=-qAm*QUfwn>eR`dt z-CSkbw_YCOGlm3VTU877ioKWq zMUU(5kv_mztdg1~KGLJZSAWNUEpItqHOprO)*zt3(^h**U%*sj-ts9kUg7f-eB@^< z>F9wQK7b+5*Do_PUMvMyvMpI>0m?Ur>0RhuJ_3H95db7P?zls-VR@MK@WAl7ag7S4 zUmHLYJ;sYz<0;#5{R@lTtX@+sAHvT55&YmNk^&S5RNLs(&4L2n_5`ueH#S@Ay#mCo zPMS8XEl!!O@7_Iii92WV(ZsP0GAgTmf1K*`E%nmSwV7-SDXv<-Twb$EF9 z8--&Onu;>l%dK=ah4yYDFuNmPU8q`g~Pn*|lu!>Y{U@3#QWhpy9VQm%V}ZhXn*F zPBh)eL`TEc`4sU!+lzZ3h-GBVmHd4?X2^w!XQ=Yw4LwY$SK1-(K7ZtI-hGqhcDC%G zKUa|hx)Ssb2cQ0Z@3pg`@Xg!{I*~8p>q7oOZ(H)MBLqI8t_MwD!h>Jx9E) z?n6K$EL`0OP?CA$J1MgEZPSfxXm`D)5^I_0)wy2Sec)UnV-1D34`?~aqPRjkm(YPR ztALb^BH_TSr(pwa@+YNjvNvX5mXurtyfiK`(Rl+sc$?^AQ3!bWBzH`>#nXAv`qk)y=0269m1s6 zOi!m4G55viRkUDT|`c92RB8B^pTA-RC&hUA1c<`f;hg~9S9-@mhQK-YP>VAG8(9C?g- z;-5RD6C{L%98-w6$9RwtvHRc4hjj5^uJTj=!E1 zWn-NfO-swp*cpWl=Nz8$SA*R?9f9rW=;()ey7s8g^>(7Y;Q;LUGf-Se1aj=@tMrIV=~E(-se22OwW zcdLzW^G6nCTl)Ui2B3kMOh>*9*VLegOIF^C@#z{Xr9Kd33=6B(%%x|TOGuQ~Kzjms zbCFER?IoQa=*h0zh4^s`dP)#iOotg zA~;MB9wGJV#Vb*3)dmzH4=ML%eA}2wwXALQL;Gm=yskDbXH^_T1VHN^s0RjpN`udV z@V3XExZEmMD6m+bAFNNN)9a+%h;xYMX#|DSr>fN!^2++4kMq(V14C;zrOo3sdMth= zy~sV0TWDx^+GZp3g2jqN(NK1~K^|m^;=2JX11lnXXK*~qic6qRFYk$;=<)$j)9_ zwgCeix9nNnnQuZKa(4!YUYEp_;GaLTE$;EYQ&8_?P*re~CBSw?0tsRaP-@B!tUGg> z%}Oz1AX(40l}3*Z1Js;Q=Z_|nW|1vf?iHi{mhOTp6sPCOjq^d1&T+hR*P422hKEr zi)imXK3`yLZi%mOnd)M6tiCF?p}9NrbzbkEM_jo}TUO#xGhk*pLF1Ay)1{mDf@kQV zR|nteb+Y&~*7zN|?_bvJ6!~Gc)S9JGY4Cv!$KWHn{^RpoYuXMq{qWwbZ8jvd{kK}u zQGnaQ9j=dYc~&b=i^`<6$|}RVK!rMGXaM{EsZ<}e`P|5sy5wOih+>Vq`ftr$CM^7Z%xHT97csY@wo>XJuvOjubgL>z)kfiH3&@MTg4Nr^PSef@BS7>K}=p z$6&WW9;*AC+K4AuJxF{(B9l9wdkQoX9*PD3Ta@lCk)*5TO_Th(%Bm_oewgAMf#NRskwXlNicqK2y~D!{3L77A$K z^$qk|2m*KGFW(|i28x6e{n_|j-4@#*RgbDX7D0+^u}THn9LZ+YnOx&Gr*-@GQy*bQ zYALfHuj+&8ZB;<`Z@Fq&RzYCjCZO>vYQ#>Ak4N*Nw{Z(%ASD%CkQg1^SMF%Cg4+)f z+2K5$2Rx|HoB{R?Hy4+b@8<6!)4EUbHKlbeq2Gb7FEwP}m!^sPuD>b1l!~;D_Kwzo zY}d<|FA1BQs_*!3V^L{bCg9eB)=or3L~?SnaitqL-~7F$g~7tON3baN2cR`*p=b+r zTr3*j_4oHDb+lSRTEmKvExCSZ{Zp&`3@|d9clNwI4T2hvzmfqPwUOvq@mH=uE&V9i z6#Os{LGpjlx`(2!(LabFk*c5du578MYOcy)xa7{a>UCDoo&lgDo{g~*Ugq&n@iz{S zVXk>MOCIp;u-QF7x~y0JiWaFC`K8t)Eu4v@fP+L~q#%#Z;MIOBm(UJ722YR(AZXTj zQamH!8mPQqOIIgmFrVC(bG&w1vSfsICkBpF{ya<}9m>k*uFTo=;U;t5jb&HWQ+6yU z{?ec`&ZOpl&xl#cp=M=h#y$et?QDTV6Q)us-P>;JP_c$s04*H~dSDL<3roafNYSZ3 z#pvPx6&@j}zCSZZj$EKp{FUrGaJ}}sG5-i70lnK-$i&M_RbuAZkthKS1!Y-T*}bVa zlPnXH@F@Z>|tO`RdqpS3? z2E|64>TrW5EcEo{U|AaE>nkg_q}~oJVRONQ#x6@UNyvYsMaY5Xz3L|#8X9$o<=)Pg z;9^H>YhKNpP|)260h*!6$;nC0bcL%70Q$SS&XF^V+PwoP6&E*n&`5VzMc)fF;MJa| zx-1DgjG)ihWC!770Zg?mUZ_+NZ1)CIMgZREw6!{gLRghs_cja~V#@7=pMe;jxz zr>LmF0123UiSwxS2al(wrUEM~=;cdhNw@nj{ix;(%*=t^^5N@J7Z6Iq%hsNczHu z?fvcT`2bO>GDp;ft$Xege}R2gQBk41*O#N6Z&Gdr6D7#_`Jq{0cz4TgJRe}fQ9N3fFKAeN`LL>xhHK7dhKqAkH(AVC%S zS4#}nJl=NTll5Q$I#VT1A@ow|<8MbZ`426%x|aM~`uZ0Syp-p*vo6nrg7_YNp2VUM z<4K^wox5zq(IV)n*J^BRY%c&Z>`3IQC!SjRcfj7a`tR*f#EaWA@Xl)u7+@`Cr}$Tb z9y+Le%44t2vC{EDKB2iKB(`d|m(szqGsOouQ5uJVP$@|I7Pr*}QByODs+RUM%lNdt z7s>nhaTWw6K>SIob$54nV;N(TiJF<3+S=M`2Qd-j(B+AVbpN}Ki@!j(7bGuYCR+1v z0p&>0u57X0)7I9`{B1%c#2H*G=LY4k&mrpV+f?5nbx$(DC60_;Dq$kQ*p}M?jL~uH zrw+}J*1&-83fxyvic%sY(~RW@L3GkkwjYDA3SX6iaoP=hLhZN9KV9AdIXkVOrl$Me zd7y)DJ6Hg8Ac%*Byn3~{HIo9&O1ZARN_7WrrkRxauFJr#*}nPsAZx3^$;XGBiZ!(0 zg#x2~Hgfu4B%^hWywUK#I~ zo4pVCVQK0hWah-24FFWJ64uGH0tAx<#Y%R<^NF&(uhh-{QcWWs$*8`^{J`a+FF3m z>`_X{>X!G0!>36$q&W>e{YDdO_^Wcz--vIYwrq{Pr>MAK2mk%+HXKd_dFLp-csY1& zb@caBP$Ymb08ao7@}z1(4r-z?ynG^Li19@56bNm`g8&a0s$6fi|K$CmhJ?`k(tx{Ql9{7!DHGz%~ciS!Tz#hLaDXvR0YQ)UUFMAD_K?^(vq-Jr1s*aXQ_e z0MaxaVM3q}0P04PsJ+2pfibQ%?^8uRIHT?N2W&S{KJ5GV@4&uLymzmnsw(Kk3mIIW zHlf!+Osu@8N6Y@Tm&Zgy=)>0NYQez4Rj{A(Qu_5aLLT~C@%?e^SbRHlclD;NgEL`4g;32NER$AAxaKtTYcFNFoUy;KECqG6d`w}TLf$Hemvdn)Y4 z@5XTHQ87z{Bur<2p58*7-XqswSoh^w_I*kafM|9GI}*%z&e`$X^nGl>_=0O_U>gs1 zIu4JAfY58@1Mtzfej*pGN~WZ?!JT<+Z}P?IcJ@f@d%e-%rCVa(_G5Nu_!QmT)kLL&v%3Vg0pJ|2S9E$-hFI1><(ORz4)ST^4z|`6FmBnE)12Ybc`T)MR5Iqdp zeKTj9DvRDu}CW9+BbXe7!{0)R)@MOlCtK-@*cVyd33d zK1BRFI~u;TlR(H-0$DjAoizRct3Ke8e0)n`^9m{*k0ev8&Y6OY(m}JikkvE#s$vTa%=}x@!qdmZ-^kz+qz0N@z zVuW*@F;@a!gXzY8s&nZ(IdoheKz^jaz0!4%L**=9$ znZtVLswL6)+W)=n;6on5uZ@HL$=MExsZIAZ%+$BGSZC5J<4+ugQlFSVM44gO;=kXt_t1g+UnSuO1! z11fFjI{|r4kV|W6YP!M0BjfZdK=M%`KO)ZX>G(M4$N>>~;k^C9DPc*$!bw*bm%gp? zHeCNIz}(!ae?-PlQFPb3{nb#60`GkB>pu9qJ*P>DwXTee3)w8O`AOG$mXF^T(@m!Ss9sv0$JNg1_)#BI{1V1I><|a;9_sq z1ML7jG(GPb zN=n?yN<6>mv&wl0cIP!m@`*2REc(LN|1)Ek$uA&Kd@q=8K4=ScY5!JbBQHq4(JB$dCwooT+S|&;81vJb@@YhExa9xw^G5YWbi~_$BQrQpOuW z!JmQ27F%LpgDEZCIi7FRt!M?}dcdrsO#MQY!`!PJE~UkI&#Ej$>R)Zd4V#oIA!#2@ zwUmJy?CB_D6Lo?jNM(aoA`|*h+bn_UZ|hDIyV3mN>?D@WtFEW_wb>bSlF^oW#mE@+ z6vIJJ(bzp}wdwXD0X&0U2M8nPwu=N&yZu%piH>&M0zKBOLA{~X{J)!w`)Ro*9r(V< z3fRYfL;tU!HK~XnrmvZX|9>fm4ZOs~IVT4_(-sd1sRnH2(6q76bE6^ka|4)g=6E1( zz>Q;%%E6Dre|+2)<~a8K1(=xt#03OGx%v3?L~n3$B}jX_fM)ouTLBi6nKrS1sWA8M z)a61jrP7dJJ~}@(=G@9tZilXX$jyK4+IKWshMj#Hm=BWu1-_@6n=+~ahXAI+=WPH5 zI4*9zbui|#y}1e0hKfkv#Pb6Imu~@qi~{q*Lg=rV7`0l(DOm6C&Hw9dU~$&!P)rO-o)d;%vn(0 z{plC<5Dh9YaNy?cy>k;_OtOIaOB??~vA4Kphho2a^>*UJ9T?Er?e9exg`aeRC_qe{ z=cqTBBID`69!#rQZXAx+AioP)b?ESFaO#D#Hy9ZiePS;AOM=dTY!wiFUz{hnAPDF+NdNinXV0Dq3JNN#s7#IClZuH`Zh8GA3e?0~lr#(s@d*ielj=#5{gv*v zB7D6ls}w)oJ!Oap0tg_^twi8$UAifxS84?9w5hOzug_}cJ$&8F!A2*jXP9%%!~g>a*t`aoKja; zAG!{PrZnna5v}ER9r`{-5MtXrGc87|y$zP5w>|=#)lZ#&LDy=z4Wy9HAlBa7I)6xX z$szwcZA&T31Tgh9WIFyD*oi>^uk2AZMB+W+*i-xyKq4;zEnZmYc#zo6aBu{eycZyP z5`4MIK9%pd;ZkBd*kLEC$BO~oJ5_xEex$jlTv-l}QedmO=i0QkL13y=3D`>lFz_Lr zPT5v1v|;;Cw>39+^L816uL|*DLfQSxYx8&=fOng3gNzp@p}3mtzmaAWE?)fjC#DPQ zDdfL=a)2a>nv;iN4{;+f$^*paDmfx8Ep6DLYH_VoE^dc{#kZ@%)oN-3W2OgQ(Bg^T zCE&D?^U0F#xdhrTs@~ZT5}@5XAUX7qt40A)uD{rlT~t<8dP+S-*0|;t>J}RE$?w}Qx02*Js#y-?#2)A+6Ss+a6QKMZ`hD9BI2v{7b3n*^OnI)yj|9qt zj>kZ!;u0n{jxNu5>%xEsSTBo&EisI*HX7oJpWK2xUqHsu=bPRd_N4v)gC;f})L-u>e!AAllFzr7r6Y z7|Nh=8W^%#n~@{N2lTO;sBgk-(l5@Pe&iV{7fX0F^;$Mv*Kfb|aO%56CX>h$Ja4R1 zZ4(C@+c64?Z6gdeLXo)`KOyr`sP>{bTfYdQw}aX)#9O?#(B#G8-;$BtdO)aA(NkJ* zWM1(`atxGU9qrSo-^m1g(@+pQ|61p2zT5Ivv;9VPMg6CfA*4i%`7zS0?n@5%!cxp2 z=EM7cN5)T3k)BnXTDN{s>4?Suh7Z7%^qIc@T}Pze_qIyQwwLCW=12?nXQK|%FrWru zil^7bMsnn0z`!H(zGKK3b6_ogS~c|EH#rVx5AU-4Whk=zzozB>IW>giY5c#E#nMxj zYHDg=aN@F+Th84VE`W{!$`jR`VFIyi&tJUfnR@67G{my<@-Bh9a?X*9i%Te>y)~96 zI4o=cwFkmqAeRO3oug0?zUS-B`GU61A2bC3K?RXGJx^a;zaFS&j*SeZVrS`E!8ni* z=pumV#$4Rr+R3l|>%N`PaWzP_&@$J48=tNio-E;-Rb~OkhRhp6x(%j5?(Qf#x!g8Z zXb}qR8U0e5n9Ek5da%GM=WQ~@wc6)+Pu0v)k%~9j8TEh8_L#Ne1x@4tAuhWMc~hDf z%NfRe{`_a#LEc4CP=f46tG6m#XRzYtF;F<@o>=#0xi1bq+&2bB_Kkr$ZGC-laMB+? z-thF?C@Lz-PWDlT4=qkl>lzs3@|^2*0CPJw@sWDa-v+Y89akKJFrQr9vW$I^ntBC+ z`b4^zJWIz&YHE#SpC%?I0(jyFP+$yo0X$hddPw+aMQZeK!pqYcjPR%-gxJ2TMnUoD|nOF~j5nzPl(6 z%%s}&=f+f3!Ja&s{qdRx07@RR6-;=7ul#}UJ_IcjkL6MO?f^>!Xy>lP=s}vBn!5Zo zOwdB}zE*`G51=l$I8uR7t5_5%ETTt0b$EuN@dGqWJ}d=(6O5ot1kcQmrCY~W0UYVx zkEH<&wr^h9ZF#_0DU*m2vTTM%EHnJuIMx2kY~8A*`9p&+Qa)Hv;d)sIR%MUoru$ee>A{-hd-)I z$-N!;GR%h=5L?tNS|Iy&1jxUy-e16YNc@!JoSb*5Iwlm|m~S)6h@=Q0i0${npSwDd z1Tw)L4Yc{gqT##Ivs6uY&(61Fx6aoCd3|+KgM%;7P1u(Opu&)1ab-Z&Q+KK)Kq!i|ET56l9u*u+>TN_ncWXr;oZCIJ=DjA z;~+2GTOXmlca{3~KB43$9X7IIx*P&#WO|J^y>x8sT*j2R^no!hH!CG2l&1*7Lvpgs z>3VhuyJy2Mzm#116cF6#;DQf^s311|ByN@KYMLK6b}#9!pBO5gUsD(X`yB4@6RLI(l?cm_bZxvU-f|RBw2D$kY_VTX~F8m@R?t z&p3pWzEalU;{YzflVTKQ;>$oxujt}R3ahX_wU)7Y>(=90Muw0rhU5IceeA-5%=RoS z9O4p_k}g=iAVW62$SHp$aeGY?unL?7MRnC@i(qcPTVB4&Tl?#X4HU0!18iXflAe$- z^H-EOh>>I7U8SB@f=|-`RGAzeCPOA*N*m&wiQ}dCv+uI#F5tF+k`F06a0-=0ge7WK zR7$|ecP6Fw8C20UEw6lX{ezP^ijgy*9i1b3!UJ@HPz0y+32jJ$6VEJ{XT440vZ2$F zE5{E#`za#;c>x`!3JdKy0so(w&%uN0{z31nb$kE+4dhTz_&Y!ko+NDqoej6(8zZuh zzz7|_O{(Epnwv)hooOQTB+CFdg~^IoJkNTa5lr(T!E9I3_6y75-CwWjXoIGoKif9H z^+dQFa1clQ*0M5;6LHP@^5Otep}%PC`$_vz+Y3+$SbhwY>XkKOVjyuK)?NJ@WHB_> zVi(aC=;yK6D6B@|U#rpd!x@eCuixlNSfM-uCv;yck+3=6rQ5x%jWMTHAT->rfJ^FL zHifyy=_guyZB}+WLi;dwt;0ZfU$Ds+dwU=4>-es!Ay;R-Sl968;fKC%JwJR0@@#g~ z=PbiEl~)iTcm4RLOh@iP0)IyD`Y({td%FDjH^t~5KFY}0zt!^Z)#S&pRMH1(9=%LBOcja6PJgzJ77J1Ssl42_Rn*~8 zbJ8i-6I*V{U$>oj-!$#FW@_*z-tATSwvL6-+PLfMin0@? zzF{jaC47nz0s*Ef2EjrPy*+%|ZvTo-1hkJZSOys-=yjYgMno84tPv%7|L}ASV(F@Q zUk+rd)FFj>l__Gu@!kI~Bu67AT@QBbB(bEO>EN9uv|?Um8lAEFhYy66CR-Eqtc$VG znStdZAjDlA_K9zRkULhuMk}yN|Cel1wpL|jrCrmRd2=?VOc4>4vD@xJA=%)rin(`B zN^Rf`r0wYG0*s3dud~Q;lYY>ez!0j6e3ReP%bAXj+;{A`F)rsWhU+HP$kxuwb7@W0 zIGT2)pa6IA;?oa9_zGWq+gOy8qHZM}h)a3rJ0{WiIR$>PDeL|SF(su>G1lbJl(^#Mtm~+zL%;tllZ)!KGn67XIQ3Uxf^{*=>8)K84H5`ysDD9qYj4?6fHBSTsOBN*jR41;qm1q zhC@-PqhX!a6gz=WtjI4rADClSHThVe>ZlGr*fq>sOeVczpjTU#DT@RSnZ&tC_BR}| z?hEr3jIL9hId2_45jmKUFB2K_M>*k`7r1HW+pRWr`oL=GtDJ^|Yq{XV^MZvzgz*KK z{mpMxjqH4Qcmvwa{&b~1%v^S)R{Mi1+cF3!CFphok`)n zOhJ@5T1=FOtl*T*uqrt81eF|gVk1cwwKQ9BQvICeXjyk=7_XV{6O<;()OUZKXXVmD zp@bAD$Oah0$Zg2xf%J^y3rYh%5&!RBF`Jtb)LJ7;Z8A} zcVe5ZZdN~5|KZejdvchdcH!kBAEz{RddsuMMk{`Onxr$sj8zlQo6%84f4<*8f#pal z=&%j2xjnyDJM{r>&%?@7GmaYXFi`5hP_4rk$-5ek6_3q7``Wu$5|J_79j)S~42PqV zMVozk8i93E7?Sa569r(b2&2ze_B1=!)xg<@{U>ts6S)eRC(9KBX>tW!(2jTl2JxP0 zhEz$zMBCp`yx5d+un!WE?obiXWFvMpIFN_Epx^P z%drEZM{xu{%2>c~Pcdp|bCb)Q2-V3a|BmrXKV8Lh7Wz#0cG07(k6Oho;~j*W8ejQ+ z{SO72_Lg_^Z0hRJ1;>5t1{W76w{)3rgdXuyu~#=zR-q#t3G`BqC}LPhm;6F0T*Pp9 zeW4UGl47U5kYiq2xGK`kn^jXo*RPL^c-&1s8@Vh{HLgt%BXy zDG>r=aQ@>H<3z>0=77nnbjh^+sl?c|-+#n-*5;@zl$2|l?|qn|Zzczu$b)nI*BLE+ z72@_SW}7qeT)STHl)`&i(yXqp$DeRov_}u50S(sp<6nl@8xb?D#Jp2H2j#*p4uED! zKr8f>ISXAZ^EyK17m@&QfpdGiS16UazTg0Sa3XYBFijKZFdxdQCgj3dF#w>k20?4c zfT>(K@0Qq0-(67&{qG}XZB~qmqZ%|3$-l0QvRFp}KQ?quW5|AXtsbZqm!dOPBbdrrTj7}f8iKyR+KQ7?p3waS$f$9Iut@e|#v+n938x+QJQ zMUN+&N8(TD=r~L-zBo&pt8(@vu@gm`nTX(srH>rn4cxMd3No2&At7P?@fZzFPk4S@ zC8s@TkvBU#)Z82uT99=tb7@@F@{%0452@yeAX|Fm^5RrWqcwGQh&T1VVMH*~O+HI{zk1_&eIV4DkS69HU*KGX}6~5Lnj`wHFhG?Oem(J@Sn-D5I#<$qQBP-=3?EToQ(M&^=>uW zc%cWn6#1`SCSl$b|9RE<<15zN)eJiz!QIoO0YS+Vrm=UyK6C=E92Alk|K|mH5dH3f gW%J;=h#!s}&0?Po?E4cs$nTDt9Y0)bdgkhX0mUtHJpcdz literal 0 HcmV?d00001 diff --git a/docs/source/DeveloperGuide/TornadoServices/index.rst b/docs/source/DeveloperGuide/TornadoServices/index.rst index 7ec6b27aa16..36200cdbbd5 100644 --- a/docs/source/DeveloperGuide/TornadoServices/index.rst +++ b/docs/source/DeveloperGuide/TornadoServices/index.rst @@ -2,6 +2,9 @@ HTTPS Services with Tornado =========================== +.. contents:: + + ************ Presentation ************ @@ -227,3 +230,223 @@ Because for now Tornado does not have "Real" services, you must use some fakes s You need tornadoCredDict, diracCredDict, User, UserDirac to run tests. Each test explain how to configure in its docstring. The only service available is the Configuration/Server, it will work with HTTPS and DISET services who needs to load configuration with a Configuration/Server. + + + + + + +********************** +How to install Tornado +********************** + + +To install and run service on Tornado you should install DIRAC first. You can install DIRAC in the standard way. But for now you don't need to configure it and generate certificates. You just have to install DIRAC:: + + mkdir -p /opt/dirac + useradd dirac + chown dirac:dirac /opt/dirac + su - dirac + cd /opt/dirac + curl -O -L https://raw.githubusercontent.com/DIRACGrid/DIRAC/integration/Core/scripts/dirac-install.py + chmod +x dirac-install.py + ./dirac-install.py -r v6r20p4 -t server + +Once dirac installed you can add Tornado with the following steps: + + +Installing requirements +*********************** +To install and compile some elements used by Tornado you may install some packages with ``yum``: ``python-devel``, ``m2crypto``, ``gcc``. If you want to do some performance tests please check if ``nscd`` is running on your machine to avoid too many DNS query (on openstack, it is not enabled with SLC6). + +Then you need to install Tornado and M2Crypto (for python), but not from official repo:: + + pip --trusted-host=files.pythonhosted.org --trusted-host=pypi.org --trusted-host=pypi.python.org install git+https://gitlab.com/chaen/m2crypto.git@tmpUntilSwigUpdated + pip --trusted-host=files.pythonhosted.org --trusted-host=pypi.org --trusted-host=pypi.python.org install git+https://github.com/chaen/tornado.git@iostreamConfigurable + pip --trusted-host=files.pythonhosted.org --trusted-host=pypi.org --trusted-host=pypi.python.org install git+https://github.com/chaen/tornado_m2crypto.git + pip --trusted-host=files.pythonhosted.org --trusted-host=pypi.org --trusted-host=pypi.python.org install -r /opt/dirac/DIRAC/requirements.txt + + +Adding Tornado to DIRAC +*********************** + +Save the DIRAC folder somewhere then clone my GithHub repo, then switch to branch "stage_toDIRAC_clean". You can run the setup.py if ``DIRAC.Core.Tornado`` is not detected by python:: + + mv DIRAC DIRAC.old + git clone https://github.com/louisjdmartin/DIRAC.git + cd DIRAC + git checkout stage_toDIRAC_clean + python setup.py install + + + + +Generate Certificates +********************* +To use HTTPS your certificates must be generated using TLS standard, you can use following lines to generate them yourself:: + + bash + cd /tmp + git clone https://github.com/chaen/DIRAC.git + cd DIRAC + git checkout rel-v6r20_FEAT_correctCA + + export DEVROOT=/tmp + export SERVERINSTALLDIR=/opt/dirac/ + export CI_CONFIG=/tmp/DIRAC/tests/Jenkins/config/ci/ + + source /tmp/DIRAC/tests/Jenkins/utilities.sh + generateCA # automatic + generateCertificates 365 # Certificates copied to /opt/dirac/etc/grid-security + generateUserCredentials 365 # Certificates generated at /opt/dirac/user -> copy to .globus and rename them userkey.pem and usercert.pem + exit + + + +Configuration (server) +********************** +Like in DISET, check your iptable to open some ports if needed ! + +Configuration is mostly the same as before, you just have to define ``Protocol`` to ``HTTPS`` inside the Services and add a new Section for tornado. You can use this example:: + + LocalSite + { + Site = localhost + } + DIRAC + { + + Setup = DeveloperSetup + Setups + { + DeveloperSetup + { + Tornado = DevInstance + Framework = DevInstance + } + } + Security + { + UseServerCertificate=True + CertFile = /opt/dirac/etc/grid-security/hostcert.pem + KeyFile = /opt/dirac/etc/grid-security/hostkey.pem + } + } + + + LocalInstallation + { + Setup = DeveloperSetup + } + + + Systems + { + + Tornado + { + DevInstance + { + + Port = 4444 + } + } + + Framework + { + DevInstance + { + Databases + { + UserDB + { + Host = 127.0.0.1 #localhost + User = root + Password = + DBName = dirac + } + } + Services + { + User + { + # Use this handler to have a dummyService, can be used for testing without load a database + #HandlerPath = DIRAC/FrameworkSystem/Service/DummyTornadoHandler.py + Protocol = https + } + } + } + } + } + Registry + { + # [Add your registry entry, like in DISET] + } + + + + +Configuration (client) +********************** +Nothing change ! +Define your URL as DIRAC service, but use https instead of dips:: + + DIRAC + { + Setup = DeveloperSetup + Setups + { + DeveloperSetup + { + Framework = DevInstance + } + } + } + Systems + { + Framework + { + DevInstance + { + URLs + { + # DISET + #User = dips://server:9135/Framework/User + + #TORNADO + User = https://server:4444/Framework/User + } + } + } + } + + +Start the server +**************** + +To start the server you must define ``OPENSSL_ALLOW_PROXY_CERTS`` and run ``DIRAC/TornadoServices/Scripts/tornado-start-all.py`` (or ``tornado-start-CS.py`` if you try to run a configuration server):: + + OPENSSL_ALLOW_PROXY_CERTS=1 python /opt/dirac/DIRAC/TornadoServices/scripts/tornado-start-all.py + + +You can now run DIRAC services. You can check the docstring of tests file (``DIRAC/test/Integration/TornadoServices`` and ``DIRAC/TornadoServices/test``) to know how to run tests. + + + +Run performance tests +********************* +For performance test unset ``PYTHONOPTIMIZE`` if it is set in your environement:: + + unset PYTHONOPTIMIZE + + +Then you have to start some clients (adapt the port):: + + cd /opt/dirac/DIRAC/test/Integration/TornadoServices + multimech-run perf-test-ping -p 9000 -b 0.0.0.0 + +Modify first lines of ``DIRAC/TornadoServices/test/multi-mechanize/distributed-test.py`` and ``DIRAC/TornadoServices/test/multi-mechanize/plot-distributed-test.py`` (follow instruction of each files) + +On the server start ``DIRAC/test/Integration/TornadoServices/getCPUInfos`` (redirect output to a file) + +Run ``distributed-test.py [NameOfYourTest]`` at the end of execution, the command to plot is given. Before executing command, copy output of ``getCPUInfos`` on ``/tmp/results.txt`` (on your local machine). \ No newline at end of file diff --git a/docs/source/DeveloperGuide/TornadoServices/installTornado.rst b/docs/source/DeveloperGuide/TornadoServices/installTornado.rst deleted file mode 100644 index 6149e1cf0cc..00000000000 --- a/docs/source/DeveloperGuide/TornadoServices/installTornado.rst +++ /dev/null @@ -1,214 +0,0 @@ -********************** -How to install Tornado -********************** - - -To install and run service on Tornado you should install DIRAC first. You can install DIRAC in the standard way. But for now you don't need to configure it and generate certificates. You just have to install DIRAC:: - - mkdir -p /opt/dirac - useradd dirac - chown dirac:dirac /opt/dirac - su - dirac - cd /opt/dirac - curl -O -L https://raw.githubusercontent.com/DIRACGrid/DIRAC/integration/Core/scripts/dirac-install.py - chmod +x dirac-install.py - ./dirac-install.py -r v6r20p4 -t server - -Once dirac installed you can add Tornado with the following steps: - -*********************** -Installing requirements -*********************** -To install and compile some elements used by Tornado you may install some packages with ``yum``: ``python-devel``, ``m2crypto``, ``gcc``. If you want to do some performance tests please check if ``nscd`` is running on your machine to avoid too many DNS query (on openstack, it is not enabled with SLC6). - -Then you need to install Tornado and M2Crypto (for python), but not from official repo:: - - pip --trusted-host=files.pythonhosted.org --trusted-host=pypi.org --trusted-host=pypi.python.org install git+https://gitlab.com/chaen/m2crypto.git@tmpUntilSwigUpdated - pip --trusted-host=files.pythonhosted.org --trusted-host=pypi.org --trusted-host=pypi.python.org install git+https://github.com/chaen/tornado.git@iostreamConfigurable - pip --trusted-host=files.pythonhosted.org --trusted-host=pypi.org --trusted-host=pypi.python.org install git+https://github.com/chaen/tornado_m2crypto.git - pip --trusted-host=files.pythonhosted.org --trusted-host=pypi.org --trusted-host=pypi.python.org install -r /opt/dirac/DIRAC/requirements.txt - -*********************** -Adding Tornado to DIRAC -*********************** - -Save the DIRAC folder somewhere then clone my GithHub repo, then switch to branch "stage_toDIRAC_clean". You can run the setup.py if ``DIRAC.Core.Tornado`` is not detected by python:: - - mv DIRAC DIRAC.old - git clone https://github.com/louisjdmartin/DIRAC.git - cd DIRAC - git checkout stage_toDIRAC_clean - python setup.py install - - - -********************* -Generate Certificates -********************* -To use HTTPS your certificates must be generated using TLS standard, you can use following lines to generate them yourself:: - - bash - cd /tmp - git clone https://github.com/chaen/DIRAC.git - cd DIRAC - git checkout rel-v6r20_FEAT_correctCA - - export DEVROOT=/tmp - export SERVERINSTALLDIR=/opt/dirac/ - export CI_CONFIG=/tmp/DIRAC/tests/Jenkins/config/ci/ - - source /tmp/DIRAC/tests/Jenkins/utilities.sh - generateCA # automatic - generateCertificates 365 # Certificates copied to /opt/dirac/etc/grid-security - generateUserCredentials 365 # Certificates generated at /opt/dirac/user -> copy to .globus and rename them userkey.pem and usercert.pem - exit - - -********************** -Configuration (server) -********************** -Like in DISET, check your iptable to open some ports if needed ! - -Configuration is mostly the same as before, you just have to define ``Protocol`` to ``HTTPS`` inside the Services and add a new Section for tornado. You can use this example:: - - LocalSite - { - Site = localhost - } - DIRAC - { - - Setup = DeveloperSetup - Setups - { - DeveloperSetup - { - Tornado = DevInstance - Framework = DevInstance - } - } - Security - { - UseServerCertificate=True - CertFile = /opt/dirac/etc/grid-security/hostcert.pem - KeyFile = /opt/dirac/etc/grid-security/hostkey.pem - } - } - - - LocalInstallation - { - Setup = DeveloperSetup - } - - - Systems - { - - Tornado - { - DevInstance - { - - Port = 4444 - } - } - - Framework - { - DevInstance - { - Databases - { - UserDB - { - Host = 127.0.0.1 #localhost - User = root - Password = - DBName = dirac - } - } - Services - { - User - { - # Use this handler to have a dummyService, can be used for testing without load a database - #HandlerPath = DIRAC/FrameworkSystem/Service/DummyTornadoHandler.py - Protocol = https - } - } - } - } - } - Registry - { - # [Add your registry entry, like in DISET] - } - - - -********************** -Configuration (client) -********************** -Nothing change ! -Define your URL as DIRAC service, but use https instead of dips:: - - DIRAC - { - Setup = DeveloperSetup - Setups - { - DeveloperSetup - { - Framework = DevInstance - } - } - } - Systems - { - Framework - { - DevInstance - { - URLs - { - # DISET - #User = dips://server:9135/Framework/User - - #TORNADO - User = https://server:4444/Framework/User - } - } - } - } - -**************** -Start the server -**************** - -To start the server you must define ``OPENSSL_ALLOW_PROXY_CERTS`` and run ``DIRAC/TornadoServices/Scripts/tornado-start-all.py`` (or ``tornado-start-CS.py`` if you try to run a configuration server):: - - OPENSSL_ALLOW_PROXY_CERTS=1 python /opt/dirac/DIRAC/TornadoServices/scripts/tornado-start-all.py - - -You can now run DIRAC services. You can check the docstring of tests file (``DIRAC/test/Integration/TornadoServices`` and ``DIRAC/TornadoServices/test``) to know how to run tests. - - -********************* -Run performance tests -********************* -For performance test unset ``PYTHONOPTIMIZE`` if it is set in your environement:: - - unset PYTHONOPTIMIZE - - -Then you have to start some clients (adapt the port):: - - cd /opt/dirac/DIRAC/test/Integration/TornadoServices - multimech-run perf-test-ping -p 9000 -b 0.0.0.0 - -Modify first lines of ``DIRAC/TornadoServices/test/multi-mechanize/distributed-test.py`` and ``DIRAC/TornadoServices/test/multi-mechanize/plot-distributed-test.py`` (follow instruction of each files) - -On the server start ``DIRAC/test/Integration/TornadoServices/getCPUInfos`` (redirect output to a file) - -Run ``distributed-test.py [NameOfYourTest]`` at the end of execution, the command to plot is given. Before executing command, copy output of ``getCPUInfos`` on ``/tmp/results.txt`` (on your local machine). \ No newline at end of file diff --git a/docs/source/DeveloperGuide/monitoring.rst b/docs/source/DeveloperGuide/monitoring.rst deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/docs/source/UserGuide/CommandReference/DataManagement/dirac-dms-clean-directory.rst b/docs/source/UserGuide/CommandReference/DataManagement/dirac-dms-clean-directory.rst index e410530daba..011414a5a31 100644 --- a/docs/source/UserGuide/CommandReference/DataManagement/dirac-dms-clean-directory.rst +++ b/docs/source/UserGuide/CommandReference/DataManagement/dirac-dms-clean-directory.rst @@ -10,7 +10,7 @@ file catalogs. Usage:: - dirac-dms-clean-directory + dirac-dms-clean-directory Example:: diff --git a/docs/source/UserGuide/CommandReference/DataManagement/dirac-dms-find-lfns.rst b/docs/source/UserGuide/CommandReference/DataManagement/dirac-dms-find-lfns.rst index 999a00c6319..48092d7eea7 100644 --- a/docs/source/UserGuide/CommandReference/DataManagement/dirac-dms-find-lfns.rst +++ b/docs/source/UserGuide/CommandReference/DataManagement/dirac-dms-find-lfns.rst @@ -12,7 +12,7 @@ Usage:: Arguments:: - metaspec: metadata index specification (of the form: "meta=value" or "meta Date: Mon, 13 Jul 2020 10:41:40 +0200 Subject: [PATCH 14/25] Move Tornado deprecated tests --- Core/Tornado/test/migrationTests/README | 2 ++ .../TestClientReturnedValuesAndCredentialsExtraction.py | 0 .../{ => migrationTests}/TestClientSelectionAndInterface.py | 0 Core/Tornado/test/{ => migrationTests}/TestServerIntegration.py | 0 .../{ => migrationTests}/multi-mechanize/distributed-test.py | 0 .../test/{ => migrationTests}/multi-mechanize/ping/config.cfg | 0 .../multi-mechanize/ping/test_scripts/v_perf.py | 0 .../multi-mechanize/plot-distributedTest.py | 0 Core/Tornado/test/{ => migrationTests}/multi-mechanize/plot.py | 0 .../{ => migrationTests}/multi-mechanize/service/config.cfg | 0 .../multi-mechanize/service/test_scripts/v_perf.py | 0 11 files changed, 2 insertions(+) create mode 100644 Core/Tornado/test/migrationTests/README rename Core/Tornado/test/{ => migrationTests}/TestClientReturnedValuesAndCredentialsExtraction.py (100%) rename Core/Tornado/test/{ => migrationTests}/TestClientSelectionAndInterface.py (100%) rename Core/Tornado/test/{ => migrationTests}/TestServerIntegration.py (100%) rename Core/Tornado/test/{ => migrationTests}/multi-mechanize/distributed-test.py (100%) rename Core/Tornado/test/{ => migrationTests}/multi-mechanize/ping/config.cfg (100%) rename Core/Tornado/test/{ => migrationTests}/multi-mechanize/ping/test_scripts/v_perf.py (100%) rename Core/Tornado/test/{ => migrationTests}/multi-mechanize/plot-distributedTest.py (100%) rename Core/Tornado/test/{ => migrationTests}/multi-mechanize/plot.py (100%) rename Core/Tornado/test/{ => migrationTests}/multi-mechanize/service/config.cfg (100%) rename Core/Tornado/test/{ => migrationTests}/multi-mechanize/service/test_scripts/v_perf.py (100%) diff --git a/Core/Tornado/test/migrationTests/README b/Core/Tornado/test/migrationTests/README new file mode 100644 index 00000000000..37869667ec6 --- /dev/null +++ b/Core/Tornado/test/migrationTests/README @@ -0,0 +1,2 @@ +These tests were writen in order to compare the existing DISET based services with the new Tornado/HTTPs based services. +They were ran by hand, and are kept here just in case for the time being. diff --git a/Core/Tornado/test/TestClientReturnedValuesAndCredentialsExtraction.py b/Core/Tornado/test/migrationTests/TestClientReturnedValuesAndCredentialsExtraction.py similarity index 100% rename from Core/Tornado/test/TestClientReturnedValuesAndCredentialsExtraction.py rename to Core/Tornado/test/migrationTests/TestClientReturnedValuesAndCredentialsExtraction.py diff --git a/Core/Tornado/test/TestClientSelectionAndInterface.py b/Core/Tornado/test/migrationTests/TestClientSelectionAndInterface.py similarity index 100% rename from Core/Tornado/test/TestClientSelectionAndInterface.py rename to Core/Tornado/test/migrationTests/TestClientSelectionAndInterface.py diff --git a/Core/Tornado/test/TestServerIntegration.py b/Core/Tornado/test/migrationTests/TestServerIntegration.py similarity index 100% rename from Core/Tornado/test/TestServerIntegration.py rename to Core/Tornado/test/migrationTests/TestServerIntegration.py diff --git a/Core/Tornado/test/multi-mechanize/distributed-test.py b/Core/Tornado/test/migrationTests/multi-mechanize/distributed-test.py similarity index 100% rename from Core/Tornado/test/multi-mechanize/distributed-test.py rename to Core/Tornado/test/migrationTests/multi-mechanize/distributed-test.py diff --git a/Core/Tornado/test/multi-mechanize/ping/config.cfg b/Core/Tornado/test/migrationTests/multi-mechanize/ping/config.cfg similarity index 100% rename from Core/Tornado/test/multi-mechanize/ping/config.cfg rename to Core/Tornado/test/migrationTests/multi-mechanize/ping/config.cfg diff --git a/Core/Tornado/test/multi-mechanize/ping/test_scripts/v_perf.py b/Core/Tornado/test/migrationTests/multi-mechanize/ping/test_scripts/v_perf.py similarity index 100% rename from Core/Tornado/test/multi-mechanize/ping/test_scripts/v_perf.py rename to Core/Tornado/test/migrationTests/multi-mechanize/ping/test_scripts/v_perf.py diff --git a/Core/Tornado/test/multi-mechanize/plot-distributedTest.py b/Core/Tornado/test/migrationTests/multi-mechanize/plot-distributedTest.py similarity index 100% rename from Core/Tornado/test/multi-mechanize/plot-distributedTest.py rename to Core/Tornado/test/migrationTests/multi-mechanize/plot-distributedTest.py diff --git a/Core/Tornado/test/multi-mechanize/plot.py b/Core/Tornado/test/migrationTests/multi-mechanize/plot.py similarity index 100% rename from Core/Tornado/test/multi-mechanize/plot.py rename to Core/Tornado/test/migrationTests/multi-mechanize/plot.py diff --git a/Core/Tornado/test/multi-mechanize/service/config.cfg b/Core/Tornado/test/migrationTests/multi-mechanize/service/config.cfg similarity index 100% rename from Core/Tornado/test/multi-mechanize/service/config.cfg rename to Core/Tornado/test/migrationTests/multi-mechanize/service/config.cfg diff --git a/Core/Tornado/test/multi-mechanize/service/test_scripts/v_perf.py b/Core/Tornado/test/migrationTests/multi-mechanize/service/test_scripts/v_perf.py similarity index 100% rename from Core/Tornado/test/multi-mechanize/service/test_scripts/v_perf.py rename to Core/Tornado/test/migrationTests/multi-mechanize/service/test_scripts/v_perf.py From 3b8d5ef396606025b0af58d8286dd27ee5f16778 Mon Sep 17 00:00:00 2001 From: Christophe Haen Date: Mon, 13 Jul 2020 16:27:51 +0200 Subject: [PATCH 15/25] Addapt a bit the doc --- ConfigurationSystem/private/Refresher.py | 2 +- Core/Tornado/Client/TornadoClient.py | 5 +- .../Client/private/TornadoBaseClient.py | 21 +- FrameworkSystem/Client/MonitoringClient.py | 2 +- .../Systems/Configuration/index.rst | 22 +- .../AdministratorGuide/technologyPreviews.rst | 4 + .../Systems/ConfigurationSystem.rst | 26 -- docs/source/DeveloperGuide/Systems/index.rst | 1 - .../DeveloperGuide/TornadoServices/index.rst | 260 ++++-------------- 9 files changed, 88 insertions(+), 255 deletions(-) delete mode 100644 docs/source/DeveloperGuide/Systems/ConfigurationSystem.rst diff --git a/ConfigurationSystem/private/Refresher.py b/ConfigurationSystem/private/Refresher.py index 9f59032c4ea..daac558b1dc 100755 --- a/ConfigurationSystem/private/Refresher.py +++ b/ConfigurationSystem/private/Refresher.py @@ -347,7 +347,7 @@ def daemonize(self): # USE_TORNADO_IOLOOP is defined by starting scripts -if os.environ.get('USE_TORNADO_IOLOOP', 'false').lower() == 'true': +if os.environ.get('USE_TORNADO_IOLOOP', 'false').lower() in ('yes', 'true'): gRefresher = TornadoRefresher() else: gRefresher = Refresher() diff --git a/Core/Tornado/Client/TornadoClient.py b/Core/Tornado/Client/TornadoClient.py index d04d8c1a69d..258da09ccff 100644 --- a/Core/Tornado/Client/TornadoClient.py +++ b/Core/Tornado/Client/TornadoClient.py @@ -49,7 +49,7 @@ def call(*args): # Name from RPCClient Interface def executeRPC(self, method, *args): """ - This function call a remote service + Calls a remote service :param str procedure: remote procedure name :param args: list of arguments @@ -58,8 +58,7 @@ def executeRPC(self, method, *args): rpcCall = {'method': method, 'args': encode(args)} # Start request retVal = self._request(**rpcCall) - # Should this line bellow go ? I guess yes - retVal['rpcStub'] = (self._getBaseStub(), method, args) + retVal['rpcStub'] = (self._getBaseStub(), method, list(args)) return retVal def receiveFile(self, destFile, *args): diff --git a/Core/Tornado/Client/private/TornadoBaseClient.py b/Core/Tornado/Client/private/TornadoBaseClient.py index 4cb43318c39..041dafbcbee 100644 --- a/Core/Tornado/Client/private/TornadoBaseClient.py +++ b/Core/Tornado/Client/private/TornadoBaseClient.py @@ -1,11 +1,11 @@ """ - TornadoBaseClient contain all low-levels functionnalities and initilization methods + TornadoBaseClient contains all the low-levels functionnalities and initilization methods It must be instanciated from :py:class:`~DIRAC.Core.Tornado.Client.TornadoClient` - Requests library manage himself retry when connection failed, so the __nbOfRetry attribute is removed from DIRAC + Requests library manage itself retry when connection failed, so the __nbOfRetry attribute is removed from DIRAC (For each URL requests manage retries himself, if it still fail, we try next url) KeepAlive lapse is also removed because managed by request, - see http://docs.python-requests.org/en/master/user/advanced/#keep-alive + see https://requests.readthedocs.io/en/latest/user/advanced/#keep-alive If necessary this class can be modified to define number of retry in requests, documentation does not give lot of informations but you can see this simple solution from StackOverflow. @@ -93,13 +93,9 @@ def __init__(self, serviceName, **kwargs): self.__initStatus = S_OK() self.__idDict = {} self.__extraCredentials = "" - self.__retry = 0 - self.__retryDelay = 0 # by default we always have 1 url for example: # RPCClient('dips://volhcb38.cern.ch:9162/Framework/SystemAdministrator') self.__nbOfUrls = 1 - # self.__nbOfRetry removed in https, see note at the begining of the class - self.__retryCounter = 1 self.__bannedUrls = [] # For pylint... @@ -324,7 +320,7 @@ def __findServiceURL(self): # Load the Gateways URLs for the current site Name gatewayURL = False - if self.KW_IGNORE_GATEWAYS not in self.kwargs or not self.kwargs[self.KW_IGNORE_GATEWAYS]: + if not self.kwargs.get(self.KW_IGNORE_GATEWAYS): dRetVal = gConfig.getOption("/DIRAC/Gateways/%s" % DIRAC.siteName()) if dRetVal['OK']: rawGatewayURL = List.randomize(List.fromChar(dRetVal['Value'], ","))[0] @@ -372,7 +368,7 @@ def __findServiceURL(self): self.__bannedUrls = [] # retry all urls gLogger.debug("Retrying again all URLs") - if len(self.__bannedUrls) > 0 and len(urlsList) > 1: + if self.__bannedUrls and len(urlsList) > 1: # we have host which is not accessible. We remove that host from the list. # We only remove if we have more than one instance for i in self.__bannedUrls: @@ -387,7 +383,8 @@ def __findServiceURL(self): # If we have banned URLs, and several URLs at disposals, we make sure that the selected sURL # is not on a host which is banned. If it is, we take the next one in the list using __selectUrl - if len(self.__bannedUrls) > 0 and self.__nbOfUrls > 2: + if self.__bannedUrls and self.__nbOfUrls > 2: # when we have multiple services then we can + # have a situation when two services are running on the same machine with different ports... retVal = Network.splitURL(sURL) nexturl = None if retVal['OK']: @@ -550,8 +547,8 @@ def _request(self, retry=0, stream=False, outputFile=None, **kwargs): return decode(contentFile.getvalue())[0] except Exception as e: - # CHRIS TODO review this part. - # Is this list ever cleaned ? + # CHRIS TODO review this part: catch specific exceptions + # self.__bannedUrls is emptied in findServiceURLs if url not in self.__bannedUrls: self.__bannedUrls += [url] if retry < self.__nbOfUrls - 1: diff --git a/FrameworkSystem/Client/MonitoringClient.py b/FrameworkSystem/Client/MonitoringClient.py index a16f03d8ff5..8faa28816ba 100755 --- a/FrameworkSystem/Client/MonitoringClient.py +++ b/FrameworkSystem/Client/MonitoringClient.py @@ -86,7 +86,7 @@ def registerMonitoringClient(self, mc): # USE_TORNADO_IOLOOP is defined by starting scripts -if os.environ.get('USE_TORNADO_IOLOOP', 'false').lower() == 'true': +if os.environ.get('USE_TORNADO_IOLOOP', 'false').lower() in ('yes', 'true'): from DIRAC.FrameworkSystem.Client.MonitoringClientIOLoop import MonitoringFlusherTornado gMonitoringFlusher = MonitoringFlusherTornado() else: diff --git a/docs/source/AdministratorGuide/Systems/Configuration/index.rst b/docs/source/AdministratorGuide/Systems/Configuration/index.rst index 097d6cab7e6..8bf8dd40204 100644 --- a/docs/source/AdministratorGuide/Systems/Configuration/index.rst +++ b/docs/source/AdministratorGuide/Systems/Configuration/index.rst @@ -2,5 +2,23 @@ Configuration System ==================== -.. contents:: Table of contents - :depth: 3 +The configuration system serves the configuration to any other client (be it another server or a standard client). +The infrastructure is master/slave based. + +****** +Master +****** + +The master Server holds the central configuration in a local file. This file is then served to the clients, and synchronized with the slave servers. + +the master server also regularly pings the slave servers to make sure they are still alive. If not, they are removed from the list of CS. + +When changes are committed to the master, a backup of the existing configuration file is made in ``etc/csbackup``. + +****** +Slaves +****** + +Slave server registers themselves to the master when starting. +They synchronize their configuration on a regular bases (every 5 minutes by default). +Note that the slave CS do not hold the configuration in a local file, but only in memory. \ No newline at end of file diff --git a/docs/source/AdministratorGuide/technologyPreviews.rst b/docs/source/AdministratorGuide/technologyPreviews.rst index 8c4403f1291..89899308093 100644 --- a/docs/source/AdministratorGuide/technologyPreviews.rst +++ b/docs/source/AdministratorGuide/technologyPreviews.rst @@ -27,3 +27,7 @@ The changes from one stage to the next is controlled by environment variables, a The last stage (JSON only) will be the default of the following release, so before upgrading you will have to go through the previous steps. +HTTPS Services +============== + +The aim is to replace the DISET services with HTTPS services. The changes should be almost transparent for users/admins. However, because it is still very much in the state of preview, we do not yet describe how/what to change. If you really want to play around, please check :ref:`httpsTornado`. diff --git a/docs/source/DeveloperGuide/Systems/ConfigurationSystem.rst b/docs/source/DeveloperGuide/Systems/ConfigurationSystem.rst deleted file mode 100644 index 07192a8076d..00000000000 --- a/docs/source/DeveloperGuide/Systems/ConfigurationSystem.rst +++ /dev/null @@ -1,26 +0,0 @@ -==================== -Configuration System -==================== - -The configuration system works with master server and slave server. - -Master & slave can receive requests from another master or slave and from any client who need the configuration. - -****** -Master -****** -The master Server is the only server who have a local configuration plus a configuration file with shared configuration. He can distribute the shared configuration to every client or synchronize it with a slave server. - -Master server also check if slaves server are alive, in fact he just verify the date of the last registration request to determine if they are considered dead or alive. - -When a slave want to register in the master server (or renew registration) he call ``publishSlaveServer``, then before answering to slave, the master will ping the slave and if ping work, he register the slave and returns ``S_OK`` to slave. - -Master Server also disable the refresher, because he have the configuration and he manage it, so he did'nt need the refresher in the background. - -****** -Slaves -****** -The slave server distribute the shared configuration to every client who need it. -He force the refresher to run in background and to actualize configuration after a few time (every 5 minutes by default). - -At initialization, slave register hitself to the master and get the configuration, then with every refresh he renew its registration to master. diff --git a/docs/source/DeveloperGuide/Systems/index.rst b/docs/source/DeveloperGuide/Systems/index.rst index 6dbc1ef892c..22f7778ba7b 100644 --- a/docs/source/DeveloperGuide/Systems/index.rst +++ b/docs/source/DeveloperGuide/Systems/index.rst @@ -9,7 +9,6 @@ Here the reader can find technical documentation for developing DIRAC systems .. toctree:: :maxdepth: 2 - ConfigurationSystem Framework/index Transformation/index Monitoring/index diff --git a/docs/source/DeveloperGuide/TornadoServices/index.rst b/docs/source/DeveloperGuide/TornadoServices/index.rst index 36200cdbbd5..a6958c185e3 100644 --- a/docs/source/DeveloperGuide/TornadoServices/index.rst +++ b/docs/source/DeveloperGuide/TornadoServices/index.rst @@ -1,3 +1,5 @@ +.. _httpsTornado: + =========================== HTTPS Services with Tornado =========================== @@ -8,7 +10,7 @@ HTTPS Services with Tornado ************ Presentation ************ -This page summarize changes between DISET and HTTPS. You can all also see these presentations: +This page summarizes the changes between DISET and HTTPS. You can all also see these presentations: - `Presentation of HTTPS in DIRAC `_. - `Presentation of HTTPS migration `_. @@ -23,7 +25,7 @@ Service digraph { TornadoServer -> YourServiceHandler [label=use]; YourServiceHandler -> TornadoService[label=inherit]; - + TornadoServer [shape=polygon,sides=4, label = "DIRAC.Core.Tornado.Server.TornadoServer"]; TornadoService [shape=polygon,sides=4, label = "DIRAC.Core.Tornado.Server.TornadoService"]; @@ -48,7 +50,8 @@ Internal structure How to write service ******************** -Nothing better than example:: + +Nothing better than an example:: from DIRAC.Core.Tornado.Server.TornadoService import TornadoService class yourServiceHandler(TornadoService): @@ -66,22 +69,32 @@ Nothing better than example:: ## Returned value may be an S_OK/S_ERROR ## You don't need to serialize in JSON, Tornado will do it -Write a service is similar in tornado and diset. You have to define your method starting with ``export_``, your initialization method is a class method called ``initializeHandler`` +Writing a service for tornado and diset is similar. You have to define your method starting with ``export_``, and your initialization method is a class method called ``initializeHandler``. Main changes in tornado are: - Service are initialized at first request -- You **should not** write method called ``initialize`` because Tornado already use it, so the ``initialize`` from diset handlers became ``initializeRequest`` -- infosDict, arguments of initializedHandler is not really the same as one from diset, all things relative to transport are removed, to write on the transport you can use self.write() but I recommend to avoid his usage, Tornado will encode and write what you return. -- Variables likes ``types_yourMethod`` are ignored, but you can still define ``auth_yourMethod`` if you want. - -Based on DISET request handler, you still have access to some getters in your handler, getters have the same names as DISET, which includes: -``getCSOption``, ``getRemoteAddress``, ``getRemoteCredentials``, ``srv_getCSOption``, ``srv_getRemoteAddress``, ``srv_getRemoteCredentials``, ``srv_getFormattedRemoteCredentials``, ``srv_getServiceName`` and ``srv_getURL``. +- You **should not** write a method called ``initialize`` because Tornado already use that name, so the ``initialize`` from diset handlers became ``initializeRequest`` +- ``infosDict``, arguments of initializedHandler is not really the same as for diset: all transport related matters are removed. +- There is no parameter type check any more: attributes like ``types_yourMethod`` are ignored. +- Auth attributes are still there (``auth_yourMethod``). + +The interface of the DISET request handler was preserved, in particular: +* ``getCSOption`` +* ``getRemoteAddress`` +* ``getRemoteCredentials`` +* ``srv_getCSOption`` +* ``srv_getRemoteAddress`` +* ``srv_getRemoteCredentials`` +* ``srv_getFormattedRemoteCredentials`` +* ``srv_getServiceName`` +* ``srv_getURL``. How to start server ******************* + The easy way, use ``DIRAC/Core/Tornado/script/tornado-start-all.py`` it will start all services registered in configuration ! To register a service you just have to add the service in the CS and ``Protocol = https``. It may look like this:: - + DIRAC { Setups @@ -120,13 +133,13 @@ But you can also control more settings by launching tornado yourself:: serverToLaunch = TornadoServer(youroptions) serverToLaunch.startTornado() -Options availlable are: +Options available are: - services, should be a list, to start only these services - debugSSL, True or False, activate debug mode of Tornado (includes autoreload) and SSL, for extra logs use -ddd in the command line - port, int, if you want to override value from config. If it's also not defined in config, it use 443. -This start method can bu usefull for developing new service or create starting script for a specific service, like the Configuration System (as master). +This start method can be useful for developing new service or create starting script for a specific service, like the Configuration System (as master). ****** Client @@ -143,9 +156,9 @@ Client Requests [shape=polygon,sides=4] } -This diagram present what is behind TornadoClient, but you should use :py:class:`DIRAC.Core.Base.Client` ! The new client integrate a selection system which select for you between HTTPS and DISET client. +This diagram present what is behind TornadoClient, but you should use :py:class:`DIRAC.Core.Base.Client` ! The new client integrate a selection system which select for you between HTTPS and DISET client. -In your client module when you inherit from :py:class:`DIRAC.Core.Base.Client` you can define `httpsClient` with another client, it can be usefull when you can't serialize some data in JSON. Here the step to create and use a JSON patch: +In your client module when you inherit from :py:class:`DIRAC.Core.Base.Client` you can define `httpsClient` with another client, it can be useful when you can't serialize some data in JSON. Here the step to create and use a JSON patch: - Create a class which inherit from :py:class:`~DIRAC.Core.Tornado.Client.TornadoClient` - For every method who need a JSON patch create a method with the same name as the service @@ -204,7 +217,7 @@ Internal structure - :py:class:`~DIRAC.Core.DISET.private.innerRPCClient` and :py:class:`~DIRAC.Core.DISET.RPCClient` are now a single class: :py:class:`~DIRAC.Core.Tornado.Client.TornadoClient`. Interface and usage stay the same. - :py:class:`~DIRAC.Core.Tornado.Client.private.TornadoBaseClient` is the new :py:class:`~DIRAC.Core.DISET.private.BaseClient`. Most of code is copied from :py:class:`~DIRAC.Core.DISET.private.BaseClient` but some method have been rewrited to use `Requests `_ instead of Transports. Code duplication is done to fully separate DISET and HTTPS but later, some parts can be merged by using a new common class between DISET and HTTPS (these parts are explicitly given in the docstrings). -- :py:class:`~DIRAC.Core.DISET.private.Transports.BaseTransport`, :py:class:`~DIRAC.Core.DISET.private.Transports.PlainTransport` and :py:class:`~DIRAC.Core.DISET.private.Transports.SSLTransport` are replaced by `Requests `_ +- :py:class:`~DIRAC.Core.DISET.private.Transports.BaseTransport`, :py:class:`~DIRAC.Core.DISET.private.Transports.PlainTransport` and :py:class:`~DIRAC.Core.DISET.private.Transports.SSLTransport` are replaced by `Requests `_ - keepAliveLapse is removed from rpcStub returned by Client because `Requests `_ manage it himself. - Due to JSON limitation you can write some specifics clients who inherit from :py:class:`~DIRAC.Core.Tornado.Client.TornadoClient`, there is a simple example with :py:class:`~DIRAC.Core.Tornado.Client.SpecificClient.ConfigurationClient` who transfer data in base64 to overcome JSON limitations @@ -216,24 +229,7 @@ Connections and certificates - Server certificate **must** have subject alternative names. Requests also check the hostname and you can have connection errors when using "localhost" for example. To avoid them add subject alternative name in certificate. (You can also see https://github.com/shazow/urllib3/issues/497 ). - If server certificates are used by clients, you must add clientAuth in the extendedKeyUsage (requests also check that). - In server side M2Crypto is used instead of GSI and conflict are possible between GSI and M2Crypto, to avoid them you can comment 4 lasts lines at ``DIRAC/Core/Security/__init__.py`` -- ``_connect()``, ``_disconnect()`` and ``_purposeAction()`` are removed, ``_connect``/``_disconnect`` are now managed by `requests `_ and ``_purposeAction`` is no longer used is in HTTPS protocol. - - - -************ -Launch tests -************ - -pytest -****** -Because for now Tornado does not have "Real" services, you must use some fakes services to compare and test with DISET. -You need tornadoCredDict, diracCredDict, User, UserDirac to run tests. Each test explain how to configure in its docstring. - -The only service available is the Configuration/Server, it will work with HTTPS and DISET services who needs to load configuration with a Configuration/Server. - - - - +- ``_connect()``, ``_disconnect()`` and ``_purposeAction()`` are removed, ``_connect``/``_disconnect`` are now managed by `requests `_ and ``_purposeAction`` is no longer used is in HTTPS protocol. ********************** @@ -241,195 +237,41 @@ How to install Tornado ********************** -To install and run service on Tornado you should install DIRAC first. You can install DIRAC in the standard way. But for now you don't need to configure it and generate certificates. You just have to install DIRAC:: - - mkdir -p /opt/dirac - useradd dirac - chown dirac:dirac /opt/dirac - su - dirac - cd /opt/dirac - curl -O -L https://raw.githubusercontent.com/DIRACGrid/DIRAC/integration/Core/scripts/dirac-install.py - chmod +x dirac-install.py - ./dirac-install.py -r v6r20p4 -t server - -Once dirac installed you can add Tornado with the following steps: - - -Installing requirements -*********************** -To install and compile some elements used by Tornado you may install some packages with ``yum``: ``python-devel``, ``m2crypto``, ``gcc``. If you want to do some performance tests please check if ``nscd`` is running on your machine to avoid too many DNS query (on openstack, it is not enabled with SLC6). - -Then you need to install Tornado and M2Crypto (for python), but not from official repo:: - - pip --trusted-host=files.pythonhosted.org --trusted-host=pypi.org --trusted-host=pypi.python.org install git+https://gitlab.com/chaen/m2crypto.git@tmpUntilSwigUpdated - pip --trusted-host=files.pythonhosted.org --trusted-host=pypi.org --trusted-host=pypi.python.org install git+https://github.com/chaen/tornado.git@iostreamConfigurable - pip --trusted-host=files.pythonhosted.org --trusted-host=pypi.org --trusted-host=pypi.python.org install git+https://github.com/chaen/tornado_m2crypto.git - pip --trusted-host=files.pythonhosted.org --trusted-host=pypi.org --trusted-host=pypi.python.org install -r /opt/dirac/DIRAC/requirements.txt - - -Adding Tornado to DIRAC -*********************** - -Save the DIRAC folder somewhere then clone my GithHub repo, then switch to branch "stage_toDIRAC_clean". You can run the setup.py if ``DIRAC.Core.Tornado`` is not detected by python:: - - mv DIRAC DIRAC.old - git clone https://github.com/louisjdmartin/DIRAC.git - cd DIRAC - git checkout stage_toDIRAC_clean - python setup.py install - - - - -Generate Certificates -********************* -To use HTTPS your certificates must be generated using TLS standard, you can use following lines to generate them yourself:: - - bash - cd /tmp - git clone https://github.com/chaen/DIRAC.git - cd DIRAC - git checkout rel-v6r20_FEAT_correctCA - - export DEVROOT=/tmp - export SERVERINSTALLDIR=/opt/dirac/ - export CI_CONFIG=/tmp/DIRAC/tests/Jenkins/config/ci/ - - source /tmp/DIRAC/tests/Jenkins/utilities.sh - generateCA # automatic - generateCertificates 365 # Certificates copied to /opt/dirac/etc/grid-security - generateUserCredentials 365 # Certificates generated at /opt/dirac/user -> copy to .globus and rename them userkey.pem and usercert.pem - exit - - - -Configuration (server) -********************** -Like in DISET, check your iptable to open some ports if needed ! - -Configuration is mostly the same as before, you just have to define ``Protocol`` to ``HTTPS`` inside the Services and add a new Section for tornado. You can use this example:: - - LocalSite - { - Site = localhost - } - DIRAC - { - - Setup = DeveloperSetup - Setups - { - DeveloperSetup - { - Tornado = DevInstance - Framework = DevInstance - } - } - Security - { - UseServerCertificate=True - CertFile = /opt/dirac/etc/grid-security/hostcert.pem - KeyFile = /opt/dirac/etc/grid-security/hostkey.pem - } - } - - - LocalInstallation - { - Setup = DeveloperSetup - } +Requirements +************ +Two special python packages are needed: +* git+https://github.com/chaen/tornado.git@iostreamConfigurable : in place of the standard tornado. This adds configurable feature to tornado +* git+https://github.com/chaen/tornado_m2crypto.git: this allows to use tornado with M2Crypto - Systems - { - - Tornado - { - DevInstance - { - - Port = 4444 - } - } - - Framework - { - DevInstance - { - Databases - { - UserDB - { - Host = 127.0.0.1 #localhost - User = root - Password = - DBName = dirac - } - } - Services - { - User - { - # Use this handler to have a dummyService, can be used for testing without load a database - #HandlerPath = DIRAC/FrameworkSystem/Service/DummyTornadoHandler.py - Protocol = https - } - } - } - } - } - Registry - { - # [Add your registry entry, like in DISET] - } +Install a service +***************** +`dirac-install-tornado-service` is your friend. This will install a runit component running `tornado-start-all`. +Nothing is ready yet to install specific tornado service, like the master CS. +Start the server +**************** -Configuration (client) -********************** -Nothing change ! -Define your URL as DIRAC service, but use https instead of dips:: +To start the server you must define ``OPENSSL_ALLOW_PROXY_CERTS`` and run ``DIRAC/TornadoServices/Scripts/tornado-start-all.py`` (or ``tornado-start-CS.py`` if you try to run a configuration server):: - DIRAC - { - Setup = DeveloperSetup - Setups - { - DeveloperSetup - { - Framework = DevInstance - } - } - } - Systems - { - Framework - { - DevInstance - { - URLs - { - # DISET - #User = dips://server:9135/Framework/User + OPENSSL_ALLOW_PROXY_CERTS=1 python /opt/dirac/DIRAC/TornadoServices/scripts/tornado-start-all.py - #TORNADO - User = https://server:4444/Framework/User - } - } - } - } -Start the server -**************** -To start the server you must define ``OPENSSL_ALLOW_PROXY_CERTS`` and run ``DIRAC/TornadoServices/Scripts/tornado-start-all.py`` (or ``tornado-start-CS.py`` if you try to run a configuration server):: +************ +Launch tests +************ - OPENSSL_ALLOW_PROXY_CERTS=1 python /opt/dirac/DIRAC/TornadoServices/scripts/tornado-start-all.py +pytest +****** +Because for now Tornado does not have "Real" services, you must use some fakes services to compare and test with DISET. +You need tornadoCredDict, diracCredDict, User, UserDirac to run tests. Each test explain how to configure in its docstring. +The only service available is the Configuration/Server, it will work with HTTPS and DISET services who needs to load configuration with a Configuration/Server. -You can now run DIRAC services. You can check the docstring of tests file (``DIRAC/test/Integration/TornadoServices`` and ``DIRAC/TornadoServices/test``) to know how to run tests. From bb8c03a6ec723574793b6e3d1c9bcbecb6a0813b Mon Sep 17 00:00:00 2001 From: Christophe Haen Date: Wed, 5 Aug 2020 14:09:36 +0200 Subject: [PATCH 16/25] Implement minor review comments --- .../Client/ConfigurationClient.py | 7 +- ConfigurationSystem/private/Refresher.py | 33 ++-- .../private/ServiceInterfaceBase.py | 16 +- Core/DISET/RequestHandler.py | 2 +- Core/DISET/private/BaseClient.py | 2 +- Core/DISET/private/Service.py | 2 +- Core/Tornado/Client/RPCClientSelector.py | 40 ++--- Core/Tornado/Client/TornadoClient.py | 15 +- Core/Tornado/Client/TransferClientSelector.py | 40 ++--- .../Client/private/TornadoBaseClient.py | 20 ++- Core/Tornado/Server/HandlerManager.py | 20 ++- Core/Tornado/Server/TornadoServer.py | 17 +-- Core/Tornado/Server/TornadoService.py | 53 ++++--- Core/Tornado/scripts/tornado-start-CS.py | 7 +- Core/Tornado/scripts/tornado-start-all.py | 7 +- Core/Tornado/test/migrationTests/README | 2 - .../Service/TornadoFileCatalogHandler.py | 141 +----------------- FrameworkSystem/Client/MonitoringClient.py | 4 +- .../DeveloperGuide/TornadoServices/index.rst | 2 +- environment.yml | 4 + tests/CI/install_server.sh | 2 +- tests/CI/run_docker_setup.sh | 1 + tests/DISET_HTTPS_migration/README | 2 + ...tReturnedValuesAndCredentialsExtraction.py | 0 .../TestClientSelectionAndInterface.py | 0 .../TestServerIntegration.py | 0 .../multi-mechanize/distributed-test.py | 0 .../multi-mechanize/ping/config.cfg | 0 .../ping/test_scripts/v_perf.py | 0 .../multi-mechanize/plot-distributedTest.py | 0 .../multi-mechanize/plot.py | 0 .../multi-mechanize/service/config.cfg | 0 .../service/test_scripts/v_perf.py | 0 .../DataManagementSystem/Test_Client_DFC.py | 1 + tests/py3CheckDirs.txt | 1 + 35 files changed, 167 insertions(+), 274 deletions(-) delete mode 100644 Core/Tornado/test/migrationTests/README create mode 100644 tests/DISET_HTTPS_migration/README rename {Core/Tornado/test/migrationTests => tests/DISET_HTTPS_migration}/TestClientReturnedValuesAndCredentialsExtraction.py (100%) rename {Core/Tornado/test/migrationTests => tests/DISET_HTTPS_migration}/TestClientSelectionAndInterface.py (100%) rename {Core/Tornado/test/migrationTests => tests/DISET_HTTPS_migration}/TestServerIntegration.py (100%) rename {Core/Tornado/test/migrationTests => tests/DISET_HTTPS_migration}/multi-mechanize/distributed-test.py (100%) rename {Core/Tornado/test/migrationTests => tests/DISET_HTTPS_migration}/multi-mechanize/ping/config.cfg (100%) rename {Core/Tornado/test/migrationTests => tests/DISET_HTTPS_migration}/multi-mechanize/ping/test_scripts/v_perf.py (100%) rename {Core/Tornado/test/migrationTests => tests/DISET_HTTPS_migration}/multi-mechanize/plot-distributedTest.py (100%) rename {Core/Tornado/test/migrationTests => tests/DISET_HTTPS_migration}/multi-mechanize/plot.py (100%) rename {Core/Tornado/test/migrationTests => tests/DISET_HTTPS_migration}/multi-mechanize/service/config.cfg (100%) rename {Core/Tornado/test/migrationTests => tests/DISET_HTTPS_migration}/multi-mechanize/service/test_scripts/v_perf.py (100%) diff --git a/ConfigurationSystem/Client/ConfigurationClient.py b/ConfigurationSystem/Client/ConfigurationClient.py index dd282f31351..b92134b91a3 100644 --- a/ConfigurationSystem/Client/ConfigurationClient.py +++ b/ConfigurationSystem/Client/ConfigurationClient.py @@ -2,7 +2,7 @@ Custom TornadoClient for Configuration System Used like a normal client, should be instanciated if and only if we use the configuration service - Because of limitation with JSON some datas are encoded in base64 + because JSON cannot ship binary data, we encode it in base64 """ @@ -18,7 +18,7 @@ class CSJSONClient(TornadoClient): To avoid JSON limitation the HTTPS handler encode data in base64 before sending them, this class only decode the base64 - An exception is made with CommitNewData wich ENCODE in base64 + An exception is made with CommitNewData which ENCODE in base64 """ def getCompressedData(self): @@ -26,7 +26,8 @@ def getCompressedData(self): Transmit request to service and get data in base64, it decode base64 before returning - :returns str: Configuration data, compressed + :returns: Configuration data, compressed + :rtype: str """ retVal = self.executeRPC('getCompressedData') if retVal['OK']: diff --git a/ConfigurationSystem/private/Refresher.py b/ConfigurationSystem/private/Refresher.py index daac558b1dc..f1a53fbfdc0 100755 --- a/ConfigurationSystem/private/Refresher.py +++ b/ConfigurationSystem/private/Refresher.py @@ -40,7 +40,7 @@ def _updateFromRemoteLocation(serviceClient): return retVal -class RefresherBase(): +class RefresherBase(object): """ Code factorisation for the refresher """ @@ -182,8 +182,8 @@ def __init__(self): def _refreshInThread(self): """ - Refreshing configration in the background. By default it use a thread but it can be - also runned in the IOLoop + Refreshing configuration in the background. By default it uses a thread but it can be + run also in the IOLoop """ retVal = self._refresh() if not retVal['OK']: @@ -191,7 +191,7 @@ def _refreshInThread(self): def refreshConfigurationIfNeeded(self): """ - Refresh the configuration if automatic update are disabled, refresher is enabled and servers are defined + Refresh the configuration if automatic updates are disabled, refresher is enabled and servers are defined """ if not self._refreshEnabled or self._automaticUpdate or not gConfigurationData.getServers(): return @@ -251,20 +251,10 @@ class TornadoRefresher(RefresherBase): tasks, so it work with Tornado (HTTPS server). """ - # For pylint... - gen = None - - try: - from tornado import gen - # We change the import name otherwise sphinx tries - # to compile the tornado doc and fails - from tornado.ioloop import IOLoop as _IOLoop - - except ImportError: - gLogger.fatal("You should install tornado to use this refresher, please unset USE_TORNADO_IOLOOP") - - def __init__(self): - RefresherBase.__init__(self) + from tornado import gen + # We change the import name otherwise sphinx tries + # to compile the tornado doc and fails + from tornado.ioloop import IOLoop as _IOLoop def refreshConfigurationIfNeeded(self): """ @@ -279,7 +269,6 @@ def refreshConfigurationIfNeeded(self): return self._lastUpdateTime = time.time() self._IOLoop.current().run_in_executor(None, self._refresh) # pylint: disable=no-member - return def autoRefreshAndPublish(self, sURL): """ @@ -307,7 +296,7 @@ def __refreshLoop(self): Trigger the autorefresh when configuration is expired This task must use Tornado utilities to avoid blocking the ioloop and - pottentialy deadlock the server. + potentialy deadlock the server. See http://www.tornadoweb.org/en/stable/guide/coroutines.html#looping for official documentation about this type of method. @@ -346,8 +335,8 @@ def daemonize(self): # but background tasks will be delayed until IOLoop start. -# USE_TORNADO_IOLOOP is defined by starting scripts -if os.environ.get('USE_TORNADO_IOLOOP', 'false').lower() in ('yes', 'true'): +# DIRAC_USE_TORNADO_IOLOOP is defined by starting scripts +if os.environ.get('DIRAC_USE_TORNADO_IOLOOP', 'false').lower() in ('yes', 'true'): gRefresher = TornadoRefresher() else: gRefresher = Refresher() diff --git a/ConfigurationSystem/private/ServiceInterfaceBase.py b/ConfigurationSystem/private/ServiceInterfaceBase.py index beee793f7ce..f31e31ec6a4 100644 --- a/ConfigurationSystem/private/ServiceInterfaceBase.py +++ b/ConfigurationSystem/private/ServiceInterfaceBase.py @@ -9,13 +9,13 @@ import zlib import DIRAC -from DIRAC.Core.Utilities.File import mkDir from DIRAC.ConfigurationSystem.Client.ConfigurationClient import ConfigurationClient from DIRAC.ConfigurationSystem.Client.ConfigurationData import gConfigurationData, ConfigurationData from DIRAC.ConfigurationSystem.private.Refresher import gRefresher -from DIRAC.FrameworkSystem.Client.Logger import gLogger -from DIRAC.Core.Utilities.ReturnValues import S_OK, S_ERROR from DIRAC.Core.Base.Client import Client +from DIRAC.Core.Utilities.File import mkDir +from DIRAC.Core.Utilities.ReturnValues import S_OK, S_ERROR +from DIRAC.FrameworkSystem.Client.Logger import gLogger class ServiceInterfaceBase(object): @@ -109,20 +109,20 @@ def _checkSlavesStatus(self, forceWriteConfiguration=False): """ Check if Slaves server are still availlable - :param forceWriteConfiguration=False: Force rewriting configuration after checking slaves + :param forceWriteConfiguration: (default False) Force rewriting configuration after checking slaves """ gLogger.info("Checking status of slave servers") iGraceTime = gConfigurationData.getSlavesGraceTime() bModifiedSlaveServers = False - for sSlaveURL in self.dAliveSlaveServers.keys(): + for sSlaveURL in list(self.dAliveSlaveServers): if time.time() - self.dAliveSlaveServers[sSlaveURL] > iGraceTime: - gLogger.info("Found dead slave", sSlaveURL) + gLogger.warn("Found dead slave", sSlaveURL) del self.dAliveSlaveServers[sSlaveURL] bModifiedSlaveServers = True if bModifiedSlaveServers or forceWriteConfiguration: gConfigurationData.setServers("%s, %s" % (self.sURL, - ", ".join(self.dAliveSlaveServers.keys()))) + ", ".join(list(self.dAliveSlaveServers)))) self.__generateNewVersion() @staticmethod @@ -201,7 +201,7 @@ def updateConfiguration(self, sBuffer, committer="", updateVersionOption=False): # Test that remote and new versions are the same sRemoteVersion = oRemoteConfData.getVersion() sLocalVersion = gConfigurationData.getVersion() - gLogger.info("Checking versions\nremote: %s\nlocal: %s" % (sRemoteVersion, sLocalVersion)) + gLogger.info("Checking versions\n", "remote: %s\nlocal: %s" % (sRemoteVersion, sLocalVersion)) if sRemoteVersion != sLocalVersion: if not gConfigurationData.mergingEnabled(): return S_ERROR("Local and remote versions differ (%s vs %s). Cannot commit." % (sLocalVersion, sRemoteVersion)) diff --git a/Core/DISET/RequestHandler.py b/Core/DISET/RequestHandler.py index 2b91ee2641a..bbe1c94b009 100755 --- a/Core/DISET/RequestHandler.py +++ b/Core/DISET/RequestHandler.py @@ -504,7 +504,7 @@ def export_ping(self): def export_whoami(self): """ - A simple whoami, returns all credential dictionnary, except certificate chain object. + A simple whoami, returns all credential dictionary, except certificate chain object. """ credDict = self.srv_getRemoteCredentials() if 'x509Chain' in credDict: diff --git a/Core/DISET/private/BaseClient.py b/Core/DISET/private/BaseClient.py index d5ac0a2eebc..1361617d46e 100755 --- a/Core/DISET/private/BaseClient.py +++ b/Core/DISET/private/BaseClient.py @@ -250,7 +250,7 @@ def __discoverExtraCredentials(self): :return: S_OK()/S_ERROR() """ - # Wich extra credentials to use? + # which extra credentials to use? self.__extraCredentials = self.VAL_EXTRA_CREDENTIALS_HOST if self.__useCertificates else "" if self.KW_EXTRA_CREDENTIALS in self.kwargs: self.__extraCredentials = self.kwargs[self.KW_EXTRA_CREDENTIALS] diff --git a/Core/DISET/private/Service.py b/Core/DISET/private/Service.py index e007b49d61c..1e540535681 100644 --- a/Core/DISET/private/Service.py +++ b/Core/DISET/private/Service.py @@ -350,7 +350,7 @@ def handleConnection(self, clientTransport): The method stacks openened connection in a queue, another thread read this queue and handle connection. - :param clientTransport: Object wich describe opened connection (PlainTransport or SSLTransport) + :param clientTransport: Object which describes opened connection (PlainTransport or SSLTransport) """ self._stats['connections'] += 1 self._monitor.setComponentExtraParam('queries', self._stats['connections']) diff --git a/Core/Tornado/Client/RPCClientSelector.py b/Core/Tornado/Client/RPCClientSelector.py index a24888d495d..f230860044a 100644 --- a/Core/Tornado/Client/RPCClientSelector.py +++ b/Core/Tornado/Client/RPCClientSelector.py @@ -1,6 +1,6 @@ """ RPCClientSelector can replace RPCClient (with import RPCClientSelector as RPCClient) - to migrate from DISET to Tornado. This method chooses and returns the client wich should be + to migrate from DISET to Tornado. This method chooses and returns the client which should be used for a service. If the url of the service uses HTTPS, TornadoClient is returned, else it returns RPCClient Example:: @@ -10,18 +10,16 @@ myService.doSomething() """ +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function from DIRAC.Core.Tornado.Client.TornadoClient import TornadoClient from DIRAC.Core.DISET.RPCClient import RPCClient from DIRAC.ConfigurationSystem.Client.PathFinder import getServiceURL from DIRAC import gLogger - -def isURL(url): - """ - Just a test to check if URL is already given or not - """ - return url.startswith('http') or url.startswith('dip') +sLog = gLogger.getSubLogger(__name__) def RPCClientSelector(*args, **kwargs): # We use same interface as RPCClient @@ -32,29 +30,31 @@ def RPCClientSelector(*args, **kwargs): # We use same interface as RPCClient """ # We detect if we need to use a specific class for the HTTPS client - if 'httpsClient' in kwargs: - TornadoRPCClient = kwargs.pop('httpsClient') - else: - TornadoRPCClient = TornadoClient - # We have to make URL resolution BEFORE the RPCClient or TornadoClient to determine wich one we want to use + TornadoRPCClient = kwargs.pop('httpsClient', TornadoClient) + + # We have to make URL resolution BEFORE the RPCClient or TornadoClient to determine which one we want to use # URL is defined as first argument (called serviceName) in RPCClient try: serviceName = args[0] - gLogger.verbose("Trying to autodetect client for %s" % serviceName) - if not isURL(serviceName): - completeUrl = getServiceURL(serviceName) - gLogger.verbose("URL resolved: %s" % completeUrl) - else: + sLog.verbose("Trying to autodetect client for %s" % serviceName) + + # If we are not already given a URL, resolve it + if serviceName.startswith(('http', 'dip')): completeUrl = serviceName + else: + completeUrl = getServiceURL(serviceName) + sLog.verbose("URL resolved: %s" % completeUrl) + if completeUrl.startswith("http"): - gLogger.info("Using HTTPS for service %s" % serviceName) + sLog.info("Using HTTPS for service %s" % serviceName) rpc = TornadoRPCClient(*args, **kwargs) else: rpc = RPCClient(*args, **kwargs) - except Exception: + except Exception as e: # pylint: disable=broad-except # If anything went wrong in the resolution, we return default RPCClient - # So the comportement is exactly the same as before implementation of Tornado + # So the behaviour is exactly the same as before implementation of Tornado + sLog.warn("Could not select RPC or Tornado client", "%s" % repr(e)) rpc = RPCClient(*args, **kwargs) return rpc diff --git a/Core/Tornado/Client/TornadoClient.py b/Core/Tornado/Client/TornadoClient.py index 258da09ccff..15350265f8f 100644 --- a/Core/Tornado/Client/TornadoClient.py +++ b/Core/Tornado/Client/TornadoClient.py @@ -8,9 +8,10 @@ It also exposes the same interface for receiving file than the TransferClient. Main changes: - - KeepAliveLapse is removed, requests library manage it itself. - - nbOfRetry (defined as private attribute) is removed, requests library manage it hitself. - - Underneath it use HTTP POST protocol and JSON + + - KeepAliveLapse is removed, requests library manages it itself. + - nbOfRetry (defined as private attribute) is removed, requests library manage it itself. + - Underneath it use HTTP POST protocol and JSON Example:: @@ -20,6 +21,10 @@ """ +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + # pylint: disable=broad-except from DIRAC.Core.Utilities.JEncode import encode @@ -51,7 +56,7 @@ def executeRPC(self, method, *args): """ Calls a remote service - :param str procedure: remote procedure name + :param str method: remote procedure name :param args: list of arguments :returns: decoded response from server, server may return S_OK or S_ERROR """ @@ -63,7 +68,7 @@ def executeRPC(self, method, *args): def receiveFile(self, destFile, *args): """ - Equivalent of the :py:meth`~DIRAC.Core.DISET.TransferClient.TransferClient.receiveFile + Equivalent of :py:meth`~DIRAC.Core.DISET.TransferClient.TransferClient.receiveFile In practice, it calls the remote method `streamToClient` and stores the raw result in a file diff --git a/Core/Tornado/Client/TransferClientSelector.py b/Core/Tornado/Client/TransferClientSelector.py index 9f3b2504216..f7ea5ab1006 100644 --- a/Core/Tornado/Client/TransferClientSelector.py +++ b/Core/Tornado/Client/TransferClientSelector.py @@ -1,6 +1,6 @@ """ TransferClientSelector can replace TransferClient (with import TransferClientSelector as TransferClient) - to migrate from DISET to Tornado. This method chooses and returns the client wich should be + to migrate from DISET to Tornado. This method chooses and returns the client which should be used for a service. If the url of the service uses HTTPS, TornadoClient is returned, else it returns TransferClient Example:: @@ -10,18 +10,16 @@ myService.doSomething() """ +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function from DIRAC.Core.Tornado.Client.TornadoClient import TornadoClient from DIRAC.Core.DISET.TransferClient import TransferClient from DIRAC.ConfigurationSystem.Client.PathFinder import getServiceURL from DIRAC import gLogger - -def isURL(url): - """ - Just a test to check if URL is already given or not - """ - return url.startswith('http') or url.startswith('dip') +sLog = gLogger.getSubLogger(__name__) def TransferClientSelector(*args, **kwargs): # We use same interface as TransferClient @@ -32,29 +30,31 @@ def TransferClientSelector(*args, **kwargs): # We use same interface as Transfe """ # We detect if we need to use a specific class for the HTTPS client - if 'httpsClient' in kwargs: - TornadoTransClient = kwargs.pop('httpsClient') - else: - TornadoTransClient = TornadoClient - # We have to make URL resolution BEFORE the TransferClient or TornadoClient to determine wich one we want to use + TornadoTransClient = kwargs.pop('httpsClient', TornadoClient) + + # We have to make URL resolution BEFORE the TransferClient or TornadoClient to determine which one we want to use # URL is defined as first argument (called serviceName) in TransferClient try: serviceName = args[0] - gLogger.verbose("Trying to autodetect client for %s" % serviceName) - if not isURL(serviceName): - completeUrl = getServiceURL(serviceName) - gLogger.verbose("URL resolved: %s" % completeUrl) - else: + sLog.verbose("Trying to autodetect client for %s" % serviceName) + + # If we are not already given a URL, resolve it + if serviceName.startswith(('http', 'dip')): completeUrl = serviceName + else: + completeUrl = getServiceURL(serviceName) + sLog.verbose("URL resolved: %s" % completeUrl) + if completeUrl.startswith("http"): - gLogger.info("Using HTTPS for service %s" % serviceName) + sLog.info("Using HTTPS for service %s" % serviceName) transClient = TornadoTransClient(*args, **kwargs) else: transClient = TransferClient(*args, **kwargs) - except Exception: + except Exception as e: # pylint: disable=broad-except # If anything went wrong in the resolution, we return default TransferClient - # So the comportement is exactly the same as before implementation of Tornado + # So the behavior is exactly the same as before implementation of Tornado + sLog.warn("Could not select RPC or Tornado client", "%s" % repr(e)) transClient = TransferClient(*args, **kwargs) return transClient diff --git a/Core/Tornado/Client/private/TornadoBaseClient.py b/Core/Tornado/Client/private/TornadoBaseClient.py index 041dafbcbee..484cfe8d848 100644 --- a/Core/Tornado/Client/private/TornadoBaseClient.py +++ b/Core/Tornado/Client/private/TornadoBaseClient.py @@ -12,17 +12,25 @@ After some tests request seems to retry 3 times by default. https://stackoverflow.com/questions/15431044/can-i-set-max-retries-for-requests-request - *WARNING*: If you use your own certificates, it's like in dips, please take a look at - :ref:`using_own_CA` + .. warning:: + If you use your own certificates, it's like in dips, please take a look at :ref:`using_own_CA` - *WARNING*: Lots of method are copy-paste from :py:class:`~DIRAC.Core.DISET.private.BaseClient`. - And some methods are copy-paste AND modifications, for now it permit to fully separate DISET and HTTPS. + .. warning:: + Lots of method are copy-paste from :py:class:`~DIRAC.Core.DISET.private.BaseClient`. + And some methods are copy-paste AND modifications, for now it permit to fully separate DISET and HTTPS. """ # pylint: disable=broad-except +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +from io import open +import six + import cStringIO import requests import DIRAC @@ -78,7 +86,7 @@ def __init__(self, serviceName, **kwargs): :param keepAliveLapse: Duration for keepAliveLapse (heartbeat like) (now managed by requests) """ - if not isinstance(serviceName, basestring): + if not isinstance(serviceName, six.string_types): raise TypeError("Service name expected to be a string. Received %s type %s" % (str(serviceName), type(serviceName))) @@ -246,7 +254,7 @@ def __discoverExtraCredentials(self): WARNING: COPY/PASTE FROM Core/Diset/private/BaseClient """ - # Wich extra credentials to use? + # which extra credentials to use? if self.__useCertificates: self.__extraCredentials = self.VAL_EXTRA_CREDENTIALS_HOST else: diff --git a/Core/Tornado/Server/HandlerManager.py b/Core/Tornado/Server/HandlerManager.py index 82373c6ca1a..8da6fea30d7 100644 --- a/Core/Tornado/Server/HandlerManager.py +++ b/Core/Tornado/Server/HandlerManager.py @@ -7,6 +7,10 @@ """ +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + from tornado.web import url as TornadoURL, RequestHandler from DIRAC.Core.Utilities.ObjectLoader import ObjectLoader @@ -19,16 +23,15 @@ def urlFinder(module): """ Try to guess the url with module name - :param module: path writed like import (e.g. "DIRAC.something.something") + :param module: path written like import (e.g. "DIRAC.something.something") """ sections = module.split('.') for section in sections: # This condition is a bit long # We search something which look like <...>.System.<...>.Handler # If find we return // - if(section.find("System") > 0) and (sections[-1].find('Handler') > 0): - return "/%s/%s" % (section.replace("System", ""), sections[-1].replace("Handler", "")) - return None + if section.endswith("System") and sections[-1].endswith("Handler"): + return "/".join(["", section[:-len("System")], sections[-1][:-len("Handler")]]) class HandlerManager(object): @@ -44,7 +47,8 @@ def __init__(self, autoDiscovery=True): discovery of handler. If disabled you can use loadHandlersByServiceName() to load your handlers or loadHandlerInHandlerManager() - :param autoDiscovery=False: Disable the automatic discovery, can be used to choose service we want to load. + :param autoDiscovery: (default True) Disable the automatic discovery, + can be used to choose service we want to load. """ self.__handlers = {} self.__objectLoader = ObjectLoader() @@ -86,7 +90,7 @@ def __addHandler(self, handlerTuple, url=None): gLogger.info("New handler: %s with URL %s" % (handlerTuple[0], url)) else: gLogger.debug("Handler already loaded %s" % (handlerTuple[0])) - return S_OK + return S_OK() def discoverHandlers(self): """ @@ -160,9 +164,9 @@ def getHandlersURLs(self): def getHandlersDict(self): """ - Return all handler dictionnary + Return all handler dictionary - :returns: dictionnary with absolute url as key ("/System/Service") + :returns: dictionary with absolute url as key ("/System/Service") and tornado.web.url object as value """ if not self.__handlers and self.__autoDiscovery: diff --git a/Core/Tornado/Server/TornadoServer.py b/Core/Tornado/Server/TornadoServer.py index 3579e01f947..058d2054f7e 100644 --- a/Core/Tornado/Server/TornadoServer.py +++ b/Core/Tornado/Server/TornadoServer.py @@ -3,8 +3,9 @@ It may work better with TornadoClient but as it accepts HTTPS you can create your own client """ -__RCSID__ = "$Id$" - +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function import time import datetime @@ -68,7 +69,6 @@ def __init__(self, services=None, debugSSL=False, port=None): :param int port: Used to change port, default is 443 """ - # If port not precised, get it from config if port is None: port = gConfig.getValue("/Systems/Tornado/%s/Port" % PathFinder.getSystemInstance('Tornado'), 443) @@ -93,7 +93,7 @@ def __init__(self, services=None, debugSSL=False, port=None): # if no service list is given, load services from configuration handlerDict = self.handlerManager.getHandlersDict() - for item in handlerDict.iteritems(): + for item in handlerDict.items(): # handlerDict[key].initializeService(key) self.urls.append(url(item[0], item[1], dict(debug=self.debugSSL))) # If there is no services loaded: @@ -115,7 +115,7 @@ def startTornado(self, multiprocess=False): if self.debugSSL: gLogger.warn("Server is running in debug mode") - router = Application(self.urls, debug=self.debugSSL) + router = Application(self.urls, debug=self.debugSSL, compress_response=True) certs = Locations.getHostCertificateAndKeyLocation() if certs is False: @@ -137,14 +137,14 @@ def startTornado(self, multiprocess=False): tornado.ioloop.PeriodicCallback(self.__reportToMonitoring, self.__monitoringLoopDelay * 1000).start() # Start server - server = HTTPServer(router, ssl_options=ssl_options) + server = HTTPServer(router, ssl_options=ssl_options, decompress_request=True) try: if multiprocess: server.bind(self.port) else: server.listen(self.port) - except socketerror as e: - gLogger.fatal(e) + except Exception as e: # pylint: disable=broad-except + gLogger.exception("Exception starting HTTPServer", e) return S_ERROR() gLogger.always("Listening on port %s" % self.port) for service in self.urls: @@ -153,7 +153,6 @@ def startTornado(self, multiprocess=False): if multiprocess: server.start(0) IOLoop.current().start() - return True # Never called because we are stuck in the IOLoop, but to make pylint happy def _initMonitoring(self): """ diff --git a/Core/Tornado/Server/TornadoService.py b/Core/Tornado/Server/TornadoService.py index 07896e57c2c..0529b2326d2 100644 --- a/Core/Tornado/Server/TornadoService.py +++ b/Core/Tornado/Server/TornadoService.py @@ -24,6 +24,12 @@ def export_someMethod(self): """ +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +from io import open + import os import time from datetime import datetime @@ -133,23 +139,23 @@ def __initializeService(cls, relativeUrl, absoluteUrl, debug): @classmethod def initializeHandler(cls, serviceInfoDict): """ - This may be overwrited when you write a DIRAC service handler + This may be overwritten when you write a DIRAC service handler And it must be a class method. This method is called only one time, at the first request :param dict ServiceInfoDict: infos about services, it contains 'serviceName', 'serviceSectionPath', - 'csPaths'and 'URL' + 'csPaths' and 'URL' """ pass def initializeRequest(self): """ - Called at every request, may be overwrited in your handler. + Called at every request, may be overwritten in your handler. """ pass - # This function is designed to be overwrited as we want in Tornado + # This function is designed to be overwritten as we want in Tornado # It's why we should disable pylint for this one def initialize(self, debug): # pylint: disable=arguments-differ """ @@ -203,7 +209,7 @@ def prepare(self): except Exception: # pylint: disable=broad-except # If an error occur when reading certificates we close connection # It can be strange but the RFC, for HTTP, say's that when error happend - # before authenfication we return 401 UNAUTHORIZED instead of 403 FORBIDDEN + # before authentication we return 401 UNAUTHORIZED instead of 403 FORBIDDEN self.reportUnauthorizedAccess(HTTPErrorCodes.HTTP_UNAUTHORIZED) try: @@ -232,7 +238,7 @@ def post(self): # pylint: disable=arguments-differ self.__write_return(retVal.result()) self.finish() - # This nice idea of streaming to the client cannot work because we are ran in an eecutor + # This nice idea of streaming to the client cannot work because we are ran in an executor # and we should not write back to the client in a different thread. # See https://www.tornadoweb.org/en/branch5.1/web.html#thread-safety-notes # def export_streamToClient(self, filename): @@ -296,22 +302,26 @@ def __executeMethod(self): return retVal - def __write_return(self, dictionnary): + def __write_return(self, dictionary): """ Write to client what we wan't to return to client - It must be a dictionnary + It must be a dictionary """ # In case of error in server side we hide server CallStack to client - if 'CallStack' in dictionnary: - del dictionnary['CallStack'] + if 'CallStack' in dictionary: + del dictionary['CallStack'] # Write status code before writing, by default error code is "200 OK" self.set_status(self._httpError) - # See 4.5.1 http://www.rfc-editor.org/rfc/rfc2046.txt - self.set_header("Content-Type", "application/octet-stream") - returnedData = dictionnary if self.rawContent else encode(dictionnary) + if self.rawContent: + # See 4.5.1 http://www.rfc-editor.org/rfc/rfc2046.txt + self.set_header("Content-Type", "application/octet-stream") + else: + self.set_header("Content-Type", "application/json") + + returnedData = dictionary if self.rawContent else encode(dictionary) self.write(returnedData) def reportUnauthorizedAccess(self, errorCode=401): @@ -343,7 +353,7 @@ def _gatherPeerCredentials(self): """ Load client certchain in DIRAC and extract informations. - The dictionnary returned is designed to work with the AuthManager, + The dictionary returned is designed to work with the AuthManager, already written for DISET and re-used for HTTPS. """ @@ -395,10 +405,10 @@ def export_ping(self): dInfo['time'] = datetime.utcnow() # Uptime try: - with open("/proc/uptime") as oFD: - iUptime = long(float(oFD.readline().split()[0].strip())) + with open("/proc/uptime", 'rt') as oFD: + iUptime = int(float(oFD.readline().split()[0].strip())) dInfo['host uptime'] = iUptime - except BaseException: + except Exception: # pylint: disable=broad-except pass startTime = self._startTime dInfo['service start time'] = self._startTime @@ -406,10 +416,9 @@ def export_ping(self): dInfo['service uptime'] = serviceUptime.days * 3600 + serviceUptime.seconds # Load average try: - with open("/proc/loadavg") as oFD: - sLine = oFD.readline() - dInfo['load'] = " ".join(sLine.split()[:3]) - except BaseException: + with open("/proc/loadavg", 'rt') as oFD: + dInfo['load'] = " ".join(oFD.read().split()[:3]) + except Exception: # pylint: disable=broad-except pass dInfo['name'] = self._serviceInfoDict['serviceName'] stTimes = os.times() @@ -435,7 +444,7 @@ def export_echo(data): def export_whoami(self): """ - A simple whoami, returns all credential dictionnary, except certificate chain object. + A simple whoami, returns all credential dictionary, except certificate chain object. """ credDict = self.srv_getRemoteCredentials() if 'x509Chain' in credDict: diff --git a/Core/Tornado/scripts/tornado-start-CS.py b/Core/Tornado/scripts/tornado-start-CS.py index 8b444c3f1b7..dd502f37fd8 100644 --- a/Core/Tornado/scripts/tornado-start-CS.py +++ b/Core/Tornado/scripts/tornado-start-CS.py @@ -6,13 +6,14 @@ # Just run this script to start Tornado and CS service # Use dirac.cfg (or other cfg given in the command line) to change port -__RCSID__ = "$Id$" - +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function # Must be define BEFORE any dirac import import os import sys -os.environ['USE_TORNADO_IOLOOP'] = "True" +os.environ['DIRAC_USE_TORNADO_IOLOOP'] = "True" from DIRAC.FrameworkSystem.Client.Logger import gLogger diff --git a/Core/Tornado/scripts/tornado-start-all.py b/Core/Tornado/scripts/tornado-start-all.py index 01ed73e7c88..eab0ca8b613 100644 --- a/Core/Tornado/scripts/tornado-start-all.py +++ b/Core/Tornado/scripts/tornado-start-all.py @@ -6,13 +6,14 @@ # Just run this script to start Tornado and all services # Use CS to change port -__RCSID__ = "$Id$" - +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function # Must be define BEFORE any dirac import import os import sys -os.environ['USE_TORNADO_IOLOOP'] = "True" +os.environ['DIRAC_USE_TORNADO_IOLOOP'] = "True" from DIRAC.FrameworkSystem.Client.Logger import gLogger diff --git a/Core/Tornado/test/migrationTests/README b/Core/Tornado/test/migrationTests/README deleted file mode 100644 index 37869667ec6..00000000000 --- a/Core/Tornado/test/migrationTests/README +++ /dev/null @@ -1,2 +0,0 @@ -These tests were writen in order to compare the existing DISET based services with the new Tornado/HTTPs based services. -They were ran by hand, and are kept here just in case for the time being. diff --git a/DataManagementSystem/Service/TornadoFileCatalogHandler.py b/DataManagementSystem/Service/TornadoFileCatalogHandler.py index b4210afe606..242be4f0fef 100644 --- a/DataManagementSystem/Service/TornadoFileCatalogHandler.py +++ b/DataManagementSystem/Service/TornadoFileCatalogHandler.py @@ -14,10 +14,10 @@ # imports import six -import cStringIO import csv import os -from types import IntType, LongType, DictType, StringTypes, BooleanType, ListType + +from io import BytesIO # from DIRAC from DIRAC.Core.DISET.RequestHandler import getServiceOption @@ -114,22 +114,16 @@ def initializeHandler(cls, serviceInfo): ######################################################################## # Path operations (not updated) # - types_changePathOwner = [[ListType, DictType] + list(StringTypes)] - def export_changePathOwner(self, lfns, recursive=False): """ Get replica info for the given list of LFNs """ return self.gFileCatalogDB.changePathOwner(lfns, self.getRemoteCredentials(), recursive) - types_changePathGroup = [[ListType, DictType] + list(StringTypes)] - def export_changePathGroup(self, lfns, recursive=False): """ Get replica info for the given list of LFNs """ return self.gFileCatalogDB.changePathGroup(lfns, self.getRemoteCredentials(), recursive) - types_changePathMode = [[ListType, DictType] + list(StringTypes)] - def export_changePathMode(self, lfns, recursive=False): """ Get replica info for the given list of LFNs """ @@ -138,15 +132,11 @@ def export_changePathMode(self, lfns, recursive=False): ######################################################################## # ACL Operations # - types_getPathPermissions = [[ListType, DictType] + list(StringTypes)] - def export_getPathPermissions(self, lfns): """ Determine the ACL information for a supplied path """ return self.gFileCatalogDB.getPathPermissions(lfns, self.getRemoteCredentials()) - types_hasAccess = [[basestring, dict], [basestring, list, dict]] - def export_hasAccess(self, paths, opType): """ Determine if the given op can be performed on the paths The OpType is all the operations exported @@ -168,8 +158,6 @@ def export_hasAccess(self, paths, opType): # isOK # - types_isOK = [] - @classmethod def export_isOK(cls): """ returns S_OK if DB is connected @@ -183,26 +171,18 @@ def export_isOK(cls): # User/Group write operations # - types_addUser = [StringTypes] - def export_addUser(self, userName): """ Add a new user to the File Catalog """ return self.gFileCatalogDB.addUser(userName, self.getRemoteCredentials()) - types_deleteUser = [StringTypes] - def export_deleteUser(self, userName): """ Delete user from the File Catalog """ return self.gFileCatalogDB.deleteUser(userName, self.getRemoteCredentials()) - types_addGroup = [StringTypes] - def export_addGroup(self, groupName): """ Add a new group to the File Catalog """ return self.gFileCatalogDB.addGroup(groupName, self.getRemoteCredentials()) - types_deleteGroup = [StringTypes] - def export_deleteGroup(self, groupName): """ Delete group from the File Catalog """ return self.gFileCatalogDB.deleteGroup(groupName, self.getRemoteCredentials()) @@ -212,14 +192,10 @@ def export_deleteGroup(self, groupName): # User/Group read operations # - types_getUsers = [] - def export_getUsers(self): """ Get all the users defined in the File Catalog """ return self.gFileCatalogDB.getUsers(self.getRemoteCredentials()) - types_getGroups = [] - def export_getGroups(self): """ Get all the groups defined in the File Catalog """ return self.gFileCatalogDB.getGroups(self.getRemoteCredentials()) @@ -229,8 +205,6 @@ def export_getGroups(self): # Path read operations # - types_exists = [[ListType, DictType] + list(StringTypes)] - def export_exists(self, lfns): """ Check whether the supplied paths exists """ return self.gFileCatalogDB.exists(lfns, self.getRemoteCredentials()) @@ -240,8 +214,6 @@ def export_exists(self, lfns): # File write operations # - types_addFile = [[ListType, DictType] + list(StringTypes)] - def export_addFile(self, lfns): """ Register supplied files """ gMonitor.addMark("AddFile", 1) @@ -252,8 +224,6 @@ def export_addFile(self, lfns): return res - types_removeFile = [[ListType, DictType] + list(StringTypes)] - def export_removeFile(self, lfns): """ Remove the supplied lfns """ gMonitor.addMark("RemoveFile", 1) @@ -264,14 +234,10 @@ def export_removeFile(self, lfns): return res - types_setFileStatus = [DictType] - def export_setFileStatus(self, lfns): """ Remove the supplied lfns """ return self.gFileCatalogDB.setFileStatus(lfns, self.getRemoteCredentials()) - types_addReplica = [[ListType, DictType] + list(StringTypes)] - def export_addReplica(self, lfns): """ Register supplied replicas """ gMonitor.addMark("AddReplica", 1) @@ -282,8 +248,6 @@ def export_addReplica(self, lfns): return res - types_removeReplica = [[ListType, DictType] + list(StringTypes)] - def export_removeReplica(self, lfns): """ Remove the supplied replicas """ gMonitor.addMark("RemoveReplica", 1) @@ -294,20 +258,14 @@ def export_removeReplica(self, lfns): return res - types_setReplicaStatus = [[ListType, DictType] + list(StringTypes)] - def export_setReplicaStatus(self, lfns): """ Set the status for the supplied replicas """ return self.gFileCatalogDB.setReplicaStatus(lfns, self.getRemoteCredentials()) - types_setReplicaHost = [[ListType, DictType] + list(StringTypes)] - def export_setReplicaHost(self, lfns): """ Change the registered SE for the supplied replicas """ return self.gFileCatalogDB.setReplicaHost(lfns, self.getRemoteCredentials()) - types_addFileAncestors = [DictType] - def export_addFileAncestors(self, lfns): """ Add file ancestor information for the given list of LFNs """ return self.gFileCatalogDB.addFileAncestors(lfns, self.getRemoteCredentials()) @@ -317,58 +275,42 @@ def export_addFileAncestors(self, lfns): # File read operations # - types_isFile = [[ListType, DictType] + list(StringTypes)] - def export_isFile(self, lfns): """ Check whether the supplied lfns are files """ return self.gFileCatalogDB.isFile(lfns, self.getRemoteCredentials()) - types_getFileSize = [[ListType, DictType] + list(StringTypes)] - def export_getFileSize(self, lfns): """ Get the size associated to supplied lfns """ return self.gFileCatalogDB.getFileSize(lfns, self.getRemoteCredentials()) - types_getFileMetadata = [[ListType, DictType] + list(StringTypes)] - def export_getFileMetadata(self, lfns): """ Get the metadata associated to supplied lfns """ return self.gFileCatalogDB.getFileMetadata(lfns, self.getRemoteCredentials()) - types_getReplicas = [[ListType, DictType] + list(StringTypes), BooleanType] - def export_getReplicas(self, lfns, allStatus=False): """ Get replicas for supplied lfns """ return self.gFileCatalogDB.getReplicas(lfns, allStatus, self.getRemoteCredentials()) - types_getReplicaStatus = [[ListType, DictType] + list(StringTypes)] - def export_getReplicaStatus(self, lfns): """ Get the status for the supplied replicas """ return self.gFileCatalogDB.getReplicaStatus(lfns, self.getRemoteCredentials()) - types_getFileAncestors = [[ListType, DictType], [ListType, IntType, LongType]] - def export_getFileAncestors(self, lfns, depths): """ Get the status for the supplied replicas """ dList = depths - if not isinstance(dList, ListType): + if not isinstance(dList, list): dList = [depths] lfnDict = dict.fromkeys(lfns, True) return self.gFileCatalogDB.getFileAncestors(lfnDict, dList, self.getRemoteCredentials()) - types_getFileDescendents = [[ListType, DictType], [ListType, IntType, LongType]] - def export_getFileDescendents(self, lfns, depths): """ Get the status for the supplied replicas """ dList = depths - if not isinstance(dList, ListType): + if not isinstance(dList, list): dList = [depths] lfnDict = dict.fromkeys(lfns, True) return self.gFileCatalogDB.getFileDescendents(lfnDict, dList, self.getRemoteCredentials()) - types_getLFNForGUID = [[ListType, DictType] + list(StringTypes)] - def export_getLFNForGUID(self, guids): """Get the matching lfns for given guids""" return self.gFileCatalogDB.getLFNForGUID(guids, self.getRemoteCredentials()) @@ -378,14 +320,10 @@ def export_getLFNForGUID(self, guids): # Directory write operations # - types_createDirectory = [[ListType, DictType] + list(StringTypes)] - def export_createDirectory(self, lfns): """ Create the supplied directories """ return self.gFileCatalogDB.createDirectory(lfns, self.getRemoteCredentials()) - types_removeDirectory = [[ListType, DictType] + list(StringTypes)] - def export_removeDirectory(self, lfns): """ Remove the supplied directories """ return self.gFileCatalogDB.removeDirectory(lfns, self.getRemoteCredentials()) @@ -395,33 +333,23 @@ def export_removeDirectory(self, lfns): # Directory read operations # - types_listDirectory = [[ListType, DictType] + list(StringTypes), BooleanType] - def export_listDirectory(self, lfns, verbose): """ List the contents of supplied directories """ gMonitor.addMark('ListDirectory', 1) return self.gFileCatalogDB.listDirectory(lfns, self.getRemoteCredentials(), verbose=verbose) - types_isDirectory = [[ListType, DictType] + list(StringTypes)] - def export_isDirectory(self, lfns): """ Determine whether supplied path is a directory """ return self.gFileCatalogDB.isDirectory(lfns, self.getRemoteCredentials()) - types_getDirectoryMetadata = [[ListType, DictType] + list(StringTypes)] - def export_getDirectoryMetadata(self, lfns): """ Get the size of the supplied directory """ return self.gFileCatalogDB.getDirectoryMetadata(lfns, self.getRemoteCredentials()) - types_getDirectorySize = [[ListType, DictType] + list(StringTypes)] - def export_getDirectorySize(self, lfns, longOut=False, fromFiles=False): """ Get the size of the supplied directory """ return self.gFileCatalogDB.getDirectorySize(lfns, longOut, fromFiles, self.getRemoteCredentials()) - types_getDirectoryReplicas = [[ListType, DictType] + list(StringTypes), BooleanType] - def export_getDirectoryReplicas(self, lfns, allStatus=False): """ Get replicas for files in the supplied directory """ return self.gFileCatalogDB.getDirectoryReplicas(lfns, allStatus, self.getRemoteCredentials()) @@ -431,21 +359,15 @@ def export_getDirectoryReplicas(self, lfns, allStatus=False): # Administrative database operations # - types_getCatalogCounters = [] - def export_getCatalogCounters(self): """ Get the number of registered directories, files and replicas in various tables """ return self.gFileCatalogDB.getCatalogCounters(self.getRemoteCredentials()) - types_rebuildDirectoryUsage = [] - @staticmethod def export_rebuildDirectoryUsage(self): """ Rebuild DirectoryUsage table from scratch """ return self.gFileCatalogDB.rebuildDirectoryUsage() - types_repairCatalog = [] - def export_repairCatalog(self): """ Repair the catalog inconsistencies """ return self.gFileCatalogDB.repairCatalog(self.getRemoteCredentials()) @@ -454,8 +376,6 @@ def export_repairCatalog(self): # Metadata Catalog Operations # - types_addMetadataField = [StringTypes, StringTypes, StringTypes] - def export_addMetadataField(self, fieldName, fieldType, metaType='-d'): """ Add a new metadata field of the given type """ @@ -468,8 +388,6 @@ def export_addMetadataField(self, fieldName, fieldType, metaType='-d'): else: return S_ERROR('Unknown metadata type %s' % metaType) - types_deleteMetadataField = [StringTypes] - def export_deleteMetadataField(self, fieldName): """ Delete the metadata field """ @@ -484,8 +402,6 @@ def export_deleteMetadataField(self, fieldName): return result - types_getMetadataFields = [] - def export_getMetadataFields(self): """ Get all the metadata fields """ @@ -499,51 +415,37 @@ def export_getMetadataFields(self): return S_OK({'DirectoryMetaFields': resultDir['Value'], 'FileMetaFields': resultFile['Value']}) - types_setMetadata = [StringTypes, DictType] - def export_setMetadata(self, path, metadatadict): """ Set metadata parameter for the given path """ return self.gFileCatalogDB.setMetadata(path, metadatadict, self.getRemoteCredentials()) - types_setMetadataBulk = [DictType] - def export_setMetadataBulk(self, pathMetadataDict): """ Set metadata parameter for the given path """ return self.gFileCatalogDB.setMetadataBulk(pathMetadataDict, self.getRemoteCredentials()) - types_removeMetadata = [DictType] - def export_removeMetadata(self, pathMetadataDict): """ Remove the specified metadata for the given path """ return self.gFileCatalogDB.removeMetadata(pathMetadataDict, self.getRemoteCredentials()) - types_getDirectoryUserMetadata = [StringTypes] - def export_getDirectoryUserMetadata(self, path): """ Get all the metadata valid for the given directory path """ return self.gFileCatalogDB.dmeta.getDirectoryMetadata(path, self.getRemoteCredentials()) - types_getFileUserMetadata = [StringTypes] - def export_getFileUserMetadata(self, path): """ Get all the metadata valid for the given file """ return self.gFileCatalogDB.fmeta.getFileUserMetadata(path, self.getRemoteCredentials()) - types_findDirectoriesByMetadata = [DictType] - def export_findDirectoriesByMetadata(self, metaDict, path='/'): """ Find all the directories satisfying the given metadata set """ return self.gFileCatalogDB.dmeta.findDirectoriesByMetadata( metaDict, path, self.getRemoteCredentials()) - types_findFilesByMetadata = [DictType, StringTypes] - def export_findFilesByMetadata(self, metaDict, path='/'): """ Find all the files satisfying the given metadata set """ @@ -553,8 +455,6 @@ def export_findFilesByMetadata(self, metaDict, path='/'): lfns = result['Value'].values() return S_OK(lfns) - types_getReplicasByMetadata = [DictType, StringTypes, BooleanType] - def export_getReplicasByMetadata(self, metaDict, path='/', allStatus=False): """ Find all the files satisfying the given metadata set """ @@ -563,8 +463,6 @@ def export_getReplicasByMetadata(self, metaDict, path='/', allStatus=False): allStatus, self.getRemoteCredentials()) - types_findFilesByMetadataDetailed = [DictType, StringTypes] - def export_findFilesByMetadataDetailed(self, metaDict, path='/'): """ Find all the files satisfying the given metadata set """ @@ -575,8 +473,6 @@ def export_findFilesByMetadataDetailed(self, metaDict, path='/'): lfns = result['Value'].values() return self.gFileCatalogDB.getFileDetails(lfns, self.getRemoteCredentials()) - types_findFilesByMetadataWeb = [DictType, StringTypes, [IntType, LongType], [IntType, LongType]] - def export_findFilesByMetadataWeb(self, metaDict, path, startItem, maxItems): """ Find files satisfying the given metadata set """ @@ -628,22 +524,16 @@ def findFilesByMetadataWeb(self, metaDict, path, startItem, maxItems): result = S_OK({"TotalRecords": totalRecords, "Records": resultDetails['Value']}) return result - types_getCompatibleMetadata = [DictType, StringTypes] - def export_getCompatibleMetadata(self, metaDict, path='/'): """ Get metadata values compatible with the given metadata subset """ return self.gFileCatalogDB.dmeta.getCompatibleMetadata(metaDict, path, self.getRemoteCredentials()) - types_addMetadataSet = [StringTypes, DictType] - def export_addMetadataSet(self, setName, setDict): """ Add a new metadata set """ return self.gFileCatalogDB.dmeta.addMetadataSet(setName, setDict, self.getRemoteCredentials()) - types_getMetadataSet = [StringTypes, BooleanType] - def export_getMetadataSet(self, setName, expandFlag): """ Add a new metadata set """ @@ -653,79 +543,58 @@ def export_getMetadataSet(self, setName, expandFlag): # # Dataset manipulation methods # - types_addDataset = [DictType] def export_addDataset(self, datasets): """ Add a new dynamic dataset defined by its meta query """ return self.gFileCatalogDB.datasetManager.addDataset(datasets, self.getRemoteCredentials()) - types_addDatasetAnnotation = [DictType] - def export_addDatasetAnnotation(self, datasetDict): """ Add annotation to an already created dataset """ return self.gFileCatalogDB.datasetManager.addDatasetAnnotation( datasetDict, self.getRemoteCredentials()) - types_removeDataset = [DictType] - def export_removeDataset(self, datasets): """ Check the given dynamic dataset for changes since its definition """ return self.gFileCatalogDB.datasetManager.removeDataset(datasets, self.getRemoteCredentials()) - types_checkDataset = [DictType] - def export_checkDataset(self, datasets): """ Check the given dynamic dataset for changes since its definition """ return self.gFileCatalogDB.datasetManager.checkDataset(datasets, self.getRemoteCredentials()) - types_updateDataset = [DictType] - def export_updateDataset(self, datasets): """ Update the given dynamic dataset for changes since its definition """ return self.gFileCatalogDB.datasetManager.updateDataset(datasets, self.getRemoteCredentials()) - types_getDatasets = [DictType] - def export_getDatasets(self, datasets): """ Get parameters of the given dynamic dataset as they are stored in the database """ return self.gFileCatalogDB.datasetManager.getDatasets(datasets, self.getRemoteCredentials()) - types_getDatasetParameters = [DictType] - def export_getDatasetParameters(self, datasets): """ Get parameters of the given dynamic dataset as they are stored in the database """ return self.gFileCatalogDB.datasetManager.getDatasetParameters(datasets, self.getRemoteCredentials()) - types_getDatasetAnnotation = [DictType] - def export_getDatasetAnnotation(self, datasets): """ Get annotation of the given datasets """ return self.gFileCatalogDB.datasetManager.getDatasetAnnotation(datasets, self.getRemoteCredentials()) - types_freezeDataset = [DictType] - def export_freezeDataset(self, datasets): """ Freeze the contents of the dataset making it effectively static """ return self.gFileCatalogDB.datasetManager.freezeDataset(datasets, self.getRemoteCredentials()) - types_releaseDataset = [DictType] - def export_releaseDataset(self, datasets): """ Release the contents of the frozen dataset allowing changes in its contents """ return self.gFileCatalogDB.datasetManager.releaseDataset(datasets, self.getRemoteCredentials()) - types_getDatasetFiles = [DictType] - def export_getDatasetFiles(self, datasets): """ Get lfns in the given dataset """ @@ -755,7 +624,7 @@ def export_streamToClient(self, seName): retVal = self.getSEDump(seName) try: - csvOutput = cStringIO.StringIO() + csvOutput = BytesIO() writer = csv.writer(csvOutput, delimiter='|') for lfn in retVal: writer.writerow(lfn) diff --git a/FrameworkSystem/Client/MonitoringClient.py b/FrameworkSystem/Client/MonitoringClient.py index 8faa28816ba..2004e711180 100755 --- a/FrameworkSystem/Client/MonitoringClient.py +++ b/FrameworkSystem/Client/MonitoringClient.py @@ -85,8 +85,8 @@ def registerMonitoringClient(self, mc): self.__mcList.append(mc) -# USE_TORNADO_IOLOOP is defined by starting scripts -if os.environ.get('USE_TORNADO_IOLOOP', 'false').lower() in ('yes', 'true'): +# DIRAC_USE_TORNADO_IOLOOP is defined by starting scripts +if os.environ.get('DIRAC_USE_TORNADO_IOLOOP', 'false').lower() in ('yes', 'true'): from DIRAC.FrameworkSystem.Client.MonitoringClientIOLoop import MonitoringFlusherTornado gMonitoringFlusher = MonitoringFlusherTornado() else: diff --git a/docs/source/DeveloperGuide/TornadoServices/index.rst b/docs/source/DeveloperGuide/TornadoServices/index.rst index a6958c185e3..f09464e04ba 100644 --- a/docs/source/DeveloperGuide/TornadoServices/index.rst +++ b/docs/source/DeveloperGuide/TornadoServices/index.rst @@ -171,7 +171,7 @@ You can also see this example:: The specific client for configuration system. To avoid JSON limitation the HTTPS handler encode data in base64 before sending them, this class only decode the base64 - An exception is made with CommitNewData wich ENCODE in base64 + An exception is made with CommitNewData which ENCODE in base64 """ def getCompressedData(self): """ diff --git a/environment.yml b/environment.yml index 73dc63fbe44..b7d13789797 100644 --- a/environment.yml +++ b/environment.yml @@ -66,5 +66,9 @@ dependencies: - openssl <1.1 - pip: - diraccfg + # This is a fork of tornado with a patch to allow for configurable iostream + # It should eventually be part of DIRACGrid - git+https://github.com/chaen/tornado.git@iostreamConfigurable + # This is an extension of Tornado to use M2Crypto + # It should eventually be part of DIRACGrid - git+https://github.com/chaen/tornado_m2crypto diff --git a/tests/CI/install_server.sh b/tests/CI/install_server.sh index 47baf143a62..256103169ce 100755 --- a/tests/CI/install_server.sh +++ b/tests/CI/install_server.sh @@ -73,7 +73,7 @@ echo -e "*** $(date -u) **** DONE Adding S3-INDIRECT SERVER CONFIGURATION" # Here, if we are testing HTTPS services, we install the equivalent services and replace the URL in the CS -if [[ "${TESTBRANCH:-No}" = "Yes" ]]; +if [[ "${TEST_HTTPS:-No}" = "Yes" ]]; then echo -e "*** $(date -u) **** Installing Tornado based services" diff --git a/tests/CI/run_docker_setup.sh b/tests/CI/run_docker_setup.sh index 46d9fb34bd7..618c09ed7f3 100755 --- a/tests/CI/run_docker_setup.sh +++ b/tests/CI/run_docker_setup.sh @@ -97,6 +97,7 @@ prepareEnvironment() { echo "" echo "# Test specific variables" echo "export WORKSPACE=${WORKSPACE}" + echo "export TEST_HTTPS=${TEST_HTTPS}" echo "" echo "# Optional parameters" diff --git a/tests/DISET_HTTPS_migration/README b/tests/DISET_HTTPS_migration/README new file mode 100644 index 00000000000..2e8dea36dbb --- /dev/null +++ b/tests/DISET_HTTPS_migration/README @@ -0,0 +1,2 @@ +These tests were written in order to compare the existing DISET based services with the new Tornado/HTTPs based services. +They were ran by hand, and are kept here just in case for the time being. diff --git a/Core/Tornado/test/migrationTests/TestClientReturnedValuesAndCredentialsExtraction.py b/tests/DISET_HTTPS_migration/TestClientReturnedValuesAndCredentialsExtraction.py similarity index 100% rename from Core/Tornado/test/migrationTests/TestClientReturnedValuesAndCredentialsExtraction.py rename to tests/DISET_HTTPS_migration/TestClientReturnedValuesAndCredentialsExtraction.py diff --git a/Core/Tornado/test/migrationTests/TestClientSelectionAndInterface.py b/tests/DISET_HTTPS_migration/TestClientSelectionAndInterface.py similarity index 100% rename from Core/Tornado/test/migrationTests/TestClientSelectionAndInterface.py rename to tests/DISET_HTTPS_migration/TestClientSelectionAndInterface.py diff --git a/Core/Tornado/test/migrationTests/TestServerIntegration.py b/tests/DISET_HTTPS_migration/TestServerIntegration.py similarity index 100% rename from Core/Tornado/test/migrationTests/TestServerIntegration.py rename to tests/DISET_HTTPS_migration/TestServerIntegration.py diff --git a/Core/Tornado/test/migrationTests/multi-mechanize/distributed-test.py b/tests/DISET_HTTPS_migration/multi-mechanize/distributed-test.py similarity index 100% rename from Core/Tornado/test/migrationTests/multi-mechanize/distributed-test.py rename to tests/DISET_HTTPS_migration/multi-mechanize/distributed-test.py diff --git a/Core/Tornado/test/migrationTests/multi-mechanize/ping/config.cfg b/tests/DISET_HTTPS_migration/multi-mechanize/ping/config.cfg similarity index 100% rename from Core/Tornado/test/migrationTests/multi-mechanize/ping/config.cfg rename to tests/DISET_HTTPS_migration/multi-mechanize/ping/config.cfg diff --git a/Core/Tornado/test/migrationTests/multi-mechanize/ping/test_scripts/v_perf.py b/tests/DISET_HTTPS_migration/multi-mechanize/ping/test_scripts/v_perf.py similarity index 100% rename from Core/Tornado/test/migrationTests/multi-mechanize/ping/test_scripts/v_perf.py rename to tests/DISET_HTTPS_migration/multi-mechanize/ping/test_scripts/v_perf.py diff --git a/Core/Tornado/test/migrationTests/multi-mechanize/plot-distributedTest.py b/tests/DISET_HTTPS_migration/multi-mechanize/plot-distributedTest.py similarity index 100% rename from Core/Tornado/test/migrationTests/multi-mechanize/plot-distributedTest.py rename to tests/DISET_HTTPS_migration/multi-mechanize/plot-distributedTest.py diff --git a/Core/Tornado/test/migrationTests/multi-mechanize/plot.py b/tests/DISET_HTTPS_migration/multi-mechanize/plot.py similarity index 100% rename from Core/Tornado/test/migrationTests/multi-mechanize/plot.py rename to tests/DISET_HTTPS_migration/multi-mechanize/plot.py diff --git a/Core/Tornado/test/migrationTests/multi-mechanize/service/config.cfg b/tests/DISET_HTTPS_migration/multi-mechanize/service/config.cfg similarity index 100% rename from Core/Tornado/test/migrationTests/multi-mechanize/service/config.cfg rename to tests/DISET_HTTPS_migration/multi-mechanize/service/config.cfg diff --git a/Core/Tornado/test/migrationTests/multi-mechanize/service/test_scripts/v_perf.py b/tests/DISET_HTTPS_migration/multi-mechanize/service/test_scripts/v_perf.py similarity index 100% rename from Core/Tornado/test/migrationTests/multi-mechanize/service/test_scripts/v_perf.py rename to tests/DISET_HTTPS_migration/multi-mechanize/service/test_scripts/v_perf.py diff --git a/tests/Integration/DataManagementSystem/Test_Client_DFC.py b/tests/Integration/DataManagementSystem/Test_Client_DFC.py index dc7cb99e3f7..5c56485cfe8 100644 --- a/tests/Integration/DataManagementSystem/Test_Client_DFC.py +++ b/tests/Integration/DataManagementSystem/Test_Client_DFC.py @@ -143,6 +143,7 @@ def test_fileOperations(self): 'GUID': '1000', 'Checksum': '0'}}) self.assertTrue(result['OK'], "addFile failed when adding new file %s" % result) + self.assertTrue(testFile in result['Value'].get('Successful', {}), result) result = self.dfc.exists(testFile) self.assertTrue(result['OK']) diff --git a/tests/py3CheckDirs.txt b/tests/py3CheckDirs.txt index e69de29bb2d..2b7a7b1732c 100644 --- a/tests/py3CheckDirs.txt +++ b/tests/py3CheckDirs.txt @@ -0,0 +1 @@ +Core/Tornado From b2a6edf540d9e0865c4aae42519de14af6f09064 Mon Sep 17 00:00:00 2001 From: Christophe Haen Date: Fri, 7 Aug 2020 17:42:24 +0200 Subject: [PATCH 17/25] refactor the stream --- Core/Tornado/Client/TornadoClient.py | 4 +- .../Client/private/TornadoBaseClient.py | 43 +++++++++---------- Core/Tornado/Server/TornadoService.py | 7 ++- .../DeveloperGuide/TornadoServices/index.rst | 36 ++++++++++++++++ 4 files changed, 63 insertions(+), 27 deletions(-) diff --git a/Core/Tornado/Client/TornadoClient.py b/Core/Tornado/Client/TornadoClient.py index 15350265f8f..8d7364d8015 100644 --- a/Core/Tornado/Client/TornadoClient.py +++ b/Core/Tornado/Client/TornadoClient.py @@ -76,9 +76,9 @@ def receiveFile(self, destFile, *args): :param args: list of arguments :returns: S_OK/S_ERROR """ - rpcCall = {'method': 'streamToClient', 'args': encode(args), 'rawContent': True} + rpcCall = {'method': 'streamToClient', 'args': encode(args)} # Start request - retVal = self._request(stream=True, outputFile=destFile, **rpcCall) + retVal = self._request(outputFile=destFile, **rpcCall) return retVal diff --git a/Core/Tornado/Client/private/TornadoBaseClient.py b/Core/Tornado/Client/private/TornadoBaseClient.py index 484cfe8d848..4169a361451 100644 --- a/Core/Tornado/Client/private/TornadoBaseClient.py +++ b/Core/Tornado/Client/private/TornadoBaseClient.py @@ -31,7 +31,6 @@ from io import open import six -import cStringIO import requests import DIRAC @@ -46,6 +45,10 @@ from DIRAC.Core.Security import Locations from DIRAC.Core.DISET.ThreadConfig import ThreadConfig +# TODO: refactor all the messy `discover` methods +# I do not do it now because I want first to decide +# whether we go with code copy of fatorization + class TornadoBaseClient(object): """ @@ -468,21 +471,18 @@ def _getBaseStub(self): del newKwargs['useCertificates'] return (self._destinationSrv, newKwargs) - def _request(self, retry=0, stream=False, outputFile=None, **kwargs): + def _request(self, retry=0, outputFile=None, **kwargs): """ Sends the request to server :param retry: internal parameters for recursive call. TODO: remove ? - :param stream: (default False) if True, we will receive the output in several chunks. - It is usefull when receiving big amount of data. Note that this is not optimzied - on the server side (see TornadoService) - It shows primarily useful to receive files. :param outputFile: (default None) path to a file where to store the received data. + If set, the server response will be streamed for optimization + purposes, and the response data will not go through the + JDecode process :param **kwargs: Any argument there is used as a post parameter. They are detailed bellow. :param method: (mandatory) name of the distant method :param args: (mandatory) json serialized list of argument for the procedure - :param rawContent: if set to True, the data is transmitted and returned from this method - without going through the JEncode serialization. @@ -490,8 +490,6 @@ def _request(self, retry=0, stream=False, outputFile=None, **kwargs): """ - rawContent = kwargs.get('rawContent') - # Adding some informations to send if self.__extraCredentials: kwargs[self.KW_EXTRA_CREDENTIALS] = encode(self.__extraCredentials) @@ -518,7 +516,8 @@ def _request(self, retry=0, stream=False, outputFile=None, **kwargs): try: rawText = None - if not stream: + # Default case, just return the result + if not outputFile: call = requests.post(url, data=kwargs, timeout=self.timeout, verify=verify, cert=cert) @@ -534,25 +533,23 @@ def _request(self, retry=0, stream=False, outputFile=None, **kwargs): call.raise_for_status() return decode(rawText)[0] else: + # Instruct the server not to encode the response + kwargs['rawContent'] = True + rawText = None # Stream download # https://requests.readthedocs.io/en/latest/user/advanced/#body-content-workflow with requests.post(url, data=kwargs, timeout=self.timeout, verify=verify, cert=cert, stream=True) as r: + rawText = r.text r.raise_for_status() - contentFile = outputFile if outputFile else cStringIO.StringIO() - - with open(contentFile, 'wb') as f: - for chunk in r.iter_content(): - if chunk: # filter out keep-alive new chuncks - f.write(chunk) - if outputFile: - return S_OK() + with open(outputFile, 'wb') as f: + for chunk in r.iter_content(4096): + # if chunk: # filter out keep-alive new chuncks + f.write(chunk) - if rawContent: - return S_OK(contentFile.getvalue()) - return decode(contentFile.getvalue())[0] + return S_OK() except Exception as e: # CHRIS TODO review this part: catch specific exceptions @@ -560,7 +557,7 @@ def _request(self, retry=0, stream=False, outputFile=None, **kwargs): if url not in self.__bannedUrls: self.__bannedUrls += [url] if retry < self.__nbOfUrls - 1: - self._request(retry=retry + 1, stream=stream, outputFile=outputFile, **kwargs) + self._request(retry=retry + 1, outputFile=outputFile, **kwargs) errStr = "%s: %s" % (str(e), rawText) return S_ERROR(errStr) diff --git a/Core/Tornado/Server/TornadoService.py b/Core/Tornado/Server/TornadoService.py index 0529b2326d2..b76e3b01904 100644 --- a/Core/Tornado/Server/TornadoService.py +++ b/Core/Tornado/Server/TornadoService.py @@ -191,7 +191,7 @@ def initialize(self, debug): # pylint: disable=arguments-differ def prepare(self): """ - prepare the request, it read certificates and check authorizations. + prepare the request, it reads certificates and check authorizations. """ self.method = self.get_argument("method") self.rawContent = self.get_argument('rawContent', default=False) @@ -315,13 +315,16 @@ def __write_return(self, dictionary): # Write status code before writing, by default error code is "200 OK" self.set_status(self._httpError) + # This is basically only used for file download through + # the 'streamToClient' method. if self.rawContent: # See 4.5.1 http://www.rfc-editor.org/rfc/rfc2046.txt self.set_header("Content-Type", "application/octet-stream") + returnedData = dictionary else: self.set_header("Content-Type", "application/json") + returnedData = encode(dictionary) - returnedData = dictionary if self.rawContent else encode(dictionary) self.write(returnedData) def reportUnauthorizedAccess(self, errorCode=401): diff --git a/docs/source/DeveloperGuide/TornadoServices/index.rst b/docs/source/DeveloperGuide/TornadoServices/index.rst index f09464e04ba..81e94e40298 100644 --- a/docs/source/DeveloperGuide/TornadoServices/index.rst +++ b/docs/source/DeveloperGuide/TornadoServices/index.rst @@ -141,6 +141,42 @@ Options available are: This start method can be useful for developing new service or create starting script for a specific service, like the Configuration System (as master). +TransferClient +************** + +There is no specific client for transfering files anymore. In fact, the whole idea of directly serving file will eventually disapear and be replaced with redirections to real content streaming server. In the meantine, in order to keep some compatibility, the features were implemented, but require some changes on the server side: + +- ``transfer_toClient`` needs to be renamed ``export_streamToClient`` +- It needs to return the whole file content at once +- The parameter ``fileHelper`` is removed + + +For example:: + + def transfer_toClient(self, myFileToSend, token, fileHelper): + + # Do whatever with the token + + with open(myFileToSend, 'r') as fd: + ret = fileHelper.DataSourceToNetwork(fd) + return ret + +Simply becomes:: + + def export_streamToClient(self, myFileToSend, token): + + # Do whatever with the token + + with open(myFileToSend, 'r') as fd: + return fd.read() + + +From the client side, no change is needed since :py:meth:`DIRAC.Core.Tornado.Client.TornadoClient.TornadoClient.receiveFile` keeps the interface + +This procedure is not optimized server side (see commented ``export_streamToClient`` implementation in :py:class`DIRAC.Core.Tornado.Server.TornadoService.TornadoService`). + +The ``transfer_fromClient`` equivalent has not yet been implemented as it concerns only very few cases (basically DIRAC SE and SandboxStore) + ****** Client ****** From 0d903b0b5e0af729394e819229fd92c40438b3a3 Mon Sep 17 00:00:00 2001 From: Christophe Haen Date: Tue, 11 Aug 2020 15:13:59 +0200 Subject: [PATCH 18/25] Implement minor review comments and py3 compatibility --- .../Client/ConfigurationClient.py | 7 + .../Service/TornadoConfigurationHandler.py | 10 +- ConfigurationSystem/private/Refresher.py | 6 +- .../private/ServiceInterfaceBase.py | 4 + .../private/ServiceInterfaceTornado.py | 7 + Core/Base/Client.py | 5 +- Core/DISET/RequestHandler.py | 5 +- Core/DISET/private/BaseClient.py | 12 +- Core/Security/m2crypto/X509Certificate.py | 3 +- Core/Security/m2crypto/X509Chain.py | 10 +- Core/Tornado/Client/RPCClientSelector.py | 25 +- .../Tornado/Client/SpecificClient/__init__.py | 0 Core/Tornado/Client/TornadoClient.py | 12 +- Core/Tornado/Client/TransferClientSelector.py | 11 +- .../Client/private/TornadoBaseClient.py | 18 +- Core/Tornado/Server/HandlerManager.py | 35 +- Core/Tornado/Server/TornadoServer.py | 115 ++++--- Core/Tornado/Server/TornadoService.py | 320 +++++++++++++----- Core/Tornado/Utilities/HTTPErrorCodes.py | 20 -- Core/Tornado/Utilities/__init__.py | 3 - Core/Tornado/scripts/tornado-start-CS.py | 15 +- Core/Tornado/scripts/tornado-start-all.py | 13 +- .../Service/TornadoFileCatalogHandler.py | 4 + FrameworkSystem/Client/ComponentInstaller.py | 10 +- .../Client/MonitoringClientIOLoop.py | 6 + .../scripts/dirac-install-tornado-service.py | 4 + Resources/Storage/GFAL2_StorageBase.py | 16 +- .../DeveloperGuide/TornadoServices/index.rst | 60 +++- tests/py3CheckDirs.txt | 19 ++ 29 files changed, 526 insertions(+), 249 deletions(-) delete mode 100644 Core/Tornado/Client/SpecificClient/__init__.py delete mode 100644 Core/Tornado/Utilities/HTTPErrorCodes.py delete mode 100644 Core/Tornado/Utilities/__init__.py diff --git a/ConfigurationSystem/Client/ConfigurationClient.py b/ConfigurationSystem/Client/ConfigurationClient.py index b92134b91a3..4d8c90cb91e 100644 --- a/ConfigurationSystem/Client/ConfigurationClient.py +++ b/ConfigurationSystem/Client/ConfigurationClient.py @@ -6,6 +6,13 @@ """ +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +__RCSID__ = "$Id$" + + from base64 import b64encode, b64decode from DIRAC.Core.Tornado.Client.TornadoClient import TornadoClient diff --git a/ConfigurationSystem/Service/TornadoConfigurationHandler.py b/ConfigurationSystem/Service/TornadoConfigurationHandler.py index 8ffb465d48e..beb9d9b6acb 100644 --- a/ConfigurationSystem/Service/TornadoConfigurationHandler.py +++ b/ConfigurationSystem/Service/TornadoConfigurationHandler.py @@ -6,15 +6,21 @@ """ +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + __RCSID__ = "$Id$" from base64 import b64encode, b64decode -from DIRAC.Core.Utilities.ReturnValues import S_OK, S_ERROR +from DIRAC import S_OK, S_ERROR, gLogger from DIRAC.ConfigurationSystem.private.ServiceInterfaceTornado import ServiceInterfaceTornado as ServiceInterface from DIRAC.Core.Utilities import DErrno from DIRAC.Core.Tornado.Server.TornadoService import TornadoService +sLog = gLogger.getSubLogger(__name__) + class TornadoConfigurationHandler(TornadoService): """ @@ -86,7 +92,7 @@ def export_commitNewData(self, sData): # dependency on the git client preinstalled on all the servers running CS slaves from DIRAC.WorkloadManagementSystem.Utilities.PilotCStoJSONSynchronizer import PilotCStoJSONSynchronizer except ImportError as exc: - self.log.exception("Failed to import PilotCStoJSONSynchronizer", repr(exc)) + sLog.exception("Failed to import PilotCStoJSONSynchronizer", repr(exc)) return S_ERROR(DErrno.EIMPERR, 'Failed to import PilotCStoJSONSynchronizer') self.PilotSynchronizer = PilotCStoJSONSynchronizer() return self.PilotSynchronizer.sync() diff --git a/ConfigurationSystem/private/Refresher.py b/ConfigurationSystem/private/Refresher.py index f1a53fbfdc0..20a98537cb3 100755 --- a/ConfigurationSystem/private/Refresher.py +++ b/ConfigurationSystem/private/Refresher.py @@ -8,7 +8,11 @@ __RCSID__ = "$Id$" import threading -import thread +try: + import thread +except ImportError: # python 3 compatibility + import _thread as thread + import time import random import os diff --git a/ConfigurationSystem/private/ServiceInterfaceBase.py b/ConfigurationSystem/private/ServiceInterfaceBase.py index f31e31ec6a4..b430c48a199 100644 --- a/ConfigurationSystem/private/ServiceInterfaceBase.py +++ b/ConfigurationSystem/private/ServiceInterfaceBase.py @@ -1,5 +1,9 @@ """Service interface is the service which provide config for client and synchronize Master/Slave servers""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + __RCSID__ = "$Id$" import os diff --git a/ConfigurationSystem/private/ServiceInterfaceTornado.py b/ConfigurationSystem/private/ServiceInterfaceTornado.py index e1b845df0ee..3789147f94b 100644 --- a/ConfigurationSystem/private/ServiceInterfaceTornado.py +++ b/ConfigurationSystem/private/ServiceInterfaceTornado.py @@ -1,6 +1,13 @@ """ Service interface adapted to work with tornado, must be used only by tornado service handlers """ + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +__RCSID__ = "$Id$" + from tornado import gen from tornado.ioloop import IOLoop from DIRAC.ConfigurationSystem.private.ServiceInterfaceBase import ServiceInterfaceBase diff --git a/Core/Base/Client.py b/Core/Base/Client.py index 237280d63a0..f7abd70f83e 100644 --- a/Core/Base/Client.py +++ b/Core/Base/Client.py @@ -11,13 +11,14 @@ import ast import os +from io import open + from DIRAC.Core.Tornado.Client.RPCClientSelector import RPCClientSelector from DIRAC.Core.Tornado.Client.TornadoClient import TornadoClient from DIRAC.Core.Utilities.Decorators import deprecated from DIRAC.Core.DISET import DEFAULT_RPC_TIMEOUT - class Client(object): """ Simple class to redirect unknown actions directly to the server. Arguments to the constructor are passed to the RPCClient constructor as they are. @@ -168,7 +169,7 @@ def addFunctions(clientCls): fullHandlerClassPath = '%s.%s' % (location, handlerClassPath) if not os.path.exists(fullPath): continue - with open(fullPath) as moduleFile: + with open(fullPath, 'rt') as moduleFile: # parse the handler module into abstract syntax tree handlerAst = ast.parse(moduleFile.read(), fullPath) diff --git a/Core/DISET/RequestHandler.py b/Core/DISET/RequestHandler.py index bbe1c94b009..ea1df2c109c 100755 --- a/Core/DISET/RequestHandler.py +++ b/Core/DISET/RequestHandler.py @@ -7,6 +7,7 @@ __RCSID__ = "$Id$" import os +import six import time import psutil @@ -344,7 +345,7 @@ def __checkExpectedArgumentTypes(self, method, args): # #### - __connectionCallbackTypes = {'new': [basestring, dict], + __connectionCallbackTypes = {'new': [six.string_types, dict], 'connected': [], 'drop': []} @@ -511,7 +512,7 @@ def export_whoami(self): del credDict['x509Chain'] return S_OK(credDict) - types_echo = [basestring] + types_echo = [six.string_types] @staticmethod def export_echo(data): diff --git a/Core/DISET/private/BaseClient.py b/Core/DISET/private/BaseClient.py index 1361617d46e..79a15e84a7e 100755 --- a/Core/DISET/private/BaseClient.py +++ b/Core/DISET/private/BaseClient.py @@ -9,7 +9,11 @@ import six import time -import thread +try: + import thread +except ImportError: # python 3 compatibility + import _thread as thread + import DIRAC from DIRAC.Core.DISET.private.Protocols import gProtocolDict from DIRAC.FrameworkSystem.Client.Logger import gLogger @@ -645,8 +649,12 @@ def _getBaseStub(self): del newKwargs['useCertificates'] return [self._destinationSrv, newKwargs] - def __nonzero__(self): + def __bool__(self): return True + # python 2 and 3 compatibility + # https://portingguide.readthedocs.io/en/latest/core-obj-misc.html#customizing-truthiness-bool + __nonzero__ = __bool__ + def __str__(self): return "" % (self.serviceURL, self.__extraCredentials) diff --git a/Core/Security/m2crypto/X509Certificate.py b/Core/Security/m2crypto/X509Certificate.py index 8f8b8addb34..538f9393f75 100644 --- a/Core/Security/m2crypto/X509Certificate.py +++ b/Core/Security/m2crypto/X509Certificate.py @@ -18,6 +18,7 @@ import M2Crypto +from io import open from DIRAC import S_OK, S_ERROR from DIRAC.Core.Utilities import DErrno @@ -171,7 +172,7 @@ def loadFromFile(self, certLocation): """ try: - with open(certLocation, 'r') as fd: + with open(certLocation, 'rb') as fd: pemData = fd.read() return self.loadFromString(pemData) except IOError: diff --git a/Core/Security/m2crypto/X509Chain.py b/Core/Security/m2crypto/X509Chain.py index 1140db59178..34e2021edff 100644 --- a/Core/Security/m2crypto/X509Chain.py +++ b/Core/Security/m2crypto/X509Chain.py @@ -23,6 +23,8 @@ import M2Crypto +from io import open + from DIRAC import S_OK, S_ERROR from DIRAC.Core.Utilities import DErrno from DIRAC.Core.Utilities.Decorators import executeOnlyIf, deprecated @@ -216,7 +218,7 @@ def loadChainFromFile(self, chainLocation): :returns: S_OK/S_ERROR """ try: - with open(chainLocation) as fd: + with open(chainLocation, 'rb') as fd: pemData = fd.read() except IOError as e: return S_ERROR(DErrno.EOF, "%s: %s" % (chainLocation, repr(e).replace(',)', ')'))) @@ -274,7 +276,7 @@ def loadKeyFromFile(self, chainLocation, password=False): :returns: S_OK / S_ERROR """ try: - with open(chainLocation) as fd: + with open(chainLocation, 'rb') as fd: pemData = fd.read() except BaseException as e: return S_ERROR(DErrno.EOF, "%s: %s" % (chainLocation, repr(e).replace(',)', ')'))) @@ -314,7 +316,7 @@ def loadProxyFromFile(self, chainLocation): :returns: S_OK / S_ERROR """ try: - with open(chainLocation) as fd: + with open(chainLocation, 'rb') as fd: pemData = fd.read() except BaseException as e: return S_ERROR(DErrno.EOF, "%s: %s" % (chainLocation, repr(e).replace(',)', ')'))) @@ -486,7 +488,7 @@ def generateProxyToFile(self, filePath, lifetime, diracGroup=False, strength=102 if not retVal['OK']: return retVal try: - with open(filePath, 'w') as fd: + with open(filePath, 'wb') as fd: fd.write(retVal['Value']) except BaseException as e: return S_ERROR(DErrno.EWF, "%s :%s" % (filePath, repr(e).replace(',)', ')'))) diff --git a/Core/Tornado/Client/RPCClientSelector.py b/Core/Tornado/Client/RPCClientSelector.py index f230860044a..c274a815f75 100644 --- a/Core/Tornado/Client/RPCClientSelector.py +++ b/Core/Tornado/Client/RPCClientSelector.py @@ -1,6 +1,6 @@ """ - RPCClientSelector can replace RPCClient (with import RPCClientSelector as RPCClient) - to migrate from DISET to Tornado. This method chooses and returns the client which should be + RPCClientSelector can replace RPCClient (with ``import RPCClientSelector as RPCClient``) + to migrate from DISET to Tornado. This function choses and returns the client which should be used for a service. If the url of the service uses HTTPS, TornadoClient is returned, else it returns RPCClient Example:: @@ -14,19 +14,32 @@ from __future__ import division from __future__ import print_function -from DIRAC.Core.Tornado.Client.TornadoClient import TornadoClient -from DIRAC.Core.DISET.RPCClient import RPCClient -from DIRAC.ConfigurationSystem.Client.PathFinder import getServiceURL +__RCSID__ = "$Id$" + from DIRAC import gLogger +from DIRAC.ConfigurationSystem.Client.PathFinder import getServiceURL +from DIRAC.Core.DISET.RPCClient import RPCClient +from DIRAC.Core.Tornado.Client.TornadoClient import TornadoClient + sLog = gLogger.getSubLogger(__name__) +# TODO CHRIS: factorize RPCClientSelector and TransferClientSelector (using functool partial ?) + def RPCClientSelector(*args, **kwargs): # We use same interface as RPCClient """ - Select the correct RPCClient, instanciate it, and return it + Select the correct RPCClient, instantiate it, and return it. + In principle, the only place for this class to be used is in + :py:class:`DIRAC.Core.Base.Client.Client`, since it is the only + one supposed to instantiate an :py:class:`DIRAC.Core.Base.DISET.RPCClient.RPCClient` :param args: URL can be just "system/service" or "dips://domain:port/system/service" + :param kwargs: This can contain: + + * Whatever :py:class:`DIRAC.Core.Base.DISET.RPCClient.RPCClient` takes. + * httpsClient: specific class inheriting from TornadoClient + """ # We detect if we need to use a specific class for the HTTPS client diff --git a/Core/Tornado/Client/SpecificClient/__init__.py b/Core/Tornado/Client/SpecificClient/__init__.py deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/Core/Tornado/Client/TornadoClient.py b/Core/Tornado/Client/TornadoClient.py index 8d7364d8015..f74da79cae0 100644 --- a/Core/Tornado/Client/TornadoClient.py +++ b/Core/Tornado/Client/TornadoClient.py @@ -1,17 +1,17 @@ """ TornadoClient is equivalent of the RPCClient but in HTTPS. - Usage of TornadoClient is the same as RPCClient, you can instanciate TornadoClient with - complete url (https://domain/component/service) or just "component/service". Like RPCClient + Usage of TornadoClient is the same as RPCClient, you can instantiate TornadoClient with + complete url (https://host:port/System/Component) or just "System/Component". Like RPCClient you can use all method defined in your service, your call will be automatically transformed in RPC. - It also exposes the same interface for receiving file than the TransferClient. + It also exposes the same interface for receiving file as the TransferClient. Main changes: - KeepAliveLapse is removed, requests library manages it itself. - nbOfRetry (defined as private attribute) is removed, requests library manage it itself. - - Underneath it use HTTP POST protocol and JSON + - Underneath it uses HTTP POST protocol and JSON. See :ref:`httpsTornado` for details Example:: @@ -25,10 +25,12 @@ from __future__ import division from __future__ import print_function +__RCSID__ = "$Id$" + # pylint: disable=broad-except -from DIRAC.Core.Utilities.JEncode import encode from DIRAC.Core.Tornado.Client.private.TornadoBaseClient import TornadoBaseClient +from DIRAC.Core.Utilities.JEncode import encode class TornadoClient(TornadoBaseClient): diff --git a/Core/Tornado/Client/TransferClientSelector.py b/Core/Tornado/Client/TransferClientSelector.py index f7ea5ab1006..3328319e0bd 100644 --- a/Core/Tornado/Client/TransferClientSelector.py +++ b/Core/Tornado/Client/TransferClientSelector.py @@ -14,17 +14,20 @@ from __future__ import division from __future__ import print_function -from DIRAC.Core.Tornado.Client.TornadoClient import TornadoClient -from DIRAC.Core.DISET.TransferClient import TransferClient -from DIRAC.ConfigurationSystem.Client.PathFinder import getServiceURL +__RCSID__ = "$Id$" + from DIRAC import gLogger +from DIRAC.ConfigurationSystem.Client.PathFinder import getServiceURL +from DIRAC.Core.DISET.TransferClient import TransferClient +from DIRAC.Core.Tornado.Client.TornadoClient import TornadoClient + sLog = gLogger.getSubLogger(__name__) def TransferClientSelector(*args, **kwargs): # We use same interface as TransferClient """ - Select the correct TransferClient, instanciate it, and return it + Select the correct TransferClient, instantiate it, and return it :param args: URL can be just "system/service" or "dips://domain:port/system/service" """ diff --git a/Core/Tornado/Client/private/TornadoBaseClient.py b/Core/Tornado/Client/private/TornadoBaseClient.py index 4169a361451..a03a7865100 100644 --- a/Core/Tornado/Client/private/TornadoBaseClient.py +++ b/Core/Tornado/Client/private/TornadoBaseClient.py @@ -1,6 +1,6 @@ """ - TornadoBaseClient contains all the low-levels functionnalities and initilization methods - It must be instanciated from :py:class:`~DIRAC.Core.Tornado.Client.TornadoClient` + TornadoBaseClient contains all the low-levels functionalities and initilization methods + It must be instantiated from :py:class:`~DIRAC.Core.Tornado.Client.TornadoClient` Requests library manage itself retry when connection failed, so the __nbOfRetry attribute is removed from DIRAC (For each URL requests manage retries himself, if it still fail, we try next url) @@ -28,24 +28,28 @@ from __future__ import division from __future__ import print_function +__RCSID__ = "$Id$" + from io import open import six import requests import DIRAC -from DIRAC.ConfigurationSystem.Client.Config import gConfig -from DIRAC.Core.Utilities import List, Network from DIRAC import S_OK, S_ERROR, gLogger + +from DIRAC.ConfigurationSystem.Client.Config import gConfig from DIRAC.ConfigurationSystem.Client.Helpers.CSGlobals import skipCACheck from DIRAC.ConfigurationSystem.Client.Helpers.Registry import findDefaultGroupForDN from DIRAC.ConfigurationSystem.Client.PathFinder import getServiceURL, getServiceFailoverURL -from DIRAC.Core.Utilities.JEncode import decode, encode -from DIRAC.Core.Security import Locations from DIRAC.Core.DISET.ThreadConfig import ThreadConfig +from DIRAC.Core.Security import Locations +from DIRAC.Core.Utilities import List, Network +from DIRAC.Core.Utilities.JEncode import decode, encode + -# TODO: refactor all the messy `discover` methods +# TODO CHRIS: refactor all the messy `discover` methods # I do not do it now because I want first to decide # whether we go with code copy of fatorization diff --git a/Core/Tornado/Server/HandlerManager.py b/Core/Tornado/Server/HandlerManager.py index 8da6fea30d7..4ab54715703 100644 --- a/Core/Tornado/Server/HandlerManager.py +++ b/Core/Tornado/Server/HandlerManager.py @@ -1,29 +1,31 @@ """ - - HandlerManager for tornado - This class search in the CS all services which use HTTPS and load them. - - Must be used with the TornadoServer - + This module contains the necessary tools to discover and load + the handlers for serving HTTPS """ from __future__ import absolute_import from __future__ import division from __future__ import print_function +__RCSID__ = "$Id$" + from tornado.web import url as TornadoURL, RequestHandler -from DIRAC.Core.Utilities.ObjectLoader import ObjectLoader -from DIRAC import gLogger, S_ERROR, S_OK, gConfig -from DIRAC.Core.Base.private.ModuleLoader import ModuleLoader +from DIRAC import gConfig, gLogger, S_ERROR, S_OK from DIRAC.ConfigurationSystem.Client import PathFinder +from DIRAC.Core.Base.private.ModuleLoader import ModuleLoader +from DIRAC.Core.Utilities.ObjectLoader import ObjectLoader def urlFinder(module): """ - Try to guess the url with module name + Tries to guess the url from module name. + The URL would be of the form ``/System/Component`` (e.g. ``DataManagement/FileCatalog``) + We search something which looks like ``<...>.System.<...>.Handler`` - :param module: path written like import (e.g. "DIRAC.something.something") + :param module: full module name (e.g. "DIRAC.something.something") + + :returns: the deduced URL or None """ sections = module.split('.') for section in sections: @@ -36,9 +38,16 @@ def urlFinder(module): class HandlerManager(object): """ - This class is designed to work with Tornado + This utility class allows to load the handlers, generate the appropriate route, + and discover the handlers based on the CS. + In order for a service to be considered as using HTTPS, it must have + ``protocol = https`` as an option. + + Each of the Handler will have one associated route to it: - It search and loads the handlers of services + * Directly specified as ``LOCATION`` in the handler module + * automatically deduced from the module name, of the form + ``System/Component`` (e.g. ``DataManagement/FileCatalog``) """ def __init__(self, autoDiscovery=True): diff --git a/Core/Tornado/Server/TornadoServer.py b/Core/Tornado/Server/TornadoServer.py index 058d2054f7e..3707cea6673 100644 --- a/Core/Tornado/Server/TornadoServer.py +++ b/Core/Tornado/Server/TornadoServer.py @@ -7,12 +7,12 @@ from __future__ import division from __future__ import print_function +__RCSID__ = "$Id$" + import time import datetime import os -from socket import error as socketerror - import M2Crypto import tornado.iostream @@ -25,21 +25,30 @@ import tornado.ioloop import DIRAC -from DIRAC.Core.Tornado.Server.HandlerManager import HandlerManager -from DIRAC import gLogger, S_ERROR, S_OK, gConfig -from DIRAC.FrameworkSystem.Client.MonitoringClient import MonitoringClient +from DIRAC import gConfig, gLogger from DIRAC.ConfigurationSystem.Client import PathFinder from DIRAC.Core.Security import Locations +from DIRAC.Core.Tornado.Server.HandlerManager import HandlerManager from DIRAC.Core.Utilities import MemStat +from DIRAC.FrameworkSystem.Client.MonitoringClient import MonitoringClient + +sLog = gLogger.getSubLogger(__name__) class TornadoServer(object): """ Tornado webserver - Initialize and run a HTTPS Server for DIRAC services. - By default it load all services from configuration, but you can also give an explicit list. - If you gave explicit list of services, only these ones are loaded + Initialize and run an HTTPS Server for DIRAC services. + By default it load all https services defined in the CS, + but you can also give an explicit list. + + The listening port is either: + + * Given as parameter + * Loaded from the CS ``/Systems/Tornado//Port`` + * Default to 8443 + Example 1: Easy way to start tornado:: @@ -52,74 +61,73 @@ class TornadoServer(object): Example 2:We want to debug service1 and service2 only, and use another port for that :: services = ['component/service1', 'component/service2'] - serverToLaunch = TornadoServer(services=services, port=1234, debugSSL=True) + serverToLaunch = TornadoServer(services=services, port=1234) serverToLaunch.startTornado() - - **WARNING:** debug=True enable SSL debug and Tornado autoreload, - for extra logging use -ddd in your command line """ - def __init__(self, services=None, debugSSL=False, port=None): + def __init__(self, services=None, port=None): """ - Basic instanciation, set some variables - :param list services: List of services you want to start, start all by default - :param str debug: Activate debug mode of Tornado (autoreload server + more errors display) and M2Crypto - :param int port: Used to change port, default is 443 + :param list services: (default None) List of service handlers to load. If ``None``, loads all + :param int port: Port to listen to. If None, the port is resolved following the logic + described in the class documentation """ if port is None: - port = gConfig.getValue("/Systems/Tornado/%s/Port" % PathFinder.getSystemInstance('Tornado'), 443) + port = gConfig.getValue("/Systems/Tornado/%s/Port" % PathFinder.getSystemInstance('Tornado'), 8443) if services and not isinstance(services, list): services = [services] - # URLs for services: 1URL/Service + # URLs for services. + # Contains Tornado :py:class:`tornado.web.url` object self.urls = [] # Other infos - self.debugSSL = debugSSL # Used by tornado and M2Crypto self.port = port self.handlerManager = HandlerManager() + + # Monitoring attributes + self._monitor = MonitoringClient() + # temp value for computation, used by the monitoring + self.__report = None + # Last update time stamp + self.__monitorLastStatsUpdate = None self.__monitoringLoopDelay = 60 # In secs # If services are defined, load only these ones (useful for debug purpose or specific services) if services: retVal = self.handlerManager.loadHandlersByServiceName(services) if not retVal['OK']: - gLogger.error(retVal['Message']) + sLog.error(retVal['Message']) raise ImportError("Some services can't be loaded, check the service names and configuration.") # if no service list is given, load services from configuration handlerDict = self.handlerManager.getHandlersDict() for item in handlerDict.items(): # handlerDict[key].initializeService(key) - self.urls.append(url(item[0], item[1], dict(debug=self.debugSSL))) + self.urls.append(url(item[0], item[1])) # If there is no services loaded: if not self.urls: raise ImportError("There is no services loaded, please check your configuration") - self.__report = None - self.__monitorLastStatsUpdate = None - def startTornado(self, multiprocess=False): + def startTornado(self): """ - Start the tornado server when ready. - The script is blocked in the Tornado IOLoop. - Multiprocess option is available but may be used with caution + Starts the tornado server when ready. + This method never returns. """ - gLogger.debug("Starting Tornado") + sLog.debug("Starting Tornado") self._initMonitoring() - if self.debugSSL: - gLogger.warn("Server is running in debug mode") - - router = Application(self.urls, debug=self.debugSSL, compress_response=True) + router = Application(self.urls, + debug=False, + compress_response=True) certs = Locations.getHostCertificateAndKeyLocation() if certs is False: - gLogger.fatal("Host certificates not found ! Can't start the Server") + sLog.fatal("Host certificates not found ! Can't start the Server") raise ImportError("Unable to load certificates") ca = Locations.getCAsLocation() ssl_options = { @@ -127,7 +135,7 @@ def startTornado(self, multiprocess=False): 'keyfile': certs[1], 'cert_reqs': M2Crypto.SSL.verify_peer, 'ca_certs': ca, - 'sslDebug': self.debugSSL + 'sslDebug': False, # Set to true if you want to see the TLS debug messages } self.__monitorLastStatsUpdate = time.time() @@ -139,24 +147,19 @@ def startTornado(self, multiprocess=False): # Start server server = HTTPServer(router, ssl_options=ssl_options, decompress_request=True) try: - if multiprocess: - server.bind(self.port) - else: - server.listen(self.port) + server.listen(self.port) except Exception as e: # pylint: disable=broad-except - gLogger.exception("Exception starting HTTPServer", e) - return S_ERROR() - gLogger.always("Listening on port %s" % self.port) + sLog.exception("Exception starting HTTPServer", e) + raise + sLog.always("Listening on port %s" % self.port) for service in self.urls: - gLogger.debug("Available service: %s" % service) + sLog.debug("Available service: %s" % service) - if multiprocess: - server.start(0) IOLoop.current().start() def _initMonitoring(self): """ - Init extra bits of monitoring + Initialize the monitoring """ self._monitor.setComponentType(MonitoringClient.COMPONENT_TORNADO) @@ -169,12 +172,10 @@ def _initMonitoring(self): self._monitor.setComponentExtraParam('DIRACVersion', DIRAC.version) self._monitor.setComponentExtraParam('platform', DIRAC.getPlatform()) self._monitor.setComponentExtraParam('startTime', datetime.datetime.utcnow()) - return S_OK() def __reportToMonitoring(self): """ - *Called every minute* - Every minutes we determine CPU and Memory usage + Periodically report to the monitoring of the CPU and MEM """ # Calculate CPU usage by comparing realtime and cpu time since last report @@ -185,7 +186,14 @@ def __reportToMonitoring(self): def __startReportToMonitoringLoop(self): """ - Get time to prepare CPU usage monitoring and send memory usage to monitor + Snapshot of resources to be taken at the beginning + of a monitoring cycle. + Also sends memory snapshot to the monitoring. + + This is basically copy/paste of Service.py + + :returns: tuple (`_. + + For compatibility with the existing :py:class:`DIRAC.Core.DISET.TransferClient.TransferClient`, + the handler can define a method ``export_streamToClient``. This is the method that will be called + whenever ``TransferClient.receiveFile`` is called. It is the equivalent of the DISET + ``transfer_toClient``. + Note that this is here only for compatibility, and we discourage using it for new purposes, as it is + bound to disappear. + + The handler only define the ``post`` verb. Please refer to :py:meth:`.post` for the details. + + In order to create a handler for your service, it has to + follow a certain skeleton:: + + from DIRAC.Core.Tornado.Server.TornadoService import TornadoService + class yourServiceHandler(TornadoService): + + ## Called only once when the first + ## request for this handler arrives + ## Useful for initializing DB or so. + ## You don't need to use super or to call any parents method, it's managed by the server + @classmethod + def initializeHandler(cls, infosDict): + '''Called only once when the first + request for this handler arrives + Useful for initializing DB or so. + You don't need to use super or to call any parents method, it's managed by the server + ''' + pass + + + def initializeRequest(self): + ''' + Called at the beginning of each request + ''' + pass + + # Specify the default permission for the method + # See :py:class:`DIRAC.Core.DISET.AuthManager.AuthManager` + auth_someMethod = ['authenticated'] + + + def export_someMethod(self): + '''The method you want to export. + It must start with ``export_`` + and it must return an S_OK/S_ERROR structure + ''' + return S_ERROR() + + + def export_streamToClient(self, myDataToSend, token): + ''' Automatically called when ``Transfer.receiveFile`` is called. + Contrary to the other ``export_`` methods, it does not need + to return a DIRAC structure. + ''' + + # Do whatever with the token + + with open(myFileToSend, 'r') as fd: + return fd.read() + + + Note that because we inherit from :py:class:`tornado.web.RequestHandler` + and we are running using executors, the methods you export cannot write + back directly to the client. Please see inline comments for more details. + + In order to pass information around and keep some states, we use instance attributes. + These are initialized in the :py:meth:`.initialize` method. + """ # Because we initialize at first request, we use a flag to know if it's already done @@ -67,7 +125,10 @@ class TornadoService(RequestHandler): # pylint: disable=abstract-method @classmethod def _initMonitoring(cls, serviceName, fullUrl): """ - Init monitoring specific to service + Initialize the monitoring specific to this handler + + :param serviceName: relative URL ``//`` + :param fullUrl: full URl like ``https://://`` """ # Init extra bits of monitoring @@ -95,21 +156,22 @@ def _initMonitoring(cls, serviceName, fullUrl): return S_OK() @classmethod - def __initializeService(cls, relativeUrl, absoluteUrl, debug): + def __initializeService(cls, relativeUrl, absoluteUrl): """ Initialize a service, called at first request - :param relativeUrl: the url, something like "/component/service" - :param absoluteUrl: the url, something like "https://dirac.cern.ch:1234/component/service" - :param debug: boolean which indicate if server running in debug mode + :param relativeUrl: relative URL, e.g. ``//`` + :param absoluteUrl: full URL e.g. ``https://://`` + + .. warning:: + This method is not thread safe nor re-entrant, while it should be. + TODO for Chris: do it !!! """ # Url starts with a "/", we just remove it serviceName = relativeUrl[1:] - cls.debug = debug - cls.log = gLogger cls._startTime = datetime.utcnow() - cls.log.info("First use of %s, initializing service..." % relativeUrl) + sLog.info("First use of %s, initializing service..." % relativeUrl) cls._authManager = AuthManager("%s/Authorization" % PathFinder.getServiceSection(serviceName)) cls._initMonitoring(serviceName, absoluteUrl) @@ -130,7 +192,7 @@ def __initializeService(cls, relativeUrl, absoluteUrl, debug): # If anything happen during initialization, we return the error # broad-except is necessary because we can't really control the exception in the handlers except Exception as e: # pylint: disable=broad-except - gLogger.error(e) + sLog.error(e) return S_ERROR('Error while initializing') cls.__FLAG_INIT_DONE = True @@ -155,33 +217,44 @@ def initializeRequest(self): """ pass - # This function is designed to be overwritten as we want in Tornado - # It's why we should disable pylint for this one - def initialize(self, debug): # pylint: disable=arguments-differ + # This is a Tornado magic method + def initialize(self): # pylint: disable=arguments-differ """ - initialize, called at every request + Initialize the handler, called at every request. + + If it is the first time this handler is instantiated, it will + call :py:meth:`.__initializeService` + ..warning:: DO NOT REWRITE THIS FUNCTION IN YOUR HANDLER ==> initialize in DISET became initializeRequest in HTTPS ! """ - self.debug = debug - self.authorized = False + + # "method" argument of the POST call. + # This resolves into the ``export_`` method + # on the handler side self.method = None + + # If set to true, do not JEncode the return of the RPC call + # It's only purpose so far is to stream data out. + self.rawContent = False + + # Used as timestamp for monitoring self.requestStartTime = time.time() + + # Credential dict as gathered from _gatherPeerCredentials self.credDict = None - self.authorized = False - self.method = None - # On internet you can find "HTTP Error Code" or "HTTP Status Code" for that. - # In fact code>=400 is an error (like "404 Not Found"), code<400 is a status (like "200 OK") - self._httpError = HTTPErrorCodes.HTTP_OK + # HTTP status code of the response + self._httpStatus = httplib.OK + if not self.__FLAG_INIT_DONE: - init = self.__initializeService(self.srv_getURL(), self.request.full_url(), debug) + init = self.__initializeService(self.srv_getURL(), self.request.full_url()) if not init['OK']: - self._httpError = HTTPErrorCodes.HTTP_INTERNAL_SERVER_ERROR - gLogger.error("Error during initalization on %s" % self.request.full_url()) - gLogger.debug(init) + self._httpStatus = httplib.INTERNAL_SERVER_ERROR + sLog.error("Error during initalization on %s" % self.request.full_url()) + sLog.debug(init) return False self._stats['requests'] += 1 @@ -192,10 +265,16 @@ def initialize(self, debug): # pylint: disable=arguments-differ def prepare(self): """ prepare the request, it reads certificates and check authorizations. + + It expects to find a ``method`` parameter in the arguments """ + # name of the method self.method = self.get_argument("method") + + # Return the raw return of the method self.rawContent = self.get_argument('rawContent', default=False) - self.log.notice("Incoming request on /%s: %s" % (self._serviceName, self.method)) + + sLog.notice("Incoming request on /%s: %s" % (self._serviceName, self.method)) # Init of service must be checked here, because if it have crashed we are # not able to end request at initialization (can't write on client) @@ -210,31 +289,70 @@ def prepare(self): # If an error occur when reading certificates we close connection # It can be strange but the RFC, for HTTP, say's that when error happend # before authentication we return 401 UNAUTHORIZED instead of 403 FORBIDDEN - self.reportUnauthorizedAccess(HTTPErrorCodes.HTTP_UNAUTHORIZED) + self.reportUnauthorizedAccess(httplib.UNAUTHORIZED) + # Resolves the hard coded authorization requirements try: hardcodedAuth = getattr(self, 'auth_' + self.method) except AttributeError: hardcodedAuth = None - self.authorized = self._authManager.authQuery(self.method, self.credDict, hardcodedAuth) - if not self.authorized: + # Check whether we are authorized to perform the query + authorized = self._authManager.authQuery(self.method, self.credDict, hardcodedAuth) + if not authorized: self.reportUnauthorizedAccess() + # Make post a coroutine. + # See https://www.tornadoweb.org/en/branch5.1/guide/coroutines.html#coroutines + # for details @gen.coroutine def post(self): # pylint: disable=arguments-differ """ - HTTP POST, used for RPC - Call the remote method, client may send his method via "method" argument - and list of arguments in JSON in "args" argument - """ - - # Execute the method - # First argument is "None", it's because we let Tornado manage the executor + Method to handle incoming ``POST`` requests. + Note that all the arguments are already prepared in the :py:meth:`.prepare` + method. + + The ``POST`` arguments expected are: + + * `method`: name of the method to call + * `args`: JSON encoded arguments for the method + * `extraCredentials`: (if applies) Extra informations to authenticate client + + Example of call using ``requests``:: + + In [20]: url = 'https://server:8443/DataManagement/TornadoFileCatalog' + ...: cert = '/tmp/x509up_u1000' + ...: kwargs = {'method':'whoami'} + ...: caPath = '/home/dirac/ClientInstallDIR/etc/grid-security/certificates/' + ...: with requests.post(url, data=kwargs, cert=cert, verify=caPath) as r: + ...: print r.json() + ...: + {u'OK': True, + u'Value': {u'DN': u'/C=ch/O=DIRAC/OU=DIRAC CI/CN=ciuser/emailAddress=lhcb-dirac-ci@cern.ch', + u'group': u'dirac_user', + u'identity': u'/C=ch/O=DIRAC/OU=DIRAC CI/CN=ciuser/emailAddress=lhcb-dirac-ci@cern.ch', + u'isLimitedProxy': False, + u'isProxy': True, + u'issuer': u'/C=ch/O=DIRAC/OU=DIRAC CI/CN=ciuser/emailAddress=lhcb-dirac-ci@cern.ch', + u'properties': [u'NormalUser'], + u'secondsLeft': 85441, + u'subject': u'/C=ch/O=DIRAC/OU=DIRAC CI/CN=ciuser/emailAddress=lhcb-dirac-ci@cern.ch/CN=2409820262', + u'username': u'adminusername', + u'validDN': False, + u'validGroup': False}} + """ + + # Execute the method in an executor (basically a separate thread) + # Because of that, we cannot calls certain methods like `self.write` + # in __executeMethod. This is because these methods are not threadsafe + # https://www.tornadoweb.org/en/branch5.1/web.html#thread-safety-notes + # However, we can still rely on instance attributes to store what should + # be sent back (like self._httpStatus) (reminder: there is an instance + # of this class created for each request) retVal = yield IOLoop.current().run_in_executor(None, self.__executeMethod) - # Tornado recommend to write in main thread - # TODO CHRIS READ THAT https://www.tornadoweb.org/en/branch5.1/web.html#thread-safety-notes + # Here it is safe to write back to the client, because we are not + # in a thread anymore self.__write_return(retVal.result()) self.finish() @@ -273,18 +391,25 @@ def post(self): # pylint: disable=arguments-differ @gen.coroutine def __executeMethod(self): """ - Execute the method called, this method is executed in an executor - We have several try except to catch the different problem who can occurs + Execute the method called, this method is ran in an executor + We have several try except to catch the different problem which can occur - First, the method does not exist => Attribute error, return an error to client - second, anything happend during execution => General Exception, send error to client + + .. warning:: + This method is called in an executor, and so cannot use methods like self.write + See https://www.tornadoweb.org/en/branch5.1/web.html#thread-safety-notes """ + # CHRIS maybe better to use https://www.tornadoweb.org/en/stable/guide/structure.html#error-handling + # getting method try: + # For compatibility reasons with DISET, the methods are still called ``export_*`` method = getattr(self, 'export_%s' % self.method) except AttributeError as e: - self._httpError = HTTPErrorCodes.HTTP_NOT_IMPLEMENTED + self._httpStatus = httplib.NOT_IMPLEMENTED return S_ERROR("Unknown method %s" % self.method) # Decode args @@ -296,61 +421,74 @@ def __executeMethod(self): self.initializeRequest() retVal = method(*args) except Exception as e: # pylint: disable=broad-except - gLogger.exception("Exception serving request", "%s:%s" % (str(e), repr(e))) + sLog.exception("Exception serving request", "%s:%s" % (str(e), repr(e))) retVal = S_ERROR(repr(e)) - self._httpError = HTTPErrorCodes.HTTP_INTERNAL_SERVER_ERROR + self._httpStatus = httplib.INTERNAL_SERVER_ERROR return retVal - def __write_return(self, dictionary): + def __write_return(self, retVal): """ - Write to client what we wan't to return to client - It must be a dictionary + Write back to the client and return. + It sets some headers (status code, ``Content-Type``). + If raw content was requested by the client, the ``Content-Type`` + is ``application/octet-stream``, otherwise we set it to ``application/json`` + and JEncode retVal. + + If ``retVal`` is a dictionary that contains a ``Callstack`` item, + it is removed, not to leak internal information. + + :param retVal: anything that can be serialized in json. """ # In case of error in server side we hide server CallStack to client - if 'CallStack' in dictionary: - del dictionary['CallStack'] + try: + if 'CallStack' in retVal: + del retVal['CallStack'] + except TypeError: + pass - # Write status code before writing, by default error code is "200 OK" - self.set_status(self._httpError) + # Set the status + self.set_status(self._httpStatus) # This is basically only used for file download through # the 'streamToClient' method. if self.rawContent: # See 4.5.1 http://www.rfc-editor.org/rfc/rfc2046.txt self.set_header("Content-Type", "application/octet-stream") - returnedData = dictionary + returnedData = retVal else: self.set_header("Content-Type", "application/json") - returnedData = encode(dictionary) + returnedData = encode(retVal) self.write(returnedData) - def reportUnauthorizedAccess(self, errorCode=401): + def reportUnauthorizedAccess(self, errorCode=httplib.UNAUTHORIZED): """ - This method stop the current request and return an error to client + This method stops the current request and return an error to the client :param int errorCode: Error code, 403 is "Forbidden" and 401 is "Unauthorized" """ error = S_ERROR(ENOAUTH, "Unauthorized query") - gLogger.error( + sLog.error( "Unauthorized access to %s: %s from %s" % (self.request.path, self.credDict['DN'], self.request.remote_ip)) - self._httpError = errorCode + self._httpStatus = errorCode self.__write_return(error) self.finish() def on_finish(self): """ - Called after the end of HTTP request + Called after the end of HTTP request. + Log the request duration """ + # TODO CHRIS: log better than that ! Look at what is in RequestHandler requestDuration = time.time() - self.requestStartTime - gLogger.notice("Ending request to %s after %fs" % (self.srv_getURL(), requestDuration)) + sLog.notice("Ending request to %s after %fs" % (self.srv_getURL(), requestDuration)) def _gatherPeerCredentials(self): """ @@ -358,9 +496,11 @@ def _gatherPeerCredentials(self): The dictionary returned is designed to work with the AuthManager, already written for DISET and re-used for HTTPS. + + :returns: a dict containing the return of :py:meth:`DIRAC.Core.Security.X509Chain.X509Chain.getCredentials` + (not a DIRAC structure !) """ - # This line get certificates, it must be change when M2Crypto will be fully integrated in tornado chainAsText = self.request.get_ssl_certificate().as_pem() peerChain = X509Chain() diff --git a/Core/Tornado/Utilities/HTTPErrorCodes.py b/Core/Tornado/Utilities/HTTPErrorCodes.py deleted file mode 100644 index a82959b89e4..00000000000 --- a/Core/Tornado/Utilities/HTTPErrorCodes.py +++ /dev/null @@ -1,20 +0,0 @@ -""" - List of HTTP codes - - https://tools.ietf.org/html/rfc2616#section-6.1.1 -""" - -# Success code -HTTP_OK = 200 - - -# Client error code -HTTP_UNAUTHORIZED = 401 -HTTP_FORBIDDEN = 403 -HTTP_NOT_FOUND = 404 -HTTP_IM_A_TEAPOT = 418 # see https://tools.ietf.org/html/rfc2324 - - -# Server error code -HTTP_INTERNAL_SERVER_ERROR = 500 -HTTP_NOT_IMPLEMENTED = 501 diff --git a/Core/Tornado/Utilities/__init__.py b/Core/Tornado/Utilities/__init__.py deleted file mode 100644 index cae5bc3f37c..00000000000 --- a/Core/Tornado/Utilities/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -""" -DIRAC.TornadosServices -""" diff --git a/Core/Tornado/scripts/tornado-start-CS.py b/Core/Tornado/scripts/tornado-start-CS.py index dd502f37fd8..fb4e1811e26 100644 --- a/Core/Tornado/scripts/tornado-start-CS.py +++ b/Core/Tornado/scripts/tornado-start-CS.py @@ -10,22 +10,19 @@ from __future__ import division from __future__ import print_function +__RCSID__ = "$Id$" + # Must be define BEFORE any dirac import import os import sys os.environ['DIRAC_USE_TORNADO_IOLOOP'] = "True" - -from DIRAC.FrameworkSystem.Client.Logger import gLogger -from DIRAC.Core.Tornado.Server.TornadoServer import TornadoServer -from DIRAC.Core.Base import Script - -from DIRAC.ConfigurationSystem.Client.LocalConfiguration import LocalConfiguration - -from DIRAC.Core.Utilities.DErrno import includeExtensionErrors - from DIRAC.ConfigurationSystem.Client.ConfigurationData import gConfigurationData +from DIRAC.ConfigurationSystem.Client.LocalConfiguration import LocalConfiguration from DIRAC.ConfigurationSystem.private.Refresher import gRefresher +from DIRAC.Core.Utilities.DErrno import includeExtensionErrors +from DIRAC.Core.Tornado.Server.TornadoServer import TornadoServer +from DIRAC.FrameworkSystem.Client.Logger import gLogger if gConfigurationData.isMaster(): gRefresher.disable() diff --git a/Core/Tornado/scripts/tornado-start-all.py b/Core/Tornado/scripts/tornado-start-all.py index eab0ca8b613..729165e08bf 100644 --- a/Core/Tornado/scripts/tornado-start-all.py +++ b/Core/Tornado/scripts/tornado-start-all.py @@ -10,22 +10,21 @@ from __future__ import division from __future__ import print_function +__RCSID__ = "$Id$" + # Must be define BEFORE any dirac import import os import sys os.environ['DIRAC_USE_TORNADO_IOLOOP'] = "True" -from DIRAC.FrameworkSystem.Client.Logger import gLogger -from DIRAC.Core.Tornado.Server.TornadoServer import TornadoServer -from DIRAC.Core.Base import Script - -from DIRAC.ConfigurationSystem.Client.LocalConfiguration import LocalConfiguration -from DIRAC.ConfigurationSystem.Client.ConfigurationData import gConfigurationData from DIRAC import gConfig from DIRAC.ConfigurationSystem.Client import PathFinder - +from DIRAC.ConfigurationSystem.Client.ConfigurationData import gConfigurationData +from DIRAC.ConfigurationSystem.Client.LocalConfiguration import LocalConfiguration +from DIRAC.Core.Tornado.Server.TornadoServer import TornadoServer from DIRAC.Core.Utilities.DErrno import includeExtensionErrors +from DIRAC.FrameworkSystem.Client.Logger import gLogger # We check if there is no configuration server started as master diff --git a/DataManagementSystem/Service/TornadoFileCatalogHandler.py b/DataManagementSystem/Service/TornadoFileCatalogHandler.py index 242be4f0fef..ff72722c195 100644 --- a/DataManagementSystem/Service/TornadoFileCatalogHandler.py +++ b/DataManagementSystem/Service/TornadoFileCatalogHandler.py @@ -10,6 +10,10 @@ """ +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + __RCSID__ = "$Id$" # imports diff --git a/FrameworkSystem/Client/ComponentInstaller.py b/FrameworkSystem/Client/ComponentInstaller.py index c08552e2509..b7274b6e75d 100644 --- a/FrameworkSystem/Client/ComponentInstaller.py +++ b/FrameworkSystem/Client/ComponentInstaller.py @@ -54,9 +54,11 @@ /LocalInstallation/VirtualOrganization: Name of the main Virtual Organization (default: None) """ - -from __future__ import print_function, absolute_import +from __future__ import absolute_import from __future__ import division +from __future__ import print_function + +__RCSID__ = "$Id$" import os import io @@ -2530,9 +2532,9 @@ def installTornado(self): self._createRunitLog(runitCompDir) runFile = os.path.join(runitCompDir, 'run') - with open(runFile, 'w') as fd: + with io.open(runFile, 'wt') as fd: fd.write( - """#!/bin/bash + u"""#!/bin/bash rcfile=%(bashrc)s [ -e $rcfile ] && source $rcfile # diff --git a/FrameworkSystem/Client/MonitoringClientIOLoop.py b/FrameworkSystem/Client/MonitoringClientIOLoop.py index 41d18b309e1..a757b260020 100644 --- a/FrameworkSystem/Client/MonitoringClientIOLoop.py +++ b/FrameworkSystem/Client/MonitoringClientIOLoop.py @@ -3,6 +3,12 @@ """ +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +__RCSID__ = "$Id$" + import tornado.ioloop from DIRAC import gLogger diff --git a/FrameworkSystem/scripts/dirac-install-tornado-service.py b/FrameworkSystem/scripts/dirac-install-tornado-service.py index c6d06a761a1..f857fb196b4 100755 --- a/FrameworkSystem/scripts/dirac-install-tornado-service.py +++ b/FrameworkSystem/scripts/dirac-install-tornado-service.py @@ -4,6 +4,10 @@ """ from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +__RCSID__ = "$Id$" from DIRAC import gConfig, gLogger, S_OK from DIRAC.ConfigurationSystem.Client.Helpers import getCSExtensions diff --git a/Resources/Storage/GFAL2_StorageBase.py b/Resources/Storage/GFAL2_StorageBase.py index b8c7e572f83..ee7eb250df4 100644 --- a/Resources/Storage/GFAL2_StorageBase.py +++ b/Resources/Storage/GFAL2_StorageBase.py @@ -265,7 +265,7 @@ def putFile(self, path, sourceSize=0): failed = {} successful = {} - for dest_url, src_file in urls.iteritems(): + for dest_url, src_file in urls.items(): if not src_file: errStr = "GFAL2_StorageBase.putFile: Source file not set. Argument must be a dictionary \ (or a list of a dictionary) {url : local path}" @@ -796,7 +796,7 @@ def prestageFileStatus(self, path): failed = {} successful = {} - for path, token in urls.iteritems(): + for path, token in urls.items(): res = self.__prestageSingleFileStatus(path, token) if not res['OK']: @@ -929,7 +929,7 @@ def releaseFile(self, path): failed = {} successful = {} - for path, token in urls.iteritems(): + for path, token in urls.items(): res = self.__releaseSingleFile(path, token) if not res['OK']: @@ -1193,7 +1193,7 @@ def listDirectory(self, path): directories = [] - for url, isDirectory in res['Value']['Successful'].iteritems(): + for url, isDirectory in res['Value']['Successful'].items(): if isDirectory: directories.append(url) else: @@ -1438,7 +1438,7 @@ def putDirectory(self, path): successful = {} failed = {} - for destDir, sourceDir in urls.iteritems(): + for destDir, sourceDir in urls.items(): if not sourceDir: errStr = 'No source directory set, make sure the input format is correct { dest. dir : source dir }' return S_ERROR(errStr) @@ -1518,7 +1518,7 @@ def __putSingleDirectory(self, src_directory, dest_directory): log.debug('Failed to put files to storage.', res['Message']) allSuccessful = False else: - for fileSize in res['Value']['Successful'].itervalues(): + for fileSize in res['Value']['Successful'].values(): filesPut += 1 sizePut += fileSize if res['Value']['Failed']: @@ -1715,7 +1715,7 @@ def __getSingleDirectorySize(self, path): directorySize = 0 directoryFiles = 0 # itervalues returns a list of values of the dictionnary - for fileDict in res['Value']['Files'].itervalues(): + for fileDict in res['Value']['Files'].values(): directorySize += fileDict['Size'] directoryFiles += 1 @@ -1780,7 +1780,7 @@ def _getExtendedAttributes(self, path, attributes=None): """ Get all the available extended attributes of path :param self: self reference - :param str path: path of which we wan't extended attributes + :param str path: path of which we want extended attributes :param str list attributes: list of extended attributes we want to receive :return: S_OK( attributeDict ) if successful. Where the keys of the dict are the attributes and values the respective values diff --git a/docs/source/DeveloperGuide/TornadoServices/index.rst b/docs/source/DeveloperGuide/TornadoServices/index.rst index 81e94e40298..ba37b84def3 100644 --- a/docs/source/DeveloperGuide/TornadoServices/index.rst +++ b/docs/source/DeveloperGuide/TornadoServices/index.rst @@ -10,10 +10,13 @@ HTTPS Services with Tornado ************ Presentation ************ + This page summarizes the changes between DISET and HTTPS. You can all also see these presentations: - `Presentation of HTTPS in DIRAC `_. - `Presentation of HTTPS migration `_. +- `Summary presentation (latest) `. + ******* @@ -35,6 +38,8 @@ Service Service returns to Client S_OK/S_ERROR encoded in JSON +Each service exposes only one route of the form ``System/Component`` with a ``POST`` handler. The semantic of the ``POST`` call is described bellow. + ********************************************************* Important changes between DISET server and Tornado Server ********************************************************* @@ -69,12 +74,12 @@ Nothing better than an example:: ## Returned value may be an S_OK/S_ERROR ## You don't need to serialize in JSON, Tornado will do it -Writing a service for tornado and diset is similar. You have to define your method starting with ``export_``, and your initialization method is a class method called ``initializeHandler``. +Writing a service for tornado and DISET is similar. You have to define your method starting with ``export_``, and your initialization method is a class method called ``initializeHandler``. Main changes in tornado are: - Service are initialized at first request -- You **should not** write a method called ``initialize`` because Tornado already use that name, so the ``initialize`` from diset handlers became ``initializeRequest`` -- ``infosDict``, arguments of initializedHandler is not really the same as for diset: all transport related matters are removed. +- You **should not** write a method called ``initialize`` because Tornado already use that name, so the ``initialize`` from DISET handlers became ``initializeRequest`` +- ``infosDict``, arguments of initializedHandler is not really the same as for DISET: all transport related matters are removed. - There is no parameter type check any more: attributes like ``types_yourMethod`` are ignored. - Auth attributes are still there (``auth_yourMethod``). @@ -194,7 +199,7 @@ Client This diagram present what is behind TornadoClient, but you should use :py:class:`DIRAC.Core.Base.Client` ! The new client integrate a selection system which select for you between HTTPS and DISET client. -In your client module when you inherit from :py:class:`DIRAC.Core.Base.Client` you can define `httpsClient` with another client, it can be useful when you can't serialize some data in JSON. Here the step to create and use a JSON patch: +In your client module when you inherit from :py:class:`DIRAC.Core.Base.Client` you can define ``httpsClient`` with another client, it can be useful when you can't serialize some data in JSON. Here the step to create and use a JSON patch: - Create a class which inherit from :py:class:`~DIRAC.Core.Tornado.Client.TornadoClient` - For every method who need a JSON patch create a method with the same name as the service @@ -226,14 +231,53 @@ You can also see this example:: Behind :py:class:`~DIRAC.Core.Tornado.Client.TornadoClient` the `requests `_ library sends a HTTP POST request with: -- procedure: str with procedure name +- method : str with method name - args: your arguments encoded in JSON - clientVO: The VO of client - extraCredentials: (if apply) Extra informations to authenticate client Service is determined by server thanks to URL rooting, not with port like in DISET. -By default server listen on port 443, default port for HTTPS. +By default server listen on port 8443. + +Contacting the service using ``DIRAC``:: + + In [7]: from DIRAC.Resources.Catalog.FileCatalogClient import FileCatalogClient + ...: FileCatalogClient().whoami() + ...: + Out[7]: + {u'OK': True, + u'Value': {u'DN': u'/C=ch/O=DIRAC/OU=DIRAC CI/CN=ciuser/emailAddress=lhcb-dirac-ci@cern.ch', + u'group': u'dirac_user', + u'identity': u'/C=ch/O=DIRAC/OU=DIRAC CI/CN=ciuser/emailAddress=lhcb-dirac-ci@cern.ch', + u'isLimitedProxy': False, + u'isProxy': True, + u'issuer': u'/C=ch/O=DIRAC/OU=DIRAC CI/CN=ciuser/emailAddress=lhcb-dirac-ci@cern.ch', + u'properties': [u'NormalUser'], + u'secondsLeft': 86141, + u'subject': u'/C=ch/O=DIRAC/OU=DIRAC CI/CN=ciuser/emailAddress=lhcb-dirac-ci@cern.ch/CN=2409820262', + u'username': u'adminusername', + u'validDN': False, + u'validGroup': False}, + 'rpcStub': (('DataManagement/FileCatalog', + {'skipCACheck': True, 'timeout': 600}), + 'whoami', + [])} + + + + +Contacting the service using ``requests``:: + + In [20]: url = 'https://server:8443/DataManagement/TornadoFileCatalog' + ...: cert = '/tmp/x509up_u1000' + ...: kwargs = {'method':'whoami'} + ...: caPath = '/home/dirac/ClientInstallDIR/etc/grid-security/certificates/' + ...: with requests.post(url, data=kwargs, cert=cert, verify=caPath) as r: + ...: print r.json() + ...: + {u'OK': True, u'Value': {u'DN': u'/C=ch/O=DIRAC/OU=DIRAC CI/CN=ciuser/emailAddress=lhcb-dirac-ci@cern.ch', u'username': u'adminusername', u'secondsLeft': 85846, u'group': u'dirac_user', u'isProxy': True, u'validGroup': False, u'validDN': False, u'issuer': u'/C=ch/O=DIRAC/OU=DIRAC CI/CN=ciuser/emailAddress=lhcb-dirac-ci@cern.ch', u'isLimitedProxy': False, u'properties': [u'NormalUser'], u'identity': u'/C=ch/O=DIRAC/OU=DIRAC CI/CN=ciuser/emailAddress=lhcb-dirac-ci@cern.ch', u'subject': u'/C=ch/O=DIRAC/OU=DIRAC CI/CN=ciuser/emailAddress=lhcb-dirac-ci@cern.ch/CN=2409820262'}} + ***************************** @@ -255,7 +299,7 @@ Internal structure - :py:class:`~DIRAC.Core.Tornado.Client.private.TornadoBaseClient` is the new :py:class:`~DIRAC.Core.DISET.private.BaseClient`. Most of code is copied from :py:class:`~DIRAC.Core.DISET.private.BaseClient` but some method have been rewrited to use `Requests `_ instead of Transports. Code duplication is done to fully separate DISET and HTTPS but later, some parts can be merged by using a new common class between DISET and HTTPS (these parts are explicitly given in the docstrings). - :py:class:`~DIRAC.Core.DISET.private.Transports.BaseTransport`, :py:class:`~DIRAC.Core.DISET.private.Transports.PlainTransport` and :py:class:`~DIRAC.Core.DISET.private.Transports.SSLTransport` are replaced by `Requests `_ - keepAliveLapse is removed from rpcStub returned by Client because `Requests `_ manage it himself. -- Due to JSON limitation you can write some specifics clients who inherit from :py:class:`~DIRAC.Core.Tornado.Client.TornadoClient`, there is a simple example with :py:class:`~DIRAC.Core.Tornado.Client.SpecificClient.ConfigurationClient` who transfer data in base64 to overcome JSON limitations +- Due to JSON limitation you can write some specifics clients who inherit from :py:class:`~DIRAC.Core.Tornado.Client.TornadoClient`, there is a simple example with :py:class:`~DIRAC.ConfigurationSystem.Client.ConfigurationClient.CSJSONClient` who transfer data in base64 to overcome JSON limitations Connections and certificates @@ -284,7 +328,7 @@ Two special python packages are needed: Install a service ***************** -`dirac-install-tornado-service` is your friend. This will install a runit component running `tornado-start-all`. +``dirac-install-tornado-service`` is your friend. This will install a runit component running ``tornado-start-all``. Nothing is ready yet to install specific tornado service, like the master CS. Start the server diff --git a/tests/py3CheckDirs.txt b/tests/py3CheckDirs.txt index 2b7a7b1732c..f1fd71bdd22 100644 --- a/tests/py3CheckDirs.txt +++ b/tests/py3CheckDirs.txt @@ -1 +1,20 @@ +ConfigurationSystem/Client/ConfigurationClient.py +ConfigurationSystem/Client/CSAPI.py +ConfigurationSystem/private/Refresher.py +ConfigurationSystem/private/ServiceInterfaceBase.py +ConfigurationSystem/private/ServiceInterface.py +ConfigurationSystem/private/ServiceInterfaceTornado.py +ConfigurationSystem/Service/TornadoConfigurationHandler.py +Core/Base/Client.py +Core/DISET/private/BaseClient.py +Core/DISET/private/Service.py +Core/DISET/RequestHandler.py +Core/Security/m2crypto/X509Chain.py Core/Tornado +DataManagementSystem/Service/TornadoFileCatalogHandler.py +FrameworkSystem/Client/ComponentInstaller.py +FrameworkSystem/Client/MonitoringClientIOLoop.py +FrameworkSystem/Client/MonitoringClient.py +FrameworkSystem/scripts/dirac-install-tornado-service.py +Resources/Catalog/FileCatalogClient.py +Resources/Storage/GFAL2_StorageBase.py From 62a900f03ee686f94d1607c695455a799027ce36 Mon Sep 17 00:00:00 2001 From: Christophe Haen Date: Fri, 21 Aug 2020 17:11:05 +0200 Subject: [PATCH 19/25] Better use of HTTPError --- .../Client/private/TornadoBaseClient.py | 93 +++++++++------ Core/Tornado/Server/TornadoService.py | 107 +++++++----------- 2 files changed, 99 insertions(+), 101 deletions(-) diff --git a/Core/Tornado/Client/private/TornadoBaseClient.py b/Core/Tornado/Client/private/TornadoBaseClient.py index a03a7865100..b85cb83d470 100644 --- a/Core/Tornado/Client/private/TornadoBaseClient.py +++ b/Core/Tornado/Client/private/TornadoBaseClient.py @@ -31,9 +31,12 @@ __RCSID__ = "$Id$" from io import open +import errno +import requests import six +from six.moves import http_client + -import requests import DIRAC from DIRAC import S_OK, S_ERROR, gLogger @@ -516,44 +519,60 @@ def _request(self, retry=0, outputFile=None, **kwargs): else: cert = self.__proxy_location - # Do the request + # We have a try/except for all the exceptions + # whose default behavior is to try again, + # maybe to different server try: - rawText = None - - # Default case, just return the result - if not outputFile: - call = requests.post(url, data=kwargs, - timeout=self.timeout, verify=verify, - cert=cert) - # raising the exception for status here - # means essentialy that we are losing here the information of what is returned by the server - # as error message, since it is not passed to the exception - # However, we can store the text and return it raw as an error, - # since there is no guarantee that it is any JEncoded text - # Note that we would get an exception only if there is an exception on the server side which - # is not handled. - # Any standard S_ERROR will be transfered as an S_ERROR with a correct code. - rawText = call.text - call.raise_for_status() - return decode(rawText)[0] - else: - # Instruct the server not to encode the response - kwargs['rawContent'] = True - + # And we have a second block to handle specific exceptions + # which makes it not worth retrying + try: rawText = None - # Stream download - # https://requests.readthedocs.io/en/latest/user/advanced/#body-content-workflow - with requests.post(url, data=kwargs, timeout=self.timeout, verify=verify, - cert=cert, stream=True) as r: - rawText = r.text - r.raise_for_status() - - with open(outputFile, 'wb') as f: - for chunk in r.iter_content(4096): - # if chunk: # filter out keep-alive new chuncks - f.write(chunk) - - return S_OK() + + # Default case, just return the result + if not outputFile: + call = requests.post(url, data=kwargs, + timeout=self.timeout, verify=verify, + cert=cert) + # raising the exception for status here + # means essentialy that we are losing here the information of what is returned by the server + # as error message, since it is not passed to the exception + # However, we can store the text and return it raw as an error, + # since there is no guarantee that it is any JEncoded text + # Note that we would get an exception only if there is an exception on the server side which + # is not handled. + # Any standard S_ERROR will be transfered as an S_ERROR with a correct code. + rawText = call.text + call.raise_for_status() + return decode(rawText)[0] + else: + # Instruct the server not to encode the response + kwargs['rawContent'] = True + + rawText = None + # Stream download + # https://requests.readthedocs.io/en/latest/user/advanced/#body-content-workflow + with requests.post(url, data=kwargs, timeout=self.timeout, verify=verify, + cert=cert, stream=True) as r: + rawText = r.text + r.raise_for_status() + + with open(outputFile, 'wb') as f: + for chunk in r.iter_content(4096): + # if chunk: # filter out keep-alive new chuncks + f.write(chunk) + + return S_OK() + + # Some HTTPError are not worth retrying + except requests.exceptions.HTTPError as e: + status_code = e.response.status_code + if status_code == http_client.NOT_IMPLEMENTED: + return S_ERROR(errno.ENOSYS, "%s is not implemented" % kwargs.get('method')) + elif status_code in (http_client.FORBIDDEN, http_client.UNAUTHORIZED): + return S_ERROR(errno.EACCES, "No access to %s" % url) + + # if it is something else, retry + raise except Exception as e: # CHRIS TODO review this part: catch specific exceptions diff --git a/Core/Tornado/Server/TornadoService.py b/Core/Tornado/Server/TornadoService.py index f44a43655dd..6988d8cd7ae 100644 --- a/Core/Tornado/Server/TornadoService.py +++ b/Core/Tornado/Server/TornadoService.py @@ -13,12 +13,9 @@ import os import time -try: - import httplib -except ImportError: # python 3 compatibility - import http.client as httplib from datetime import datetime -from tornado.web import RequestHandler +from six.moves import http_client +from tornado.web import RequestHandler, HTTPError from tornado import gen import tornado.ioloop from tornado.ioloop import IOLoop @@ -29,7 +26,6 @@ from DIRAC.ConfigurationSystem.Client import PathFinder from DIRAC.Core.DISET.AuthManager import AuthManager from DIRAC.Core.Security.X509Chain import X509Chain # pylint: disable=import-error -from DIRAC.Core.Utilities.DErrno import ENOAUTH from DIRAC.Core.Utilities.JEncode import decode, encode from DIRAC.FrameworkSystem.Client.MonitoringClient import MonitoringClient @@ -187,13 +183,7 @@ def __initializeService(cls, relativeUrl, absoluteUrl): cls.__monitorLastStatsUpdate = time.time() - try: - cls.initializeHandler(serviceInfo) - # If anything happen during initialization, we return the error - # broad-except is necessary because we can't really control the exception in the handlers - except Exception as e: # pylint: disable=broad-except - sLog.error(e) - return S_ERROR('Error while initializing') + cls.initializeHandler(serviceInfo) cls.__FLAG_INIT_DONE = True return S_OK() @@ -225,21 +215,15 @@ def initialize(self): # pylint: disable=arguments-differ If it is the first time this handler is instantiated, it will call :py:meth:`.__initializeService` + If anything goes wrong, the client will get ``Connection aborted`` + error. See details inside the method. + ..warning:: DO NOT REWRITE THIS FUNCTION IN YOUR HANDLER ==> initialize in DISET became initializeRequest in HTTPS ! """ - # "method" argument of the POST call. - # This resolves into the ``export_`` method - # on the handler side - self.method = None - - # If set to true, do not JEncode the return of the RPC call - # It's only purpose so far is to stream data out. - self.rawContent = False - # Used as timestamp for monitoring self.requestStartTime = time.time() @@ -247,49 +231,55 @@ def initialize(self): # pylint: disable=arguments-differ self.credDict = None # HTTP status code of the response - self._httpStatus = httplib.OK + self._httpStatus = http_client.OK + # Only initialize once if not self.__FLAG_INIT_DONE: - init = self.__initializeService(self.srv_getURL(), self.request.full_url()) - if not init['OK']: - self._httpStatus = httplib.INTERNAL_SERVER_ERROR - sLog.error("Error during initalization on %s" % self.request.full_url()) - sLog.debug(init) - return False + # Ideally, if something goes wrong, we would like to return a Server Error 500 + # but this method cannot write back to the client as per the + # `tornado doc `_. + # So the client will get a ``Connection aborted``` + try: + res = self.__initializeService(self.srv_getURL(), self.request.full_url()) + if not res['OK']: + raise Exception(res['Message']) + except Exception as e: + sLog.error("Error in initialization", repr(e)) + raise self._stats['requests'] += 1 self._monitor.setComponentExtraParam('queries', self._stats['requests']) self._monitor.addMark("Queries") - return True def prepare(self): """ - prepare the request, it reads certificates and check authorizations. + Prepare the request. It reads certificates and check authorizations. + We make the assumption that there is always going to be a ``method`` argument + regardless of the HTTP method used - It expects to find a ``method`` parameter in the arguments """ - # name of the method + + # "method" argument of the POST call. + # This resolves into the ``export_`` method + # on the handler side self.method = self.get_argument("method") - # Return the raw return of the method + # If set to true, do not JEncode the return of the RPC call + # It's only purpose so far is to stream data out. self.rawContent = self.get_argument('rawContent', default=False) sLog.notice("Incoming request on /%s: %s" % (self._serviceName, self.method)) - # Init of service must be checked here, because if it have crashed we are - # not able to end request at initialization (can't write on client) - if not self.__FLAG_INIT_DONE: - error = encode("Service can't be initialized ! Check logs on the server for more informations.") - self.__write_return(error) - self.finish() - try: self.credDict = self._gatherPeerCredentials() except Exception: # pylint: disable=broad-except # If an error occur when reading certificates we close connection # It can be strange but the RFC, for HTTP, say's that when error happend # before authentication we return 401 UNAUTHORIZED instead of 403 FORBIDDEN - self.reportUnauthorizedAccess(httplib.UNAUTHORIZED) + sLog.error( + "Error gathering credentials", "IP %s; path %s" % + (self.request.remote_ip, self.request.path)) + raise HTTPError(status_code=http_client.UNAUTHORIZED) # Resolves the hard coded authorization requirements try: @@ -300,7 +290,13 @@ def prepare(self): # Check whether we are authorized to perform the query authorized = self._authManager.authQuery(self.method, self.credDict, hardcodedAuth) if not authorized: - self.reportUnauthorizedAccess() + sLog.error( + "Unauthorized access", "IP %s; path %s; DN %s" % + (self.request.remote_ip, + self.request.path, + self.credDict['DN'], + )) + raise HTTPError(status_code=http_client.UNAUTHORIZED) # Make post a coroutine. # See https://www.tornadoweb.org/en/branch5.1/guide/coroutines.html#coroutines @@ -409,8 +405,7 @@ def __executeMethod(self): # For compatibility reasons with DISET, the methods are still called ``export_*`` method = getattr(self, 'export_%s' % self.method) except AttributeError as e: - self._httpStatus = httplib.NOT_IMPLEMENTED - return S_ERROR("Unknown method %s" % self.method) + raise HTTPError(status_code=http_client.NOT_IMPLEMENTED) # Decode args args_encoded = self.get_body_argument('args', default=encode([])) @@ -422,8 +417,7 @@ def __executeMethod(self): retVal = method(*args) except Exception as e: # pylint: disable=broad-except sLog.exception("Exception serving request", "%s:%s" % (str(e), repr(e))) - retVal = S_ERROR(repr(e)) - self._httpStatus = httplib.INTERNAL_SERVER_ERROR + raise HTTPError(http_client.INTERNAL_SERVER_ERROR) return retVal @@ -463,23 +457,8 @@ def __write_return(self, retVal): self.write(returnedData) - def reportUnauthorizedAccess(self, errorCode=httplib.UNAUTHORIZED): - """ - This method stops the current request and return an error to the client - - - :param int errorCode: Error code, 403 is "Forbidden" and 401 is "Unauthorized" - """ - error = S_ERROR(ENOAUTH, "Unauthorized query") - sLog.error( - "Unauthorized access to %s: %s from %s" % - (self.request.path, - self.credDict['DN'], - self.request.remote_ip)) - - self._httpStatus = errorCode - self.__write_return(error) - self.finish() + # TODO CHRIS ADD THAT + # set_default_headers def on_finish(self): """ From ed821bb75335686e4dffedcbf88462393e575e8b Mon Sep 17 00:00:00 2001 From: Christophe Haen Date: Fri, 21 Aug 2020 18:49:38 +0200 Subject: [PATCH 20/25] Factorize --- Core/Tornado/Server/TornadoService.py | 224 ++++++++++++++++---------- 1 file changed, 143 insertions(+), 81 deletions(-) diff --git a/Core/Tornado/Server/TornadoService.py b/Core/Tornado/Server/TornadoService.py index 6988d8cd7ae..97f2f055e11 100644 --- a/Core/Tornado/Server/TornadoService.py +++ b/Core/Tornado/Server/TornadoService.py @@ -57,10 +57,10 @@ class TornadoService(RequestHandler): # pylint: disable=abstract-method from DIRAC.Core.Tornado.Server.TornadoService import TornadoService class yourServiceHandler(TornadoService): - ## Called only once when the first - ## request for this handler arrives - ## Useful for initializing DB or so. - ## You don't need to use super or to call any parents method, it's managed by the server + # Called only once when the first + # request for this handler arrives + # Useful for initializing DB or so. + # You don't need to use super or to call any parents method, it's managed by the server @classmethod def initializeHandler(cls, infosDict): '''Called only once when the first @@ -224,16 +224,7 @@ def initialize(self): # pylint: disable=arguments-differ ==> initialize in DISET became initializeRequest in HTTPS ! """ - # Used as timestamp for monitoring - self.requestStartTime = time.time() - - # Credential dict as gathered from _gatherPeerCredentials - self.credDict = None - - # HTTP status code of the response - self._httpStatus = http_client.OK - - # Only initialize once + # Only initialized once if not self.__FLAG_INIT_DONE: # Ideally, if something goes wrong, we would like to return a Server Error 500 # but this method cannot write back to the client as per the @@ -247,10 +238,6 @@ def initialize(self): # pylint: disable=arguments-differ sLog.error("Error in initialization", repr(e)) raise - self._stats['requests'] += 1 - self._monitor.setComponentExtraParam('queries', self._stats['requests']) - self._monitor.addMark("Queries") - def prepare(self): """ Prepare the request. It reads certificates and check authorizations. @@ -262,13 +249,13 @@ def prepare(self): # "method" argument of the POST call. # This resolves into the ``export_`` method # on the handler side + # If the argument is not available, the method exists + # and an error 400 ``Bad Request`` is returned to the client self.method = self.get_argument("method") - # If set to true, do not JEncode the return of the RPC call - # It's only purpose so far is to stream data out. - self.rawContent = self.get_argument('rawContent', default=False) - - sLog.notice("Incoming request on /%s: %s" % (self._serviceName, self.method)) + self._stats['requests'] += 1 + self._monitor.setComponentExtraParam('queries', self._stats['requests']) + self._monitor.addMark("Queries") try: self.credDict = self._gatherPeerCredentials() @@ -277,8 +264,8 @@ def prepare(self): # It can be strange but the RFC, for HTTP, say's that when error happend # before authentication we return 401 UNAUTHORIZED instead of 403 FORBIDDEN sLog.error( - "Error gathering credentials", "IP %s; path %s" % - (self.request.remote_ip, self.request.path)) + "Error gathering credentials", "%s; path %s" % + (self.getRemoteAddress(), self.request.path)) raise HTTPError(status_code=http_client.UNAUTHORIZED) # Resolves the hard coded authorization requirements @@ -288,11 +275,12 @@ def prepare(self): hardcodedAuth = None # Check whether we are authorized to perform the query + # Note that performing the authQuery modifies the credDict... authorized = self._authManager.authQuery(self.method, self.credDict, hardcodedAuth) if not authorized: sLog.error( - "Unauthorized access", "IP %s; path %s; DN %s" % - (self.request.remote_ip, + "Unauthorized access", "Identity %s; path %s; DN %s" % + (self.srv_getFormattedRemoteCredentials, self.request.path, self.credDict['DN'], )) @@ -310,9 +298,19 @@ def post(self): # pylint: disable=arguments-differ The ``POST`` arguments expected are: - * `method`: name of the method to call - * `args`: JSON encoded arguments for the method - * `extraCredentials`: (if applies) Extra informations to authenticate client + * ``method``: name of the method to call + * ``args``: JSON encoded arguments for the method + * ``extraCredentials``: (optional) Extra informations to authenticate client + * ``rawContent``: (optionnal, default False) If set to True, return the raw output + of the method called. + + If ``rawContent`` was requested by the client, the ``Content-Type`` + is ``application/octet-stream``, otherwise we set it to ``application/json`` + and JEncode retVal. + + If ``retVal`` is a dictionary that contains a ``Callstack`` item, + it is removed, not to leak internal information. + Example of call using ``requests``:: @@ -338,18 +336,41 @@ def post(self): # pylint: disable=arguments-differ u'validGroup': False}} """ + sLog.notice( + "Incoming request %s /%s: %s" % + (self.srv_getFormattedRemoteCredentials(), + self._serviceName, + self.method)) + # Execute the method in an executor (basically a separate thread) # Because of that, we cannot calls certain methods like `self.write` # in __executeMethod. This is because these methods are not threadsafe # https://www.tornadoweb.org/en/branch5.1/web.html#thread-safety-notes # However, we can still rely on instance attributes to store what should - # be sent back (like self._httpStatus) (reminder: there is an instance + # be sent back (reminder: there is an instance # of this class created for each request) retVal = yield IOLoop.current().run_in_executor(None, self.__executeMethod) + # retVal is :py:class:`tornado.concurrent.Future` + self.result = retVal.result() + # Here it is safe to write back to the client, because we are not # in a thread anymore - self.__write_return(retVal.result()) + + # If set to true, do not JEncode the return of the RPC call + # This is basically only used for file download through + # the 'streamToClient' method. + rawContent = self.get_argument('rawContent', default=False) + + if rawContent: + # See 4.5.1 http://www.rfc-editor.org/rfc/rfc2046.txt + self.set_header("Content-Type", "application/octet-stream") + result = self.result + else: + self.set_header("Content-Type", "application/json") + result = encode(self.result) + + self.write(result) self.finish() # This nice idea of streaming to the client cannot work because we are ran in an executor @@ -398,13 +419,12 @@ def __executeMethod(self): See https://www.tornadoweb.org/en/branch5.1/web.html#thread-safety-notes """ - # CHRIS maybe better to use https://www.tornadoweb.org/en/stable/guide/structure.html#error-handling - # getting method try: # For compatibility reasons with DISET, the methods are still called ``export_*`` method = getattr(self, 'export_%s' % self.method) except AttributeError as e: + sLog.error("Invalid method", self.method) raise HTTPError(status_code=http_client.NOT_IMPLEMENTED) # Decode args @@ -421,53 +441,64 @@ def __executeMethod(self): return retVal - def __write_return(self, retVal): - """ - Write back to the client and return. - It sets some headers (status code, ``Content-Type``). - If raw content was requested by the client, the ``Content-Type`` - is ``application/octet-stream``, otherwise we set it to ``application/json`` - and JEncode retVal. - - If ``retVal`` is a dictionary that contains a ``Callstack`` item, - it is removed, not to leak internal information. - - :param retVal: anything that can be serialized in json. - """ - - # In case of error in server side we hide server CallStack to client - try: - if 'CallStack' in retVal: - del retVal['CallStack'] - except TypeError: - pass - - # Set the status - self.set_status(self._httpStatus) - - # This is basically only used for file download through - # the 'streamToClient' method. - if self.rawContent: - # See 4.5.1 http://www.rfc-editor.org/rfc/rfc2046.txt - self.set_header("Content-Type", "application/octet-stream") - returnedData = retVal - else: - self.set_header("Content-Type", "application/json") - returnedData = encode(retVal) - - self.write(returnedData) - - # TODO CHRIS ADD THAT - # set_default_headers + # def __write_return(self, retVal): + # """ + # Write back to the client and return. + # It sets some headers (status code, ``Content-Type``). + # If raw content was requested by the client, the ``Content-Type`` + # is ``application/octet-stream``, otherwise we set it to ``application/json`` + # and JEncode retVal. + + # If ``retVal`` is a dictionary that contains a ``Callstack`` item, + # it is removed, not to leak internal information. + + # :param retVal: anything that can be serialized in json. + # """ + + # # In case of error in server side we hide server CallStack to client + # try: + # if 'CallStack' in retVal: + # del retVal['CallStack'] + # except TypeError: + # pass + + # # Set the status + # self.set_status(self._httpStatus) + + # # This is basically only used for file download through + # # the 'streamToClient' method. + # if self.rawContent: + # # See 4.5.1 http://www.rfc-editor.org/rfc/rfc2046.txt + # self.set_header("Content-Type", "application/octet-stream") + # returnedData = retVal + # else: + # self.set_header("Content-Type", "application/json") + # returnedData = encode(retVal) + + # self.write(returnedData) def on_finish(self): """ Called after the end of HTTP request. Log the request duration """ - # TODO CHRIS: log better than that ! Look at what is in RequestHandler - requestDuration = time.time() - self.requestStartTime - sLog.notice("Ending request to %s after %fs" % (self.srv_getURL(), requestDuration)) + elapsedTime = 1000.0 * self.request.request_time() + + try: + if self.result['OK']: + argsString = "OK" + else: + argsString = "ERROR: %s" % self.result['Message'] + except (AttributeError, KeyError): # In case it is not a DIRAC structure + if self._reason == 'OK': + argsString = 'OK' + else: + argsString = 'ERROR %s' % self._reason + + argsString = "ERROR: %s" % self._reason + sLog.notice("Returning response", "%s %s (%.2f ms) %s" % (self.srv_getFormattedRemoteCredentials(), + self._serviceName, + elapsedTime, argsString)) def _gatherPeerCredentials(self): """ @@ -488,8 +519,6 @@ def _gatherPeerCredentials(self): for cert in cert_chain: chainAsText += cert.as_pem() - # And we let some utilities do the job... - # Following lines just get the right info, at the right place peerChain.loadChainFromString(chainAsText) # Retrieve the credentials @@ -611,7 +640,16 @@ def srv_getRemoteAddress(self): :return: Address of remote peer. """ - return self.request.remote_ip + + remote_ip = self.request.remote_ip + # Although it would be trivial to add this attribute in _HTTPRequestContext, + # Tornado won't release anymore 5.1 series, so go the hacky way + try: + remote_port = self.request.connection.stream.socket.getpeername()[1] + except Exception: # pylint: disable=broad-except + remote_port = 0 + + return (remote_ip, remote_port) def getRemoteAddress(self): """ @@ -638,11 +676,35 @@ def getRemoteCredentials(self): def srv_getFormattedRemoteCredentials(self): """ Return the DN of user + + Mostly copy paste from + :py:meth:`DIRAC.Core.DISET.private.Transports.BaseTransport.BaseTransport.getFormattedCredentials` + + Note that the information will be complete only once the AuthManager was called """ + address = self.getRemoteAddress() + peerId = "" + # Depending on where this is call, it may be that credDict is not yet filled. + # (reminder: AuthQuery fills part of it..) try: - return self.credDict['DN'] - except KeyError: # Called before reading certificate chain - return "unknown" + peerId = "[%s:%s]" % (self.credDict['group'], self.credDict['username']) + except AttributeError: + pass + + if address[0].find(":") > -1: + return "([%s]:%s)%s" % (address[0], address[1], peerId) + return "(%s:%s)%s" % (address[0], address[1], peerId) + +# def getFormattedCredentials(self): +# peerCreds = self.getConnectingCredentials() +# address = self.getRemoteAddress() +# if 'username' in peerCreds: +# peerId = "[%s:%s]" % (peerCreds['group'], peerCreds['username']) +# else: +# peerId = "" +# if address[0].find(":") > -1: +# return "([%s]:%s)%s" % (address[0], address[1], peerId) +# return "(%s:%s)%s" % (address[0], address[1], peerId) def srv_getServiceName(self): """ From 4233c58aa35cc4861ca2d4b0ef581d8b172cb1cd Mon Sep 17 00:00:00 2001 From: Christophe Haen Date: Tue, 25 Aug 2020 15:17:42 +0200 Subject: [PATCH 21/25] Tornado: make service initialization thread safe --- ConfigurationSystem/private/Refresher.py | 6 +- Core/DISET/private/BaseClient.py | 6 +- .../Client/private/TornadoBaseClient.py | 3 +- Core/Tornado/Server/TornadoService.py | 70 ++++++++++++------- 4 files changed, 48 insertions(+), 37 deletions(-) diff --git a/ConfigurationSystem/private/Refresher.py b/ConfigurationSystem/private/Refresher.py index 20a98537cb3..baa892bd6ab 100755 --- a/ConfigurationSystem/private/Refresher.py +++ b/ConfigurationSystem/private/Refresher.py @@ -8,15 +8,11 @@ __RCSID__ = "$Id$" import threading -try: - import thread -except ImportError: # python 3 compatibility - import _thread as thread - import time import random import os +from six.moves import _thread as thread from DIRAC.ConfigurationSystem.Client.ConfigurationData import gConfigurationData from DIRAC.ConfigurationSystem.Client.PathFinder import getGatewayURLs diff --git a/Core/DISET/private/BaseClient.py b/Core/DISET/private/BaseClient.py index 79a15e84a7e..24187779a90 100755 --- a/Core/DISET/private/BaseClient.py +++ b/Core/DISET/private/BaseClient.py @@ -9,10 +9,8 @@ import six import time -try: - import thread -except ImportError: # python 3 compatibility - import _thread as thread + +from six.moves import _thread as thread import DIRAC from DIRAC.Core.DISET.private.Protocols import gProtocolDict diff --git a/Core/Tornado/Client/private/TornadoBaseClient.py b/Core/Tornado/Client/private/TornadoBaseClient.py index b85cb83d470..b949699a30d 100644 --- a/Core/Tornado/Client/private/TornadoBaseClient.py +++ b/Core/Tornado/Client/private/TornadoBaseClient.py @@ -574,8 +574,9 @@ def _request(self, retry=0, outputFile=None, **kwargs): # if it is something else, retry raise + # Whatever exception we have here, we deem worth retrying except Exception as e: - # CHRIS TODO review this part: catch specific exceptions + # CHRIS TODO review this part: retry logic is fishy # self.__bannedUrls is emptied in findServiceURLs if url not in self.__bannedUrls: self.__bannedUrls += [url] diff --git a/Core/Tornado/Server/TornadoService.py b/Core/Tornado/Server/TornadoService.py index 97f2f055e11..c545e340568 100644 --- a/Core/Tornado/Server/TornadoService.py +++ b/Core/Tornado/Server/TornadoService.py @@ -13,6 +13,7 @@ import os import time +import threading from datetime import datetime from six.moves import http_client from tornado.web import RequestHandler, HTTPError @@ -112,7 +113,9 @@ def export_streamToClient(self, myDataToSend, token): """ # Because we initialize at first request, we use a flag to know if it's already done - __FLAG_INIT_DONE = False + __init_done = False + # Lock to make sure that two threads are not initializing at the same time + __init_lock = threading.RLock() # MonitoringClient, we don't use gMonitor which is not thread-safe # We also need to add specific attributes for each service @@ -122,6 +125,8 @@ def export_streamToClient(self, myDataToSend, token): def _initMonitoring(cls, serviceName, fullUrl): """ Initialize the monitoring specific to this handler + This has to be called only by :py:meth:`.__initializeService` + to ensure thread safety and unicity of the call. :param serviceName: relative URL ``//`` :param fullUrl: full URl like ``https://://`` @@ -154,39 +159,52 @@ def _initMonitoring(cls, serviceName, fullUrl): @classmethod def __initializeService(cls, relativeUrl, absoluteUrl): """ - Initialize a service, called at first request + Initialize a service. + The work is only perform once at the first request. :param relativeUrl: relative URL, e.g. ``//`` :param absoluteUrl: full URL e.g. ``https://://`` - .. warning:: - This method is not thread safe nor re-entrant, while it should be. - TODO for Chris: do it !!! + :returns: S_OK """ - # Url starts with a "/", we just remove it - serviceName = relativeUrl[1:] + # If the initialization was already done successfuly, + # we can just return + if cls.__init_done: + return S_OK() - cls._startTime = datetime.utcnow() - sLog.info("First use of %s, initializing service..." % relativeUrl) - cls._authManager = AuthManager("%s/Authorization" % PathFinder.getServiceSection(serviceName)) + # Otherwise, do the work but with a lock + with cls.__init_lock: - cls._initMonitoring(serviceName, absoluteUrl) + # Check again that the initialization was not done by another thread + # while we were waiting for the lock + if cls.__init_done: + return S_OK() - cls._serviceName = serviceName - cls._validNames = [serviceName] - serviceInfo = {'serviceName': serviceName, - 'serviceSectionPath': PathFinder.getServiceSection(serviceName), - 'csPaths': [PathFinder.getServiceSection(serviceName)], - 'URL': absoluteUrl - } - cls._serviceInfoDict = serviceInfo + # Url starts with a "/", we just remove it + serviceName = relativeUrl[1:] - cls.__monitorLastStatsUpdate = time.time() + cls._startTime = datetime.utcnow() + sLog.info("First use of %s, initializing service..." % relativeUrl) + cls._authManager = AuthManager("%s/Authorization" % PathFinder.getServiceSection(serviceName)) - cls.initializeHandler(serviceInfo) + cls._initMonitoring(serviceName, absoluteUrl) - cls.__FLAG_INIT_DONE = True - return S_OK() + cls._serviceName = serviceName + cls._validNames = [serviceName] + serviceInfo = {'serviceName': serviceName, + 'serviceSectionPath': PathFinder.getServiceSection(serviceName), + 'csPaths': [PathFinder.getServiceSection(serviceName)], + 'URL': absoluteUrl + } + cls._serviceInfoDict = serviceInfo + + cls.__monitorLastStatsUpdate = time.time() + + cls.initializeHandler(serviceInfo) + + cls.__init_done = True + + return S_OK() @classmethod def initializeHandler(cls, serviceInfoDict): @@ -212,20 +230,18 @@ def initialize(self): # pylint: disable=arguments-differ """ Initialize the handler, called at every request. - If it is the first time this handler is instantiated, it will - call :py:meth:`.__initializeService` + It just calls :py:meth:`.__initializeService` If anything goes wrong, the client will get ``Connection aborted`` error. See details inside the method. - ..warning:: DO NOT REWRITE THIS FUNCTION IN YOUR HANDLER ==> initialize in DISET became initializeRequest in HTTPS ! """ # Only initialized once - if not self.__FLAG_INIT_DONE: + if not self.__init_done: # Ideally, if something goes wrong, we would like to return a Server Error 500 # but this method cannot write back to the client as per the # `tornado doc `_. From 624bfacb7076130b3ad154c35c57fe24920913df Mon Sep 17 00:00:00 2001 From: Christophe Haen Date: Tue, 25 Aug 2020 17:17:40 +0200 Subject: [PATCH 22/25] Factorize RPCClientSelector and TransferClientSelector into ClientSelector --- Core/Base/Client.py | 2 +- ...RPCClientSelector.py => ClientSelector.py} | 52 ++++++++++----- Core/Tornado/Client/TransferClientSelector.py | 63 ------------------- Resources/Catalog/FileCatalogClient.py | 2 +- .../TestClientSelectionAndInterface.py | 2 +- .../Test_TornadoAndDISETmixed.py | 2 +- .../perf-test-DB/test_scripts/v_perf.py | 2 +- .../perf-test-ping/test_scripts/v_perf.py | 2 +- 8 files changed, 43 insertions(+), 84 deletions(-) rename Core/Tornado/Client/{RPCClientSelector.py => ClientSelector.py} (50%) delete mode 100644 Core/Tornado/Client/TransferClientSelector.py diff --git a/Core/Base/Client.py b/Core/Base/Client.py index f7abd70f83e..c41e9e4b8f3 100644 --- a/Core/Base/Client.py +++ b/Core/Base/Client.py @@ -13,7 +13,7 @@ from io import open -from DIRAC.Core.Tornado.Client.RPCClientSelector import RPCClientSelector +from DIRAC.Core.Tornado.Client.ClientSelector import RPCClientSelector from DIRAC.Core.Tornado.Client.TornadoClient import TornadoClient from DIRAC.Core.Utilities.Decorators import deprecated from DIRAC.Core.DISET import DEFAULT_RPC_TIMEOUT diff --git a/Core/Tornado/Client/RPCClientSelector.py b/Core/Tornado/Client/ClientSelector.py similarity index 50% rename from Core/Tornado/Client/RPCClientSelector.py rename to Core/Tornado/Client/ClientSelector.py index c274a815f75..edb415fbcc2 100644 --- a/Core/Tornado/Client/RPCClientSelector.py +++ b/Core/Tornado/Client/ClientSelector.py @@ -1,11 +1,10 @@ """ - RPCClientSelector can replace RPCClient (with ``import RPCClientSelector as RPCClient``) - to migrate from DISET to Tornado. This function choses and returns the client which should be - used for a service. If the url of the service uses HTTPS, TornadoClient is returned, else it returns RPCClient + This modules defines two functions that can be used in place of the ``RPCClient`` and + ``TransferClient`` to transparently switch to ``https``. Example:: - from DIRAC.Core.Tornado.Client.RPCClientSelector import RPCClientSelector as RPCClient + from DIRAC.Core.Tornado.Client.ClientSelector import RPCClientSelector as RPCClient myService = RPCClient("Framework/MyService") myService.doSomething() """ @@ -16,35 +15,51 @@ __RCSID__ = "$Id$" +import functools + from DIRAC import gLogger from DIRAC.ConfigurationSystem.Client.PathFinder import getServiceURL from DIRAC.Core.DISET.RPCClient import RPCClient +from DIRAC.Core.DISET.TransferClient import TransferClient from DIRAC.Core.Tornado.Client.TornadoClient import TornadoClient sLog = gLogger.getSubLogger(__name__) -# TODO CHRIS: factorize RPCClientSelector and TransferClientSelector (using functool partial ?) - -def RPCClientSelector(*args, **kwargs): # We use same interface as RPCClient +def ClientSelector(disetClient, *args, **kwargs): # We use same interface as RPCClient """ - Select the correct RPCClient, instantiate it, and return it. + Select the correct Client (either RPC or Transfer ), instantiate it, and return it. + + The selection is based on the URL: + + * either it contains the protocol, in which case we make a choice + * or it is in the form , in which case we resolve first + + This is a generic function. You should rather use :py:class:`.RPCClientSelector` + or :py:class:`.TransferClientSelector` + + In principle, the only place for this class to be used is in :py:class:`DIRAC.Core.Base.Client.Client`, since it is the only one supposed to instantiate an :py:class:`DIRAC.Core.Base.DISET.RPCClient.RPCClient` - :param args: URL can be just "system/service" or "dips://domain:port/system/service" + :params disetClient: the DISET class to be instantiated, so either + :py:class:`DIRAC.Core.Base.DISET.RPCClient.RPCClient` + or :py:class:`DIRAC.Core.Base.DISET.TransferClient.TransferClient` + :param args: Whatever ``disetClient`` takes as args, but the first one is + always the URL we want to rely on. + It can be either "system/service" or "dips://domain:port/system/service" :param kwargs: This can contain: - * Whatever :py:class:`DIRAC.Core.Base.DISET.RPCClient.RPCClient` takes. + * Whatever ``disetClient`` takes. * httpsClient: specific class inheriting from TornadoClient """ # We detect if we need to use a specific class for the HTTPS client - TornadoRPCClient = kwargs.pop('httpsClient', TornadoClient) + tornadoClient = kwargs.pop('httpsClient', TornadoClient) # We have to make URL resolution BEFORE the RPCClient or TornadoClient to determine which one we want to use # URL is defined as first argument (called serviceName) in RPCClient @@ -62,12 +77,19 @@ def RPCClientSelector(*args, **kwargs): # We use same interface as RPCClient if completeUrl.startswith("http"): sLog.info("Using HTTPS for service %s" % serviceName) - rpc = TornadoRPCClient(*args, **kwargs) + rpc = tornadoClient(*args, **kwargs) else: - rpc = RPCClient(*args, **kwargs) + rpc = disetClient(*args, **kwargs) except Exception as e: # pylint: disable=broad-except # If anything went wrong in the resolution, we return default RPCClient # So the behaviour is exactly the same as before implementation of Tornado - sLog.warn("Could not select RPC or Tornado client", "%s" % repr(e)) - rpc = RPCClient(*args, **kwargs) + sLog.warn("Could not select DISET or Tornado client", "%s" % repr(e)) + rpc = disetClient(*args, **kwargs) return rpc + + +# Client to use for RPC selection +RPCClientSelector = functools.partial(ClientSelector, RPCClient) + +# Client to use for Transfer selection +TransferClientSelector = functools.partial(ClientSelector, TransferClient) diff --git a/Core/Tornado/Client/TransferClientSelector.py b/Core/Tornado/Client/TransferClientSelector.py deleted file mode 100644 index 3328319e0bd..00000000000 --- a/Core/Tornado/Client/TransferClientSelector.py +++ /dev/null @@ -1,63 +0,0 @@ -""" - TransferClientSelector can replace TransferClient (with import TransferClientSelector as TransferClient) - to migrate from DISET to Tornado. This method chooses and returns the client which should be - used for a service. If the url of the service uses HTTPS, TornadoClient is returned, else it returns TransferClient - - Example:: - - from DIRAC.Core.Tornado.Client.TransferClientSelector import TransferClientSelector as TransferClient - myService = TransferClient("Framework/MyService") - myService.doSomething() -""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -__RCSID__ = "$Id$" - -from DIRAC import gLogger -from DIRAC.ConfigurationSystem.Client.PathFinder import getServiceURL -from DIRAC.Core.DISET.TransferClient import TransferClient -from DIRAC.Core.Tornado.Client.TornadoClient import TornadoClient - - -sLog = gLogger.getSubLogger(__name__) - - -def TransferClientSelector(*args, **kwargs): # We use same interface as TransferClient - """ - Select the correct TransferClient, instantiate it, and return it - - :param args: URL can be just "system/service" or "dips://domain:port/system/service" - """ - - # We detect if we need to use a specific class for the HTTPS client - - TornadoTransClient = kwargs.pop('httpsClient', TornadoClient) - - # We have to make URL resolution BEFORE the TransferClient or TornadoClient to determine which one we want to use - # URL is defined as first argument (called serviceName) in TransferClient - - try: - serviceName = args[0] - sLog.verbose("Trying to autodetect client for %s" % serviceName) - - # If we are not already given a URL, resolve it - if serviceName.startswith(('http', 'dip')): - completeUrl = serviceName - else: - completeUrl = getServiceURL(serviceName) - sLog.verbose("URL resolved: %s" % completeUrl) - - if completeUrl.startswith("http"): - sLog.info("Using HTTPS for service %s" % serviceName) - transClient = TornadoTransClient(*args, **kwargs) - else: - transClient = TransferClient(*args, **kwargs) - except Exception as e: # pylint: disable=broad-except - # If anything went wrong in the resolution, we return default TransferClient - # So the behavior is exactly the same as before implementation of Tornado - sLog.warn("Could not select RPC or Tornado client", "%s" % repr(e)) - transClient = TransferClient(*args, **kwargs) - return transClient diff --git a/Resources/Catalog/FileCatalogClient.py b/Resources/Catalog/FileCatalogClient.py index 56c3dceef1f..e8f33717855 100644 --- a/Resources/Catalog/FileCatalogClient.py +++ b/Resources/Catalog/FileCatalogClient.py @@ -7,7 +7,7 @@ import os from DIRAC import S_OK, S_ERROR -from DIRAC.Core.Tornado.Client.TransferClientSelector import TransferClientSelector as TransferClient +from DIRAC.Core.Tornado.Client.ClientSelector import TransferClientSelector as TransferClient from DIRAC.Core.Security.ProxyInfo import getVOfromProxyGroup from DIRAC.ConfigurationSystem.Client.Helpers.Registry import getVOMSAttributeForGroup, getDNForUsername diff --git a/tests/DISET_HTTPS_migration/TestClientSelectionAndInterface.py b/tests/DISET_HTTPS_migration/TestClientSelectionAndInterface.py index c3e427770f6..9119aa7656d 100644 --- a/tests/DISET_HTTPS_migration/TestClientSelectionAndInterface.py +++ b/tests/DISET_HTTPS_migration/TestClientSelectionAndInterface.py @@ -21,7 +21,7 @@ from pytest import mark, fixture -from DIRAC.Core.Tornado.Client.RPCClientSelector import RPCClientSelector +from DIRAC.Core.Tornado.Client.ClientSelector import RPCClientSelector from DIRAC.Core.Tornado.Client.TornadoClient import TornadoClient from DIRAC.Core.DISET.RPCClient import RPCClient from DIRAC.ConfigurationSystem.private.ConfigurationClient import ConfigurationClient diff --git a/tests/Integration/TornadoServices/Test_TornadoAndDISETmixed.py b/tests/Integration/TornadoServices/Test_TornadoAndDISETmixed.py index d19d7552489..1dfddd9ef17 100644 --- a/tests/Integration/TornadoServices/Test_TornadoAndDISETmixed.py +++ b/tests/Integration/TornadoServices/Test_TornadoAndDISETmixed.py @@ -123,7 +123,7 @@ from hypothesis import given, settings, unlimited from hypothesis.strategies import text -from DIRAC.Core.Tornado.Client.RPCClientSelector import RPCClientSelector as RPCClient +from DIRAC.Core.Tornado.Client.ClientSelector import RPCClientSelector as RPCClient from DIRAC.Core.Utilities.DErrno import ENOAUTH from DIRAC import S_ERROR from DIRAC.ConfigurationSystem.Client.CSAPI import CSAPI diff --git a/tests/Integration/TornadoServices/perf-test-DB/test_scripts/v_perf.py b/tests/Integration/TornadoServices/perf-test-DB/test_scripts/v_perf.py index 589d0d72a78..39d136895f7 100644 --- a/tests/Integration/TornadoServices/perf-test-DB/test_scripts/v_perf.py +++ b/tests/Integration/TornadoServices/perf-test-DB/test_scripts/v_perf.py @@ -4,7 +4,7 @@ Configuration and how to run it are explained in the documentation """ -from DIRAC.Core.Tornado.Client.RPCClientSelector import RPCClientSelector +from DIRAC.Core.Tornado.Client.ClientSelector import RPCClientSelector from time import time from random import random import sys diff --git a/tests/Integration/TornadoServices/perf-test-ping/test_scripts/v_perf.py b/tests/Integration/TornadoServices/perf-test-ping/test_scripts/v_perf.py index ca20640831f..8abec97aaca 100644 --- a/tests/Integration/TornadoServices/perf-test-ping/test_scripts/v_perf.py +++ b/tests/Integration/TornadoServices/perf-test-ping/test_scripts/v_perf.py @@ -4,7 +4,7 @@ Configuration and how to run it are explained in the documentation """ -from DIRAC.Core.Tornado.Client.RPCClientSelector import RPCClientSelector +from DIRAC.Core.Tornado.Client.ClientSelector import RPCClientSelector from time import time from random import randint import sys From a0230d7250bc45c7ac6fd03da9845a2e37b9d04f Mon Sep 17 00:00:00 2001 From: chaen Date: Fri, 28 Aug 2020 11:16:00 +0200 Subject: [PATCH 23/25] Typo in comment Co-authored-by: Andre Sailer --- ConfigurationSystem/Client/ConfigurationClient.py | 10 +--------- Core/Tornado/Client/TornadoClient.py | 2 +- 2 files changed, 2 insertions(+), 10 deletions(-) diff --git a/ConfigurationSystem/Client/ConfigurationClient.py b/ConfigurationSystem/Client/ConfigurationClient.py index 4d8c90cb91e..59dd90745b4 100644 --- a/ConfigurationSystem/Client/ConfigurationClient.py +++ b/ConfigurationSystem/Client/ConfigurationClient.py @@ -1,11 +1,3 @@ -""" - Custom TornadoClient for Configuration System - Used like a normal client, should be instanciated if and only if we use the configuration service - - because JSON cannot ship binary data, we encode it in base64 - -""" - from __future__ import absolute_import from __future__ import division from __future__ import print_function @@ -21,7 +13,7 @@ class CSJSONClient(TornadoClient): """ - The specific client for configuration system. + The specific Tornado client for configuration system. To avoid JSON limitation the HTTPS handler encode data in base64 before sending them, this class only decode the base64 diff --git a/Core/Tornado/Client/TornadoClient.py b/Core/Tornado/Client/TornadoClient.py index f74da79cae0..c189bb3ec8f 100644 --- a/Core/Tornado/Client/TornadoClient.py +++ b/Core/Tornado/Client/TornadoClient.py @@ -70,7 +70,7 @@ def executeRPC(self, method, *args): def receiveFile(self, destFile, *args): """ - Equivalent of :py:meth`~DIRAC.Core.DISET.TransferClient.TransferClient.receiveFile + Equivalent of :py:meth:`~DIRAC.Core.DISET.TransferClient.TransferClient.receiveFile` In practice, it calls the remote method `streamToClient` and stores the raw result in a file From a0012a70e0a1a20e085f1df229dfe81bd6d1c74e Mon Sep 17 00:00:00 2001 From: Christophe Haen Date: Thu, 8 Oct 2020 15:32:35 +0200 Subject: [PATCH 24/25] Point to DIRACGrid instead of chaen repo --- docs/source/DeveloperGuide/TornadoServices/index.rst | 6 +++--- environment.yml | 4 ++-- requirements.txt | 5 ++--- 3 files changed, 7 insertions(+), 8 deletions(-) diff --git a/docs/source/DeveloperGuide/TornadoServices/index.rst b/docs/source/DeveloperGuide/TornadoServices/index.rst index ba37b84def3..a4995edbcf9 100644 --- a/docs/source/DeveloperGuide/TornadoServices/index.rst +++ b/docs/source/DeveloperGuide/TornadoServices/index.rst @@ -321,8 +321,8 @@ Requirements ************ Two special python packages are needed: -* git+https://github.com/chaen/tornado.git@iostreamConfigurable : in place of the standard tornado. This adds configurable feature to tornado -* git+https://github.com/chaen/tornado_m2crypto.git: this allows to use tornado with M2Crypto +* git+https://github.com/DIRACGrid/tornado.git@iostreamConfigurable : in place of the standard tornado. This adds configurable feature to tornado +* git+https://github.com/DIRACGrid/tornado_m2crypto.git: this allows to use tornado with M2Crypto Install a service @@ -371,4 +371,4 @@ Modify first lines of ``DIRAC/TornadoServices/test/multi-mechanize/distributed-t On the server start ``DIRAC/test/Integration/TornadoServices/getCPUInfos`` (redirect output to a file) -Run ``distributed-test.py [NameOfYourTest]`` at the end of execution, the command to plot is given. Before executing command, copy output of ``getCPUInfos`` on ``/tmp/results.txt`` (on your local machine). \ No newline at end of file +Run ``distributed-test.py [NameOfYourTest]`` at the end of execution, the command to plot is given. Before executing command, copy output of ``getCPUInfos`` on ``/tmp/results.txt`` (on your local machine). diff --git a/environment.yml b/environment.yml index b7d13789797..34fc44977c3 100644 --- a/environment.yml +++ b/environment.yml @@ -68,7 +68,7 @@ dependencies: - diraccfg # This is a fork of tornado with a patch to allow for configurable iostream # It should eventually be part of DIRACGrid - - git+https://github.com/chaen/tornado.git@iostreamConfigurable + - git+https://github.com/DIRACGrid/tornado.git@iostreamConfigurable # This is an extension of Tornado to use M2Crypto # It should eventually be part of DIRACGrid - - git+https://github.com/chaen/tornado_m2crypto + - git+https://github.com/DIRACGrid/tornado_m2crypto diff --git a/requirements.txt b/requirements.txt index 9accdf9daf4..b647b641349 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,9 +2,8 @@ fts3-rest #Patch for tornado -git+https://gitlab.com/chaen/m2crypto.git@tmpUntilSwigUpdated -git+https://github.com/chaen/tornado.git@iostreamConfigurable -git+https://github.com/chaen/tornado_m2crypto.git +git+https://github.com/DIRACGrid/tornado.git@iostreamConfigurable +git+https://github.com/DIRACGrid/tornado_m2crypto.git # From pypi boto3 From 916ced64ca3ae7c59206bc7947028926551ee4e1 Mon Sep 17 00:00:00 2001 From: Christophe Haen Date: Fri, 9 Oct 2020 16:49:17 +0200 Subject: [PATCH 25/25] Update environments.yml and requirements.txt to point to the correct tornado patch repo --- environment.yml | 2 -- requirements.txt | 1 - 2 files changed, 3 deletions(-) diff --git a/environment.yml b/environment.yml index 34fc44977c3..07d030183fd 100644 --- a/environment.yml +++ b/environment.yml @@ -67,8 +67,6 @@ dependencies: - pip: - diraccfg # This is a fork of tornado with a patch to allow for configurable iostream - # It should eventually be part of DIRACGrid - git+https://github.com/DIRACGrid/tornado.git@iostreamConfigurable # This is an extension of Tornado to use M2Crypto - # It should eventually be part of DIRACGrid - git+https://github.com/DIRACGrid/tornado_m2crypto diff --git a/requirements.txt b/requirements.txt index b647b641349..e3f71da85cd 100644 --- a/requirements.txt +++ b/requirements.txt @@ -56,6 +56,5 @@ typing==3.6.6 hypothesis python-json-logger>=0.1.8 multi-mechanize>=1.2.0 -tornado>=5.0.0 caniusepython3 subprocess32