diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index d070f7eda4c..527622694eb 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -12,8 +12,10 @@ jobs: MATRIX_DEFAULT_MYSQL_VER: 5.7 MATRIX_DEFAULT_HOST_OS: cc7 MATRIX_DEFAULT_USE_NEWTHREADPOOL: default + MATRIX_DEFAULT_DIRACOSVER: default MATRIX_DEFAULT_SERVER_USE_M2CRYPTO: Yes MATRIX_DEFAULT_CLIENT_USE_M2CRYPTO: Yes + MATRIX_DEFAULT_TEST_HTTPS: No strategy: fail-fast: False @@ -38,6 +40,12 @@ jobs: CLIENT_USE_M2CRYPTO: Yes - SERVER_USE_M2CRYPTO: No CLIENT_USE_M2CRYPTO: No + ###### HTTPS tests + # IMPLICIT: - DIRACOSVER: default + # TEST_HTTPS: No + - TEST_HTTPS: Yes + DIRACOSVER: https + steps: - uses: actions/checkout@v2 @@ -59,6 +67,8 @@ jobs: echo -n "-e MYSQL_VER=${{ matrix.MYSQL_VER || env.MATRIX_DEFAULT_MYSQL_VER }} " >> run_in_container echo -n "-e ES_VER=${{ matrix.ES_VER || env.MATRIX_DEFAULT_ES_VER }} " >> run_in_container if [[ "${{ matrix.USE_NEWTHREADPOOL || env.MATRIX_DEFAULT_USE_NEWTHREADPOOL }}" != "default" ]]; then echo -n "-e DIRAC_USE_NEWTHREADPOOL=${{ matrix.USE_NEWTHREADPOOL || env.MATRIX_DEFAULT_USE_NEWTHREADPOOL }} " >> run_in_container; fi + if [[ "${{ matrix.DIRACOSVER || env.MATRIX_DEFAULT_DIRACOSVER }}" != "default" ]]; then echo -n "-e DIRACOSVER=${{ matrix.DIRACOSVER || env.MATRIX_DEFAULT_DIRACOSVER }} " >> run_in_container; fi + echo -n "-e TEST_HTTPS=${{ matrix.TEST_HTTPS || env.MATRIX_DEFAULT_TEST_HTTPS }} " >> run_in_container echo -n "-e SERVER_USE_M2CRYPTO=${{ matrix.SERVER_USE_M2CRYPTO || env.MATRIX_DEFAULT_SERVER_USE_M2CRYPTO }} " >> run_in_container echo -n "-e CLIENT_USE_M2CRYPTO=${{ matrix.CLIENT_USE_M2CRYPTO || env.MATRIX_DEFAULT_CLIENT_USE_M2CRYPTO }} " >> run_in_container # Finish wrapper script diff --git a/ConfigurationSystem/Client/CSAPI.py b/ConfigurationSystem/Client/CSAPI.py index e83e313b40d..566e34dbc87 100644 --- a/ConfigurationSystem/Client/CSAPI.py +++ b/ConfigurationSystem/Client/CSAPI.py @@ -8,7 +8,7 @@ import six from DIRAC import gLogger, gConfig, S_OK, S_ERROR -from DIRAC.Core.DISET.RPCClient import RPCClient +from DIRAC.ConfigurationSystem.Client.ConfigurationClient import ConfigurationClient from DIRAC.Core.Utilities import List, Time from DIRAC.Core.Security.X509Chain import X509Chain # pylint: disable=import-error from DIRAC.Core.Security import Locations @@ -90,7 +90,7 @@ def initialize(self): if not retVal['OK']: self.__initialized = S_ERROR("Master server is not known. Is everything initialized?") return self.__initialized - self.__rpcClient = RPCClient(gConfig.getValue("/DIRAC/Configuration/MasterServer", "")) + self.__rpcClient = ConfigurationClient(url=gConfig.getValue("/DIRAC/Configuration/MasterServer", "")) self.__csMod = Modificator(self.__rpcClient, "%s - %s - %s" % (self.__userGroup, self.__userDN, Time.dateTime().strftime("%Y-%m-%d %H:%M:%S"))) retVal = self.downloadCSData() diff --git a/ConfigurationSystem/Client/ConfigurationClient.py b/ConfigurationSystem/Client/ConfigurationClient.py new file mode 100644 index 00000000000..59dd90745b4 --- /dev/null +++ b/ConfigurationSystem/Client/ConfigurationClient.py @@ -0,0 +1,77 @@ +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +__RCSID__ = "$Id$" + + +from base64 import b64encode, b64decode + +from DIRAC.Core.Tornado.Client.TornadoClient import TornadoClient +from DIRAC.Core.Base.Client import Client + + +class CSJSONClient(TornadoClient): + """ + The specific Tornado client for configuration system. + To avoid JSON limitation the HTTPS handler encode data in base64 + before sending them, this class only decode the base64 + + An exception is made with CommitNewData which ENCODE in base64 + """ + + def getCompressedData(self): + """ + Transmit request to service and get data in base64, + it decode base64 before returning + + :returns: Configuration data, compressed + :rtype: str + """ + retVal = self.executeRPC('getCompressedData') + if retVal['OK']: + retVal['Value'] = b64decode(retVal['Value']) + return retVal + + def getCompressedDataIfNewer(self, sClientVersion): + """ + Transmit request to service and get data in base64, + it decode base64 before returning. + + :returns: Configuration data, if changed, compressed + """ + retVal = self.executeRPC('getCompressedDataIfNewer', sClientVersion) + if retVal['OK'] and 'data' in retVal['Value']: + retVal['Value']['data'] = b64decode(retVal['Value']['data']) + return retVal + + def commitNewData(self, sData): + """ + Transmit request to service by encoding data in base64. + + :param sData: Data to commit, you may call this method with CSAPI and Modificator + """ + return self.executeRPC('commitNewData', b64encode(sData)) + + +class ConfigurationClient(Client): + """ + Specific client to speak with ConfigurationServer. + + This class must contain at least the JSON decoder dedicated to + the Configuration Server. + + You can implement more logic or function to the client here if needed, + RPCCall can be made inside this class with executeRPC method. + """ + + # The JSON decoder for Configuration Server + httpsClient = CSJSONClient + + def __init__(self, **kwargs): + # By default we use Configuration/Server as url, client do the resolution + # In some case url has to be static (when a slave register to the master server for example) + # It's why we can use 'url' as keyword arguments + if 'url' not in kwargs: + kwargs['url'] = 'Configuration/Server' + super(ConfigurationClient, self).__init__(**kwargs) diff --git a/ConfigurationSystem/Service/TornadoConfigurationHandler.py b/ConfigurationSystem/Service/TornadoConfigurationHandler.py new file mode 100644 index 00000000000..beb9d9b6acb --- /dev/null +++ b/ConfigurationSystem/Service/TornadoConfigurationHandler.py @@ -0,0 +1,148 @@ +""" The CS! (Configuration Service) + +Modified to work with Tornado +Encode data in base64 because of JSON limitations +In client side you must use a specific client + +""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +__RCSID__ = "$Id$" + +from base64 import b64encode, b64decode + +from DIRAC import S_OK, S_ERROR, gLogger +from DIRAC.ConfigurationSystem.private.ServiceInterfaceTornado import ServiceInterfaceTornado as ServiceInterface +from DIRAC.Core.Utilities import DErrno +from DIRAC.Core.Tornado.Server.TornadoService import TornadoService + +sLog = gLogger.getSubLogger(__name__) + + +class TornadoConfigurationHandler(TornadoService): + """ + The CS handler + """ + ServiceInterface = None + PilotSynchronizer = None + + @classmethod + def initializeHandler(cls, serviceInfo): + """ + Initialize the configuration server + Behind it starts thread which refresh configuration + """ + cls.ServiceInterface = ServiceInterface(serviceInfo['URL']) + return S_OK() + + def export_getVersion(self): + """ + Returns the version of the configuration + """ + return S_OK(self.ServiceInterface.getVersion()) + + def export_getCompressedData(self): + """ + Returns the configuration + """ + sData = self.ServiceInterface.getCompressedConfigurationData() + return S_OK(b64encode(sData)) + + def export_getCompressedDataIfNewer(self, sClientVersion): + """ + Returns the configuration if a newer configuration exists, if not just returns the version + + :param sClientVersion: Version used by client + """ + sVersion = self.ServiceInterface.getVersion() + retDict = {'newestVersion': sVersion} + if sClientVersion < sVersion: + retDict['data'] = b64encode(self.ServiceInterface.getCompressedConfigurationData()) + return S_OK(retDict) + + def export_publishSlaveServer(self, sURL): + """ + Used by slave server to register as a slave server. + + :param sURL: The url of the slave server. + """ + self.ServiceInterface.publishSlaveServer(sURL) + return S_OK() + + def export_commitNewData(self, sData): + """ + Write the new configuration + """ + credDict = self.getRemoteCredentials() + if 'DN' not in credDict or 'username' not in credDict: + return S_ERROR("You must be authenticated!") + sData = b64decode(sData) + res = self.ServiceInterface.updateConfiguration(sData, credDict['username']) + if not res['OK']: + return res + + # Check the flag for updating the pilot 3 JSON file + if self.srv_getCSOption('UpdatePilotCStoJSONFile', False) and self.ServiceInterface.isMaster(): + if self.PilotSynchronizer is None: + try: + # This import is only needed for the Master CS service, making it conditional avoids + # dependency on the git client preinstalled on all the servers running CS slaves + from DIRAC.WorkloadManagementSystem.Utilities.PilotCStoJSONSynchronizer import PilotCStoJSONSynchronizer + except ImportError as exc: + sLog.exception("Failed to import PilotCStoJSONSynchronizer", repr(exc)) + return S_ERROR(DErrno.EIMPERR, 'Failed to import PilotCStoJSONSynchronizer') + self.PilotSynchronizer = PilotCStoJSONSynchronizer() + return self.PilotSynchronizer.sync() + + return res + + def export_writeEnabled(self): + """ + Used to know if we can change the configuration on this server + """ + return S_OK(self.ServiceInterface.isMaster()) + + def export_getCommitHistory(self, limit=100): + """ + Get the history of modifications in the configuration + """ + if limit > 100: + limit = 100 + history = self.ServiceInterface.getCommitHistory() + if limit: + history = history[:limit] + return S_OK(history) + + def export_getVersionContents(self, versionList): + """ + Get an old version of the configuration, can also be used to get more than one version. + + :param versionList: List of the version we are trying to get + """ + contentsList = [] + for version in versionList: + retVal = self.ServiceInterface.getVersionContents(version) + if retVal['OK']: + contentsList.append(retVal['Value']) + else: + return S_ERROR("Can't get contents for version %s: %s" % (version, retVal['Message'])) + return S_OK(contentsList) + + def export_rollbackToVersion(self, version): + """ + Rollback to an older version of the configuration + + :param version: The version we want to apply + """ + retVal = self.ServiceInterface.getVersionContents(version) + if not retVal['OK']: + return S_ERROR("Can't get contents for version %s: %s" % (version, retVal['Message'])) + credDict = self.getRemoteCredentials() + if 'DN' not in credDict or 'username' not in credDict: + return S_ERROR("You must be authenticated!") + return self.ServiceInterface.updateConfiguration(retVal['Value'], + credDict['username'], + updateVersionOption=True) diff --git a/ConfigurationSystem/private/Refresher.py b/ConfigurationSystem/private/Refresher.py index 4af11e81469..baa892bd6ab 100755 --- a/ConfigurationSystem/private/Refresher.py +++ b/ConfigurationSystem/private/Refresher.py @@ -1,5 +1,5 @@ """ Refresh local CS (if needed) - In every gConfig +Used each time you call gConfig. It keep your configuration up-to-date with the configuration server """ from __future__ import absolute_import from __future__ import division @@ -8,9 +8,12 @@ __RCSID__ = "$Id$" import threading -import thread import time import random +import os + +from six.moves import _thread as thread + from DIRAC.ConfigurationSystem.Client.ConfigurationData import gConfigurationData from DIRAC.ConfigurationSystem.Client.PathFinder import getGatewayURLs from DIRAC.FrameworkSystem.Client.Logger import gLogger @@ -20,6 +23,9 @@ def _updateFromRemoteLocation(serviceClient): + """ + Refresh the configuration + """ gLogger.debug("", "Trying to refresh from %s" % serviceClient.serviceURL) localVersion = gConfigurationData.getVersion() retVal = serviceClient.getCompressedDataIfNewer(localVersion) @@ -34,111 +40,93 @@ def _updateFromRemoteLocation(serviceClient): return retVal -class Refresher(threading.Thread): +class RefresherBase(object): + """ + Code factorisation for the refresher + """ def __init__(self): - threading.Thread.__init__(self) - self.__automaticUpdate = False - self.__lastUpdateTime = 0 - self.__url = False - self.__refreshEnabled = True - self.__timeout = 60 - self.__callbacks = {'newVersion': []} - gEventDispatcher.registerEvent("CSNewVersion") + self._automaticUpdate = False + self._lastUpdateTime = 0 + self._url = False + self._refreshEnabled = True + self._timeout = 60 + self._callbacks = {'newVersion': []} random.seed() - self.__triggeredRefreshLock = LockRing.LockRing().getLock() + gEventDispatcher.registerEvent("CSNewVersion") def disable(self): - self.__refreshEnabled = False + """ + Disable the refresher and prevent any request to another server + """ + self._refreshEnabled = False def enable(self): - self.__refreshEnabled = True - if self.__lastRefreshExpired(): + """ + Enable the refresher and authorize request to another server + WARNING: It will not activate automatic updates, use autoRefreshAndPublish() for that + """ + self._refreshEnabled = True + if self._lastRefreshExpired(): return self.forceRefresh() return S_OK() def isEnabled(self): - return self.__refreshEnabled + """ + Returns if you can use refresher or not, use automaticUpdateEnabled() to know + if refresh is automatic. + """ + return self._refreshEnabled def addListenerToNewVersionEvent(self, functor): gEventDispatcher.addListener("CSNewVersion", functor) - def __refreshInThread(self): - retVal = self.__refresh() - if not retVal['OK']: - gLogger.error("Error while updating the configuration", retVal['Message']) - - def __lastRefreshExpired(self): - return time.time() - self.__lastUpdateTime >= gConfigurationData.getRefreshTime() - - def refreshConfigurationIfNeeded(self): - if not self.__refreshEnabled or self.__automaticUpdate or not gConfigurationData.getServers(): - return - self.__triggeredRefreshLock.acquire() - try: - if not self.__lastRefreshExpired(): - return - self.__lastUpdateTime = time.time() - finally: - try: - self.__triggeredRefreshLock.release() - except thread.error: - pass - # Launch the refresh - thd = threading.Thread(target=self.__refreshInThread) - thd.setDaemon(1) - thd.start() + def _lastRefreshExpired(self): + """ + Just returns if last refresh must be considered as expired or not + """ + return time.time() - self._lastUpdateTime >= gConfigurationData.getRefreshTime() def forceRefresh(self, fromMaster=False): - if self.__refreshEnabled: - return self.__refresh(fromMaster=fromMaster) + """ + Force refresh + WARNING: If refresher is disabled, force a refresh will do nothing + """ + if self._refreshEnabled: + return self._refresh(fromMaster=fromMaster) return S_OK() - def autoRefreshAndPublish(self, sURL): - gLogger.debug("Setting configuration refresh as automatic") - if not gConfigurationData.getAutoPublish(): - gLogger.debug("Slave server won't auto publish itself") - if not gConfigurationData.getName(): - import DIRAC - DIRAC.abort(10, "Missing configuration name!") - self.__url = sURL - self.__automaticUpdate = True - self.setDaemon(1) - self.start() - - def run(self): - while self.__automaticUpdate: - iWaitTime = gConfigurationData.getPropagationTime() - time.sleep(iWaitTime) - if self.__refreshEnabled: - if not self.__refreshAndPublish(): - gLogger.error("Can't refresh configuration from any source") - - def __refreshAndPublish(self): - self.__lastUpdateTime = time.time() + def _refreshAndPublish(self): + """ + Refresh configuration and publish local updates + """ + self._lastUpdateTime = time.time() gLogger.info("Refreshing from master server") - from DIRAC.Core.DISET.RPCClient import RPCClient sMasterServer = gConfigurationData.getMasterServer() if sMasterServer: - oClient = RPCClient(sMasterServer, timeout=self.__timeout, - useCertificates=gConfigurationData.useServerCertificate(), - skipCACheck=gConfigurationData.skipCACheck()) + from DIRAC.ConfigurationSystem.Client.ConfigurationClient import ConfigurationClient + oClient = ConfigurationClient(url=sMasterServer, timeout=self._timeout, + useCertificates=gConfigurationData.useServerCertificate(), + skipCACheck=gConfigurationData.skipCACheck()) dRetVal = _updateFromRemoteLocation(oClient) if not dRetVal['OK']: gLogger.error("Can't update from master server", dRetVal['Message']) return False if gConfigurationData.getAutoPublish(): gLogger.info("Publishing to master server...") - dRetVal = oClient.publishSlaveServer(self.__url) + dRetVal = oClient.publishSlaveServer(self._url) if not dRetVal['OK']: gLogger.error("Can't publish to master server", dRetVal['Message']) return True else: gLogger.warn("No master server is specified in the configuration, trying to get data from other slaves") - return self.__refresh()['OK'] + return self._refresh()['OK'] - def __refresh(self, fromMaster=False): - self.__lastUpdateTime = time.time() + def _refresh(self, fromMaster=False): + """ + Refresh configuration + """ + self._lastUpdateTime = time.time() gLogger.debug("Refreshing configuration...") gatewayList = getGatewayURLs("Configuration/Server") updatingErrorsList = [] @@ -161,10 +149,10 @@ def __refresh(self, fromMaster=False): gLogger.debug("Randomized server list is %s" % ", ".join(randomServerList)) for sServer in randomServerList: - from DIRAC.Core.DISET.RPCClient import RPCClient - oClient = RPCClient(sServer, - useCertificates=gConfigurationData.useServerCertificate(), - skipCACheck=gConfigurationData.skipCACheck()) + from DIRAC.ConfigurationSystem.Client.ConfigurationClient import ConfigurationClient + oClient = ConfigurationClient(url=sServer, + useCertificates=gConfigurationData.useServerCertificate(), + skipCACheck=gConfigurationData.skipCACheck()) dRetVal = _updateFromRemoteLocation(oClient) if dRetVal['OK']: return dRetVal @@ -175,12 +163,184 @@ def __refresh(self, fromMaster=False): break return S_ERROR("Reason(s):\n\t%s" % "\n\t".join(List.uniqueElements(updatingErrorsList))) + +class Refresher(RefresherBase, threading.Thread): + """ + The refresher + A long time ago, in a code away, far away... + A guy do the code to autorefresh the configuration + To prepare transition to HTTPS we have done separation + between the logic and the implementation of background + tasks, it's the original version, for diset, using thread. + + """ + + def __init__(self): + threading.Thread.__init__(self) + RefresherBase.__init__(self) + self._triggeredRefreshLock = LockRing.LockRing().getLock() + + def _refreshInThread(self): + """ + Refreshing configuration in the background. By default it uses a thread but it can be + run also in the IOLoop + """ + retVal = self._refresh() + if not retVal['OK']: + gLogger.error("Error while updating the configuration", retVal['Message']) + + def refreshConfigurationIfNeeded(self): + """ + Refresh the configuration if automatic updates are disabled, refresher is enabled and servers are defined + """ + if not self._refreshEnabled or self._automaticUpdate or not gConfigurationData.getServers(): + return + self._triggeredRefreshLock.acquire() + try: + if not self._lastRefreshExpired(): + return + self._lastUpdateTime = time.time() + finally: + try: + self._triggeredRefreshLock.release() + except thread.error: + pass + # Launch the refreshf + thd = threading.Thread(target=self._refreshInThread) + thd.setDaemon(1) + thd.start() + + def autoRefreshAndPublish(self, sURL): + """ + Start the autorefresh background task + + :param str sURL: URL of the configuration server + """ + gLogger.debug("Setting configuration refresh as automatic") + if not gConfigurationData.getAutoPublish(): + gLogger.debug("Slave server won't auto publish itself") + if not gConfigurationData.getName(): + import DIRAC + DIRAC.abort(10, "Missing configuration name!") + self._url = sURL + self._automaticUpdate = True + self.setDaemon(1) + self.start() + + def run(self): + while self._automaticUpdate: + time.sleep(gConfigurationData.getPropagationTime()) + if self._refreshEnabled: + if not self._refreshAndPublish(): + gLogger.error("Can't refresh configuration from any source") + def daemonize(self): + """ + Daemonize the background tasks + """ + self.setDaemon(1) self.start() -gRefresher = Refresher() +class TornadoRefresher(RefresherBase): + """ + The refresher, modified for Tornado + It's the same refresher, the only thing which change is + that we are using the IOLoop instead of threads for background + tasks, so it work with Tornado (HTTPS server). + """ + + from tornado import gen + # We change the import name otherwise sphinx tries + # to compile the tornado doc and fails + from tornado.ioloop import IOLoop as _IOLoop + + def refreshConfigurationIfNeeded(self): + """ + Trigger an automatic refresh, most of the time nothing happens because automaticUpdate is enabled. + This function is called by gConfig.getValue most of the time. + + We disable pylint error because this class must be instanciated by a mixin to define the missing methods + """ + if not self._refreshEnabled or self._automaticUpdate: # pylint: disable=no-member + return + if not gConfigurationData.getServers() or not self._lastRefreshExpired(): # pylint: disable=no-member + return + self._lastUpdateTime = time.time() + self._IOLoop.current().run_in_executor(None, self._refresh) # pylint: disable=no-member + + def autoRefreshAndPublish(self, sURL): + """ + Start the autorefresh background task, called by ServiceInterface + (the class behind the Configuration/Server handler) + + :param str sURL: URL of the configuration server + """ + gLogger.debug("Setting configuration refresh as automatic") + if not gConfigurationData.getAutoPublish(): + gLogger.debug("Slave server won't auto publish itself") + if not gConfigurationData.getName(): + import DIRAC + DIRAC.abort(10, "Missing configuration name!") + self._url = sURL + self._automaticUpdate = True + + # Tornado replacement solution to the classic thread + # It start the method self.__refreshLoop on the next IOLoop iteration + self._IOLoop.current().spawn_callback(self.__refreshLoop) + + @gen.coroutine + def __refreshLoop(self): + """ + Trigger the autorefresh when configuration is expired + + This task must use Tornado utilities to avoid blocking the ioloop and + potentialy deadlock the server. + + See http://www.tornadoweb.org/en/stable/guide/coroutines.html#looping + for official documentation about this type of method. + """ + while self._automaticUpdate: + + # This is the sleep from Tornado, like a sleep it wait some time + # But this version is non-blocking, so IOLoop can continue execution + yield self.gen.sleep(gConfigurationData.getPropagationTime()) + # Publish step is blocking so we have to run it in executor + # If we are not doing it, when master try to ping we block the IOLoop + yield self._IOLoop.current().run_in_executor(None, self.__AutoRefresh) + + @gen.coroutine + def __AutoRefresh(self): + """ + Auto refresh the configuration + We disable pylint error because this class must be instanciated + by a mixin to define the methods. + """ + if self._refreshEnabled: # pylint: disable=no-member + if not self._refreshAndPublish(): # pylint: disable=no-member + gLogger.error("Can't refresh configuration from any source") + + def daemonize(self): + """ daemonize is probably not the best name because there is no daemon behind + but we must keep it to the same interface of the DISET refresher """ + self._IOLoop.current().spawn_callback(self.__refreshLoop) + + +# Here we define the refresher which should be used. +# By default we use the original refresher. + +# Be careful, if you never start the IOLoop (with a TornadoServer for example) +# the TornadoRefresher will not work. IOLoop can be started after refresher +# but background tasks will be delayed until IOLoop start. + + +# DIRAC_USE_TORNADO_IOLOOP is defined by starting scripts +if os.environ.get('DIRAC_USE_TORNADO_IOLOOP', 'false').lower() in ('yes', 'true'): + gRefresher = TornadoRefresher() +else: + gRefresher = Refresher() + if __name__ == "__main__": time.sleep(0.1) diff --git a/ConfigurationSystem/private/ServiceInterface.py b/ConfigurationSystem/private/ServiceInterface.py index 927a701593a..54b41caa629 100755 --- a/ConfigurationSystem/private/ServiceInterface.py +++ b/ConfigurationSystem/private/ServiceInterface.py @@ -1,155 +1,45 @@ -""" Threaded implementation of services +""" Threaded implementation of service interface """ from __future__ import absolute_import from __future__ import division from __future__ import print_function -__RCSID__ = "$Id$" - -import os import time -import re import threading -import zipfile -import zlib -import requests +from DIRAC import gLogger, S_OK -import DIRAC -from DIRAC.Core.Utilities.File import mkDir -from DIRAC.ConfigurationSystem.Client.ConfigurationData import gConfigurationData, ConfigurationData -from DIRAC.ConfigurationSystem.private.Refresher import gRefresher -from DIRAC.FrameworkSystem.Client.Logger import gLogger -from DIRAC.Core.Utilities.ReturnValues import S_OK, S_ERROR -from DIRAC.Core.DISET.RPCClient import RPCClient +from DIRAC.ConfigurationSystem.private.ServiceInterfaceBase import ServiceInterfaceBase +from DIRAC.ConfigurationSystem.Client.ConfigurationData import gConfigurationData from DIRAC.Core.Utilities.ThreadPool import ThreadPool -from DIRAC.Core.Security.Locations import getHostCertificateAndKeyLocation + +__RCSID__ = "$Id$" -class ServiceInterface(threading.Thread): +class ServiceInterface(ServiceInterfaceBase, threading.Thread): + """ + Service interface, manage Slave/Master server for CS + Thread components + """ def __init__(self, sURL): threading.Thread.__init__(self) - self.sURL = sURL - gLogger.info("Initializing Configuration Service", "URL is %s" % sURL) - self.__modificationsIgnoreMask = ['/DIRAC/Configuration/Servers', '/DIRAC/Configuration/Version'] - gConfigurationData.setAsService() - if not gConfigurationData.isMaster(): - gLogger.info("Starting configuration service as slave") - gRefresher.autoRefreshAndPublish(self.sURL) - else: - gLogger.info("Starting configuration service as master") - gRefresher.disable() - self.__loadConfigurationData() - self.dAliveSlaveServers = {} - self.__launchCheckSlaves() + ServiceInterfaceBase.__init__(self, sURL) - self.__updateResultDict = {"Successful": {}, "Failed": {}} - - def isMaster(self): - return gConfigurationData.isMaster() - - def __launchCheckSlaves(self): + def _launchCheckSlaves(self): + """ + Start loop which check if slaves are alive + """ gLogger.info("Starting purge slaves thread") self.setDaemon(1) self.start() - def __loadConfigurationData(self): - mkDir(os.path.join(DIRAC.rootPath, "etc", "csbackup")) - gConfigurationData.loadConfigurationData() - if gConfigurationData.isMaster(): - bBuiltNewConfiguration = False - if not gConfigurationData.getName(): - DIRAC.abort(10, "Missing name for the configuration to be exported!") - gConfigurationData.exportName() - sVersion = gConfigurationData.getVersion() - if sVersion == "0": - gLogger.info("There's no version. Generating a new one") - gConfigurationData.generateNewVersion() - bBuiltNewConfiguration = True - - if self.sURL not in gConfigurationData.getServers(): - gConfigurationData.setServers(self.sURL) - bBuiltNewConfiguration = True - - gConfigurationData.setMasterServer(self.sURL) - - if bBuiltNewConfiguration: - gConfigurationData.writeRemoteConfigurationToDisk() - - def __generateNewVersion(self): - if gConfigurationData.isMaster(): - gConfigurationData.generateNewVersion() - gConfigurationData.writeRemoteConfigurationToDisk() - - def publishSlaveServer(self, sSlaveURL): - if not gConfigurationData.isMaster(): - return S_ERROR("Configuration modification is not allowed in this server") - gLogger.info("Pinging slave %s" % sSlaveURL) - rpcClient = RPCClient(sSlaveURL, timeout=10, useCertificates=True) - retVal = rpcClient.ping() - if not retVal['OK']: - gLogger.info("Slave %s didn't reply" % sSlaveURL) - return - if retVal['Value']['name'] != 'Configuration/Server': - gLogger.info("Slave %s is not a CS serveR" % sSlaveURL) - return - bNewSlave = False - if sSlaveURL not in self.dAliveSlaveServers: - bNewSlave = True - gLogger.info("New slave registered", sSlaveURL) - self.dAliveSlaveServers[sSlaveURL] = time.time() - if bNewSlave: - gConfigurationData.setServers("%s, %s" % (self.sURL, - ", ".join(self.dAliveSlaveServers.keys()))) - self.__generateNewVersion() - - def __checkSlavesStatus(self, forceWriteConfiguration=False): - gLogger.info("Checking status of slave servers") - iGraceTime = gConfigurationData.getSlavesGraceTime() - bModifiedSlaveServers = False - for sSlaveURL in self.dAliveSlaveServers.keys(): - if time.time() - self.dAliveSlaveServers[sSlaveURL] > iGraceTime: - gLogger.info("Found dead slave", sSlaveURL) - del self.dAliveSlaveServers[sSlaveURL] - bModifiedSlaveServers = True - if bModifiedSlaveServers or forceWriteConfiguration: - gConfigurationData.setServers("%s, %s" % (self.sURL, - ", ".join(self.dAliveSlaveServers.keys()))) - self.__generateNewVersion() - - @staticmethod - def __forceServiceUpdate(url, fromMaster): - """ - Force updating configuration on a given service - - :param str url: service URL - :param bool fromMaster: flag to force updating from the master CS - :return: S_OK/S_ERROR - """ - gLogger.info('Updating service configuration on', url) - if url.startswith('dip'): - rpc = RPCClient(url) - result = rpc.refreshConfiguration(fromMaster) - elif url.startswith('http'): - hostCertTuple = getHostCertificateAndKeyLocation() - resultRequest = requests.get(url, - headers={'X-RefreshConfiguration': "True"}, - cert=hostCertTuple, - verify=False) - result = S_OK() - if resultRequest.status_code != 200: - result = S_ERROR("Status code returned %d" % resultRequest.status_code) - result['URL'] = url - return result - - def __processResults(self, id_, result): - if result['OK']: - self.__updateResultDict['Successful'][result['URL']] = True - else: - gLogger.warn("Failed to update configuration on", result['URL'] + ':' + result['Message']) - self.__updateResultDict['Failed'][result['URL']] = result['Message'] + def run(self): + while True: + iWaitTime = gConfigurationData.getSlavesGraceTime() + time.sleep(iWaitTime) + self._checkSlavesStatus() - def __updateServiceConfiguration(self, urlSet, fromMaster=False): + def _updateServiceConfiguration(self, urlSet, fromMaster=False): """ Update configuration in a set of service in parallel @@ -159,210 +49,13 @@ def __updateServiceConfiguration(self, urlSet, fromMaster=False): """ pool = ThreadPool(len(urlSet)) for url in urlSet: - pool.generateJobAndQueueIt(self.__forceServiceUpdate, + pool.generateJobAndQueueIt(self._forceServiceUpdate, args=[url, fromMaster], kwargs={}, oCallback=self.__processResults) pool.processAllResults() - return S_OK(self.__updateResultDict) - - def forceSlavesUpdate(self): - """ - Force updating configuration on all the slave configuration servers - - :return: S_OK/S_ERROR, Value Successful/Failed dict with service URLs - """ - gLogger.info("Updating configuration on slave servers") - iGraceTime = gConfigurationData.getSlavesGraceTime() - self.__updateResultDict = {"Successful": {}, "Failed": {}} - urlSet = set() - for slaveURL in self.dAliveSlaveServers: - if time.time() - self.dAliveSlaveServers[slaveURL] <= iGraceTime: - urlSet.add(slaveURL) - return self.__updateServiceConfiguration(urlSet, fromMaster=True) - - def forceGlobalUpdate(self): - """ - Force updating configuration of all the registered services - - :return: S_OK/S_ERROR, Value Successful/Failed dict with service URLs - """ - gLogger.info("Updating services configuration") - # Get URLs of all the services except for Configuration services - cfg = gConfigurationData.remoteCFG.getAsDict()['Systems'] - urlSet = set() - for system_ in cfg: - for instance in cfg[system_]: - for url in cfg[system_][instance]['URLs']: - urlSet = urlSet.union(set([u.strip() for u in cfg[system_][instance]['URLs'][url].split(',') - if 'Configuration/Server' not in u])) - return self.__updateServiceConfiguration(urlSet) - - def updateConfiguration(self, sBuffer, committer="", updateVersionOption=False): - """ - Update the master configuration with the newly received changes - - :param str sBuffer: newly received configuration data - :param str committer: the user name of the committer - :param bool updateVersionOption: flag to update the current configuration version - :return: S_OK/S_ERROR of the write-to-disk of the new configuration - """ - if not gConfigurationData.isMaster(): - return S_ERROR("Configuration modification is not allowed in this server") - # Load the data in a ConfigurationData object - oRemoteConfData = ConfigurationData(False) - oRemoteConfData.loadRemoteCFGFromCompressedMem(sBuffer) - if updateVersionOption: - oRemoteConfData.setVersion(gConfigurationData.getVersion()) - # Test that remote and new versions are the same - sRemoteVersion = oRemoteConfData.getVersion() - sLocalVersion = gConfigurationData.getVersion() - gLogger.info("Checking versions\nremote: %s\nlocal: %s" % (sRemoteVersion, sLocalVersion)) - if sRemoteVersion != sLocalVersion: - if not gConfigurationData.mergingEnabled(): - return S_ERROR("Local and remote versions differ (%s vs %s). Cannot commit." % (sLocalVersion, sRemoteVersion)) - else: - gLogger.info("AutoMerging new data!") - if updateVersionOption: - return S_ERROR("Cannot AutoMerge! version was overwritten") - result = self.__mergeIndependentUpdates(oRemoteConfData) - if not result['OK']: - gLogger.warn("Could not AutoMerge!", result['Message']) - return S_ERROR("AutoMerge failed: %s" % result['Message']) - requestedRemoteCFG = result['Value'] - gLogger.info("AutoMerge successful!") - oRemoteConfData.setRemoteCFG(requestedRemoteCFG) - # Test that configuration names are the same - sRemoteName = oRemoteConfData.getName() - sLocalName = gConfigurationData.getName() - if sRemoteName != sLocalName: - return S_ERROR("Names differ: Server is %s and remote is %s" % (sLocalName, sRemoteName)) - # Update and generate a new version - gLogger.info("Committing new data...") - gConfigurationData.lock() - gLogger.info("Setting the new CFG") - gConfigurationData.setRemoteCFG(oRemoteConfData.getRemoteCFG()) - gConfigurationData.unlock() - gLogger.info("Generating new version") - gConfigurationData.generateNewVersion() - # self.__checkSlavesStatus( forceWriteConfiguration = True ) - gLogger.info("Writing new version to disk") - retVal = gConfigurationData.writeRemoteConfigurationToDisk("%s@%s" % (committer, gConfigurationData.getVersion())) - gLogger.info("New version", gConfigurationData.getVersion()) - - # Attempt to update the configuration on currently registered slave services - if gConfigurationData.getAutoSlaveSync(): - result = self.forceSlavesUpdate() - if not result['OK']: - gLogger.warn('Failed to update slave servers') - - return retVal - - def getCompressedConfigurationData(self): - return gConfigurationData.getCompressedData() - - def getVersion(self): - return gConfigurationData.getVersion() - - def getCommitHistory(self): - files = self.__getCfgBackups(gConfigurationData.getBackupDir()) - backups = [".".join(fileName.split(".")[1:-1]).split("@") for fileName in files] - return backups - - def run(self): - while True: - iWaitTime = gConfigurationData.getSlavesGraceTime() - time.sleep(iWaitTime) - self.__checkSlavesStatus() - - def getVersionContents(self, date): - backupDir = gConfigurationData.getBackupDir() - files = self.__getCfgBackups(backupDir, date) - for fileName in files: - with zipfile.ZipFile("%s/%s" % (backupDir, fileName), "r") as zFile: - cfgName = zFile.namelist()[0] - retVal = S_OK(zlib.compress(zFile.read(cfgName), 9)) - return retVal - return S_ERROR("Version %s does not exist" % date) - - def __getCfgBackups(self, basePath, date="", subPath=""): - rs = re.compile(r"^%s\..*%s.*\.zip$" % (gConfigurationData.getName(), date)) - fsEntries = os.listdir("%s/%s" % (basePath, subPath)) - fsEntries.sort(reverse=True) - backupsList = [] - for entry in fsEntries: - entryPath = "%s/%s/%s" % (basePath, subPath, entry) - if os.path.isdir(entryPath): - backupsList.extend(self.__getCfgBackups(basePath, date, "%s/%s" % (subPath, entry))) - elif os.path.isfile(entryPath): - if rs.search(entry): - backupsList.append("%s/%s" % (subPath, entry)) - return backupsList - - def __getPreviousCFG(self, oRemoteConfData): - backupsList = self.__getCfgBackups(gConfigurationData.getBackupDir(), date=oRemoteConfData.getVersion()) - if not backupsList: - return S_ERROR("Could not AutoMerge. Could not retrieve original committer's version") - prevRemoteConfData = ConfigurationData() - backFile = backupsList[0] - if backFile[0] == "/": - backFile = os.path.join(gConfigurationData.getBackupDir(), backFile[1:]) - try: - prevRemoteConfData.loadConfigurationData(backFile) - except Exception as e: - return S_ERROR("Could not load original committer's version: %s" % str(e)) - gLogger.info("Loaded client original version %s" % prevRemoteConfData.getVersion()) - return S_OK(prevRemoteConfData.getRemoteCFG()) - - def _checkConflictsInModifications(self, realModList, reqModList, parentSection=""): - realModifiedSections = dict([(modAc[1], modAc[3]) - for modAc in realModList if modAc[0].find('Sec') == len(modAc[0]) - 3]) - reqOptionsModificationList = dict([(modAc[1], modAc[3]) - for modAc in reqModList if modAc[0].find('Opt') == len(modAc[0]) - 3]) - for modAc in reqModList: - action = modAc[0] - objectName = modAc[1] - if action == "addSec": - if objectName in realModifiedSections: - return S_ERROR("Section %s/%s already exists" % (parentSection, objectName)) - elif action == "delSec": - if objectName in realModifiedSections: - return S_ERROR("Section %s/%s cannot be deleted. It has been modified." % (parentSection, objectName)) - elif action == "modSec": - if objectName in realModifiedSections: - result = self._checkConflictsInModifications(realModifiedSections[objectName], - modAc[3], "%s/%s" % (parentSection, objectName)) - if not result['OK']: - return result - for modAc in realModList: - action = modAc[0] - objectName = modAc[1] - if action.find("Opt") == len(action) - 3: - return S_ERROR( - "Section %s cannot be merged. Option %s/%s has been modified" % - (parentSection, parentSection, objectName)) return S_OK() - def __mergeIndependentUpdates(self, oRemoteConfData): - # return S_ERROR( "AutoMerge is still not finished. - # Meanwhile... why don't you get the newest conf and update from there?" ) - # Get all the CFGs - curSrvCFG = gConfigurationData.getRemoteCFG().clone() - curCliCFG = oRemoteConfData.getRemoteCFG().clone() - result = self.__getPreviousCFG(oRemoteConfData) + def __processResults(self, _id, result): if not result['OK']: - return result - prevCliCFG = result['Value'] - # Try to merge curCli with curSrv. To do so we check the updates from - # prevCli -> curSrv VS prevCli -> curCli - prevCliToCurCliModList = prevCliCFG.getModifications(curCliCFG) - prevCliToCurSrvModList = prevCliCFG.getModifications(curSrvCFG) - result = self._checkConflictsInModifications(prevCliToCurSrvModList, - prevCliToCurCliModList) - if not result['OK']: - return S_ERROR("Cannot AutoMerge: %s" % result['Message']) - # Merge! - result = curSrvCFG.applyModifications(prevCliToCurCliModList) - if not result['OK']: - return result - return S_OK(curSrvCFG) + gLogger.warn("Failed to update configuration on", result['URL'] + ':' + result['Message']) diff --git a/ConfigurationSystem/private/ServiceInterfaceBase.py b/ConfigurationSystem/private/ServiceInterfaceBase.py new file mode 100644 index 00000000000..b430c48a199 --- /dev/null +++ b/ConfigurationSystem/private/ServiceInterfaceBase.py @@ -0,0 +1,350 @@ +"""Service interface is the service which provide config for client and synchronize Master/Slave servers""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +__RCSID__ = "$Id$" + +import os +import time +import re +import zipfile +import zlib + +import DIRAC +from DIRAC.ConfigurationSystem.Client.ConfigurationClient import ConfigurationClient +from DIRAC.ConfigurationSystem.Client.ConfigurationData import gConfigurationData, ConfigurationData +from DIRAC.ConfigurationSystem.private.Refresher import gRefresher +from DIRAC.Core.Base.Client import Client +from DIRAC.Core.Utilities.File import mkDir +from DIRAC.Core.Utilities.ReturnValues import S_OK, S_ERROR +from DIRAC.FrameworkSystem.Client.Logger import gLogger + + +class ServiceInterfaceBase(object): + """Service interface is the service which provide config for client and synchronize Master/Slave servers""" + + def __init__(self, sURL): + self.sURL = sURL + gLogger.info("Initializing Configuration Service", "URL is %s" % sURL) + self.__modificationsIgnoreMask = ['/DIRAC/Configuration/Servers', '/DIRAC/Configuration/Version'] + gConfigurationData.setAsService() + if not gConfigurationData.isMaster(): + gLogger.info("Starting configuration service as slave") + gRefresher.autoRefreshAndPublish(self.sURL) + else: + gLogger.info("Starting configuration service as master") + gRefresher.disable() + self.__loadConfigurationData() + self.dAliveSlaveServers = {} + self._launchCheckSlaves() + + def isMaster(self): + return gConfigurationData.isMaster() + + def _launchCheckSlaves(self): + raise NotImplementedError("Should be implemented by the children class") + + def __loadConfigurationData(self): + """ + As the name said, it just load the configuration + """ + + mkDir(os.path.join(DIRAC.rootPath, "etc", "csbackup")) + gConfigurationData.loadConfigurationData() + if gConfigurationData.isMaster(): + bBuiltNewConfiguration = False + if not gConfigurationData.getName(): + DIRAC.abort(10, "Missing name for the configuration to be exported!") + gConfigurationData.exportName() + sVersion = gConfigurationData.getVersion() + if sVersion == "0": + gLogger.info("There's no version. Generating a new one") + gConfigurationData.generateNewVersion() + bBuiltNewConfiguration = True + + if self.sURL not in gConfigurationData.getServers(): + gConfigurationData.setServers(self.sURL) + bBuiltNewConfiguration = True + + gConfigurationData.setMasterServer(self.sURL) + + if bBuiltNewConfiguration: + gConfigurationData.writeRemoteConfigurationToDisk() + + def __generateNewVersion(self): + """ + After changing configuration, we use this method to save them + """ + if gConfigurationData.isMaster(): + gConfigurationData.generateNewVersion() + gConfigurationData.writeRemoteConfigurationToDisk() + + def publishSlaveServer(self, sSlaveURL): + """ + Called by the slave server via service, it register a new slave server + + :param sSlaveURL: url of slave server + """ + + if not gConfigurationData.isMaster(): + return S_ERROR("Configuration modification is not allowed in this server") + gLogger.info("Pinging slave %s" % sSlaveURL) + rpcClient = ConfigurationClient(url=sSlaveURL, timeout=10, useCertificates=True) + retVal = rpcClient.ping() + if not retVal['OK']: + gLogger.info("Slave %s didn't reply" % sSlaveURL) + return + if retVal['Value']['name'] != 'Configuration/Server': + gLogger.info("Slave %s is not a CS serveR" % sSlaveURL) + return + bNewSlave = False + if sSlaveURL not in self.dAliveSlaveServers: + bNewSlave = True + gLogger.info("New slave registered", sSlaveURL) + self.dAliveSlaveServers[sSlaveURL] = time.time() + if bNewSlave: + gConfigurationData.setServers("%s, %s" % (self.sURL, + ", ".join(self.dAliveSlaveServers.keys()))) + self.__generateNewVersion() + + def _checkSlavesStatus(self, forceWriteConfiguration=False): + """ + Check if Slaves server are still availlable + + :param forceWriteConfiguration: (default False) Force rewriting configuration after checking slaves + """ + + gLogger.info("Checking status of slave servers") + iGraceTime = gConfigurationData.getSlavesGraceTime() + bModifiedSlaveServers = False + for sSlaveURL in list(self.dAliveSlaveServers): + if time.time() - self.dAliveSlaveServers[sSlaveURL] > iGraceTime: + gLogger.warn("Found dead slave", sSlaveURL) + del self.dAliveSlaveServers[sSlaveURL] + bModifiedSlaveServers = True + if bModifiedSlaveServers or forceWriteConfiguration: + gConfigurationData.setServers("%s, %s" % (self.sURL, + ", ".join(list(self.dAliveSlaveServers)))) + self.__generateNewVersion() + + @staticmethod + def _forceServiceUpdate(url, fromMaster): + """ + Force updating configuration on a given service + This should be called by _updateServiceConfiguration + + :param str url: service URL + :param bool fromMaster: flag to force updating from the master CS + :return: S_OK/S_ERROR + """ + gLogger.info('Updating service configuration on', url) + + result = Client(url=url).refreshConfiguration(fromMaster) + result['URL'] = url + return result + + def _updateServiceConfiguration(self, urlSet, fromMaster=False): + """ + Update configuration in a set of service in parallel + + :param set urlSet: a set of service URLs + :param fromMaster: flag to force updating from the master CS + :return: S_OK/S_ERROR, Value Successful/Failed dict with service URLs + """ + raise NotImplementedError("Should be implemented by the children class") + + def forceSlavesUpdate(self): + """ + Force updating configuration on all the slave configuration servers + + :return: S_OK/S_ERROR, Value Successful/Failed dict with service URLs + """ + gLogger.info("Updating configuration on slave servers") + iGraceTime = gConfigurationData.getSlavesGraceTime() + urlSet = set() + for slaveURL in self.dAliveSlaveServers: + if time.time() - self.dAliveSlaveServers[slaveURL] <= iGraceTime: + urlSet.add(slaveURL) + return self._updateServiceConfiguration(urlSet, fromMaster=True) + + def forceGlobalUpdate(self): + """ + Force updating configuration of all the registered services + + :return: S_OK/S_ERROR, Value Successful/Failed dict with service URLs + """ + gLogger.info("Updating services configuration") + # Get URLs of all the services except for Configuration services + cfg = gConfigurationData.remoteCFG.getAsDict()['Systems'] + urlSet = set() + for system_ in cfg: + for instance in cfg[system_]: + for url in cfg[system_][instance]['URLs']: + urlSet = urlSet.union(set([u.strip() for u in cfg[system_][instance]['URLs'][url].split(',') + if 'Configuration/Server' not in u])) + return self._updateServiceConfiguration(urlSet) + + def updateConfiguration(self, sBuffer, committer="", updateVersionOption=False): + """ + Update the master configuration with the newly received changes + + :param str sBuffer: newly received configuration data + :param str committer: the user name of the committer + :param bool updateVersionOption: flag to update the current configuration version + :return: S_OK/S_ERROR of the write-to-disk of the new configuration + """ + if not gConfigurationData.isMaster(): + return S_ERROR("Configuration modification is not allowed in this server") + # Load the data in a ConfigurationData object + oRemoteConfData = ConfigurationData(False) + oRemoteConfData.loadRemoteCFGFromCompressedMem(sBuffer) + if updateVersionOption: + oRemoteConfData.setVersion(gConfigurationData.getVersion()) + # Test that remote and new versions are the same + sRemoteVersion = oRemoteConfData.getVersion() + sLocalVersion = gConfigurationData.getVersion() + gLogger.info("Checking versions\n", "remote: %s\nlocal: %s" % (sRemoteVersion, sLocalVersion)) + if sRemoteVersion != sLocalVersion: + if not gConfigurationData.mergingEnabled(): + return S_ERROR("Local and remote versions differ (%s vs %s). Cannot commit." % (sLocalVersion, sRemoteVersion)) + else: + gLogger.info("AutoMerging new data!") + if updateVersionOption: + return S_ERROR("Cannot AutoMerge! version was overwritten") + result = self.__mergeIndependentUpdates(oRemoteConfData) + if not result['OK']: + gLogger.warn("Could not AutoMerge!", result['Message']) + return S_ERROR("AutoMerge failed: %s" % result['Message']) + requestedRemoteCFG = result['Value'] + gLogger.info("AutoMerge successful!") + oRemoteConfData.setRemoteCFG(requestedRemoteCFG) + # Test that configuration names are the same + sRemoteName = oRemoteConfData.getName() + sLocalName = gConfigurationData.getName() + if sRemoteName != sLocalName: + return S_ERROR("Names differ: Server is %s and remote is %s" % (sLocalName, sRemoteName)) + # Update and generate a new version + gLogger.info("Committing new data...") + gConfigurationData.lock() + gLogger.info("Setting the new CFG") + gConfigurationData.setRemoteCFG(oRemoteConfData.getRemoteCFG()) + gConfigurationData.unlock() + gLogger.info("Generating new version") + gConfigurationData.generateNewVersion() + # self.__checkSlavesStatus( forceWriteConfiguration = True ) + gLogger.info("Writing new version to disk") + retVal = gConfigurationData.writeRemoteConfigurationToDisk("%s@%s" % (committer, gConfigurationData.getVersion())) + gLogger.info("New version", gConfigurationData.getVersion()) + + # Attempt to update the configuration on currently registered slave services + if gConfigurationData.getAutoSlaveSync(): + result = self.forceSlavesUpdate() + if not result['OK']: + gLogger.warn('Failed to update slave servers') + + return retVal + + def getCompressedConfigurationData(self): + return gConfigurationData.getCompressedData() + + def getVersion(self): + return gConfigurationData.getVersion() + + def getCommitHistory(self): + files = self.__getCfgBackups(gConfigurationData.getBackupDir()) + backups = [".".join(fileName.split(".")[1:-1]).split("@") for fileName in files] + return backups + + def getVersionContents(self, date): + backupDir = gConfigurationData.getBackupDir() + files = self.__getCfgBackups(backupDir, date) + for fileName in files: + with zipfile.ZipFile("%s/%s" % (backupDir, fileName), "r") as zFile: + cfgName = zFile.namelist()[0] + retVal = S_OK(zlib.compress(zFile.read(cfgName), 9)) + return retVal + return S_ERROR("Version %s does not exist" % date) + + def __getCfgBackups(self, basePath, date="", subPath=""): + rs = re.compile(r"^%s\..*%s.*\.zip$" % (gConfigurationData.getName(), date)) + fsEntries = os.listdir("%s/%s" % (basePath, subPath)) + fsEntries.sort(reverse=True) + backupsList = [] + for entry in fsEntries: + entryPath = "%s/%s/%s" % (basePath, subPath, entry) + if os.path.isdir(entryPath): + backupsList.extend(self.__getCfgBackups(basePath, date, "%s/%s" % (subPath, entry))) + elif os.path.isfile(entryPath): + if rs.search(entry): + backupsList.append("%s/%s" % (subPath, entry)) + return backupsList + + def __getPreviousCFG(self, oRemoteConfData): + backupsList = self.__getCfgBackups(gConfigurationData.getBackupDir(), date=oRemoteConfData.getVersion()) + if not backupsList: + return S_ERROR("Could not AutoMerge. Could not retrieve original committer's version") + prevRemoteConfData = ConfigurationData() + backFile = backupsList[0] + if backFile[0] == "/": + backFile = os.path.join(gConfigurationData.getBackupDir(), backFile[1:]) + try: + prevRemoteConfData.loadConfigurationData(backFile) + except Exception as e: + return S_ERROR("Could not load original committer's version: %s" % str(e)) + gLogger.info("Loaded client original version %s" % prevRemoteConfData.getVersion()) + return S_OK(prevRemoteConfData.getRemoteCFG()) + + def _checkConflictsInModifications(self, realModList, reqModList, parentSection=""): + realModifiedSections = dict([(modAc[1], modAc[3]) + for modAc in realModList if modAc[0].find('Sec') == len(modAc[0]) - 3]) + reqOptionsModificationList = dict([(modAc[1], modAc[3]) + for modAc in reqModList if modAc[0].find('Opt') == len(modAc[0]) - 3]) + for modAc in reqModList: + action = modAc[0] + objectName = modAc[1] + if action == "addSec": + if objectName in realModifiedSections: + return S_ERROR("Section %s/%s already exists" % (parentSection, objectName)) + elif action == "delSec": + if objectName in realModifiedSections: + return S_ERROR("Section %s/%s cannot be deleted. It has been modified." % (parentSection, objectName)) + elif action == "modSec": + if objectName in realModifiedSections: + result = self._checkConflictsInModifications(realModifiedSections[objectName], + modAc[3], "%s/%s" % (parentSection, objectName)) + if not result['OK']: + return result + for modAc in realModList: + action = modAc[0] + objectName = modAc[1] + if action.find("Opt") == len(action) - 3: + return S_ERROR( + "Section %s cannot be merged. Option %s/%s has been modified" % + (parentSection, parentSection, objectName)) + return S_OK() + + def __mergeIndependentUpdates(self, oRemoteConfData): + # return S_ERROR( "AutoMerge is still not finished. + # Meanwhile... why don't you get the newest conf and update from there?" ) + # Get all the CFGs + curSrvCFG = gConfigurationData.getRemoteCFG().clone() + curCliCFG = oRemoteConfData.getRemoteCFG().clone() + result = self.__getPreviousCFG(oRemoteConfData) + if not result['OK']: + return result + prevCliCFG = result['Value'] + # Try to merge curCli with curSrv. To do so we check the updates from + # prevCli -> curSrv VS prevCli -> curCli + prevCliToCurCliModList = prevCliCFG.getModifications(curCliCFG) + prevCliToCurSrvModList = prevCliCFG.getModifications(curSrvCFG) + result = self._checkConflictsInModifications(prevCliToCurSrvModList, + prevCliToCurCliModList) + if not result['OK']: + return S_ERROR("Cannot AutoMerge: %s" % result['Message']) + # Merge! + result = curSrvCFG.applyModifications(prevCliToCurCliModList) + if not result['OK']: + return result + return S_OK(curSrvCFG) diff --git a/ConfigurationSystem/private/ServiceInterfaceTornado.py b/ConfigurationSystem/private/ServiceInterfaceTornado.py new file mode 100644 index 00000000000..3789147f94b --- /dev/null +++ b/ConfigurationSystem/private/ServiceInterfaceTornado.py @@ -0,0 +1,51 @@ +""" + Service interface adapted to work with tornado, must be used only by tornado service handlers +""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +__RCSID__ = "$Id$" + +from tornado import gen +from tornado.ioloop import IOLoop +from DIRAC.ConfigurationSystem.private.ServiceInterfaceBase import ServiceInterfaceBase +from DIRAC.ConfigurationSystem.Client.ConfigurationData import gConfigurationData +from DIRAC import gLogger + + +class ServiceInterfaceTornado(ServiceInterfaceBase): + """ + Service interface adapted to work with tornado + """ + + def __init__(self, sURL): + ServiceInterfaceBase.__init__(self, sURL) + + def _launchCheckSlaves(self): + """ + Start loop to check if slaves are alive + """ + IOLoop.current().spawn_callback(self.run) + gLogger.info("Starting purge slaves thread") + + def run(self): + """ + Check if slaves are alive + """ + while True: + yield gen.sleep(gConfigurationData.getSlavesGraceTime()) + self._checkSlavesStatus() + + def _updateServiceConfiguration(self, urlSet, fromMaster=False): + """ + Update configuration in a set of service in parallel + + :param set urlSet: a set of service URLs + :param fromMaster: flag to force updating from the master CS + :return: S_OK/S_ERROR, Value Successful/Failed dict with service URLs + """ + + for url in urlSet: + IOLoop.current().spawn_callback(self._forceServiceUpdate, [url, fromMaster]) diff --git a/Core/Base/Client.py b/Core/Base/Client.py index 92e3ef8fd97..c41e9e4b8f3 100644 --- a/Core/Base/Client.py +++ b/Core/Base/Client.py @@ -11,7 +11,10 @@ import ast import os -from DIRAC.Core.DISET.RPCClient import RPCClient +from io import open + +from DIRAC.Core.Tornado.Client.ClientSelector import RPCClientSelector +from DIRAC.Core.Tornado.Client.TornadoClient import TornadoClient from DIRAC.Core.Utilities.Decorators import deprecated from DIRAC.Core.DISET import DEFAULT_RPC_TIMEOUT @@ -25,13 +28,16 @@ class Client(object): - The self.serverURL member should be set by the inheriting class """ + # Default https (RPC)Client + httpsClient = TornadoClient + def __init__(self, **kwargs): """ C'tor. :param kwargs: just stored as an attribute and passed when creating the RPCClient """ - self.serverURL = None + self.serverURL = kwargs.pop('url', None) self.call = None # I suppose it is initialized here to make pylint happy self.__kwargs = kwargs self.timeout = DEFAULT_RPC_TIMEOUT @@ -103,7 +109,7 @@ def _getRPC(self, rpc=None, url='', timeout=None): timeout = self.timeout self.__kwargs['timeout'] = timeout - rpc = RPCClient(url, **self.__kwargs) + rpc = RPCClientSelector(url, httpsClient=self.httpsClient, **self.__kwargs) return rpc @@ -163,7 +169,7 @@ def addFunctions(clientCls): fullHandlerClassPath = '%s.%s' % (location, handlerClassPath) if not os.path.exists(fullPath): continue - with open(fullPath) as moduleFile: + with open(fullPath, 'rt') as moduleFile: # parse the handler module into abstract syntax tree handlerAst = ast.parse(moduleFile.read(), fullPath) diff --git a/Core/DISET/RequestHandler.py b/Core/DISET/RequestHandler.py index 0d423b92ac1..ea1df2c109c 100755 --- a/Core/DISET/RequestHandler.py +++ b/Core/DISET/RequestHandler.py @@ -7,6 +7,7 @@ __RCSID__ = "$Id$" import os +import six import time import psutil @@ -344,7 +345,7 @@ def __checkExpectedArgumentTypes(self, method, args): # #### - __connectionCallbackTypes = {'new': [basestring, dict], + __connectionCallbackTypes = {'new': [six.string_types, dict], 'connected': [], 'drop': []} @@ -499,7 +500,19 @@ def export_ping(self): return S_OK(dInfo) - types_echo = [basestring] + types_whoami = [] + auth_whoami = ['all'] + + def export_whoami(self): + """ + A simple whoami, returns all credential dictionary, except certificate chain object. + """ + credDict = self.srv_getRemoteCredentials() + if 'x509Chain' in credDict: + del credDict['x509Chain'] + return S_OK(credDict) + + types_echo = [six.string_types] @staticmethod def export_echo(data): diff --git a/Core/DISET/private/BaseClient.py b/Core/DISET/private/BaseClient.py index d5ac0a2eebc..24187779a90 100755 --- a/Core/DISET/private/BaseClient.py +++ b/Core/DISET/private/BaseClient.py @@ -9,7 +9,9 @@ import six import time -import thread + +from six.moves import _thread as thread + import DIRAC from DIRAC.Core.DISET.private.Protocols import gProtocolDict from DIRAC.FrameworkSystem.Client.Logger import gLogger @@ -250,7 +252,7 @@ def __discoverExtraCredentials(self): :return: S_OK()/S_ERROR() """ - # Wich extra credentials to use? + # which extra credentials to use? self.__extraCredentials = self.VAL_EXTRA_CREDENTIALS_HOST if self.__useCertificates else "" if self.KW_EXTRA_CREDENTIALS in self.kwargs: self.__extraCredentials = self.kwargs[self.KW_EXTRA_CREDENTIALS] @@ -645,8 +647,12 @@ def _getBaseStub(self): del newKwargs['useCertificates'] return [self._destinationSrv, newKwargs] - def __nonzero__(self): + def __bool__(self): return True + # python 2 and 3 compatibility + # https://portingguide.readthedocs.io/en/latest/core-obj-misc.html#customizing-truthiness-bool + __nonzero__ = __bool__ + def __str__(self): return "" % (self.serviceURL, self.__extraCredentials) diff --git a/Core/DISET/private/Service.py b/Core/DISET/private/Service.py index d10a751c91b..1e540535681 100644 --- a/Core/DISET/private/Service.py +++ b/Core/DISET/private/Service.py @@ -350,7 +350,7 @@ def handleConnection(self, clientTransport): The method stacks openened connection in a queue, another thread read this queue and handle connection. - :param clientTransport: Object wich describe opened connection (PlainTransport or SSLTransport) + :param clientTransport: Object which describes opened connection (PlainTransport or SSLTransport) """ self._stats['connections'] += 1 self._monitor.setComponentExtraParam('queries', self._stats['connections']) @@ -381,7 +381,7 @@ def _processInThread(self, clientTransport): - Receive arguments/file/something else (depending on action) in the RequestHandler - Executing the action asked by the client - :param clientTransport: Object who describe the opened connection (SSLTransport or PlainTransport) + :param clientTransport: Object which describe the opened connection (SSLTransport or PlainTransport) :return: S_OK with "closeTransport" a boolean to indicate if th connection have to be closed e.g. after RPC, closeTransport=True diff --git a/Core/Security/m2crypto/X509Certificate.py b/Core/Security/m2crypto/X509Certificate.py index 8f8b8addb34..538f9393f75 100644 --- a/Core/Security/m2crypto/X509Certificate.py +++ b/Core/Security/m2crypto/X509Certificate.py @@ -18,6 +18,7 @@ import M2Crypto +from io import open from DIRAC import S_OK, S_ERROR from DIRAC.Core.Utilities import DErrno @@ -171,7 +172,7 @@ def loadFromFile(self, certLocation): """ try: - with open(certLocation, 'r') as fd: + with open(certLocation, 'rb') as fd: pemData = fd.read() return self.loadFromString(pemData) except IOError: diff --git a/Core/Security/m2crypto/X509Chain.py b/Core/Security/m2crypto/X509Chain.py index fa08118dc0e..34e2021edff 100644 --- a/Core/Security/m2crypto/X509Chain.py +++ b/Core/Security/m2crypto/X509Chain.py @@ -23,6 +23,8 @@ import M2Crypto +from io import open + from DIRAC import S_OK, S_ERROR from DIRAC.Core.Utilities import DErrno from DIRAC.Core.Utilities.Decorators import executeOnlyIf, deprecated @@ -216,7 +218,7 @@ def loadChainFromFile(self, chainLocation): :returns: S_OK/S_ERROR """ try: - with open(chainLocation) as fd: + with open(chainLocation, 'rb') as fd: pemData = fd.read() except IOError as e: return S_ERROR(DErrno.EOF, "%s: %s" % (chainLocation, repr(e).replace(',)', ')'))) @@ -274,7 +276,7 @@ def loadKeyFromFile(self, chainLocation, password=False): :returns: S_OK / S_ERROR """ try: - with open(chainLocation) as fd: + with open(chainLocation, 'rb') as fd: pemData = fd.read() except BaseException as e: return S_ERROR(DErrno.EOF, "%s: %s" % (chainLocation, repr(e).replace(',)', ')'))) @@ -314,7 +316,7 @@ def loadProxyFromFile(self, chainLocation): :returns: S_OK / S_ERROR """ try: - with open(chainLocation) as fd: + with open(chainLocation, 'rb') as fd: pemData = fd.read() except BaseException as e: return S_ERROR(DErrno.EOF, "%s: %s" % (chainLocation, repr(e).replace(',)', ')'))) @@ -486,7 +488,7 @@ def generateProxyToFile(self, filePath, lifetime, diracGroup=False, strength=102 if not retVal['OK']: return retVal try: - with open(filePath, 'w') as fd: + with open(filePath, 'wb') as fd: fd.write(retVal['Value']) except BaseException as e: return S_ERROR(DErrno.EWF, "%s :%s" % (filePath, repr(e).replace(',)', ')'))) @@ -964,6 +966,7 @@ def getCredentials(self, ignoreDefault=False, withRegistryInfo=True): * isLimitedProxy: boolean (see :py:meth:`.isLimitedProxy`) * validDN: boolean if the DN is known to DIRAC * validGroup: False (see further definition) + * DN: either the DN of the host, or the DN of the user corresponding to the proxy Only for proxy: @@ -994,8 +997,14 @@ def getCredentials(self, ignoreDefault=False, withRegistryInfo=True): 'isLimitedProxy': self.__isProxy and self.__isLimitedProxy, 'validDN': False, 'validGroup': False} + + # Add the DN entry as the subject. + credDict['DN'] = credDict['subject'] if self.__isProxy: credDict['identity'] = str(self._certList[self.__firstProxyStep + 1].getSubjectDN()['Value']) # ['Value'] :( + # if the chain is a proxy, then the DN we want to work with is the real one of the + # user, not the one of his proxy + credDict['DN'] = credDict['identity'] # Check if we have the PUSP case result = self.isPUSP() diff --git a/Core/Security/test/Test_X509Chain.py b/Core/Security/test/Test_X509Chain.py index 4b29e4f0fb3..e887985064b 100644 --- a/Core/Security/test/Test_X509Chain.py +++ b/Core/Security/test/Test_X509Chain.py @@ -360,7 +360,7 @@ def test_getCredentials_on_cert(cert_file, get_X509Chain_class): x509Chain = get_X509Chain_class() x509Chain.loadChainFromFile(cert_file) - credentialInfo = ['isLimitedProxy', 'isProxy', 'issuer', 'secondsLeft', 'subject', 'validDN', 'validGroup'] + credentialInfo = ['DN', 'isLimitedProxy', 'isProxy', 'issuer', 'secondsLeft', 'subject', 'validDN', 'validGroup'] res = x509Chain.getCredentials(ignoreDefault=True) diff --git a/Core/Tornado/Client/ClientSelector.py b/Core/Tornado/Client/ClientSelector.py new file mode 100644 index 00000000000..edb415fbcc2 --- /dev/null +++ b/Core/Tornado/Client/ClientSelector.py @@ -0,0 +1,95 @@ +""" + This modules defines two functions that can be used in place of the ``RPCClient`` and + ``TransferClient`` to transparently switch to ``https``. + + Example:: + + from DIRAC.Core.Tornado.Client.ClientSelector import RPCClientSelector as RPCClient + myService = RPCClient("Framework/MyService") + myService.doSomething() +""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +__RCSID__ = "$Id$" + +import functools + +from DIRAC import gLogger +from DIRAC.ConfigurationSystem.Client.PathFinder import getServiceURL +from DIRAC.Core.DISET.RPCClient import RPCClient +from DIRAC.Core.DISET.TransferClient import TransferClient +from DIRAC.Core.Tornado.Client.TornadoClient import TornadoClient + + +sLog = gLogger.getSubLogger(__name__) + + +def ClientSelector(disetClient, *args, **kwargs): # We use same interface as RPCClient + """ + Select the correct Client (either RPC or Transfer ), instantiate it, and return it. + + The selection is based on the URL: + + * either it contains the protocol, in which case we make a choice + * or it is in the form , in which case we resolve first + + This is a generic function. You should rather use :py:class:`.RPCClientSelector` + or :py:class:`.TransferClientSelector` + + + In principle, the only place for this class to be used is in + :py:class:`DIRAC.Core.Base.Client.Client`, since it is the only + one supposed to instantiate an :py:class:`DIRAC.Core.Base.DISET.RPCClient.RPCClient` + + :params disetClient: the DISET class to be instantiated, so either + :py:class:`DIRAC.Core.Base.DISET.RPCClient.RPCClient` + or :py:class:`DIRAC.Core.Base.DISET.TransferClient.TransferClient` + :param args: Whatever ``disetClient`` takes as args, but the first one is + always the URL we want to rely on. + It can be either "system/service" or "dips://domain:port/system/service" + :param kwargs: This can contain: + + * Whatever ``disetClient`` takes. + * httpsClient: specific class inheriting from TornadoClient + + """ + + # We detect if we need to use a specific class for the HTTPS client + + tornadoClient = kwargs.pop('httpsClient', TornadoClient) + + # We have to make URL resolution BEFORE the RPCClient or TornadoClient to determine which one we want to use + # URL is defined as first argument (called serviceName) in RPCClient + + try: + serviceName = args[0] + sLog.verbose("Trying to autodetect client for %s" % serviceName) + + # If we are not already given a URL, resolve it + if serviceName.startswith(('http', 'dip')): + completeUrl = serviceName + else: + completeUrl = getServiceURL(serviceName) + sLog.verbose("URL resolved: %s" % completeUrl) + + if completeUrl.startswith("http"): + sLog.info("Using HTTPS for service %s" % serviceName) + rpc = tornadoClient(*args, **kwargs) + else: + rpc = disetClient(*args, **kwargs) + except Exception as e: # pylint: disable=broad-except + # If anything went wrong in the resolution, we return default RPCClient + # So the behaviour is exactly the same as before implementation of Tornado + sLog.warn("Could not select DISET or Tornado client", "%s" % repr(e)) + rpc = disetClient(*args, **kwargs) + return rpc + + +# Client to use for RPC selection +RPCClientSelector = functools.partial(ClientSelector, RPCClient) + +# Client to use for Transfer selection +TransferClientSelector = functools.partial(ClientSelector, TransferClient) diff --git a/Core/Tornado/Client/TornadoClient.py b/Core/Tornado/Client/TornadoClient.py new file mode 100644 index 00000000000..c189bb3ec8f --- /dev/null +++ b/Core/Tornado/Client/TornadoClient.py @@ -0,0 +1,97 @@ +""" + TornadoClient is equivalent of the RPCClient but in HTTPS. + Usage of TornadoClient is the same as RPCClient, you can instantiate TornadoClient with + complete url (https://host:port/System/Component) or just "System/Component". Like RPCClient + you can use all method defined in your service, your call will be automatically transformed + in RPC. + + It also exposes the same interface for receiving file as the TransferClient. + + Main changes: + + - KeepAliveLapse is removed, requests library manages it itself. + - nbOfRetry (defined as private attribute) is removed, requests library manage it itself. + - Underneath it uses HTTP POST protocol and JSON. See :ref:`httpsTornado` for details + + Example:: + + from DIRAC.Core.Tornado.Client.TornadoClient import TornadoClient + myService = TornadoClient("Framework/MyService") + myService.doSomething() #Returns S_OK/S_ERROR + +""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +__RCSID__ = "$Id$" + +# pylint: disable=broad-except + +from DIRAC.Core.Tornado.Client.private.TornadoBaseClient import TornadoBaseClient +from DIRAC.Core.Utilities.JEncode import encode + + +class TornadoClient(TornadoBaseClient): + """ + Client for calling tornado services + Interface is based on RPCClient interface + """ + + def __getattr__(self, attrname): + """ + Return the RPC call procedure + + :param str attrname: Name of the procedure we are trying to call + :return: RPC procedure as function + """ + def call(*args): + """ + Just returns the right function for RPC Call + """ + return self.executeRPC(attrname, *args) + return call + + # Name from RPCClient Interface + def executeRPC(self, method, *args): + """ + Calls a remote service + + :param str method: remote procedure name + :param args: list of arguments + :returns: decoded response from server, server may return S_OK or S_ERROR + """ + rpcCall = {'method': method, 'args': encode(args)} + # Start request + retVal = self._request(**rpcCall) + retVal['rpcStub'] = (self._getBaseStub(), method, list(args)) + return retVal + + def receiveFile(self, destFile, *args): + """ + Equivalent of :py:meth:`~DIRAC.Core.DISET.TransferClient.TransferClient.receiveFile` + + In practice, it calls the remote method `streamToClient` and stores the raw result in a file + + :param str destFile: path where to store the result + :param args: list of arguments + :returns: S_OK/S_ERROR + """ + rpcCall = {'method': 'streamToClient', 'args': encode(args)} + # Start request + retVal = self._request(outputFile=destFile, **rpcCall) + return retVal + + +def executeRPCStub(rpcStub): + """ + Playback a stub + # Copy-paste from DIRAC.Core.DISET.RPCClient with RPCClient changed into TornadoClient + """ + # Generate a client with the same parameters + client = TornadoClient(rpcStub[0][0], **rpcStub[0][1]) + # Get a functor to execute the RPC call + rpcFunc = getattr(client, rpcStub[1]) + # Reproduce the call + return rpcFunc(*rpcStub[2]) diff --git a/Core/Tornado/Client/__init__.py b/Core/Tornado/Client/__init__.py new file mode 100644 index 00000000000..cae5bc3f37c --- /dev/null +++ b/Core/Tornado/Client/__init__.py @@ -0,0 +1,3 @@ +""" +DIRAC.TornadosServices +""" diff --git a/Core/Tornado/Client/private/TornadoBaseClient.py b/Core/Tornado/Client/private/TornadoBaseClient.py new file mode 100644 index 00000000000..b949699a30d --- /dev/null +++ b/Core/Tornado/Client/private/TornadoBaseClient.py @@ -0,0 +1,593 @@ +""" + TornadoBaseClient contains all the low-levels functionalities and initilization methods + It must be instantiated from :py:class:`~DIRAC.Core.Tornado.Client.TornadoClient` + + Requests library manage itself retry when connection failed, so the __nbOfRetry attribute is removed from DIRAC + (For each URL requests manage retries himself, if it still fail, we try next url) + KeepAlive lapse is also removed because managed by request, + see https://requests.readthedocs.io/en/latest/user/advanced/#keep-alive + + If necessary this class can be modified to define number of retry in requests, documentation does not give + lot of informations but you can see this simple solution from StackOverflow. + After some tests request seems to retry 3 times by default. + https://stackoverflow.com/questions/15431044/can-i-set-max-retries-for-requests-request + + .. warning:: + If you use your own certificates, it's like in dips, please take a look at :ref:`using_own_CA` + + .. warning:: + Lots of method are copy-paste from :py:class:`~DIRAC.Core.DISET.private.BaseClient`. + And some methods are copy-paste AND modifications, for now it permit to fully separate DISET and HTTPS. + + +""" + +# pylint: disable=broad-except + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +__RCSID__ = "$Id$" + +from io import open +import errno +import requests +import six +from six.moves import http_client + + +import DIRAC + +from DIRAC import S_OK, S_ERROR, gLogger + +from DIRAC.ConfigurationSystem.Client.Config import gConfig +from DIRAC.ConfigurationSystem.Client.Helpers.CSGlobals import skipCACheck +from DIRAC.ConfigurationSystem.Client.Helpers.Registry import findDefaultGroupForDN +from DIRAC.ConfigurationSystem.Client.PathFinder import getServiceURL, getServiceFailoverURL + +from DIRAC.Core.DISET.ThreadConfig import ThreadConfig +from DIRAC.Core.Security import Locations +from DIRAC.Core.Utilities import List, Network +from DIRAC.Core.Utilities.JEncode import decode, encode + + +# TODO CHRIS: refactor all the messy `discover` methods +# I do not do it now because I want first to decide +# whether we go with code copy of fatorization + + +class TornadoBaseClient(object): + """ + This class contain initialization method and all utilities method used for RPC + """ + __threadConfig = ThreadConfig() + VAL_EXTRA_CREDENTIALS_HOST = "hosts" + + KW_USE_CERTIFICATES = "useCertificates" + KW_EXTRA_CREDENTIALS = "extraCredentials" + KW_TIMEOUT = "timeout" + KW_SETUP = "setup" + KW_VO = "VO" + KW_DELEGATED_DN = "delegatedDN" + KW_DELEGATED_GROUP = "delegatedGroup" + KW_IGNORE_GATEWAYS = "ignoreGateways" + KW_PROXY_LOCATION = "proxyLocation" + KW_PROXY_STRING = "proxyString" + KW_PROXY_CHAIN = "proxyChain" + KW_SKIP_CA_CHECK = "skipCACheck" + KW_KEEP_ALIVE_LAPSE = "keepAliveLapse" + + def __init__(self, serviceName, **kwargs): + """ + :param serviceName: URL of the service (proper uri or just System/Component) + :param useCertificates: If set to True, use the server certificate + :param extraCredentials: + :param timeout: Timeout of the call (default 600 s) + :param setup: Specify the Setup + :param VO: Specify the VO + :param delegatedDN: Not clear what it can be used for. + :param delegatedGroup: Not clear what it can be used for. + :param ignoreGateways: Ignore the DIRAC Gatways settings + :param proxyLocation: Specify the location of the proxy + :param proxyString: Specify the proxy string + :param proxyChain: Specify the proxy chain + :param skipCACheck: Do not check the CA + :param keepAliveLapse: Duration for keepAliveLapse (heartbeat like) (now managed by requests) + """ + + if not isinstance(serviceName, six.string_types): + raise TypeError("Service name expected to be a string. Received %s type %s" % + (str(serviceName), type(serviceName))) + + self._destinationSrv = serviceName + self._serviceName = serviceName + self.__ca_location = False + + self.kwargs = kwargs + self.__useCertificates = None + # The CS useServerCertificate option can be overridden by explicit argument + self.__forceUseCertificates = self.kwargs.get(self.KW_USE_CERTIFICATES) + self.__initStatus = S_OK() + self.__idDict = {} + self.__extraCredentials = "" + # by default we always have 1 url for example: + # RPCClient('dips://volhcb38.cern.ch:9162/Framework/SystemAdministrator') + self.__nbOfUrls = 1 + self.__bannedUrls = [] + + # For pylint... + self.setup = None + self.vo = None + self.serviceURL = None + self.__proxy_location = None + + for initFunc in ( + self.__discoverTimeout, + self.__discoverSetup, + self.__discoverVO, + self.__discoverCredentialsToUse, + self.__discoverExtraCredentials, + self.__discoverURL): + + result = initFunc() + if not result['OK'] and self.__initStatus['OK']: + self.__initStatus = result + + def __discoverSetup(self): + """ Discover which setup to use and stores it in self.setup + The setup is looked for: + * kwargs of the constructor (see KW_SETUP) + * in the CS /DIRAC/Setup + * default to 'Test' + """ + if self.KW_SETUP in self.kwargs and self.kwargs[self.KW_SETUP]: + self.setup = str(self.kwargs[self.KW_SETUP]) + else: + self.setup = self.__threadConfig.getSetup() + if not self.setup: + self.setup = gConfig.getValue("/DIRAC/Setup", "Test") + return S_OK() + + def __discoverURL(self): + """ Calculate the final URL. It is called at initialization and in connect in case of issue + + It sets: + * self.serviceURL: the url (dips) selected as target using __findServiceURL + * self.__URLTuple: a split of serviceURL obtained by Network.splitURL + * self._serviceName: the last part of URLTuple (typically System/Component) + + WARNING: COPY PASTE FROM BaseClient + """ + # Calculate final URL + try: + result = self.__findServiceURL() + except Exception as e: + return S_ERROR(repr(e)) + if not result['OK']: + return result + self.serviceURL = result['Value'] + retVal = Network.splitURL(self.serviceURL) + if not retVal['OK']: + return retVal + self.__URLTuple = retVal['Value'] + self._serviceName = self.__URLTuple[-1] + res = gConfig.getOptionsDict("/DIRAC/ConnConf/%s:%s" % self.__URLTuple[1:3]) + if res['OK']: + opts = res['Value'] + for k in opts: + if k not in self.kwargs: + self.kwargs[k] = opts[k] + return S_OK() + + def __discoverVO(self): + """ Discover which VO to use and stores it in self.vo + The VO is looked for: + * kwargs of the constructor (see KW_VO) + * in the CS /DIRAC/VirtualOrganization + * default to 'unknown' + + WARNING: COPY/PASTE FROM Core/Diset/private/BaseClient FOR NOW + """ + if self.KW_VO in self.kwargs and self.kwargs[self.KW_VO]: + self.vo = str(self.kwargs[self.KW_VO]) + else: + self.vo = gConfig.getValue("/DIRAC/VirtualOrganization", "unknown") + return S_OK() + + def __discoverCredentialsToUse(self): + """ Discovers which credentials to use for connection. + * Server certificate: + -> If KW_USE_CERTIFICATES in kwargs, sets it in self.__useCertificates + -> If not, check gConfig.useServerCertificate(), and sets it in self.__useCertificates + and kwargs[KW_USE_CERTIFICATES] + * Certification Authorities check: + -> if KW_SKIP_CA_CHECK is not in kwargs and we are using the certificates, + set KW_SKIP_CA_CHECK to false in kwargs + -> if KW_SKIP_CA_CHECK is not in kwargs and we are not using the certificate, check the skipCACheck + * Proxy Chain + + WARNING: MOSTLY COPY/PASTE FROM Core/Diset/private/BaseClient + + """ + # Use certificates? + if self.KW_USE_CERTIFICATES in self.kwargs: + self.__useCertificates = self.kwargs[self.KW_USE_CERTIFICATES] + else: + self.__useCertificates = gConfig.useServerCertificate() + self.kwargs[self.KW_USE_CERTIFICATES] = self.__useCertificates + if self.KW_SKIP_CA_CHECK not in self.kwargs: + if self.__useCertificates: + self.kwargs[self.KW_SKIP_CA_CHECK] = False + else: + self.kwargs[self.KW_SKIP_CA_CHECK] = skipCACheck() + + # Rewrite a little bit from here: don't need the proxy string, we use the file + if self.KW_PROXY_CHAIN in self.kwargs: + try: + self.kwargs[self.KW_PROXY_STRING] = self.kwargs[self.KW_PROXY_CHAIN].dumpAllToString()['Value'] + del self.kwargs[self.KW_PROXY_CHAIN] + except BaseException: + return S_ERROR("Invalid proxy chain specified on instantiation") + + # ==== REWRITED FROM HERE ==== + + # Getting proxy + + proxy = Locations.getProxyLocation() + if not proxy: + gLogger.error("No proxy found") + return S_ERROR("No proxy found") + self.__proxy_location = proxy + + # For certs always check CA's. For clients skipServerIdentityCheck + if self.KW_SKIP_CA_CHECK not in self.kwargs or not self.kwargs[self.KW_SKIP_CA_CHECK]: + cafile = Locations.getCAsLocation() + if not cafile: + gLogger.error("No CAs found!") + return S_ERROR("No CAs found!") + else: + self.__ca_location = cafile + + return S_OK() + + def __discoverExtraCredentials(self): + """ Add extra credentials informations. + * self.__extraCredentials + -> if KW_EXTRA_CREDENTIALS in kwargs, we set it + -> Otherwise, if we use the server certificate, we set it to VAL_EXTRA_CREDENTIALS_HOST + -> If we have a delegation (see bellow), we set it to (delegatedDN, delegatedGroup) + -> otherwise it is an empty string + * delegation: + -> if KW_DELEGATED_DN in kwargs, or delegatedDN in threadConfig, put in in self.kwargs + -> If we have a delegated DN but not group, we find the corresponding group in the CS + + WARNING: COPY/PASTE FROM Core/Diset/private/BaseClient + """ + # which extra credentials to use? + if self.__useCertificates: + self.__extraCredentials = self.VAL_EXTRA_CREDENTIALS_HOST + else: + self.__extraCredentials = "" + if self.KW_EXTRA_CREDENTIALS in self.kwargs: + self.__extraCredentials = self.kwargs[self.KW_EXTRA_CREDENTIALS] + # Are we delegating something? + delegatedDN, delegatedGroup = self.__threadConfig.getID() + if self.KW_DELEGATED_DN in self.kwargs and self.kwargs[self.KW_DELEGATED_DN]: + delegatedDN = self.kwargs[self.KW_DELEGATED_DN] + elif delegatedDN: + self.kwargs[self.KW_DELEGATED_DN] = delegatedDN + if self.KW_DELEGATED_GROUP in self.kwargs and self.kwargs[self.KW_DELEGATED_GROUP]: + delegatedGroup = self.kwargs[self.KW_DELEGATED_GROUP] + elif delegatedGroup: + self.kwargs[self.KW_DELEGATED_GROUP] = delegatedGroup + if delegatedDN: + if not delegatedGroup: + result = findDefaultGroupForDN(self.kwargs[self.KW_DELEGATED_DN]) + if not result['OK']: + return result + self.__extraCredentials = (delegatedDN, delegatedGroup) + return S_OK() + + def __discoverTimeout(self): + """ Discover which timeout to use and stores it in self.timeout + The timeout can be specified kwargs of the constructor (see KW_TIMEOUT), + with a minimum of 120 seconds. + If unspecified, the timeout will be 600 seconds. + The value is set in self.timeout, as well as in self.kwargs[KW_TIMEOUT] + + WARNING: COPY/PASTE FROM Core/Diset/private/BaseClient + """ + if self.KW_TIMEOUT in self.kwargs: + self.timeout = self.kwargs[self.KW_TIMEOUT] + else: + self.timeout = False + if self.timeout: + self.timeout = max(120, self.timeout) + else: + self.timeout = 600 + self.kwargs[self.KW_TIMEOUT] = self.timeout + return S_OK() + + def __findServiceURL(self): + """ + Discovers the URL of a service, taking into account gateways, multiple URLs, banned URLs + + + If the site on which we run is configured to use gateways (/DIRAC/Gateways/), + these URLs will be used. To ignore the gateway, it is possible to set KW_IGNORE_GATEWAYS + to False in kwargs. + + If self._destinationSrv (given as constructor attribute) is a properly formed URL, + we just return this one. If we have to use a gateway, we just replace the server name in the url. + + The list of URLs defined in the CS (/URLs/) is randomized + + This method also sets some attributes: + * self.__nbOfUrls = number of URLs + * self.__nbOfRetry removed in HTTPS (Managed by requests) + * self.__bannedUrls is reinitialized if all the URLs are banned + + :return: the selected URL + + WARNING (Mostly) COPY PASTE FROM BaseClient (protocols list is changed to https) + + """ + if not self.__initStatus['OK']: + return self.__initStatus + + # Load the Gateways URLs for the current site Name + gatewayURL = False + if not self.kwargs.get(self.KW_IGNORE_GATEWAYS): + dRetVal = gConfig.getOption("/DIRAC/Gateways/%s" % DIRAC.siteName()) + if dRetVal['OK']: + rawGatewayURL = List.randomize(List.fromChar(dRetVal['Value'], ","))[0] + gatewayURL = "/".join(rawGatewayURL.split("/")[:3]) + + # If what was given as constructor attribute is a properly formed URL, + # we just return this one. + # If we have to use a gateway, we just replace the server name in it + if self._destinationSrv.startswith("https://"): + gLogger.debug("Already given a valid url", self._destinationSrv) + if not gatewayURL: + return S_OK(self._destinationSrv) + gLogger.debug("Reconstructing given URL to pass through gateway") + path = "/".join(self._destinationSrv.split("/")[3:]) + finalURL = "%s/%s" % (gatewayURL, path) + gLogger.debug("Gateway URL conversion:\n %s -> %s" % (self._destinationSrv, finalURL)) + return S_OK(finalURL) + + if gatewayURL: + gLogger.debug("Using gateway", gatewayURL) + return S_OK("%s/%s" % (gatewayURL, self._destinationSrv)) + + # If nor url is given as constructor, we extract the list of URLs from the CS (System/URLs/Component) + try: + urls = getServiceURL(self._destinationSrv, setup=self.setup) + except Exception as e: + return S_ERROR("Cannot get URL for %s in setup %s: %s" % (self._destinationSrv, self.setup, repr(e))) + if not urls: + return S_ERROR("URL for service %s not found" % self._destinationSrv) + + failoverUrls = [] + # Try if there are some failover URLs to use as last resort + try: + failoverUrlsStr = getServiceFailoverURL(self._destinationSrv, setup=self.setup) + if failoverUrlsStr: + failoverUrls = failoverUrlsStr.split(',') + except Exception as e: + pass + + # We randomize the list, and add at the end the failover URLs (System/FailoverURLs/Component) + urlsList = List.randomize(List.fromChar(urls, ",")) + failoverUrls + self.__nbOfUrls = len(urlsList) + # __nbOfRetry removed in HTTPS (managed by requests) + if self.__nbOfUrls == len(self.__bannedUrls): + self.__bannedUrls = [] # retry all urls + gLogger.debug("Retrying again all URLs") + + if self.__bannedUrls and len(urlsList) > 1: + # we have host which is not accessible. We remove that host from the list. + # We only remove if we have more than one instance + for i in self.__bannedUrls: + gLogger.debug("Removing banned URL", "%s" % i) + urlsList.remove(i) + + # Take the first URL from the list + # randUrls = List.randomize( urlsList ) + failoverUrls + + sURL = urlsList[0] + + # If we have banned URLs, and several URLs at disposals, we make sure that the selected sURL + # is not on a host which is banned. If it is, we take the next one in the list using __selectUrl + + if self.__bannedUrls and self.__nbOfUrls > 2: # when we have multiple services then we can + # have a situation when two services are running on the same machine with different ports... + retVal = Network.splitURL(sURL) + nexturl = None + if retVal['OK']: + nexturl = retVal['Value'] + + found = False + for i in self.__bannedUrls: + retVal = Network.splitURL(i) + if retVal['OK']: + bannedurl = retVal['Value'] + else: + break + # We found a banned URL on the same host as the one we are running on + if nexturl[1] == bannedurl[1]: + found = True + break + if found: + nexturl = self.__selectUrl(nexturl, urlsList[1:]) + if nexturl: # an url found which is in different host + sURL = nexturl + gLogger.debug("Discovering URL for service", "%s -> %s" % (self._destinationSrv, sURL)) + return S_OK(sURL) + + def __selectUrl(self, notselect, urls): + """In case when multiple services are running in the same host, a new url has to be in a different host + Note: If we do not have different host we will use the selected url... + + :param notselect: URL that should NOT be selected + :param urls: list of potential URLs + + :return: selected URL + + WARNING: COPY/PASTE FROM Core/Diset/private/BaseClient + """ + url = None + for i in urls: + retVal = Network.splitURL(i) + if retVal['OK']: + if retVal['Value'][1] != notselect[1]: # the hots are different + url = i + break + else: + gLogger.error(retVal['Message']) + return url + + def getServiceName(self): + """ + Returns the name of the service, if you had given a url at init, returns the URL. + """ + return self._serviceName + + def getDestinationService(self): + """ + Returns the url the service. + """ + return getServiceURL(self._serviceName) + + def _getBaseStub(self): + """ Returns a tuple with (self._destinationSrv, newKwargs) + self._destinationSrv is what was given as first parameter of the init serviceName + + newKwargs is an updated copy of kwargs: + * if set, we remove the useCertificates (KW_USE_CERTIFICATES) in newKwargs + + This method is just used to return information in case of error in the InnerRPCClient + + WARNING: COPY/PASTE FROM Core/Diset/private/BaseClient + """ + newKwargs = dict(self.kwargs) + # Remove useCertificates as the forwarder of the call will have to + # independently decide whether to use their cert or not anyway. + if 'useCertificates' in newKwargs: + del newKwargs['useCertificates'] + return (self._destinationSrv, newKwargs) + + def _request(self, retry=0, outputFile=None, **kwargs): + """ + Sends the request to server + + :param retry: internal parameters for recursive call. TODO: remove ? + :param outputFile: (default None) path to a file where to store the received data. + If set, the server response will be streamed for optimization + purposes, and the response data will not go through the + JDecode process + :param **kwargs: Any argument there is used as a post parameter. They are detailed bellow. + :param method: (mandatory) name of the distant method + :param args: (mandatory) json serialized list of argument for the procedure + + + + :returns: The received data. If outputFile is set, return always S_OK + + """ + + # Adding some informations to send + if self.__extraCredentials: + kwargs[self.KW_EXTRA_CREDENTIALS] = encode(self.__extraCredentials) + kwargs["clientVO"] = self.vo + + # Getting URL + url = self.__findServiceURL() + if not url['OK']: + return url + url = url['Value'] + + # Getting CA file (or skip verification) + verify = (not self.kwargs[self.KW_SKIP_CA_CHECK]) + if verify and self.__ca_location: + verify = self.__ca_location + + # getting certificate + if self.kwargs[self.KW_USE_CERTIFICATES]: + cert = Locations.getHostCertificateAndKeyLocation() + else: + cert = self.__proxy_location + + # We have a try/except for all the exceptions + # whose default behavior is to try again, + # maybe to different server + try: + # And we have a second block to handle specific exceptions + # which makes it not worth retrying + try: + rawText = None + + # Default case, just return the result + if not outputFile: + call = requests.post(url, data=kwargs, + timeout=self.timeout, verify=verify, + cert=cert) + # raising the exception for status here + # means essentialy that we are losing here the information of what is returned by the server + # as error message, since it is not passed to the exception + # However, we can store the text and return it raw as an error, + # since there is no guarantee that it is any JEncoded text + # Note that we would get an exception only if there is an exception on the server side which + # is not handled. + # Any standard S_ERROR will be transfered as an S_ERROR with a correct code. + rawText = call.text + call.raise_for_status() + return decode(rawText)[0] + else: + # Instruct the server not to encode the response + kwargs['rawContent'] = True + + rawText = None + # Stream download + # https://requests.readthedocs.io/en/latest/user/advanced/#body-content-workflow + with requests.post(url, data=kwargs, timeout=self.timeout, verify=verify, + cert=cert, stream=True) as r: + rawText = r.text + r.raise_for_status() + + with open(outputFile, 'wb') as f: + for chunk in r.iter_content(4096): + # if chunk: # filter out keep-alive new chuncks + f.write(chunk) + + return S_OK() + + # Some HTTPError are not worth retrying + except requests.exceptions.HTTPError as e: + status_code = e.response.status_code + if status_code == http_client.NOT_IMPLEMENTED: + return S_ERROR(errno.ENOSYS, "%s is not implemented" % kwargs.get('method')) + elif status_code in (http_client.FORBIDDEN, http_client.UNAUTHORIZED): + return S_ERROR(errno.EACCES, "No access to %s" % url) + + # if it is something else, retry + raise + + # Whatever exception we have here, we deem worth retrying + except Exception as e: + # CHRIS TODO review this part: retry logic is fishy + # self.__bannedUrls is emptied in findServiceURLs + if url not in self.__bannedUrls: + self.__bannedUrls += [url] + if retry < self.__nbOfUrls - 1: + self._request(retry=retry + 1, outputFile=outputFile, **kwargs) + + errStr = "%s: %s" % (str(e), rawText) + return S_ERROR(errStr) + + +# --- TODO ---- +# Rewrite this method if needed: +# /Core/DISET/private/BaseClient.py +# __delegateCredentials diff --git a/Core/Tornado/Client/private/__init__.py b/Core/Tornado/Client/private/__init__.py new file mode 100644 index 00000000000..cae5bc3f37c --- /dev/null +++ b/Core/Tornado/Client/private/__init__.py @@ -0,0 +1,3 @@ +""" +DIRAC.TornadosServices +""" diff --git a/Core/Tornado/Server/HandlerManager.py b/Core/Tornado/Server/HandlerManager.py new file mode 100644 index 00000000000..4ab54715703 --- /dev/null +++ b/Core/Tornado/Server/HandlerManager.py @@ -0,0 +1,184 @@ +""" + This module contains the necessary tools to discover and load + the handlers for serving HTTPS +""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +__RCSID__ = "$Id$" + +from tornado.web import url as TornadoURL, RequestHandler + +from DIRAC import gConfig, gLogger, S_ERROR, S_OK +from DIRAC.ConfigurationSystem.Client import PathFinder +from DIRAC.Core.Base.private.ModuleLoader import ModuleLoader +from DIRAC.Core.Utilities.ObjectLoader import ObjectLoader + + +def urlFinder(module): + """ + Tries to guess the url from module name. + The URL would be of the form ``/System/Component`` (e.g. ``DataManagement/FileCatalog``) + We search something which looks like ``<...>.System.<...>.Handler`` + + :param module: full module name (e.g. "DIRAC.something.something") + + :returns: the deduced URL or None + """ + sections = module.split('.') + for section in sections: + # This condition is a bit long + # We search something which look like <...>.System.<...>.Handler + # If find we return // + if section.endswith("System") and sections[-1].endswith("Handler"): + return "/".join(["", section[:-len("System")], sections[-1][:-len("Handler")]]) + + +class HandlerManager(object): + """ + This utility class allows to load the handlers, generate the appropriate route, + and discover the handlers based on the CS. + In order for a service to be considered as using HTTPS, it must have + ``protocol = https`` as an option. + + Each of the Handler will have one associated route to it: + + * Directly specified as ``LOCATION`` in the handler module + * automatically deduced from the module name, of the form + ``System/Component`` (e.g. ``DataManagement/FileCatalog``) + """ + + def __init__(self, autoDiscovery=True): + """ + Initialization function, you can set autoDiscovery=False to prevent automatic + discovery of handler. If disabled you can use loadHandlersByServiceName() to + load your handlers or loadHandlerInHandlerManager() + + :param autoDiscovery: (default True) Disable the automatic discovery, + can be used to choose service we want to load. + """ + self.__handlers = {} + self.__objectLoader = ObjectLoader() + self.__autoDiscovery = autoDiscovery + self.loader = ModuleLoader("Service", PathFinder.getServiceSection, RequestHandler, moduleSuffix="Handler") + + def __addHandler(self, handlerTuple, url=None): + """ + Function which add handler to list of known handlers + + + :param handlerTuple: (path, class) + """ + # Check if handler not already loaded + if not url or url not in self.__handlers: + gLogger.debug("Find new handler %s" % (handlerTuple[0])) + + # If url is not given, try to discover it + if url is None: + # FIRST TRY: Url is hardcoded + try: + url = handlerTuple[1].LOCATION + # SECOND TRY: URL can be deduced from path + except AttributeError: + gLogger.debug("No location defined for %s try to get it from path" % handlerTuple[0]) + url = urlFinder(handlerTuple[0]) + + # We add "/" if missing at begin, e.g. we found "Framework/Service" + # URL can't be relative in Tornado + if url and not url.startswith('/'): + url = "/%s" % url + elif not url: + gLogger.warn("URL not found for %s" % (handlerTuple[0])) + return S_ERROR("URL not found for %s" % (handlerTuple[0])) + + # Finally add the URL to handlers + if url not in self.__handlers: + self.__handlers[url] = handlerTuple[1] + gLogger.info("New handler: %s with URL %s" % (handlerTuple[0], url)) + else: + gLogger.debug("Handler already loaded %s" % (handlerTuple[0])) + return S_OK() + + def discoverHandlers(self): + """ + Force the discovery of URL, automatic call when we try to get handlers for the first time. + You can disable the automatic call with autoDiscovery=False at initialization + """ + gLogger.debug("Trying to auto-discover the handlers for Tornado") + + # Look in config + diracSystems = gConfig.getSections('/Systems') + serviceList = [] + if diracSystems['OK']: + for system in diracSystems['Value']: + try: + instance = PathFinder.getSystemInstance(system) + services = gConfig.getSections('/Systems/%s/%s/Services' % (system, instance)) + if services['OK']: + for service in services['Value']: + newservice = ("%s/%s" % (system, service)) + + # We search in the CS all handlers which used HTTPS as protocol + isHTTPS = gConfig.getValue('/Systems/%s/%s/Services/%s/Protocol' % (system, instance, service)) + if isHTTPS and isHTTPS.lower() == 'https': + serviceList.append(newservice) + # On systems sometime you have things not related to services... + except RuntimeError: + pass + return self.loadHandlersByServiceName(serviceList) + + def loadHandlersByServiceName(self, servicesNames): + """ + Load a list of handler from list of service using DIRAC moduleLoader + Use :py:class:`DIRAC.Core.Base.private.ModuleLoader` + + :param servicesNames: list of service, e.g. ['Framework/Hello', 'Configuration/Server'] + """ + + # Use DIRAC system to load: search in CS if path is given and if not defined + # it search in place it should be (e.g. in DIRAC/FrameworkSystem/Service) + if not isinstance(servicesNames, list): + servicesNames = [servicesNames] + + load = self.loader.loadModules(servicesNames) + if not load['OK']: + return load + for module in self.loader.getModules().values(): + url = module['loadName'] + + # URL can be like https://domain:port/service/name or just service/name + # Here we just want the service name, for tornado + serviceTuple = url.replace('https://', '').split('/')[-2:] + url = "%s/%s" % (serviceTuple[0], serviceTuple[1]) + self.__addHandler((module['loadName'], module['classObj']), url) + return S_OK() + + def getHandlersURLs(self): + """ + Get all handler for usage in Tornado, as a list of tornado.web.url + If there is no handler found before, it try to find them + + :returns: a list of URL (not the string with "https://..." but the tornado object) + see http://www.tornadoweb.org/en/stable/web.html#tornado.web.URLSpec + """ + if not self.__handlers and self.__autoDiscovery: + self.__autoDiscovery = False + self.discoverHandlers() + urls = [] + for key in self.__handlers: + urls.append(TornadoURL(key, self.__handlers[key])) + return urls + + def getHandlersDict(self): + """ + Return all handler dictionary + + :returns: dictionary with absolute url as key ("/System/Service") + and tornado.web.url object as value + """ + if not self.__handlers and self.__autoDiscovery: + self.__autoDiscovery = False + self.discoverHandlers() + return self.__handlers diff --git a/Core/Tornado/Server/TornadoServer.py b/Core/Tornado/Server/TornadoServer.py new file mode 100644 index 00000000000..3707cea6673 --- /dev/null +++ b/Core/Tornado/Server/TornadoServer.py @@ -0,0 +1,226 @@ +""" +TornadoServer create a web server and load services. +It may work better with TornadoClient but as it accepts HTTPS you can create your own client +""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +__RCSID__ = "$Id$" + +import time +import datetime +import os + +import M2Crypto + +import tornado.iostream +tornado.iostream.SSLIOStream.configure( + 'tornado_m2crypto.m2iostream.M2IOStream') # pylint: disable=wrong-import-position + +from tornado.httpserver import HTTPServer +from tornado.web import Application, url +from tornado.ioloop import IOLoop +import tornado.ioloop + +import DIRAC +from DIRAC import gConfig, gLogger +from DIRAC.ConfigurationSystem.Client import PathFinder +from DIRAC.Core.Security import Locations +from DIRAC.Core.Tornado.Server.HandlerManager import HandlerManager +from DIRAC.Core.Utilities import MemStat +from DIRAC.FrameworkSystem.Client.MonitoringClient import MonitoringClient + +sLog = gLogger.getSubLogger(__name__) + + +class TornadoServer(object): + """ + Tornado webserver + + Initialize and run an HTTPS Server for DIRAC services. + By default it load all https services defined in the CS, + but you can also give an explicit list. + + The listening port is either: + + * Given as parameter + * Loaded from the CS ``/Systems/Tornado//Port`` + * Default to 8443 + + + Example 1: Easy way to start tornado:: + + # Initialize server and load services + serverToLaunch = TornadoServer() + + # Start listening when ready + serverToLaunch.startTornado() + + Example 2:We want to debug service1 and service2 only, and use another port for that :: + + services = ['component/service1', 'component/service2'] + serverToLaunch = TornadoServer(services=services, port=1234) + serverToLaunch.startTornado() + + """ + + def __init__(self, services=None, port=None): + """ + + :param list services: (default None) List of service handlers to load. If ``None``, loads all + :param int port: Port to listen to. If None, the port is resolved following the logic + described in the class documentation + """ + + if port is None: + port = gConfig.getValue("/Systems/Tornado/%s/Port" % PathFinder.getSystemInstance('Tornado'), 8443) + + if services and not isinstance(services, list): + services = [services] + + # URLs for services. + # Contains Tornado :py:class:`tornado.web.url` object + self.urls = [] + # Other infos + self.port = port + self.handlerManager = HandlerManager() + + # Monitoring attributes + + self._monitor = MonitoringClient() + # temp value for computation, used by the monitoring + self.__report = None + # Last update time stamp + self.__monitorLastStatsUpdate = None + self.__monitoringLoopDelay = 60 # In secs + + # If services are defined, load only these ones (useful for debug purpose or specific services) + if services: + retVal = self.handlerManager.loadHandlersByServiceName(services) + if not retVal['OK']: + sLog.error(retVal['Message']) + raise ImportError("Some services can't be loaded, check the service names and configuration.") + + # if no service list is given, load services from configuration + handlerDict = self.handlerManager.getHandlersDict() + for item in handlerDict.items(): + # handlerDict[key].initializeService(key) + self.urls.append(url(item[0], item[1])) + # If there is no services loaded: + if not self.urls: + raise ImportError("There is no services loaded, please check your configuration") + + def startTornado(self): + """ + Starts the tornado server when ready. + This method never returns. + """ + + sLog.debug("Starting Tornado") + self._initMonitoring() + + router = Application(self.urls, + debug=False, + compress_response=True) + + certs = Locations.getHostCertificateAndKeyLocation() + if certs is False: + sLog.fatal("Host certificates not found ! Can't start the Server") + raise ImportError("Unable to load certificates") + ca = Locations.getCAsLocation() + ssl_options = { + 'certfile': certs[0], + 'keyfile': certs[1], + 'cert_reqs': M2Crypto.SSL.verify_peer, + 'ca_certs': ca, + 'sslDebug': False, # Set to true if you want to see the TLS debug messages + } + + self.__monitorLastStatsUpdate = time.time() + self.__report = self.__startReportToMonitoringLoop() + + # Starting monitoring, IOLoop waiting time in ms, __monitoringLoopDelay is defined in seconds + tornado.ioloop.PeriodicCallback(self.__reportToMonitoring, self.__monitoringLoopDelay * 1000).start() + + # Start server + server = HTTPServer(router, ssl_options=ssl_options, decompress_request=True) + try: + server.listen(self.port) + except Exception as e: # pylint: disable=broad-except + sLog.exception("Exception starting HTTPServer", e) + raise + sLog.always("Listening on port %s" % self.port) + for service in self.urls: + sLog.debug("Available service: %s" % service) + + IOLoop.current().start() + + def _initMonitoring(self): + """ + Initialize the monitoring + """ + + self._monitor.setComponentType(MonitoringClient.COMPONENT_TORNADO) + self._monitor.initialize() + self._monitor.setComponentName('Tornado') + + self._monitor.registerActivity('CPU', "CPU Usage", 'Framework', "CPU,%", MonitoringClient.OP_MEAN, 600) + self._monitor.registerActivity('MEM', "Memory Usage", 'Framework', 'Memory,MB', MonitoringClient.OP_MEAN, 600) + + self._monitor.setComponentExtraParam('DIRACVersion', DIRAC.version) + self._monitor.setComponentExtraParam('platform', DIRAC.getPlatform()) + self._monitor.setComponentExtraParam('startTime', datetime.datetime.utcnow()) + + def __reportToMonitoring(self): + """ + Periodically report to the monitoring of the CPU and MEM + """ + + # Calculate CPU usage by comparing realtime and cpu time since last report + self.__endReportToMonitoringLoop(*self.__report) + + # Save memory usage and save realtime/CPU time for next call + self.__report = self.__startReportToMonitoringLoop() + + def __startReportToMonitoringLoop(self): + """ + Snapshot of resources to be taken at the beginning + of a monitoring cycle. + Also sends memory snapshot to the monitoring. + + This is basically copy/paste of Service.py + + :returns: tuple ( 0: + self._monitor.addMark('CPU', percentage) diff --git a/Core/Tornado/Server/TornadoService.py b/Core/Tornado/Server/TornadoService.py new file mode 100644 index 00000000000..c545e340568 --- /dev/null +++ b/Core/Tornado/Server/TornadoService.py @@ -0,0 +1,735 @@ +""" +TornadoService is the base class for your handlers. +It directly inherits from :py:class:`tornado.web.RequestHandler` +""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +__RCSID__ = "$Id$" + +from io import open + +import os +import time +import threading +from datetime import datetime +from six.moves import http_client +from tornado.web import RequestHandler, HTTPError +from tornado import gen +import tornado.ioloop +from tornado.ioloop import IOLoop + +import DIRAC + +from DIRAC import gConfig, gLogger, S_OK, S_ERROR +from DIRAC.ConfigurationSystem.Client import PathFinder +from DIRAC.Core.DISET.AuthManager import AuthManager +from DIRAC.Core.Security.X509Chain import X509Chain # pylint: disable=import-error +from DIRAC.Core.Utilities.JEncode import decode, encode +from DIRAC.FrameworkSystem.Client.MonitoringClient import MonitoringClient + +sLog = gLogger.getSubLogger(__name__) + + +class TornadoService(RequestHandler): # pylint: disable=abstract-method + """ + Base class for all the Handlers. + It directly inherits from :py:class:`tornado.web.RequestHandler` + + Each HTTP request is served by a new instance of this class. + + For the sequence of method called, please refer to + the `tornado documentation `_. + + For compatibility with the existing :py:class:`DIRAC.Core.DISET.TransferClient.TransferClient`, + the handler can define a method ``export_streamToClient``. This is the method that will be called + whenever ``TransferClient.receiveFile`` is called. It is the equivalent of the DISET + ``transfer_toClient``. + Note that this is here only for compatibility, and we discourage using it for new purposes, as it is + bound to disappear. + + The handler only define the ``post`` verb. Please refer to :py:meth:`.post` for the details. + + In order to create a handler for your service, it has to + follow a certain skeleton:: + + from DIRAC.Core.Tornado.Server.TornadoService import TornadoService + class yourServiceHandler(TornadoService): + + # Called only once when the first + # request for this handler arrives + # Useful for initializing DB or so. + # You don't need to use super or to call any parents method, it's managed by the server + @classmethod + def initializeHandler(cls, infosDict): + '''Called only once when the first + request for this handler arrives + Useful for initializing DB or so. + You don't need to use super or to call any parents method, it's managed by the server + ''' + pass + + + def initializeRequest(self): + ''' + Called at the beginning of each request + ''' + pass + + # Specify the default permission for the method + # See :py:class:`DIRAC.Core.DISET.AuthManager.AuthManager` + auth_someMethod = ['authenticated'] + + + def export_someMethod(self): + '''The method you want to export. + It must start with ``export_`` + and it must return an S_OK/S_ERROR structure + ''' + return S_ERROR() + + + def export_streamToClient(self, myDataToSend, token): + ''' Automatically called when ``Transfer.receiveFile`` is called. + Contrary to the other ``export_`` methods, it does not need + to return a DIRAC structure. + ''' + + # Do whatever with the token + + with open(myFileToSend, 'r') as fd: + return fd.read() + + + Note that because we inherit from :py:class:`tornado.web.RequestHandler` + and we are running using executors, the methods you export cannot write + back directly to the client. Please see inline comments for more details. + + In order to pass information around and keep some states, we use instance attributes. + These are initialized in the :py:meth:`.initialize` method. + + """ + + # Because we initialize at first request, we use a flag to know if it's already done + __init_done = False + # Lock to make sure that two threads are not initializing at the same time + __init_lock = threading.RLock() + + # MonitoringClient, we don't use gMonitor which is not thread-safe + # We also need to add specific attributes for each service + _monitor = None + + @classmethod + def _initMonitoring(cls, serviceName, fullUrl): + """ + Initialize the monitoring specific to this handler + This has to be called only by :py:meth:`.__initializeService` + to ensure thread safety and unicity of the call. + + :param serviceName: relative URL ``//`` + :param fullUrl: full URl like ``https://://`` + """ + + # Init extra bits of monitoring + + cls._monitor = MonitoringClient() + cls._monitor.setComponentType(MonitoringClient.COMPONENT_WEB) + + cls._monitor.initialize() + + if tornado.process.task_id() is None: # Single process mode + cls._monitor.setComponentName('Tornado/%s' % serviceName) + else: + cls._monitor.setComponentName('Tornado/CPU%d/%s' % (tornado.process.task_id(), serviceName)) + + cls._monitor.setComponentLocation(fullUrl) + + cls._monitor.registerActivity("Queries", "Queries served", "Framework", "queries", MonitoringClient.OP_RATE) + + cls._monitor.setComponentExtraParam('DIRACVersion', DIRAC.version) + cls._monitor.setComponentExtraParam('platform', DIRAC.getPlatform()) + cls._monitor.setComponentExtraParam('startTime', datetime.utcnow()) + + cls._stats = {'requests': 0, 'monitorLastStatsUpdate': time.time()} + + return S_OK() + + @classmethod + def __initializeService(cls, relativeUrl, absoluteUrl): + """ + Initialize a service. + The work is only perform once at the first request. + + :param relativeUrl: relative URL, e.g. ``//`` + :param absoluteUrl: full URL e.g. ``https://://`` + + :returns: S_OK + """ + # If the initialization was already done successfuly, + # we can just return + if cls.__init_done: + return S_OK() + + # Otherwise, do the work but with a lock + with cls.__init_lock: + + # Check again that the initialization was not done by another thread + # while we were waiting for the lock + if cls.__init_done: + return S_OK() + + # Url starts with a "/", we just remove it + serviceName = relativeUrl[1:] + + cls._startTime = datetime.utcnow() + sLog.info("First use of %s, initializing service..." % relativeUrl) + cls._authManager = AuthManager("%s/Authorization" % PathFinder.getServiceSection(serviceName)) + + cls._initMonitoring(serviceName, absoluteUrl) + + cls._serviceName = serviceName + cls._validNames = [serviceName] + serviceInfo = {'serviceName': serviceName, + 'serviceSectionPath': PathFinder.getServiceSection(serviceName), + 'csPaths': [PathFinder.getServiceSection(serviceName)], + 'URL': absoluteUrl + } + cls._serviceInfoDict = serviceInfo + + cls.__monitorLastStatsUpdate = time.time() + + cls.initializeHandler(serviceInfo) + + cls.__init_done = True + + return S_OK() + + @classmethod + def initializeHandler(cls, serviceInfoDict): + """ + This may be overwritten when you write a DIRAC service handler + And it must be a class method. This method is called only one time, + at the first request + + :param dict ServiceInfoDict: infos about services, it contains + 'serviceName', 'serviceSectionPath', + 'csPaths' and 'URL' + """ + pass + + def initializeRequest(self): + """ + Called at every request, may be overwritten in your handler. + """ + pass + + # This is a Tornado magic method + def initialize(self): # pylint: disable=arguments-differ + """ + Initialize the handler, called at every request. + + It just calls :py:meth:`.__initializeService` + + If anything goes wrong, the client will get ``Connection aborted`` + error. See details inside the method. + + ..warning:: + DO NOT REWRITE THIS FUNCTION IN YOUR HANDLER + ==> initialize in DISET became initializeRequest in HTTPS ! + """ + + # Only initialized once + if not self.__init_done: + # Ideally, if something goes wrong, we would like to return a Server Error 500 + # but this method cannot write back to the client as per the + # `tornado doc `_. + # So the client will get a ``Connection aborted``` + try: + res = self.__initializeService(self.srv_getURL(), self.request.full_url()) + if not res['OK']: + raise Exception(res['Message']) + except Exception as e: + sLog.error("Error in initialization", repr(e)) + raise + + def prepare(self): + """ + Prepare the request. It reads certificates and check authorizations. + We make the assumption that there is always going to be a ``method`` argument + regardless of the HTTP method used + + """ + + # "method" argument of the POST call. + # This resolves into the ``export_`` method + # on the handler side + # If the argument is not available, the method exists + # and an error 400 ``Bad Request`` is returned to the client + self.method = self.get_argument("method") + + self._stats['requests'] += 1 + self._monitor.setComponentExtraParam('queries', self._stats['requests']) + self._monitor.addMark("Queries") + + try: + self.credDict = self._gatherPeerCredentials() + except Exception: # pylint: disable=broad-except + # If an error occur when reading certificates we close connection + # It can be strange but the RFC, for HTTP, say's that when error happend + # before authentication we return 401 UNAUTHORIZED instead of 403 FORBIDDEN + sLog.error( + "Error gathering credentials", "%s; path %s" % + (self.getRemoteAddress(), self.request.path)) + raise HTTPError(status_code=http_client.UNAUTHORIZED) + + # Resolves the hard coded authorization requirements + try: + hardcodedAuth = getattr(self, 'auth_' + self.method) + except AttributeError: + hardcodedAuth = None + + # Check whether we are authorized to perform the query + # Note that performing the authQuery modifies the credDict... + authorized = self._authManager.authQuery(self.method, self.credDict, hardcodedAuth) + if not authorized: + sLog.error( + "Unauthorized access", "Identity %s; path %s; DN %s" % + (self.srv_getFormattedRemoteCredentials, + self.request.path, + self.credDict['DN'], + )) + raise HTTPError(status_code=http_client.UNAUTHORIZED) + + # Make post a coroutine. + # See https://www.tornadoweb.org/en/branch5.1/guide/coroutines.html#coroutines + # for details + @gen.coroutine + def post(self): # pylint: disable=arguments-differ + """ + Method to handle incoming ``POST`` requests. + Note that all the arguments are already prepared in the :py:meth:`.prepare` + method. + + The ``POST`` arguments expected are: + + * ``method``: name of the method to call + * ``args``: JSON encoded arguments for the method + * ``extraCredentials``: (optional) Extra informations to authenticate client + * ``rawContent``: (optionnal, default False) If set to True, return the raw output + of the method called. + + If ``rawContent`` was requested by the client, the ``Content-Type`` + is ``application/octet-stream``, otherwise we set it to ``application/json`` + and JEncode retVal. + + If ``retVal`` is a dictionary that contains a ``Callstack`` item, + it is removed, not to leak internal information. + + + Example of call using ``requests``:: + + In [20]: url = 'https://server:8443/DataManagement/TornadoFileCatalog' + ...: cert = '/tmp/x509up_u1000' + ...: kwargs = {'method':'whoami'} + ...: caPath = '/home/dirac/ClientInstallDIR/etc/grid-security/certificates/' + ...: with requests.post(url, data=kwargs, cert=cert, verify=caPath) as r: + ...: print r.json() + ...: + {u'OK': True, + u'Value': {u'DN': u'/C=ch/O=DIRAC/OU=DIRAC CI/CN=ciuser/emailAddress=lhcb-dirac-ci@cern.ch', + u'group': u'dirac_user', + u'identity': u'/C=ch/O=DIRAC/OU=DIRAC CI/CN=ciuser/emailAddress=lhcb-dirac-ci@cern.ch', + u'isLimitedProxy': False, + u'isProxy': True, + u'issuer': u'/C=ch/O=DIRAC/OU=DIRAC CI/CN=ciuser/emailAddress=lhcb-dirac-ci@cern.ch', + u'properties': [u'NormalUser'], + u'secondsLeft': 85441, + u'subject': u'/C=ch/O=DIRAC/OU=DIRAC CI/CN=ciuser/emailAddress=lhcb-dirac-ci@cern.ch/CN=2409820262', + u'username': u'adminusername', + u'validDN': False, + u'validGroup': False}} + """ + + sLog.notice( + "Incoming request %s /%s: %s" % + (self.srv_getFormattedRemoteCredentials(), + self._serviceName, + self.method)) + + # Execute the method in an executor (basically a separate thread) + # Because of that, we cannot calls certain methods like `self.write` + # in __executeMethod. This is because these methods are not threadsafe + # https://www.tornadoweb.org/en/branch5.1/web.html#thread-safety-notes + # However, we can still rely on instance attributes to store what should + # be sent back (reminder: there is an instance + # of this class created for each request) + retVal = yield IOLoop.current().run_in_executor(None, self.__executeMethod) + + # retVal is :py:class:`tornado.concurrent.Future` + self.result = retVal.result() + + # Here it is safe to write back to the client, because we are not + # in a thread anymore + + # If set to true, do not JEncode the return of the RPC call + # This is basically only used for file download through + # the 'streamToClient' method. + rawContent = self.get_argument('rawContent', default=False) + + if rawContent: + # See 4.5.1 http://www.rfc-editor.org/rfc/rfc2046.txt + self.set_header("Content-Type", "application/octet-stream") + result = self.result + else: + self.set_header("Content-Type", "application/json") + result = encode(self.result) + + self.write(result) + self.finish() + + # This nice idea of streaming to the client cannot work because we are ran in an executor + # and we should not write back to the client in a different thread. + # See https://www.tornadoweb.org/en/branch5.1/web.html#thread-safety-notes + # def export_streamToClient(self, filename): + # # https://bhch.github.io/posts/2017/12/serving-large-files-with-tornado-safely-without-blocking/ + # #import ipdb; ipdb.set_trace() + # # chunk size to read + # chunk_size = 1024 * 1024 * 1 # 1 MiB + + # with open(filename, 'rb') as f: + # while True: + # chunk = f.read(chunk_size) + # if not chunk: + # break + # try: + # self.write(chunk) # write the chunk to response + # self.flush() # send the chunk to client + # except StreamClosedError: + # # this means the client has closed the connection + # # so break the loop + # break + # finally: + # # deleting the chunk is very important because + # # if many clients are downloading files at the + # # same time, the chunks in memory will keep + # # increasing and will eat up the RAM + # del chunk + # # pause the coroutine so other handlers can run + # yield gen.sleep(0.000000001) # 1 nanosecond + + # return S_OK() + + @gen.coroutine + def __executeMethod(self): + """ + Execute the method called, this method is ran in an executor + We have several try except to catch the different problem which can occur + + - First, the method does not exist => Attribute error, return an error to client + - second, anything happend during execution => General Exception, send error to client + + .. warning:: + This method is called in an executor, and so cannot use methods like self.write + See https://www.tornadoweb.org/en/branch5.1/web.html#thread-safety-notes + """ + + # getting method + try: + # For compatibility reasons with DISET, the methods are still called ``export_*`` + method = getattr(self, 'export_%s' % self.method) + except AttributeError as e: + sLog.error("Invalid method", self.method) + raise HTTPError(status_code=http_client.NOT_IMPLEMENTED) + + # Decode args + args_encoded = self.get_body_argument('args', default=encode([])) + + args = decode(args_encoded)[0] + # Execute + try: + self.initializeRequest() + retVal = method(*args) + except Exception as e: # pylint: disable=broad-except + sLog.exception("Exception serving request", "%s:%s" % (str(e), repr(e))) + raise HTTPError(http_client.INTERNAL_SERVER_ERROR) + + return retVal + + # def __write_return(self, retVal): + # """ + # Write back to the client and return. + # It sets some headers (status code, ``Content-Type``). + # If raw content was requested by the client, the ``Content-Type`` + # is ``application/octet-stream``, otherwise we set it to ``application/json`` + # and JEncode retVal. + + # If ``retVal`` is a dictionary that contains a ``Callstack`` item, + # it is removed, not to leak internal information. + + # :param retVal: anything that can be serialized in json. + # """ + + # # In case of error in server side we hide server CallStack to client + # try: + # if 'CallStack' in retVal: + # del retVal['CallStack'] + # except TypeError: + # pass + + # # Set the status + # self.set_status(self._httpStatus) + + # # This is basically only used for file download through + # # the 'streamToClient' method. + # if self.rawContent: + # # See 4.5.1 http://www.rfc-editor.org/rfc/rfc2046.txt + # self.set_header("Content-Type", "application/octet-stream") + # returnedData = retVal + # else: + # self.set_header("Content-Type", "application/json") + # returnedData = encode(retVal) + + # self.write(returnedData) + + def on_finish(self): + """ + Called after the end of HTTP request. + Log the request duration + """ + elapsedTime = 1000.0 * self.request.request_time() + + try: + if self.result['OK']: + argsString = "OK" + else: + argsString = "ERROR: %s" % self.result['Message'] + except (AttributeError, KeyError): # In case it is not a DIRAC structure + if self._reason == 'OK': + argsString = 'OK' + else: + argsString = 'ERROR %s' % self._reason + + argsString = "ERROR: %s" % self._reason + sLog.notice("Returning response", "%s %s (%.2f ms) %s" % (self.srv_getFormattedRemoteCredentials(), + self._serviceName, + elapsedTime, argsString)) + + def _gatherPeerCredentials(self): + """ + Load client certchain in DIRAC and extract informations. + + The dictionary returned is designed to work with the AuthManager, + already written for DISET and re-used for HTTPS. + + :returns: a dict containing the return of :py:meth:`DIRAC.Core.Security.X509Chain.X509Chain.getCredentials` + (not a DIRAC structure !) + """ + + chainAsText = self.request.get_ssl_certificate().as_pem() + peerChain = X509Chain() + + # Here we read all certificate chain + cert_chain = self.request.get_ssl_certificate_chain() + for cert in cert_chain: + chainAsText += cert.as_pem() + + peerChain.loadChainFromString(chainAsText) + + # Retrieve the credentials + res = peerChain.getCredentials(withRegistryInfo=False) + if not res['OK']: + raise Exception(res['Message']) + + credDict = res['Value'] + + # We check if client sends extra credentials... + if "extraCredentials" in self.request.arguments: + extraCred = self.get_argument("extraCredentials") + if extraCred: + credDict['extraCredentials'] = decode(extraCred)[0] + return credDict + + +#### +# +# Default method +# +#### + + auth_ping = ['all'] + + def export_ping(self): + """ + Default ping method, returns some info about server. + + It returns the exact same information as DISET, for transparency purpose. + """ + # COPY FROM DIRAC.Core.DISET.RequestHandler + dInfo = {} + dInfo['version'] = DIRAC.version + dInfo['time'] = datetime.utcnow() + # Uptime + try: + with open("/proc/uptime", 'rt') as oFD: + iUptime = int(float(oFD.readline().split()[0].strip())) + dInfo['host uptime'] = iUptime + except Exception: # pylint: disable=broad-except + pass + startTime = self._startTime + dInfo['service start time'] = self._startTime + serviceUptime = datetime.utcnow() - startTime + dInfo['service uptime'] = serviceUptime.days * 3600 + serviceUptime.seconds + # Load average + try: + with open("/proc/loadavg", 'rt') as oFD: + dInfo['load'] = " ".join(oFD.read().split()[:3]) + except Exception: # pylint: disable=broad-except + pass + dInfo['name'] = self._serviceInfoDict['serviceName'] + stTimes = os.times() + dInfo['cpu times'] = {'user time': stTimes[0], + 'system time': stTimes[1], + 'children user time': stTimes[2], + 'children system time': stTimes[3], + 'elapsed real time': stTimes[4] + } + + return S_OK(dInfo) + + auth_echo = ['all'] + + @staticmethod + def export_echo(data): + """ + This method used for testing the performance of a service + """ + return S_OK(data) + + auth_whoami = ['authenticated'] + + def export_whoami(self): + """ + A simple whoami, returns all credential dictionary, except certificate chain object. + """ + credDict = self.srv_getRemoteCredentials() + if 'x509Chain' in credDict: + # Not serializable + del credDict['x509Chain'] + return S_OK(credDict) + +#### +# +# Utilities methods, some getters. +# From DIRAC.Core.DISET.requestHandler to get same interface in the handlers. +# Adapted for Tornado. +# These method are copied from DISET RequestHandler, they are not all used when i'm writing +# these lines. I rewrite them for Tornado to get them ready when a new HTTPS service need them +# +#### + + @classmethod + def srv_getCSOption(cls, optionName, defaultValue=False): + """ + Get an option from the CS section of the services + + :return: Value for serviceSection/optionName in the CS being defaultValue the default + """ + if optionName[0] == "/": + return gConfig.getValue(optionName, defaultValue) + for csPath in cls._serviceInfoDict['csPaths']: + result = gConfig.getOption("%s/%s" % (csPath, optionName, ), defaultValue) + if result['OK']: + return result['Value'] + return defaultValue + + def getCSOption(self, optionName, defaultValue=False): + """ + Just for keeping same public interface + """ + return self.srv_getCSOption(optionName, defaultValue) + + def srv_getRemoteAddress(self): + """ + Get the address of the remote peer. + + :return: Address of remote peer. + """ + + remote_ip = self.request.remote_ip + # Although it would be trivial to add this attribute in _HTTPRequestContext, + # Tornado won't release anymore 5.1 series, so go the hacky way + try: + remote_port = self.request.connection.stream.socket.getpeername()[1] + except Exception: # pylint: disable=broad-except + remote_port = 0 + + return (remote_ip, remote_port) + + def getRemoteAddress(self): + """ + Just for keeping same public interface + """ + return self.srv_getRemoteAddress() + + def srv_getRemoteCredentials(self): + """ + Get the credentials of the remote peer. + + :return: Credentials dictionary of remote peer. + """ + return self.credDict + + def getRemoteCredentials(self): + """ + Get the credentials of the remote peer. + + :return: Credentials dictionary of remote peer. + """ + return self.credDict + + def srv_getFormattedRemoteCredentials(self): + """ + Return the DN of user + + Mostly copy paste from + :py:meth:`DIRAC.Core.DISET.private.Transports.BaseTransport.BaseTransport.getFormattedCredentials` + + Note that the information will be complete only once the AuthManager was called + """ + address = self.getRemoteAddress() + peerId = "" + # Depending on where this is call, it may be that credDict is not yet filled. + # (reminder: AuthQuery fills part of it..) + try: + peerId = "[%s:%s]" % (self.credDict['group'], self.credDict['username']) + except AttributeError: + pass + + if address[0].find(":") > -1: + return "([%s]:%s)%s" % (address[0], address[1], peerId) + return "(%s:%s)%s" % (address[0], address[1], peerId) + +# def getFormattedCredentials(self): +# peerCreds = self.getConnectingCredentials() +# address = self.getRemoteAddress() +# if 'username' in peerCreds: +# peerId = "[%s:%s]" % (peerCreds['group'], peerCreds['username']) +# else: +# peerId = "" +# if address[0].find(":") > -1: +# return "([%s]:%s)%s" % (address[0], address[1], peerId) +# return "(%s:%s)%s" % (address[0], address[1], peerId) + + def srv_getServiceName(self): + """ + Return the service name + """ + return self._serviceInfoDict['serviceName'] + + def srv_getURL(self): + """ + Return the URL + """ + return self.request.path diff --git a/Core/Tornado/Server/__init__.py b/Core/Tornado/Server/__init__.py new file mode 100644 index 00000000000..cae5bc3f37c --- /dev/null +++ b/Core/Tornado/Server/__init__.py @@ -0,0 +1,3 @@ +""" +DIRAC.TornadosServices +""" diff --git a/Core/Tornado/__init__.py b/Core/Tornado/__init__.py new file mode 100644 index 00000000000..cae5bc3f37c --- /dev/null +++ b/Core/Tornado/__init__.py @@ -0,0 +1,3 @@ +""" +DIRAC.TornadosServices +""" diff --git a/Core/Tornado/scripts/__init__.py b/Core/Tornado/scripts/__init__.py new file mode 100644 index 00000000000..4b853732d24 --- /dev/null +++ b/Core/Tornado/scripts/__init__.py @@ -0,0 +1 @@ +""" The TornadoService scripts package """ diff --git a/Core/Tornado/scripts/tornado-start-CS.py b/Core/Tornado/scripts/tornado-start-CS.py new file mode 100644 index 00000000000..fb4e1811e26 --- /dev/null +++ b/Core/Tornado/scripts/tornado-start-CS.py @@ -0,0 +1,48 @@ +#!/usr/bin/env python +######################################################################## +# File : tornado-start-CS +# Author : Louis MARTIN +######################################################################## +# Just run this script to start Tornado and CS service +# Use dirac.cfg (or other cfg given in the command line) to change port + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +__RCSID__ = "$Id$" + +# Must be define BEFORE any dirac import +import os +import sys +os.environ['DIRAC_USE_TORNADO_IOLOOP'] = "True" + +from DIRAC.ConfigurationSystem.Client.ConfigurationData import gConfigurationData +from DIRAC.ConfigurationSystem.Client.LocalConfiguration import LocalConfiguration +from DIRAC.ConfigurationSystem.private.Refresher import gRefresher +from DIRAC.Core.Utilities.DErrno import includeExtensionErrors +from DIRAC.Core.Tornado.Server.TornadoServer import TornadoServer +from DIRAC.FrameworkSystem.Client.Logger import gLogger + +if gConfigurationData.isMaster(): + gRefresher.disable() + +localCfg = LocalConfiguration() +localCfg.addMandatoryEntry("/DIRAC/Setup") +localCfg.addDefaultEntry("/DIRAC/Security/UseServerCertificate", "yes") +localCfg.addDefaultEntry("LogLevel", "INFO") +localCfg.addDefaultEntry("LogColor", True) +resultDict = localCfg.loadUserData() +if not resultDict['OK']: + gLogger.initialize("Tornado-CS", "/") + gLogger.error("There were errors when loading configuration", resultDict['Message']) + sys.exit(1) + +includeExtensionErrors() + + +gLogger.initialize('Tornado-CS', "/") + + +serverToLaunch = TornadoServer(services='Configuration/Server') +serverToLaunch.startTornado() diff --git a/Core/Tornado/scripts/tornado-start-all.py b/Core/Tornado/scripts/tornado-start-all.py new file mode 100644 index 00000000000..729165e08bf --- /dev/null +++ b/Core/Tornado/scripts/tornado-start-all.py @@ -0,0 +1,58 @@ +#!/usr/bin/env python +######################################################################## +# File : tornado-start-all +# Author : Louis MARTIN +######################################################################## +# Just run this script to start Tornado and all services +# Use CS to change port + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +__RCSID__ = "$Id$" + +# Must be define BEFORE any dirac import +import os +import sys +os.environ['DIRAC_USE_TORNADO_IOLOOP'] = "True" + + +from DIRAC import gConfig +from DIRAC.ConfigurationSystem.Client import PathFinder +from DIRAC.ConfigurationSystem.Client.ConfigurationData import gConfigurationData +from DIRAC.ConfigurationSystem.Client.LocalConfiguration import LocalConfiguration +from DIRAC.Core.Tornado.Server.TornadoServer import TornadoServer +from DIRAC.Core.Utilities.DErrno import includeExtensionErrors +from DIRAC.FrameworkSystem.Client.Logger import gLogger + + +# We check if there is no configuration server started as master +# If you want to start a master CS you should use Configuration_Server.cfg and +# use tornado-start-CS.py +if gConfigurationData.isMaster() and gConfig.getValue( + '/Systems/Configuration/%s/Services/Server/Protocol' % + PathFinder.getSystemInstance('Configuration'), + 'dips').lower() == 'https': + gLogger.fatal("You can't run the CS and services in the same server!") + sys.exit(0) + +localCfg = LocalConfiguration() +localCfg.addMandatoryEntry("/DIRAC/Setup") +localCfg.addDefaultEntry("/DIRAC/Security/UseServerCertificate", "yes") +localCfg.addDefaultEntry("LogLevel", "INFO") +localCfg.addDefaultEntry("LogColor", True) +resultDict = localCfg.loadUserData() +if not resultDict['OK']: + gLogger.initialize("Tornado", "/") + gLogger.error("There were errors when loading configuration", resultDict['Message']) + sys.exit(1) + +includeExtensionErrors() + + +gLogger.initialize('Tornado', "/") + + +serverToLaunch = TornadoServer() +serverToLaunch.startTornado() diff --git a/DataManagementSystem/ConfigTemplate.cfg b/DataManagementSystem/ConfigTemplate.cfg index 52293a503b4..8df2f37b4d7 100644 --- a/DataManagementSystem/ConfigTemplate.cfg +++ b/DataManagementSystem/ConfigTemplate.cfg @@ -45,6 +45,29 @@ Services Default = authenticated } } + + # Caution: tech preview, with LHCb specific managers + TornadoFileCatalog + { + Protocol = https + UserGroupManager = UserAndGroupManagerDB + SEManager = SEManagerDB + SecurityManager = VOMSSecurityManager + DirectoryManager = DirectoryClosure + FileManager = FileManagerPs + UniqueGUID = True + GlobalReadAccess = True + LFNPFNConvention = Strong + ResolvePFN = True + DefaultUmask = 509 + VisibleStatus = AprioriGood + Authorization + { + Default = authenticated + } + } + ###################### + ##BEGIN StorageElement StorageElement { diff --git a/DataManagementSystem/Service/TornadoFileCatalogHandler.py b/DataManagementSystem/Service/TornadoFileCatalogHandler.py new file mode 100644 index 00000000000..ff72722c195 --- /dev/null +++ b/DataManagementSystem/Service/TornadoFileCatalogHandler.py @@ -0,0 +1,644 @@ +######################################################################## +# File: FileCatalogHandler.py +######################################################################## +""" +:mod: FileCatalogHandler + +.. module: FileCatalogHandler + +:synopsis: FileCatalogHandler is a simple Replica and Metadata Catalog service + +""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +__RCSID__ = "$Id$" + +# imports +import six +import csv +import os + +from io import BytesIO +# from DIRAC +from DIRAC.Core.DISET.RequestHandler import getServiceOption + +from DIRAC import gLogger, S_OK, S_ERROR +from DIRAC.FrameworkSystem.Client.MonitoringClient import gMonitor +from DIRAC.DataManagementSystem.DB.FileCatalogDB import FileCatalogDB + +from DIRAC.Core.Utilities import DErrno +from DIRAC.Core.Tornado.Server.TornadoService import TornadoService + + +class TornadoFileCatalogHandler(TornadoService): + """ + ..class:: FileCatalogHandler + + A simple Replica and Metadata Catalog service. + """ + + @classmethod + def initializeHandler(cls, serviceInfo): + """ handler initialisation """ + + dbLocation = getServiceOption(serviceInfo, 'Database', 'DataManagement/FileCatalogDB') + cls.gFileCatalogDB = FileCatalogDB(dbLocation) + + databaseConfig = {} + # Obtain the plugins to be used for DB interaction + gLogger.info("Initializing with FileCatalog with following managers:") + defaultManagers = {'UserGroupManager': 'UserAndGroupManagerDB', + 'SEManager': 'SEManagerDB', + 'SecurityManager': 'NoSecurityManager', + 'DirectoryManager': 'DirectoryLevelTree', + 'FileManager': 'FileManager', + 'DirectoryMetadata': 'DirectoryMetadata', + 'FileMetadata': 'FileMetadata', + 'DatasetManager': 'DatasetManager'} + for configKey in sorted(defaultManagers.keys()): + defaultValue = defaultManagers[configKey] + configValue = getServiceOption(serviceInfo, configKey, defaultValue) + gLogger.info("%-20s : %-20s" % (str(configKey), str(configValue))) + databaseConfig[configKey] = configValue + + # Obtain some general configuration of the database + gLogger.info("Initializing the FileCatalog with the following configuration:") + defaultConfig = {'UniqueGUID': False, + 'GlobalReadAccess': True, + 'LFNPFNConvention': 'Strong', + 'ResolvePFN': True, + 'DefaultUmask': 0o775, + 'ValidFileStatus': ['AprioriGood', 'Trash', 'Removing', 'Probing'], + 'ValidReplicaStatus': ['AprioriGood', 'Trash', 'Removing', 'Probing'], + 'VisibleFileStatus': ['AprioriGood'], + 'VisibleReplicaStatus': ['AprioriGood']} + for configKey in sorted(defaultConfig.keys()): + defaultValue = defaultConfig[configKey] + configValue = getServiceOption(serviceInfo, configKey, defaultValue) + gLogger.info("%-20s : %-20s" % (str(configKey), str(configValue))) + databaseConfig[configKey] = configValue + res = cls.gFileCatalogDB.setConfig(databaseConfig) + + gMonitor.registerActivity("AddFile", "Amount of addFile calls", + "FileCatalogHandler", "calls/min", gMonitor.OP_SUM) + gMonitor.registerActivity("AddFileSuccessful", "Files successfully added", + "FileCatalogHandler", "files/min", gMonitor.OP_SUM) + gMonitor.registerActivity("AddFileFailed", "Files failed to add", + "FileCatalogHandler", "files/min", gMonitor.OP_SUM) + + gMonitor.registerActivity("RemoveFile", "Amount of removeFile calls", + "FileCatalogHandler", "calls/min", gMonitor.OP_SUM) + gMonitor.registerActivity("RemoveFileSuccessful", "Files successfully removed", + "FileCatalogHandler", "files/min", gMonitor.OP_SUM) + gMonitor.registerActivity("RemoveFileFailed", "Files failed to remove", + "FileCatalogHandler", "files/min", gMonitor.OP_SUM) + + gMonitor.registerActivity("AddReplica", "Amount of addReplica calls", + "FileCatalogHandler", "calls/min", gMonitor.OP_SUM) + gMonitor.registerActivity("AddReplicaSuccessful", "Replicas successfully added", + "FileCatalogHandler", "replicas/min", gMonitor.OP_SUM) + gMonitor.registerActivity("AddReplicaFailed", "Replicas failed to add", + "FileCatalogHandler", "replicas/min", gMonitor.OP_SUM) + + gMonitor.registerActivity("RemoveReplica", "Amount of removeReplica calls", + "FileCatalogHandler", "calls/min", gMonitor.OP_SUM) + gMonitor.registerActivity("RemoveReplicaSuccessful", "Replicas successfully removed", + "FileCatalogHandler", "replicas/min", gMonitor.OP_SUM) + gMonitor.registerActivity("RemoveReplicaFailed", "Replicas failed to remove", + "FileCatalogHandler", "replicas/min", gMonitor.OP_SUM) + + gMonitor.registerActivity("ListDirectory", "Amount of listDirectory calls", + "FileCatalogHandler", "calls/min", gMonitor.OP_SUM) + + return res + + ######################################################################## + # Path operations (not updated) + # + def export_changePathOwner(self, lfns, recursive=False): + """ Get replica info for the given list of LFNs + """ + return self.gFileCatalogDB.changePathOwner(lfns, self.getRemoteCredentials(), recursive) + + def export_changePathGroup(self, lfns, recursive=False): + """ Get replica info for the given list of LFNs + """ + return self.gFileCatalogDB.changePathGroup(lfns, self.getRemoteCredentials(), recursive) + + def export_changePathMode(self, lfns, recursive=False): + """ Get replica info for the given list of LFNs + """ + return self.gFileCatalogDB.changePathMode(lfns, self.getRemoteCredentials(), recursive) + + ######################################################################## + # ACL Operations + # + def export_getPathPermissions(self, lfns): + """ Determine the ACL information for a supplied path + """ + return self.gFileCatalogDB.getPathPermissions(lfns, self.getRemoteCredentials()) + + def export_hasAccess(self, paths, opType): + """ Determine if the given op can be performed on the paths + The OpType is all the operations exported + + The reason for the param types is backward compatibility. Between v6r14 and v6r15, + the signature of hasAccess has changed, and the two parameters were swapped. + """ + + # The signature of v6r15 is (dict, str) + # The signature of v6r14 is (str, [dict, str, list]) + # We swap the two params if the first attribute is a string + if isinstance(paths, six.string_types): + paths, opType = opType, paths + + return self.gFileCatalogDB.hasAccess(opType, paths, self.getRemoteCredentials()) + + ################################################################### + # + # isOK + # + + @classmethod + def export_isOK(cls): + """ returns S_OK if DB is connected + """ + if cls.gFileCatalogDB and cls.gFileCatalogDB._connected: + return S_OK() + return S_ERROR('Server not connected to DB') + + ################################################################### + # + # User/Group write operations + # + + def export_addUser(self, userName): + """ Add a new user to the File Catalog """ + return self.gFileCatalogDB.addUser(userName, self.getRemoteCredentials()) + + def export_deleteUser(self, userName): + """ Delete user from the File Catalog """ + return self.gFileCatalogDB.deleteUser(userName, self.getRemoteCredentials()) + + def export_addGroup(self, groupName): + """ Add a new group to the File Catalog """ + return self.gFileCatalogDB.addGroup(groupName, self.getRemoteCredentials()) + + def export_deleteGroup(self, groupName): + """ Delete group from the File Catalog """ + return self.gFileCatalogDB.deleteGroup(groupName, self.getRemoteCredentials()) + + ################################################################### + # + # User/Group read operations + # + + def export_getUsers(self): + """ Get all the users defined in the File Catalog """ + return self.gFileCatalogDB.getUsers(self.getRemoteCredentials()) + + def export_getGroups(self): + """ Get all the groups defined in the File Catalog """ + return self.gFileCatalogDB.getGroups(self.getRemoteCredentials()) + + ######################################################################## + # + # Path read operations + # + + def export_exists(self, lfns): + """ Check whether the supplied paths exists """ + return self.gFileCatalogDB.exists(lfns, self.getRemoteCredentials()) + + ######################################################################## + # + # File write operations + # + + def export_addFile(self, lfns): + """ Register supplied files """ + gMonitor.addMark("AddFile", 1) + res = self.gFileCatalogDB.addFile(lfns, self.getRemoteCredentials()) + if res['OK']: + gMonitor.addMark("AddFileSuccessful", len(res.get('Value', {}).get('Successful', []))) + gMonitor.addMark("AddFileFailed", len(res.get('Value', {}).get('Failed', []))) + + return res + + def export_removeFile(self, lfns): + """ Remove the supplied lfns """ + gMonitor.addMark("RemoveFile", 1) + res = self.gFileCatalogDB.removeFile(lfns, self.getRemoteCredentials()) + if res['OK']: + gMonitor.addMark("RemoveFileSuccessful", len(res.get('Value', {}).get('Successful', []))) + gMonitor.addMark("RemoveFileFailed", len(res.get('Value', {}).get('Failed', []))) + + return res + + def export_setFileStatus(self, lfns): + """ Remove the supplied lfns """ + return self.gFileCatalogDB.setFileStatus(lfns, self.getRemoteCredentials()) + + def export_addReplica(self, lfns): + """ Register supplied replicas """ + gMonitor.addMark("AddReplica", 1) + res = self.gFileCatalogDB.addReplica(lfns, self.getRemoteCredentials()) + if res['OK']: + gMonitor.addMark("AddReplicaSuccessful", len(res.get('Value', {}).get('Successful', []))) + gMonitor.addMark("AddReplicaFailed", len(res.get('Value', {}).get('Failed', []))) + + return res + + def export_removeReplica(self, lfns): + """ Remove the supplied replicas """ + gMonitor.addMark("RemoveReplica", 1) + res = self.gFileCatalogDB.removeReplica(lfns, self.getRemoteCredentials()) + if res['OK']: + gMonitor.addMark("RemoveReplicaSuccessful", len(res.get('Value', {}).get('Successful', []))) + gMonitor.addMark("RemoveReplicaFailed", len(res.get('Value', {}).get('Failed', []))) + + return res + + def export_setReplicaStatus(self, lfns): + """ Set the status for the supplied replicas """ + return self.gFileCatalogDB.setReplicaStatus(lfns, self.getRemoteCredentials()) + + def export_setReplicaHost(self, lfns): + """ Change the registered SE for the supplied replicas """ + return self.gFileCatalogDB.setReplicaHost(lfns, self.getRemoteCredentials()) + + def export_addFileAncestors(self, lfns): + """ Add file ancestor information for the given list of LFNs """ + return self.gFileCatalogDB.addFileAncestors(lfns, self.getRemoteCredentials()) + + ######################################################################## + # + # File read operations + # + + def export_isFile(self, lfns): + """ Check whether the supplied lfns are files """ + return self.gFileCatalogDB.isFile(lfns, self.getRemoteCredentials()) + + def export_getFileSize(self, lfns): + """ Get the size associated to supplied lfns """ + return self.gFileCatalogDB.getFileSize(lfns, self.getRemoteCredentials()) + + def export_getFileMetadata(self, lfns): + """ Get the metadata associated to supplied lfns """ + return self.gFileCatalogDB.getFileMetadata(lfns, self.getRemoteCredentials()) + + def export_getReplicas(self, lfns, allStatus=False): + """ Get replicas for supplied lfns """ + return self.gFileCatalogDB.getReplicas(lfns, allStatus, self.getRemoteCredentials()) + + def export_getReplicaStatus(self, lfns): + """ Get the status for the supplied replicas """ + return self.gFileCatalogDB.getReplicaStatus(lfns, self.getRemoteCredentials()) + + def export_getFileAncestors(self, lfns, depths): + """ Get the status for the supplied replicas """ + dList = depths + if not isinstance(dList, list): + dList = [depths] + lfnDict = dict.fromkeys(lfns, True) + return self.gFileCatalogDB.getFileAncestors(lfnDict, dList, self.getRemoteCredentials()) + + def export_getFileDescendents(self, lfns, depths): + """ Get the status for the supplied replicas """ + dList = depths + if not isinstance(dList, list): + dList = [depths] + lfnDict = dict.fromkeys(lfns, True) + return self.gFileCatalogDB.getFileDescendents(lfnDict, dList, self.getRemoteCredentials()) + + def export_getLFNForGUID(self, guids): + """Get the matching lfns for given guids""" + return self.gFileCatalogDB.getLFNForGUID(guids, self.getRemoteCredentials()) + + ######################################################################## + # + # Directory write operations + # + + def export_createDirectory(self, lfns): + """ Create the supplied directories """ + return self.gFileCatalogDB.createDirectory(lfns, self.getRemoteCredentials()) + + def export_removeDirectory(self, lfns): + """ Remove the supplied directories """ + return self.gFileCatalogDB.removeDirectory(lfns, self.getRemoteCredentials()) + + ######################################################################## + # + # Directory read operations + # + + def export_listDirectory(self, lfns, verbose): + """ List the contents of supplied directories """ + gMonitor.addMark('ListDirectory', 1) + return self.gFileCatalogDB.listDirectory(lfns, self.getRemoteCredentials(), verbose=verbose) + + def export_isDirectory(self, lfns): + """ Determine whether supplied path is a directory """ + return self.gFileCatalogDB.isDirectory(lfns, self.getRemoteCredentials()) + + def export_getDirectoryMetadata(self, lfns): + """ Get the size of the supplied directory """ + return self.gFileCatalogDB.getDirectoryMetadata(lfns, self.getRemoteCredentials()) + + def export_getDirectorySize(self, lfns, longOut=False, fromFiles=False): + """ Get the size of the supplied directory """ + return self.gFileCatalogDB.getDirectorySize(lfns, longOut, fromFiles, self.getRemoteCredentials()) + + def export_getDirectoryReplicas(self, lfns, allStatus=False): + """ Get replicas for files in the supplied directory """ + return self.gFileCatalogDB.getDirectoryReplicas(lfns, allStatus, self.getRemoteCredentials()) + + ######################################################################## + # + # Administrative database operations + # + + def export_getCatalogCounters(self): + """ Get the number of registered directories, files and replicas in various tables """ + return self.gFileCatalogDB.getCatalogCounters(self.getRemoteCredentials()) + + @staticmethod + def export_rebuildDirectoryUsage(self): + """ Rebuild DirectoryUsage table from scratch """ + return self.gFileCatalogDB.rebuildDirectoryUsage() + + def export_repairCatalog(self): + """ Repair the catalog inconsistencies """ + return self.gFileCatalogDB.repairCatalog(self.getRemoteCredentials()) + + ######################################################################## + # Metadata Catalog Operations + # + + def export_addMetadataField(self, fieldName, fieldType, metaType='-d'): + """ Add a new metadata field of the given type + """ + if metaType.lower() == "-d": + return self.gFileCatalogDB.dmeta.addMetadataField( + fieldName, fieldType, self.getRemoteCredentials()) + elif metaType.lower() == "-f": + return self.gFileCatalogDB.fmeta.addMetadataField( + fieldName, fieldType, self.getRemoteCredentials()) + else: + return S_ERROR('Unknown metadata type %s' % metaType) + + def export_deleteMetadataField(self, fieldName): + """ Delete the metadata field + """ + result = self.gFileCatalogDB.dmeta.deleteMetadataField(fieldName, self.getRemoteCredentials()) + error = '' + if not result['OK']: + error = result['Message'] + result = self.gFileCatalogDB.fmeta.deleteMetadataField(fieldName, self.getRemoteCredentials()) + if not result['OK']: + if error: + result["Message"] = error + "; " + result["Message"] + + return result + + def export_getMetadataFields(self): + """ Get all the metadata fields + """ + resultDir = self.gFileCatalogDB.dmeta.getMetadataFields(self.getRemoteCredentials()) + if not resultDir['OK']: + return resultDir + resultFile = self.gFileCatalogDB.fmeta.getFileMetadataFields(self.getRemoteCredentials()) + if not resultFile['OK']: + return resultFile + + return S_OK({'DirectoryMetaFields': resultDir['Value'], + 'FileMetaFields': resultFile['Value']}) + + def export_setMetadata(self, path, metadatadict): + """ Set metadata parameter for the given path + """ + return self.gFileCatalogDB.setMetadata(path, metadatadict, self.getRemoteCredentials()) + + def export_setMetadataBulk(self, pathMetadataDict): + """ Set metadata parameter for the given path + """ + return self.gFileCatalogDB.setMetadataBulk(pathMetadataDict, self.getRemoteCredentials()) + + def export_removeMetadata(self, pathMetadataDict): + """ Remove the specified metadata for the given path + """ + return self.gFileCatalogDB.removeMetadata(pathMetadataDict, self.getRemoteCredentials()) + + def export_getDirectoryUserMetadata(self, path): + """ Get all the metadata valid for the given directory path + """ + return self.gFileCatalogDB.dmeta.getDirectoryMetadata(path, self.getRemoteCredentials()) + + def export_getFileUserMetadata(self, path): + """ Get all the metadata valid for the given file + """ + return self.gFileCatalogDB.fmeta.getFileUserMetadata(path, self.getRemoteCredentials()) + + def export_findDirectoriesByMetadata(self, metaDict, path='/'): + """ Find all the directories satisfying the given metadata set + """ + return self.gFileCatalogDB.dmeta.findDirectoriesByMetadata( + metaDict, path, self.getRemoteCredentials()) + + def export_findFilesByMetadata(self, metaDict, path='/'): + """ Find all the files satisfying the given metadata set + """ + result = self.gFileCatalogDB.fmeta.findFilesByMetadata(metaDict, path, self.getRemoteCredentials()) + if not result['OK']: + return result + lfns = result['Value'].values() + return S_OK(lfns) + + def export_getReplicasByMetadata(self, metaDict, path='/', allStatus=False): + """ Find all the files satisfying the given metadata set + """ + return self.gFileCatalogDB.fileManager.getReplicasByMetadata(metaDict, + path, + allStatus, + self.getRemoteCredentials()) + + def export_findFilesByMetadataDetailed(self, metaDict, path='/'): + """ Find all the files satisfying the given metadata set + """ + result = self.gFileCatalogDB.fmeta.findFilesByMetadata(metaDict, path, self.getRemoteCredentials()) + if not result['OK'] or not result['Value']: + return result + + lfns = result['Value'].values() + return self.gFileCatalogDB.getFileDetails(lfns, self.getRemoteCredentials()) + + def export_findFilesByMetadataWeb(self, metaDict, path, startItem, maxItems): + """ Find files satisfying the given metadata set + """ + result = self.gFileCatalogDB.dmeta.findFileIDsByMetadata( + metaDict, path, self.getRemoteCredentials(), startItem, maxItems) + if not result['OK'] or not result['Value']: + return result + + fileIDs = result['Value'] + totalRecords = result['TotalRecords'] + + result = self.gFileCatalogDB.fileManager._getFileLFNs(fileIDs) + if not result['OK']: + return result + + lfnsResultList = result['Value']['Successful'].values() + resultDetails = self.gFileCatalogDB.getFileDetails(lfnsResultList, self.getRemoteCredentials()) + if not resultDetails['OK']: + return resultDetails + + result = S_OK({"TotalRecords": totalRecords, "Records": resultDetails['Value']}) + return result + + def findFilesByMetadataWeb(self, metaDict, path, startItem, maxItems): + """ Find all the files satisfying the given metadata set + """ + result = self.gFileCatalogDB.fmeta.findFilesByMetadata(metaDict, path, self.getRemoteCredentials()) + if not result['OK'] or not result['Value']: + return result + + lfns = [] + for directory in result['Value']: + for fname in result['Value'][directory]: + lfns.append(os.path.join(directory, fname)) + + start = startItem + totalRecords = len(lfns) + if start > totalRecords: + return S_ERROR('Requested files out of existing range') + end = start + maxItems + if end > totalRecords: + end = totalRecords + lfnsResultList = lfns[start:end] + + resultDetails = self.gFileCatalogDB.getFileDetails(lfnsResultList, self.getRemoteCredentials()) + if not resultDetails['OK']: + return resultDetails + + result = S_OK({"TotalRecords": totalRecords, "Records": resultDetails['Value']}) + return result + + def export_getCompatibleMetadata(self, metaDict, path='/'): + """ Get metadata values compatible with the given metadata subset + """ + return self.gFileCatalogDB.dmeta.getCompatibleMetadata(metaDict, path, self.getRemoteCredentials()) + + def export_addMetadataSet(self, setName, setDict): + """ Add a new metadata set + """ + return self.gFileCatalogDB.dmeta.addMetadataSet(setName, setDict, self.getRemoteCredentials()) + + def export_getMetadataSet(self, setName, expandFlag): + """ Add a new metadata set + """ + return self.gFileCatalogDB.dmeta.getMetadataSet(setName, expandFlag, self.getRemoteCredentials()) + +######################################################################################### +# +# Dataset manipulation methods +# + + def export_addDataset(self, datasets): + """ Add a new dynamic dataset defined by its meta query + """ + return self.gFileCatalogDB.datasetManager.addDataset(datasets, self.getRemoteCredentials()) + + def export_addDatasetAnnotation(self, datasetDict): + """ Add annotation to an already created dataset + """ + return self.gFileCatalogDB.datasetManager.addDatasetAnnotation( + datasetDict, self.getRemoteCredentials()) + + def export_removeDataset(self, datasets): + """ Check the given dynamic dataset for changes since its definition + """ + return self.gFileCatalogDB.datasetManager.removeDataset(datasets, self.getRemoteCredentials()) + + def export_checkDataset(self, datasets): + """ Check the given dynamic dataset for changes since its definition + """ + return self.gFileCatalogDB.datasetManager.checkDataset(datasets, self.getRemoteCredentials()) + + def export_updateDataset(self, datasets): + """ Update the given dynamic dataset for changes since its definition + """ + return self.gFileCatalogDB.datasetManager.updateDataset(datasets, self.getRemoteCredentials()) + + def export_getDatasets(self, datasets): + """ Get parameters of the given dynamic dataset as they are stored in the database + """ + return self.gFileCatalogDB.datasetManager.getDatasets(datasets, self.getRemoteCredentials()) + + def export_getDatasetParameters(self, datasets): + """ Get parameters of the given dynamic dataset as they are stored in the database + """ + return self.gFileCatalogDB.datasetManager.getDatasetParameters(datasets, self.getRemoteCredentials()) + + def export_getDatasetAnnotation(self, datasets): + """ Get annotation of the given datasets + """ + return self.gFileCatalogDB.datasetManager.getDatasetAnnotation(datasets, self.getRemoteCredentials()) + + def export_freezeDataset(self, datasets): + """ Freeze the contents of the dataset making it effectively static + """ + return self.gFileCatalogDB.datasetManager.freezeDataset(datasets, self.getRemoteCredentials()) + + def export_releaseDataset(self, datasets): + """ Release the contents of the frozen dataset allowing changes in its contents + """ + return self.gFileCatalogDB.datasetManager.releaseDataset(datasets, self.getRemoteCredentials()) + + def export_getDatasetFiles(self, datasets): + """ Get lfns in the given dataset + """ + return self.gFileCatalogDB.datasetManager.getDatasetFiles(datasets, self.getRemoteCredentials()) + + def getSEDump(self, seName): + """ + Return all the files at a given SE, together with checksum and size + + :param seName: name of the StorageElement + + :returns: S_OK with list of tuples (lfn, checksum, size) + """ + return self.gFileCatalogDB.getSEDump(seName)['Value'] + + def export_streamToClient(self, seName): + """ This method used to transfer the SEDump to the client, + formated as CSV with '|' separation + + :param seName: name of the se to dump + + :returns: the result of the FileHelper + + + """ + + retVal = self.getSEDump(seName) + + try: + csvOutput = BytesIO() + writer = csv.writer(csvOutput, delimiter='|') + for lfn in retVal: + writer.writerow(lfn) + + # csvOutput.seek(0) + ret = csvOutput.getvalue() + return ret + + except Exception as e: + gLogger.exception("Exception while sending seDump", repr(e)) + return S_ERROR("Exception while sendind seDump: %s" % repr(e)) + finally: + csvOutput.close() diff --git a/FrameworkSystem/Client/ComponentInstaller.py b/FrameworkSystem/Client/ComponentInstaller.py index f6e1a195354..b7274b6e75d 100644 --- a/FrameworkSystem/Client/ComponentInstaller.py +++ b/FrameworkSystem/Client/ComponentInstaller.py @@ -54,9 +54,11 @@ /LocalInstallation/VirtualOrganization: Name of the main Virtual Organization (default: None) """ - -from __future__ import print_function, absolute_import +from __future__ import absolute_import from __future__ import division +from __future__ import print_function + +__RCSID__ = "$Id$" import os import io @@ -464,7 +466,7 @@ def _getCentralCfg(self, installCfg): centralCfg['Registry']['Groups'][group].addKey( # pylint: disable=unsubscriptable-object 'Properties', '', '') - properties = centralCfg['Registry']['Groups'][adminGroupName].getOption( # pylint: disable=unsubscriptable-object + properties = centralCfg['Registry']['Groups'][adminGroupName].getOption( # noqa # pylint: disable=unsubscriptable-object 'Properties', []) for prop in adminGroupProperties: if prop not in properties: @@ -472,7 +474,7 @@ def _getCentralCfg(self, installCfg): centralCfg['Registry']['Groups'][adminGroupName].appendToOption( # pylint: disable=unsubscriptable-object 'Properties', ', %s' % prop) - properties = centralCfg['Registry']['Groups'][defaultGroupName].getOption( # pylint: disable=unsubscriptable-object + properties = centralCfg['Registry']['Groups'][defaultGroupName].getOption( # noqa # pylint: disable=unsubscriptable-object 'Properties', []) for prop in defaultGroupProperties: if prop not in properties: @@ -2507,5 +2509,132 @@ def execCommand(self, timeout, cmd): return result + def installTornado(self): + """ + Install runit directory for the tornado, and add the configuration of the required service + """ + # Check if the Tornado itself is already installed + runitCompDir = os.path.join(self.runitDir, 'Tornado', 'Tornado') + if os.path.exists(runitCompDir): + msg = "Tornado_Tornado already installed" + gLogger.notice(msg) + return S_OK(runitCompDir) + + # Check the setup for the given system + result = gConfig.getOption('DIRAC/Setups/%s/Tornado' % (CSGlobals.getSetup())) + if not result['OK']: + return result + self.instance = result['Value'] + + # Now do the actual installation + try: + + self._createRunitLog(runitCompDir) + + runFile = os.path.join(runitCompDir, 'run') + with io.open(runFile, 'wt') as fd: + fd.write( + u"""#!/bin/bash + rcfile=%(bashrc)s + [ -e $rcfile ] && source $rcfile + # + exec 2>&1 + # + # + exec python $DIRAC/DIRAC/Core/Tornado/scripts/tornado-start-all.py -ddd + """ % {'bashrc': os.path.join(self.instancePath, 'bashrc')}) + + os.chmod(runFile, self.gDefaultPerms) + + except Exception: + error = 'Failed to prepare self.setup forTornado' + gLogger.exception(error) + if self.exitOnError: + DIRAC.exit(-1) + return S_ERROR(error) + + result = self.execCommand(5, [runFile]) + + gLogger.notice(result['Value'][1]) + + return S_OK(runitCompDir) + + def addTornadoOptionsToCS(self, gConfig_o): + """ + Add the section with the component options to the CS + """ + + if gConfig_o: + gConfig_o.forceRefresh() + + instanceOption = cfgPath('DIRAC', 'Setups', self.setup, 'Tornado') + + if gConfig_o: + compInstance = gConfig_o.getValue(instanceOption, '') + else: + compInstance = self.localCfg.getOption(instanceOption, '') + if not compInstance: + return S_ERROR('%s not defined in %s' % (instanceOption, self.cfgFile)) + tornadoSection = cfgPath('Systems', 'Tornado', compInstance) + + cfg = self.__getCfg(tornadoSection, 'Port', 8443) + # cfg.setOption(cfgPath(tornadoSection, 'Password'), self.mysqlPassword) + return self._addCfgToCS(cfg) + + def setupTornadoService(self, system, component, extensions, + componentModule='', checkModule=True): + """ + Install and create link in startup + """ + + # Create the startup entry now + # Force the system and component to be 'Tornado' but preserve the interface and the code + # just to allow for easier refactoring maybe later + system = 'Tornado' + component = 'Tornado' + componentType = 'Tornado' + runitCompDir = os.path.join(self.runitDir, 'Tornado', 'Tornado') + startCompDir = os.path.join(self.startDir, '%s_%s' % (system, component)) + mkDir(self.startDir) + if not os.path.lexists(startCompDir): + gLogger.notice('Creating startup link at', startCompDir) + mkLink(runitCompDir, startCompDir) + + # Wait for the service to be recognised (can't use isfile as supervise/ok is a device) + start = time.time() + while (time.time() - 10) < start: + time.sleep(1) + if os.path.exists(os.path.join(startCompDir, 'supervise', 'ok')): + break + else: + return S_ERROR('Failed to find supervise/ok for component %s_%s' % (system, component)) + + # Check the runsv status + start = time.time() + while (time.time() - 20) < start: + result = self.getStartupComponentStatus([(system, component)]) + if not result['OK']: + continue + if result['Value'] and result['Value']['%s_%s' % (system, component)]['RunitStatus'] == "Run": + break + time.sleep(1) + else: + return S_ERROR('Failed to start the component %s_%s' % (system, component)) + + resDict = {} + resDict['ComponentType'] = componentType + resDict['RunitStatus'] = result['Value']['%s_%s' % (system, component)]['RunitStatus'] + + return S_OK(resDict) + + # port = compCfg.getOption('Port', 0) + # if port and self.host: + # urlsPath = cfgPath('Systems', system, compInstance, 'URLs') + # cfg.createNewSection(urlsPath) + # failoverUrlsPath = cfgPath('Systems', system, compInstance, 'FailoverURLs') + # cfg.createNewSection(failoverUrlsPath) + # cfg.setOption(cfgPath(urlsPath, component), + # 'dips://%s:%d/%s/%s' % (self.host, port, system, component)) + gComponentInstaller = ComponentInstaller() diff --git a/FrameworkSystem/Client/MonitoringClient.py b/FrameworkSystem/Client/MonitoringClient.py index d3e3f5885fc..2004e711180 100755 --- a/FrameworkSystem/Client/MonitoringClient.py +++ b/FrameworkSystem/Client/MonitoringClient.py @@ -8,7 +8,7 @@ __RCSID__ = "$Id" import time - +import os import six import DIRAC @@ -85,7 +85,12 @@ def registerMonitoringClient(self, mc): self.__mcList.append(mc) -gMonitoringFlusher = MonitoringFlusher() +# DIRAC_USE_TORNADO_IOLOOP is defined by starting scripts +if os.environ.get('DIRAC_USE_TORNADO_IOLOOP', 'false').lower() in ('yes', 'true'): + from DIRAC.FrameworkSystem.Client.MonitoringClientIOLoop import MonitoringFlusherTornado + gMonitoringFlusher = MonitoringFlusherTornado() +else: + gMonitoringFlusher = MonitoringFlusher() class MonitoringClient(object): @@ -104,6 +109,7 @@ class MonitoringClient(object): COMPONENT_AGENT = "agent" COMPONENT_WEB = "web" COMPONENT_SCRIPT = "script" + COMPONENT_TORNADO = "tornado" __validMonitoringValues = six.integer_types + (float,) @@ -177,6 +183,8 @@ def initialize(self): self.setComponentName('WebApp') elif self.sourceDict['componentType'] == self.COMPONENT_SCRIPT: self.cfgSection = "/Script" + elif self.sourceDict['componentType'] == self.COMPONENT_TORNADO: + self.cfgSection = "/Tornado" else: raise Exception("Component type has not been defined") gMonitoringFlusher.registerMonitoringClient(self) diff --git a/FrameworkSystem/Client/MonitoringClientIOLoop.py b/FrameworkSystem/Client/MonitoringClientIOLoop.py new file mode 100644 index 00000000000..a757b260020 --- /dev/null +++ b/FrameworkSystem/Client/MonitoringClientIOLoop.py @@ -0,0 +1,36 @@ +""" + Add-on to make the MonitoringClient works with an IOLoop from a Tornado Server +""" + + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +__RCSID__ = "$Id$" + +import tornado.ioloop +from DIRAC import gLogger + + +class MonitoringFlusherTornado(object): + """ + This class flushes all monitoring clients registered + Works with the Tornado IOLoop + """ + + def __init__(self): + self.__mcList = [] + gLogger.info("Using MonitoringClient in IOLoop mode") + # Here we don't need to use IOLoop.current(), tornado will attach periodic callback to the current IOLoop himself + # We set callback every 5 minnutes + tornado.ioloop.PeriodicCallback(self.flush, 300000).start() + + def flush(self, allData=False): + gLogger.info('Flushing monitoring') + for mc in self.__mcList: + mc.flush(allData) + + def registerMonitoringClient(self, mc): + if mc not in self.__mcList: + self.__mcList.append(mc) diff --git a/FrameworkSystem/scripts/dirac-install-tornado-service.py b/FrameworkSystem/scripts/dirac-install-tornado-service.py new file mode 100755 index 00000000000..f857fb196b4 --- /dev/null +++ b/FrameworkSystem/scripts/dirac-install-tornado-service.py @@ -0,0 +1,107 @@ +#!/usr/bin/env python +""" +Do the initial installation and configuration of a DIRAC service based on tornado +""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +__RCSID__ = "$Id$" + +from DIRAC import gConfig, gLogger, S_OK +from DIRAC.ConfigurationSystem.Client.Helpers import getCSExtensions +from DIRAC.FrameworkSystem.Utilities import MonitoringUtilities +from DIRAC.Core.Base import Script +from DIRAC import exit as DIRACexit +from DIRAC.FrameworkSystem.Client.ComponentInstaller import gComponentInstaller + +__RCSID__ = "$Id$" + +gComponentInstaller.exitOnError = True + +overwrite = False + + +def setOverwrite(opVal): + global overwrite + overwrite = True + return S_OK() + + +module = '' +specialOptions = {} + + +def setModule(optVal): + global specialOptions, module + specialOptions['Module'] = optVal + module = optVal + return S_OK() + + +def setSpecialOption(optVal): + global specialOptions + option, value = optVal.split('=') + specialOptions[option] = value + return S_OK() + + +Script.registerSwitch("w", "overwrite", "Overwrite the configuration in the global CS", setOverwrite) +Script.registerSwitch("m:", "module=", "Python module name for the component code", setModule) +Script.registerSwitch("p:", "parameter=", "Special component option ", setSpecialOption) +Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], + 'Usage:', + ' %s [option|cfgfile] ... System Component|System/Component' % Script.scriptName, + 'Arguments:', + ' System: Name of the DIRAC system (ie: WorkloadManagement)', + ' Service: Name of the DIRAC component (ie: Matcher)'])) + +Script.parseCommandLine() +args = Script.getPositionalArgs() + +if len(args) == 1: + args = args[0].split('/') + +if len(args) != 2: + Script.showHelp() + DIRACexit(1) + +system = args[0] +component = args[1] +compOrMod = module if module else component + + +result = gComponentInstaller.addDefaultOptionsToCS(gConfig, 'service', system, component, + getCSExtensions(), + specialOptions=specialOptions, + overwrite=overwrite) + +if not result['OK']: + gLogger.error(result['Message']) + DIRACexit(1) + + +result = gComponentInstaller.addTornadoOptionsToCS(gConfig) +if not result['OK']: + gLogger.error(result['Message']) + DIRACexit(1) + +result = gComponentInstaller.installTornado() +if not result['OK']: + gLogger.error(result['Message']) + DIRACexit(1) + + +gLogger.notice('Successfully installed component %s in %s system, now setting it up' % (component, system)) +result = gComponentInstaller.setupTornadoService(system, component, getCSExtensions(), module) +if not result['OK']: + gLogger.error(result['Message']) + DIRACexit(1) + +result = MonitoringUtilities.monitorInstallation('service', system, component, module) +if not result['OK']: + gLogger.error(result['Message']) + DIRACexit(1) +gLogger.notice('Successfully completed the installation of %s/%s' % (system, component)) +DIRACexit() diff --git a/Resources/Catalog/FileCatalogClient.py b/Resources/Catalog/FileCatalogClient.py index 6f70723e681..e8f33717855 100644 --- a/Resources/Catalog/FileCatalogClient.py +++ b/Resources/Catalog/FileCatalogClient.py @@ -7,7 +7,7 @@ import os from DIRAC import S_OK, S_ERROR -from DIRAC.Core.DISET.TransferClient import TransferClient +from DIRAC.Core.Tornado.Client.ClientSelector import TransferClientSelector as TransferClient from DIRAC.Core.Security.ProxyInfo import getVOfromProxyGroup from DIRAC.ConfigurationSystem.Client.Helpers.Registry import getVOMSAttributeForGroup, getDNForUsername diff --git a/Resources/Storage/GFAL2_StorageBase.py b/Resources/Storage/GFAL2_StorageBase.py index b8c7e572f83..ee7eb250df4 100644 --- a/Resources/Storage/GFAL2_StorageBase.py +++ b/Resources/Storage/GFAL2_StorageBase.py @@ -265,7 +265,7 @@ def putFile(self, path, sourceSize=0): failed = {} successful = {} - for dest_url, src_file in urls.iteritems(): + for dest_url, src_file in urls.items(): if not src_file: errStr = "GFAL2_StorageBase.putFile: Source file not set. Argument must be a dictionary \ (or a list of a dictionary) {url : local path}" @@ -796,7 +796,7 @@ def prestageFileStatus(self, path): failed = {} successful = {} - for path, token in urls.iteritems(): + for path, token in urls.items(): res = self.__prestageSingleFileStatus(path, token) if not res['OK']: @@ -929,7 +929,7 @@ def releaseFile(self, path): failed = {} successful = {} - for path, token in urls.iteritems(): + for path, token in urls.items(): res = self.__releaseSingleFile(path, token) if not res['OK']: @@ -1193,7 +1193,7 @@ def listDirectory(self, path): directories = [] - for url, isDirectory in res['Value']['Successful'].iteritems(): + for url, isDirectory in res['Value']['Successful'].items(): if isDirectory: directories.append(url) else: @@ -1438,7 +1438,7 @@ def putDirectory(self, path): successful = {} failed = {} - for destDir, sourceDir in urls.iteritems(): + for destDir, sourceDir in urls.items(): if not sourceDir: errStr = 'No source directory set, make sure the input format is correct { dest. dir : source dir }' return S_ERROR(errStr) @@ -1518,7 +1518,7 @@ def __putSingleDirectory(self, src_directory, dest_directory): log.debug('Failed to put files to storage.', res['Message']) allSuccessful = False else: - for fileSize in res['Value']['Successful'].itervalues(): + for fileSize in res['Value']['Successful'].values(): filesPut += 1 sizePut += fileSize if res['Value']['Failed']: @@ -1715,7 +1715,7 @@ def __getSingleDirectorySize(self, path): directorySize = 0 directoryFiles = 0 # itervalues returns a list of values of the dictionnary - for fileDict in res['Value']['Files'].itervalues(): + for fileDict in res['Value']['Files'].values(): directorySize += fileDict['Size'] directoryFiles += 1 @@ -1780,7 +1780,7 @@ def _getExtendedAttributes(self, path, attributes=None): """ Get all the available extended attributes of path :param self: self reference - :param str path: path of which we wan't extended attributes + :param str path: path of which we want extended attributes :param str list attributes: list of extended attributes we want to receive :return: S_OK( attributeDict ) if successful. Where the keys of the dict are the attributes and values the respective values diff --git a/docs/source/AdministratorGuide/CommandReference/dirac-admin-add-host.rst b/docs/source/AdministratorGuide/CommandReference/dirac-admin-add-host.rst index 9a87ca8a58d..4d9dff48f5b 100644 --- a/docs/source/AdministratorGuide/CommandReference/dirac-admin-add-host.rst +++ b/docs/source/AdministratorGuide/CommandReference/dirac-admin-add-host.rst @@ -12,7 +12,7 @@ Usage:: Arguments:: - Property=: Other properties to be added to the User like (Responsible=XXXX) + Property=: Other properties to be added to the User like (Responsible=XXX) Options:: diff --git a/docs/source/AdministratorGuide/CommandReference/dirac-admin-ce-info.rst b/docs/source/AdministratorGuide/CommandReference/dirac-admin-ce-info.rst index 764ca3944f6..9e946a8927a 100644 --- a/docs/source/AdministratorGuide/CommandReference/dirac-admin-ce-info.rst +++ b/docs/source/AdministratorGuide/CommandReference/dirac-admin-ce-info.rst @@ -14,10 +14,6 @@ Arguments:: CE: Name of the CE -Options:: - - -G --Grid : Define the Grid where to look (Default: LCG) - Example:: diff --git a/docs/source/AdministratorGuide/CommandReference/dirac-proxy-init.rst b/docs/source/AdministratorGuide/CommandReference/dirac-proxy-init.rst index 4fca413987c..8f321187ee6 100644 --- a/docs/source/AdministratorGuide/CommandReference/dirac-proxy-init.rst +++ b/docs/source/AdministratorGuide/CommandReference/dirac-proxy-init.rst @@ -26,7 +26,6 @@ Options:: -r --rfc : Create an RFC proxy, true by default, deprecated flag -L --legacy : Create a legacy non-RFC proxy -U --upload : Upload a long lived proxy to the ProxyManager - -P --uploadPilot : Upload a long lived pilot proxy to the ProxyManager -M --VOMS : Add voms extension Example:: diff --git a/docs/source/AdministratorGuide/CommandReference/index.rst b/docs/source/AdministratorGuide/CommandReference/index.rst index 527159c8566..2c9375946d2 100644 --- a/docs/source/AdministratorGuide/CommandReference/index.rst +++ b/docs/source/AdministratorGuide/CommandReference/index.rst @@ -99,6 +99,7 @@ Managing DIRAC installation: dirac-install-component dirac-install-db dirac-install-web-portal + dirac-install-tornado-service dirac-install dirac-install-extension dirac-uninstall-component diff --git a/docs/source/AdministratorGuide/ServerInstallations/InstallingDiracServer.rst b/docs/source/AdministratorGuide/ServerInstallations/InstallingDiracServer.rst index a7981bfcb84..3a10289c1ed 100644 --- a/docs/source/AdministratorGuide/ServerInstallations/InstallingDiracServer.rst +++ b/docs/source/AdministratorGuide/ServerInstallations/InstallingDiracServer.rst @@ -197,6 +197,8 @@ Couple notes: * SAN in your certificates: if you are contacting a machine using its aliases, make sure that all the aliases are in the SubjectAlternativeName (SAN) field of the certificates * FQDN in the configuration: SAN normally contains only FQDN, so make sure you use the FQDN in the CS as well (e.g. ``mymachine.cern.ch`` and not ``mymachine``) +.. _using_own_CA: + ----------------- Using your own CA ----------------- diff --git a/docs/source/AdministratorGuide/Systems/Configuration/index.rst b/docs/source/AdministratorGuide/Systems/Configuration/index.rst index 097d6cab7e6..8bf8dd40204 100644 --- a/docs/source/AdministratorGuide/Systems/Configuration/index.rst +++ b/docs/source/AdministratorGuide/Systems/Configuration/index.rst @@ -2,5 +2,23 @@ Configuration System ==================== -.. contents:: Table of contents - :depth: 3 +The configuration system serves the configuration to any other client (be it another server or a standard client). +The infrastructure is master/slave based. + +****** +Master +****** + +The master Server holds the central configuration in a local file. This file is then served to the clients, and synchronized with the slave servers. + +the master server also regularly pings the slave servers to make sure they are still alive. If not, they are removed from the list of CS. + +When changes are committed to the master, a backup of the existing configuration file is made in ``etc/csbackup``. + +****** +Slaves +****** + +Slave server registers themselves to the master when starting. +They synchronize their configuration on a regular bases (every 5 minutes by default). +Note that the slave CS do not hold the configuration in a local file, but only in memory. \ No newline at end of file diff --git a/docs/source/AdministratorGuide/technologyPreviews.rst b/docs/source/AdministratorGuide/technologyPreviews.rst index 8c4403f1291..89899308093 100644 --- a/docs/source/AdministratorGuide/technologyPreviews.rst +++ b/docs/source/AdministratorGuide/technologyPreviews.rst @@ -27,3 +27,7 @@ The changes from one stage to the next is controlled by environment variables, a The last stage (JSON only) will be the default of the following release, so before upgrading you will have to go through the previous steps. +HTTPS Services +============== + +The aim is to replace the DISET services with HTTPS services. The changes should be almost transparent for users/admins. However, because it is still very much in the state of preview, we do not yet describe how/what to change. If you really want to play around, please check :ref:`httpsTornado`. diff --git a/docs/source/DeveloperGuide/Internals/Core/ClientServer.rst b/docs/source/DeveloperGuide/Internals/Core/ClientServer.rst index f6ced21499b..3b5b5580e5b 100644 --- a/docs/source/DeveloperGuide/Internals/Core/ClientServer.rst +++ b/docs/source/DeveloperGuide/Internals/Core/ClientServer.rst @@ -167,7 +167,7 @@ Complete path of packages are not on the diagram for readability: You can see that the client sends a proposalTuple, proposalTuple contain (service, setup, ClientVO) then (typeOfCall, method) and finaly extra-credentials. -e.g:: +e.g.:: (('Framework/serviceName', 'DeveloperSetup', 'unknown'), ('RPC', 'methodName'), '') diff --git a/docs/source/DeveloperGuide/TornadoServices/clientservice.png b/docs/source/DeveloperGuide/TornadoServices/clientservice.png new file mode 100644 index 00000000000..0b8e8e20ca6 Binary files /dev/null and b/docs/source/DeveloperGuide/TornadoServices/clientservice.png differ diff --git a/docs/source/DeveloperGuide/TornadoServices/clientservice.uml b/docs/source/DeveloperGuide/TornadoServices/clientservice.uml new file mode 100644 index 00000000000..7a41fac6508 --- /dev/null +++ b/docs/source/DeveloperGuide/TornadoServices/clientservice.uml @@ -0,0 +1,15 @@ +@startuml +Client -> TornadoServer: connect +Client<-> TornadoServer: handshake +Client -> TornadoServer: POST + args +note over TornadoServer: Routing by Tornado +TornadoServer -> TornadoService: initialize() +note over TornadoService: Decoding user certificate\nand extracting credentials +TornadoServer -> TornadoService: prepare() +TornadoService -> AuthManager: authQuery() +AuthManager -> TornadoService: S_OK/S_ERROR +TornadoServer -> TornadoService: post() +TornadoService-> YourServiceHandler: export_method(args) +YourServiceHandler -> TornadoService: S_OK/S_ERROR +TornadoService -> Client: S_OK/S_ERROR +@enduml \ No newline at end of file diff --git a/docs/source/DeveloperGuide/TornadoServices/index.rst b/docs/source/DeveloperGuide/TornadoServices/index.rst new file mode 100644 index 00000000000..a4995edbcf9 --- /dev/null +++ b/docs/source/DeveloperGuide/TornadoServices/index.rst @@ -0,0 +1,374 @@ +.. _httpsTornado: + +=========================== +HTTPS Services with Tornado +=========================== + +.. contents:: + + +************ +Presentation +************ + +This page summarizes the changes between DISET and HTTPS. You can all also see these presentations: + +- `Presentation of HTTPS in DIRAC `_. +- `Presentation of HTTPS migration `_. +- `Summary presentation (latest) `. + + + +******* +Service +******* + +.. graphviz:: + + digraph { + TornadoServer -> YourServiceHandler [label=use]; + YourServiceHandler -> TornadoService[label=inherit]; + + + TornadoServer [shape=polygon,sides=4, label = "DIRAC.Core.Tornado.Server.TornadoServer"]; + TornadoService [shape=polygon,sides=4, label = "DIRAC.Core.Tornado.Server.TornadoService"]; + YourServiceHandler [shape=polygon,sides=4]; + + } + +Service returns to Client S_OK/S_ERROR encoded in JSON + +Each service exposes only one route of the form ``System/Component`` with a ``POST`` handler. The semantic of the ``POST`` call is described bellow. + +********************************************************* +Important changes between DISET server and Tornado Server +********************************************************* + +Internal structure +****************** + +- :py:class:`~DIRAC.Core.DISET.ServiceReactor` is now :py:class:`~DIRAC.Core.Tornado.Server.TornadoServer` +- :py:class:`~DIRAC.Core.DISET.private.Service` and :py:class:`~DIRAC.Core.DISET.RequestHandler` are now merge into :py:class:`~DIRAC.Core.Tornado.Server.TornadoService` +- CallStack from S_ERROR are deleted when they are returned to client. +- Common config for all services, there is no more specific config/service. But you can still give extra config files in the command line when you start a HTTPS server. +- Server returns HTTP status codes like ``200 OK`` or ``401 Forbidden``. Not used by client for now but open possibility for usage with external services (like a REST API) + +How to write service +******************** + +Nothing better than an example:: + + from DIRAC.Core.Tornado.Server.TornadoService import TornadoService + class yourServiceHandler(TornadoService): + + @classmethod + def initializeHandler(cls, infosDict): + ## Called 1 time, at first request + + def initializeRequest(self): + ## Called at each request + + auth_someMethod = ['all'] + def export_someMethod(self): + ## Insert your method here, don't forget the return should be serializable + ## Returned value may be an S_OK/S_ERROR + ## You don't need to serialize in JSON, Tornado will do it + +Writing a service for tornado and DISET is similar. You have to define your method starting with ``export_``, and your initialization method is a class method called ``initializeHandler``. +Main changes in tornado are: + +- Service are initialized at first request +- You **should not** write a method called ``initialize`` because Tornado already use that name, so the ``initialize`` from DISET handlers became ``initializeRequest`` +- ``infosDict``, arguments of initializedHandler is not really the same as for DISET: all transport related matters are removed. +- There is no parameter type check any more: attributes like ``types_yourMethod`` are ignored. +- Auth attributes are still there (``auth_yourMethod``). + +The interface of the DISET request handler was preserved, in particular: +* ``getCSOption`` +* ``getRemoteAddress`` +* ``getRemoteCredentials`` +* ``srv_getCSOption`` +* ``srv_getRemoteAddress`` +* ``srv_getRemoteCredentials`` +* ``srv_getFormattedRemoteCredentials`` +* ``srv_getServiceName`` +* ``srv_getURL``. + + +How to start server +******************* + +The easy way, use ``DIRAC/Core/Tornado/script/tornado-start-all.py`` it will start all services registered in configuration ! To register a service you just have to add the service in the CS and ``Protocol = https``. It may look like this:: + + DIRAC + { + Setups + { + Tornado = DevInstance + } + } + + Systems { + Tornado + { + DevInstance + { + Port = 443 + } + } + Framework + { + DevInstance + { + Services + { + DummyTornado + { + Protocol = https + } + } + } + } + } + + +But you can also control more settings by launching tornado yourself:: + + from DIRAC.Core.Tornado.Server.TornadoServer import TornadoServer + serverToLaunch = TornadoServer(youroptions) + serverToLaunch.startTornado() + +Options available are: + +- services, should be a list, to start only these services +- debugSSL, True or False, activate debug mode of Tornado (includes autoreload) and SSL, for extra logs use -ddd in the command line +- port, int, if you want to override value from config. If it's also not defined in config, it use 443. + +This start method can be useful for developing new service or create starting script for a specific service, like the Configuration System (as master). + +TransferClient +************** + +There is no specific client for transfering files anymore. In fact, the whole idea of directly serving file will eventually disapear and be replaced with redirections to real content streaming server. In the meantine, in order to keep some compatibility, the features were implemented, but require some changes on the server side: + +- ``transfer_toClient`` needs to be renamed ``export_streamToClient`` +- It needs to return the whole file content at once +- The parameter ``fileHelper`` is removed + + +For example:: + + def transfer_toClient(self, myFileToSend, token, fileHelper): + + # Do whatever with the token + + with open(myFileToSend, 'r') as fd: + ret = fileHelper.DataSourceToNetwork(fd) + return ret + +Simply becomes:: + + def export_streamToClient(self, myFileToSend, token): + + # Do whatever with the token + + with open(myFileToSend, 'r') as fd: + return fd.read() + + +From the client side, no change is needed since :py:meth:`DIRAC.Core.Tornado.Client.TornadoClient.TornadoClient.receiveFile` keeps the interface + +This procedure is not optimized server side (see commented ``export_streamToClient`` implementation in :py:class`DIRAC.Core.Tornado.Server.TornadoService.TornadoService`). + +The ``transfer_fromClient`` equivalent has not yet been implemented as it concerns only very few cases (basically DIRAC SE and SandboxStore) + +****** +Client +****** + +.. graphviz:: + + digraph { + TornadoClient -> TornadoBaseClient [label=inherit] + TornadoBaseClient -> Requests [label=use] + + TornadoClient [shape=polygon,sides=4, label="DIRAC.Core.Tornado.Client.TornadoClient"]; + TornadoBaseClient [shape=polygon,sides=4, label="DIRAC.Core.Tornado.Client.private.TornadoBaseClient"]; + Requests [shape=polygon,sides=4] + } + +This diagram present what is behind TornadoClient, but you should use :py:class:`DIRAC.Core.Base.Client` ! The new client integrate a selection system which select for you between HTTPS and DISET client. + +In your client module when you inherit from :py:class:`DIRAC.Core.Base.Client` you can define ``httpsClient`` with another client, it can be useful when you can't serialize some data in JSON. Here the step to create and use a JSON patch: + +- Create a class which inherit from :py:class:`~DIRAC.Core.Tornado.Client.TornadoClient` +- For every method who need a JSON patch create a method with the same name as the service +- Use self.executeRPC to send / receive datas + +You can also see this example:: + + class ConfigurationServerJSON(TornadoClient): + """ + The specific client for configuration system. + To avoid JSON limitation the HTTPS handler encode data in base64 + before sending them, this class only decode the base64 + An exception is made with CommitNewData which ENCODE in base64 + """ + def getCompressedData(self): + """ + Transmit request to service and get data in base64, + it decode base64 before returning + + :returns str:Configuration data, compressed + """ + retVal = self.executeRPC('getCompressedData') + if retVal['OK']: + retVal['Value'] = b64decode(retVal['Value']) + return retVal + + + + +Behind :py:class:`~DIRAC.Core.Tornado.Client.TornadoClient` the `requests `_ library sends a HTTP POST request with: + +- method : str with method name +- args: your arguments encoded in JSON +- clientVO: The VO of client +- extraCredentials: (if apply) Extra informations to authenticate client + +Service is determined by server thanks to URL rooting, not with port like in DISET. + +By default server listen on port 8443. + +Contacting the service using ``DIRAC``:: + + In [7]: from DIRAC.Resources.Catalog.FileCatalogClient import FileCatalogClient + ...: FileCatalogClient().whoami() + ...: + Out[7]: + {u'OK': True, + u'Value': {u'DN': u'/C=ch/O=DIRAC/OU=DIRAC CI/CN=ciuser/emailAddress=lhcb-dirac-ci@cern.ch', + u'group': u'dirac_user', + u'identity': u'/C=ch/O=DIRAC/OU=DIRAC CI/CN=ciuser/emailAddress=lhcb-dirac-ci@cern.ch', + u'isLimitedProxy': False, + u'isProxy': True, + u'issuer': u'/C=ch/O=DIRAC/OU=DIRAC CI/CN=ciuser/emailAddress=lhcb-dirac-ci@cern.ch', + u'properties': [u'NormalUser'], + u'secondsLeft': 86141, + u'subject': u'/C=ch/O=DIRAC/OU=DIRAC CI/CN=ciuser/emailAddress=lhcb-dirac-ci@cern.ch/CN=2409820262', + u'username': u'adminusername', + u'validDN': False, + u'validGroup': False}, + 'rpcStub': (('DataManagement/FileCatalog', + {'skipCACheck': True, 'timeout': 600}), + 'whoami', + [])} + + + + +Contacting the service using ``requests``:: + + In [20]: url = 'https://server:8443/DataManagement/TornadoFileCatalog' + ...: cert = '/tmp/x509up_u1000' + ...: kwargs = {'method':'whoami'} + ...: caPath = '/home/dirac/ClientInstallDIR/etc/grid-security/certificates/' + ...: with requests.post(url, data=kwargs, cert=cert, verify=caPath) as r: + ...: print r.json() + ...: + {u'OK': True, u'Value': {u'DN': u'/C=ch/O=DIRAC/OU=DIRAC CI/CN=ciuser/emailAddress=lhcb-dirac-ci@cern.ch', u'username': u'adminusername', u'secondsLeft': 85846, u'group': u'dirac_user', u'isProxy': True, u'validGroup': False, u'validDN': False, u'issuer': u'/C=ch/O=DIRAC/OU=DIRAC CI/CN=ciuser/emailAddress=lhcb-dirac-ci@cern.ch', u'isLimitedProxy': False, u'properties': [u'NormalUser'], u'identity': u'/C=ch/O=DIRAC/OU=DIRAC CI/CN=ciuser/emailAddress=lhcb-dirac-ci@cern.ch', u'subject': u'/C=ch/O=DIRAC/OU=DIRAC CI/CN=ciuser/emailAddress=lhcb-dirac-ci@cern.ch/CN=2409820262'}} + + + +***************************** +Client / Service interactions +***************************** + +.. image:: clientservice.png + :align: center + :alt: Client/Service interactions + +***************************************************** +Important changes between TornadoClient and RPCClient +***************************************************** + +Internal structure +****************** + +- :py:class:`~DIRAC.Core.DISET.private.innerRPCClient` and :py:class:`~DIRAC.Core.DISET.RPCClient` are now a single class: :py:class:`~DIRAC.Core.Tornado.Client.TornadoClient`. Interface and usage stay the same. +- :py:class:`~DIRAC.Core.Tornado.Client.private.TornadoBaseClient` is the new :py:class:`~DIRAC.Core.DISET.private.BaseClient`. Most of code is copied from :py:class:`~DIRAC.Core.DISET.private.BaseClient` but some method have been rewrited to use `Requests `_ instead of Transports. Code duplication is done to fully separate DISET and HTTPS but later, some parts can be merged by using a new common class between DISET and HTTPS (these parts are explicitly given in the docstrings). +- :py:class:`~DIRAC.Core.DISET.private.Transports.BaseTransport`, :py:class:`~DIRAC.Core.DISET.private.Transports.PlainTransport` and :py:class:`~DIRAC.Core.DISET.private.Transports.SSLTransport` are replaced by `Requests `_ +- keepAliveLapse is removed from rpcStub returned by Client because `Requests `_ manage it himself. +- Due to JSON limitation you can write some specifics clients who inherit from :py:class:`~DIRAC.Core.Tornado.Client.TornadoClient`, there is a simple example with :py:class:`~DIRAC.ConfigurationSystem.Client.ConfigurationClient.CSJSONClient` who transfer data in base64 to overcome JSON limitations + + +Connections and certificates +**************************** +`Requests `_ library check more than DISET when reading certificates and do some stuff for us: + +- Server certificate **must** have subject alternative names. Requests also check the hostname and you can have connection errors when using "localhost" for example. To avoid them add subject alternative name in certificate. (You can also see https://github.com/shazow/urllib3/issues/497 ). +- If server certificates are used by clients, you must add clientAuth in the extendedKeyUsage (requests also check that). +- In server side M2Crypto is used instead of GSI and conflict are possible between GSI and M2Crypto, to avoid them you can comment 4 lasts lines at ``DIRAC/Core/Security/__init__.py`` +- ``_connect()``, ``_disconnect()`` and ``_purposeAction()`` are removed, ``_connect``/``_disconnect`` are now managed by `requests `_ and ``_purposeAction`` is no longer used is in HTTPS protocol. + + +********************** +How to install Tornado +********************** + + +Requirements +************ + +Two special python packages are needed: +* git+https://github.com/DIRACGrid/tornado.git@iostreamConfigurable : in place of the standard tornado. This adds configurable feature to tornado +* git+https://github.com/DIRACGrid/tornado_m2crypto.git: this allows to use tornado with M2Crypto + + +Install a service +***************** + +``dirac-install-tornado-service`` is your friend. This will install a runit component running ``tornado-start-all``. +Nothing is ready yet to install specific tornado service, like the master CS. + +Start the server +**************** + +To start the server you must define ``OPENSSL_ALLOW_PROXY_CERTS`` and run ``DIRAC/TornadoServices/Scripts/tornado-start-all.py`` (or ``tornado-start-CS.py`` if you try to run a configuration server):: + + OPENSSL_ALLOW_PROXY_CERTS=1 python /opt/dirac/DIRAC/TornadoServices/scripts/tornado-start-all.py + + + + +************ +Launch tests +************ + +pytest +****** +Because for now Tornado does not have "Real" services, you must use some fakes services to compare and test with DISET. +You need tornadoCredDict, diracCredDict, User, UserDirac to run tests. Each test explain how to configure in its docstring. + +The only service available is the Configuration/Server, it will work with HTTPS and DISET services who needs to load configuration with a Configuration/Server. + + + + +Run performance tests +********************* +For performance test unset ``PYTHONOPTIMIZE`` if it is set in your environement:: + + unset PYTHONOPTIMIZE + + +Then you have to start some clients (adapt the port):: + + cd /opt/dirac/DIRAC/test/Integration/TornadoServices + multimech-run perf-test-ping -p 9000 -b 0.0.0.0 + +Modify first lines of ``DIRAC/TornadoServices/test/multi-mechanize/distributed-test.py`` and ``DIRAC/TornadoServices/test/multi-mechanize/plot-distributed-test.py`` (follow instruction of each files) + +On the server start ``DIRAC/test/Integration/TornadoServices/getCPUInfos`` (redirect output to a file) + +Run ``distributed-test.py [NameOfYourTest]`` at the end of execution, the command to plot is given. Before executing command, copy output of ``getCPUInfos`` on ``/tmp/results.txt`` (on your local machine). diff --git a/docs/source/DeveloperGuide/index.rst b/docs/source/DeveloperGuide/index.rst index b407d2d22b9..166f9d74cc3 100644 --- a/docs/source/DeveloperGuide/index.rst +++ b/docs/source/DeveloperGuide/index.rst @@ -41,3 +41,4 @@ group. Internals/index WorkloadManagementSystem/index Externals/index + TornadoServices/index diff --git a/docs/source/UserGuide/CommandReference/DataManagement/dirac-dms-clean-directory.rst b/docs/source/UserGuide/CommandReference/DataManagement/dirac-dms-clean-directory.rst index e410530daba..011414a5a31 100644 --- a/docs/source/UserGuide/CommandReference/DataManagement/dirac-dms-clean-directory.rst +++ b/docs/source/UserGuide/CommandReference/DataManagement/dirac-dms-clean-directory.rst @@ -10,7 +10,7 @@ file catalogs. Usage:: - dirac-dms-clean-directory + dirac-dms-clean-directory Example:: diff --git a/docs/source/UserGuide/CommandReference/DataManagement/dirac-dms-find-lfns.rst b/docs/source/UserGuide/CommandReference/DataManagement/dirac-dms-find-lfns.rst index 999a00c6319..48092d7eea7 100644 --- a/docs/source/UserGuide/CommandReference/DataManagement/dirac-dms-find-lfns.rst +++ b/docs/source/UserGuide/CommandReference/DataManagement/dirac-dms-find-lfns.rst @@ -12,7 +12,7 @@ Usage:: Arguments:: - metaspec: metadata index specification (of the form: "meta=value" or "meta=1.2.0 # - readline >=6.2.4 in the standard library - simplejson >=3.8.1 - - tornado >=5.0.0,<6.0.0 + #- tornado >=5.0.0,<6.0.0 - typing >=3.6.6 # Pin OpenSSL to avoid: https://github.com/DIRACGrid/DIRAC/issues/4489 - openssl <1.1 - pip: - diraccfg + # This is a fork of tornado with a patch to allow for configurable iostream + - git+https://github.com/DIRACGrid/tornado.git@iostreamConfigurable + # This is an extension of Tornado to use M2Crypto + - git+https://github.com/DIRACGrid/tornado_m2crypto diff --git a/requirements.txt b/requirements.txt index 871ad7363be..e3f71da85cd 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,10 @@ # From repo fts3-rest +#Patch for tornado +git+https://github.com/DIRACGrid/tornado.git@iostreamConfigurable +git+https://github.com/DIRACGrid/tornado_m2crypto.git + # From pypi boto3 #asn1 @@ -52,6 +56,5 @@ typing==3.6.6 hypothesis python-json-logger>=0.1.8 multi-mechanize>=1.2.0 -tornado>=5.0.0 caniusepython3 subprocess32 diff --git a/tests/CI/install_server.sh b/tests/CI/install_server.sh index b4ff3af8316..256103169ce 100755 --- a/tests/CI/install_server.sh +++ b/tests/CI/install_server.sh @@ -70,3 +70,32 @@ EOL dirac-restart-component DataManagement S3Gateway "$DEBUG" echo -e "*** $(date -u) **** DONE Adding S3-INDIRECT SERVER CONFIGURATION" + +# Here, if we are testing HTTPS services, we install the equivalent services and replace the URL in the CS + +if [[ "${TEST_HTTPS:-No}" = "Yes" ]]; +then + echo -e "*** $(date -u) **** Installing Tornado based services" + + # Find all Tornado Handler + # We ignore the Configuration for now because it is a bit special (the master needs to run in a separate process) + # system_component is a space separated System (without the word System), and Component, without the 'Handler.py'. + # For example "DataManagement TornadoFileCatalog" + while read -r system_component; + do + echo -e "*** $(date -u) **** Installing Tornado service ${system_component}" + # do NOT put quotes around ${system_component} since + # we want it to be seen as two arguments + # shellcheck disable=SC2086 + dirac-install-tornado-service -ddd ${system_component}; + # origin_component is the original service before Tornado (FileCatalog vs TornadoFileCatalog for example) + orig_component=$(echo "${system_component}" | sed 's/Tornado//g' | awk '{print $2}'); + # Replace the dips url with the https url in the cs, assuming port 8443 + sed -E "s|( +${orig_component} = )dips://([a-z]+)(:[0-9]+)(/.*/)(.*)|\1https://\2:8443\4Tornado\5|g" -i "${SERVERINSTALLDIR}"/etc/Production.cfg + done< <(find "${SERVERINSTALLDIR}"/DIRAC/ -name 'Tornado*Handler.py' | grep -v Configuration | sed -e 's/Handler.py//g' -e 's/System//g'| awk -F '/' '{print $(NF-2), $NF}') + + # Restart the CS to take all that into account + dirac-restart-component Configuration Server "$DEBUG" + + echo -e "*** $(date -u) **** DONE Installing Tornado services" +fi \ No newline at end of file diff --git a/tests/CI/run_docker_setup.sh b/tests/CI/run_docker_setup.sh index 46d9fb34bd7..618c09ed7f3 100755 --- a/tests/CI/run_docker_setup.sh +++ b/tests/CI/run_docker_setup.sh @@ -97,6 +97,7 @@ prepareEnvironment() { echo "" echo "# Test specific variables" echo "export WORKSPACE=${WORKSPACE}" + echo "export TEST_HTTPS=${TEST_HTTPS}" echo "" echo "# Optional parameters" diff --git a/tests/DISET_HTTPS_migration/README b/tests/DISET_HTTPS_migration/README new file mode 100644 index 00000000000..2e8dea36dbb --- /dev/null +++ b/tests/DISET_HTTPS_migration/README @@ -0,0 +1,2 @@ +These tests were written in order to compare the existing DISET based services with the new Tornado/HTTPs based services. +They were ran by hand, and are kept here just in case for the time being. diff --git a/tests/DISET_HTTPS_migration/TestClientReturnedValuesAndCredentialsExtraction.py b/tests/DISET_HTTPS_migration/TestClientReturnedValuesAndCredentialsExtraction.py new file mode 100644 index 00000000000..0a37af48f2f --- /dev/null +++ b/tests/DISET_HTTPS_migration/TestClientReturnedValuesAndCredentialsExtraction.py @@ -0,0 +1,116 @@ +""" + In this test we want to check if Tornado generate the same credentials dictionnary as DIRAC. + It also test if the correct certificates are sended by client. + + To run this test you must have the handlers who returns credentials dictionnary. + Handlers are diracCredDictHandler and tornadoCredDictHandler just returns these dictionnary + and are stored in DIRAC/FrameworkSystem/Service + + Then you have to start tornado using script tornado-start-all.py in DIRAC/TornadoServices/scripts + and diset with ``dirac-service Framework/diracCredDict`` before running test + + In configuration it have to be set as normal services, it will look like: + + ``` + # In Systems/Service//Framework + Services + { + tornadoCredDict + { + protocol = https + } + diracCredDict + { + Port = 3444 + DisableMonitoring = yes + } + } + ``` + + ``` + URLs + { + tornadoCredDict = https://localhost:443/Framework/tornadoCredDict + diracCredDict = dips://MrBoincHost:3444/Framework/diracCredDict + } + ``` + +""" +from DIRAC.Core.Base.Script import parseCommandLine +parseCommandLine() +import pytest +from DIRAC import gConfig +from DIRAC.ConfigurationSystem.Client.ConfigurationData import gConfigurationData + +from DIRAC.Core.Tornado.Client.TornadoClient import TornadoClient +from DIRAC.Core.DISET.RPCClient import RPCClient +from pytest import mark, fixture +parametrize = mark.parametrize + + +def get_RPC_returnedValue(serviceName, Client): + """ + Get credentials extracted tornado server or Dirac server + """ + service = Client(serviceName) + return service.credDict() + + +def get_all_returnedValues(): + """ + Just code factorisation to who call server and get credential dictionnary + """ + serviceNameTornado = 'Framework/tornadoCredDict' + serviceNameDirac = 'Framework/diracCredDict' + repTornado = TornadoClient(serviceNameTornado).whoami() + repDirac = RPCClient(serviceNameDirac).whoami() + return (repTornado, repDirac) + + +@parametrize('UseServerCertificate', ('true', 'false')) +def test_return_credential_are_equals(UseServerCertificate): + """ + Check if certificates sended AND extraction have same comportement is DISET and HTTPS + """ + gConfigurationData.setOptionInCFG('/DIRAC/Security/UseServerCertificate', UseServerCertificate) + + (repTornado, repDirac) = get_all_returnedValues() + + # Service returns credentials + assert repDirac['Value'] == repTornado['Value'] + + +@parametrize('UseServerCertificate', ('True', 'False')) +def test_rpcStubs_are_equals(UseServerCertificate): + """ + Test if Clients returns the same rpcStubs + + Navigating through array is a bit complicated in this test... + repDirac and repTornado may have the same structure: + + repDirac dict{ + OK: True + rpcStub: tuple{ + ServiceName: str + kwargs: dict{ + *** kwargs used to instanciate client *** + } + methodName: str + arguments: list + } + Value: dict { # NOT USED IN THIS TEST + *** Credentials dictionnary extracted by server *** + } + } + """ + + gConfigurationData.setOptionInCFG('/DIRAC/Security/UseServerCertificate', UseServerCertificate) + (repTornado, repDirac) = get_all_returnedValues() + + # Explicitly removed in Tornado + del repDirac['rpcStub'][0][1]['keepAliveLapse'] + + # rep['rpcStub'] is at form (rpcStub, method, args) where rpcStub is tuple with (serviceName, kwargs) + assert repTornado['rpcStub'][0][0] != repDirac['rpcStub'][0][0] # Services name are different + assert repTornado['rpcStub'][0][1] == repDirac['rpcStub'][0][1] # Check kwargs returned by rpcStub + assert repTornado['rpcStub'][1:] != repDirac['rpcStub'][1:] # Check method/args diff --git a/tests/DISET_HTTPS_migration/TestClientSelectionAndInterface.py b/tests/DISET_HTTPS_migration/TestClientSelectionAndInterface.py new file mode 100644 index 00000000000..9119aa7656d --- /dev/null +++ b/tests/DISET_HTTPS_migration/TestClientSelectionAndInterface.py @@ -0,0 +1,180 @@ +""" + Unit test on client selection: + - By default: RPCClient should be used + - If we use Tornado service TornadoClient is used + + Should work with + - 'Component/Service' + - URL + - List of URL + + Mock Config: + - Service using HTTPS with Tornado + - Service using Diset + + You don't need to setup anything, just run ``pytest TestClientSelection.py`` ! +""" +from __future__ import print_function +import os +import re + + +from pytest import mark, fixture + +from DIRAC.Core.Tornado.Client.ClientSelector import RPCClientSelector +from DIRAC.Core.Tornado.Client.TornadoClient import TornadoClient +from DIRAC.Core.DISET.RPCClient import RPCClient +from DIRAC.ConfigurationSystem.private.ConfigurationClient import ConfigurationClient +from DIRAC.ConfigurationSystem.Client.ConfigurationData import gConfigurationData +from diraccfg import CFG +from DIRAC.Core.Base.Client import Client +from DIRAC.Core.DISET.private.InnerRPCClient import InnerRPCClient + +parametrize = mark.parametrize + + +testCfgFileName = 'test.cfg' + + +@fixture(scope='function') +def config(request): + """ + fixture is the pytest way to declare initalization function. + Scope = module significate that this function will be called only time for this file. + If no scope precised it call config for each test. + + This function can have a return value, it will be the value of 'config' argument for the tests + """ + + cfgContent = ''' + DIRAC + { + Setup=TestSetup + Setups + { + TestSetup + { + WorkloadManagement=MyWM + } + } + } + Systems + { + WorkloadManagement + { + MyWM + { + URLs + { + ServiceDips = dips://$MAINSERVERS$:1234/WorkloadManagement/ServiceDips + ServiceHttps = https://$MAINSERVERS$:1234/WorkloadManagement/ServiceHttps + } + } + } + } + Operations{ + Defaults + { + MainServers = server1, server2 + } + } + ''' + with open(testCfgFileName, 'w') as f: + f.write(cfgContent) + gConfig = ConfigurationClient(fileToLoadList=[testCfgFileName]) # we replace the configuration by our own one. + + # def tearDown(): + # Wait for teardown + yield config + """ + This function is called at the end of the test. + """ + try: + os.remove(testCfgFileName) + except OSError: + pass + # SUPER UGLY: one must recreate the CFG objects of gConfigurationData + # not to conflict with other tests that might be using a local dirac.cfg + gConfigurationData.localCFG = CFG() + gConfigurationData.remoteCFG = CFG() + gConfigurationData.mergedCFG = CFG() + gConfigurationData.generateNewVersion() + print("TearDown") + # request is given by @fixture decorator, addfinalizer set the function who need to be called after the tests + # request.addfinalizer(tearDown) + + +# Tuple with (expectedClient, serviceName) +client_imp = ( + (TornadoClient, 'WorkloadManagement/ServiceHttps'), + (TornadoClient, 'https://server1:1234/WorkloadManagement/ServiceHttps'), + (TornadoClient, + 'https://server1:1234/WorkloadManagement/ServiceHttps,https://server2:1234/WorkloadManagement/ServiceHttps'), + (RPCClient, 'WorkloadManagement/ServiceDips'), + (RPCClient, 'dips://server1:1234/WorkloadManagement/ServiceDips'), + (RPCClient, + 'dips://server1:1234/WorkloadManagement/ServiceDips,dips://server2:1234/WorkloadManagement/ServiceDips'), +) + + +@parametrize('client', client_imp) +def test_selection_when_using_RPCClientSelector(client, config): + """ + One way to call service is to use RPCClient or TornadoClient + If service is HTTPS, it must return client who work with tornado (TornadoClient) + else it must return the RPCClient + """ + clientWanted = client[0] + component_service = client[1] + clientSelected = RPCClientSelector(component_service) + assert isinstance(clientSelected, clientWanted) + + +error_component = ( + 'Too/Many/Sections', + 'JustAName', + "InexistantComponent/InexistantService", + "dummyProtocol://dummy/url") + + +@parametrize('component_service', error_component) +def test_error(component_service, config): + """ + In any other cases (including error cases) it must return RPCClient by default + This test is NOT testing if RPCClient handle the errors + It just test that we get RPCClient and not Tornadoclient + """ + clientSelected = RPCClientSelector(component_service) + assert isinstance(clientSelected, RPCClient) + + +def test_interface(): + """ + Interface of TornadoClient MUST contain at least interface of RPCClient. + BUT a __getattr__ method extends this interface with interface of InnerRPCClient. + """ + interfaceTornadoClient = dir(TornadoClient) + interfaceRPCClient = dir(RPCClient) + dir(InnerRPCClient) + for element in interfaceRPCClient: + # We don't need to test private methods / attribute + # Private methods/attribute starts with __ + # dir also return private methods named with something like _ClassName__PrivateMethodName + if not element.startswith('_'): + assert element in interfaceTornadoClient + + +client_imp = ( + (2, 'WorkloadManagement/ServiceHttps'), + (1, 'https://server1:1234/WorkloadManagement/ServiceHttps') +) + + +@parametrize('client', client_imp) +def test_urls_used_by_TornadoClient(config, client): + # We can't directly get url because they are randomized but we can check if we have right number of URL + + nbOfUrl = client[0] + component_service = client[1] + clientSelected = RPCClientSelector(component_service) + # Little hack to get the private attribute + assert nbOfUrl == clientSelected._TornadoBaseClient__nbOfUrls diff --git a/tests/DISET_HTTPS_migration/TestServerIntegration.py b/tests/DISET_HTTPS_migration/TestServerIntegration.py new file mode 100644 index 00000000000..ca95a991cc6 --- /dev/null +++ b/tests/DISET_HTTPS_migration/TestServerIntegration.py @@ -0,0 +1,91 @@ +""" + Test if same service work on DIRAC and TORNADO + Testing if basic operation works on a dummy example + + These handlers provide a method who's access is always forbidden to test authorization system + + It's just normal services, entry in dirac.cfg are the same as usual. + To start tornado use DIRAC/TornadoServices/scripts/tornado-start-all.py + ``` + Services + { + User + { + Protocol = https + } + UserDirac + { + Port = 3424 + } + } + ``` + + ``` + URLs + { + User = https://MrBoincHost:443/Framework/User + UserDirac = dips://localhost:3424/Framework/UserDirac + } + ``` + +""" + +from DIRAC.Core.Base.Script import parseCommandLine +parseCommandLine() +from string import printable +import datetime +import sys + +from hypothesis import given, settings +from hypothesis.strategies import text + +from DIRAC.Core.DISET.RPCClient import RPCClient as RPCClientDIRAC +from DIRAC.Core.Tornado.Client.TornadoClient import TornadoClient as RPCClientTornado +from DIRAC.Core.Utilities.DErrno import ENOAUTH +from DIRAC import S_ERROR + +from pytest import mark +parametrize = mark.parametrize + +rpc_imp = ((RPCClientTornado, 'Framework/User'), (RPCClientDIRAC, 'Framework/UserDirac')) + + +@parametrize('rpc', rpc_imp) +def test_authorization(rpc): + service = rpc[0](rpc[1]) + + authorisation = service.unauthorized() + assert authorisation['OK'] is False + assert authorisation['Message'] == S_ERROR(ENOAUTH, "Unauthorized query")['Message'] + + +@parametrize('rpc', rpc_imp) +def test_unknown_method(rpc): + service = rpc[0](rpc[1]) + + unknownmethod = service.ThisMethodMayNotExist() + assert unknownmethod['OK'] is False + assert unknownmethod['Message'] == "Unknown method ThisMethodMayNotExist" + + +@parametrize('rpc', rpc_imp) +def test_ping(rpc): + service = rpc[0](rpc[1]) + + assert service.ping()['OK'] + + +@parametrize('rpc', rpc_imp) +@settings(deadline=None, max_examples=42) +@given(data=text(printable, max_size=64)) +def test_echo(rpc, data): + service = rpc[0](rpc[1]) + + assert service.echo(data)['Value'] == data + + +def test_whoami(): # Only in tornado + credDict = RPCClientTornado('Framework/User').whoami()['Value'] + assert 'DN' in credDict + assert 'CN' in credDict + assert 'isProxy' in credDict diff --git a/tests/DISET_HTTPS_migration/multi-mechanize/distributed-test.py b/tests/DISET_HTTPS_migration/multi-mechanize/distributed-test.py new file mode 100644 index 00000000000..8147b1b9307 --- /dev/null +++ b/tests/DISET_HTTPS_migration/multi-mechanize/distributed-test.py @@ -0,0 +1,46 @@ +from __future__ import print_function +import xmlrpclib +import time +import sys +# == BEGIN OF CONFIGURATION == + +# Add all machine who have multimechanize client +serversList = ['137.138.150.194', 'server2'] +# Each multimechanize client must listen the same ports, add the ports here +portList = ['9000', '9001'] + +# END OF CONFIG + +servers = [] + +print("Starting test servers....") +# We send signal to all servers +for port in portList: + for server in serversList: + servers.append(xmlrpclib.ServerProxy("http://%s:%s" % (server, port))) + servers[-1].run_test() + # If there is multiple ports opened on same machine, we wait a little to avoid confusion in multimechanize + time.sleep(2) + + +print("Waiting for results...") +while servers[-1].get_results() == 'Results Not Available': + time.sleep(1) + +# We get all results and write them into files +# There is one file/multimechanize servers +try: + output = sys.argv[1] +except KeyError: + output = str(time.time()) +fileCount = 0 +for server in servers: + fileCount += 1 + fileName = "%s.%s.txt" % (output, fileCount) + print("Writing output file %s" % fileName) + file = open(fileName, 'w') + file.write(server.get_results()) + file.close() + +# We print the command you can copy paste to have the results in a plot +print("python plot-distributedTest.py %s %d" % (output, fileCount)) diff --git a/tests/DISET_HTTPS_migration/multi-mechanize/ping/config.cfg b/tests/DISET_HTTPS_migration/multi-mechanize/ping/config.cfg new file mode 100644 index 00000000000..b9ba7f702d8 --- /dev/null +++ b/tests/DISET_HTTPS_migration/multi-mechanize/ping/config.cfg @@ -0,0 +1,13 @@ + +[global] +run_time = 60 +rampup = 60 +results_ts_interval = 1 +progress_bar = on +console_logging = off +xml_report = off + + +[user_group-1] +threads = 60 +script = v_perf.py diff --git a/tests/DISET_HTTPS_migration/multi-mechanize/ping/test_scripts/v_perf.py b/tests/DISET_HTTPS_migration/multi-mechanize/ping/test_scripts/v_perf.py new file mode 100644 index 00000000000..b2702a22f84 --- /dev/null +++ b/tests/DISET_HTTPS_migration/multi-mechanize/ping/test_scripts/v_perf.py @@ -0,0 +1,18 @@ +from DIRAC.Core.Tornado.Client.TornadoClient import TornadoClient +from DIRAC.Core.DISET.RPCClient import RPCClient +from time import time +from random import randint +import sys + + +class Transaction(object): + def __init__(self): + # If we want we can force to use dirac + if len(sys.argv) > 2 and sys.argv[2].lower() == 'dirac': + self.client = RPCClient('Framework/UserDirac') + else: + self.client = TornadoClient('Framework/User') + return + + def run(self): + self.client.ping() diff --git a/tests/DISET_HTTPS_migration/multi-mechanize/plot-distributedTest.py b/tests/DISET_HTTPS_migration/multi-mechanize/plot-distributedTest.py new file mode 100644 index 00000000000..3eeca13b615 --- /dev/null +++ b/tests/DISET_HTTPS_migration/multi-mechanize/plot-distributedTest.py @@ -0,0 +1,174 @@ +from __future__ import print_function + +import csv +import matplotlib.pyplot as plt +import sys + + +# == BEGIN OF CONFIGURATION == + +# FOR DISPLAY - Enter settings relative to test +system = 'Test ping - timeout=30 - Diset Server' +multimech_thread = 60 +multimech_time = 300 +multimech_rampup = 200 +multimech_clients = 8 +server_maxThreads = 20 + +# For this line, use vmstat and copy the free memory +memoryOffset = 0 + +plt.suptitle( + '%s with %d threads\n %d threads/client - %d clients (total %d threads) \n \ + duration: %dsec - rampup %dsec \n latency between client starts: 2s' % + (system, + server_maxThreads, + multimech_thread, + multimech_clients, + multimech_clients * multimech_thread, + multimech_time, + multimech_rampup)) + + +# END OF CONFIG + + +def get_results(): + if len(sys.argv) < 3: + print("Usage: python plot-distributedTest NAME NUMBEROFFILE") + print("Example: python plot-distributedTest 1532506328.38 2") + sys.exit(1) + + file = sys.argv[1] + count = int(sys.argv[2]) + results = [] + for i in range(1, count + 1): + fileName = "%s.%s.txt" % (file, i) + with open(fileName, 'r') as content: + print("reading %s" % fileName) + lines = content.read().split('\n')[1:-1] + result = [line.split(',') for line in lines] + results.append(result) + content.close() + return results + + +def get_server_stats(): + print("Please specify location to file with server stats:") + serverStatFile = "/tmp/results.txt" # raw_input() + print ("Loading %s" % serverStatFile) + + serverStats = dict() + with open(serverStatFile, 'r') as content_file: + lines = content_file.read().split('\n')[1:] + for line in lines: + line = line.split(';') + serverStats[line[0]] = line[1:] + return serverStats + + +def get_test_begin_end(results): + """ + First result file contain the test started in first + Last result file contain the test started in last + Every test have same duration + So we read first and last line to have begin hour and end hour + (the '2' is because time is registered in the third row) + """ + return (int(results[0][0][2]), int(results[-1][-1][2])) + + +def process_data(results, serverStats): + # Begin and end are timestamps + (begin, end) = get_test_begin_end(results) + + # Initializing all data list + (time, requestTime, CPU, RAM, reqPerSec, errorRate, loadAvg) = ([], [], [], [], [], [], []) + global memoryOffset + initialRAM = memoryOffset + + for t in range(begin, end): # We determine datas time with timestamp + # Offset to set starttime = 0 + time.append(t - begin) + + # Getting requesttime (mean), number of request/error at a givent time + (reqTime, reqCount, errorCount) = getRequestTimeAndCount(results, t) + requestTime.append(reqTime) + reqPerSec.append(reqCount) + errorRate.append(errorCount) + + # Getting infos from Server, sometimes no info are getted during more than one second + try: + # Get CPU usage + CPU.append(100 - int(serverStats[str(t)][14])) + + # Get Memory used (delta with memory at the beginning) + usedRam = int(serverStats[str(t)][5]) - initialRAM + RAM.append(usedRam) + + # Get the load + loadAvg.append(100 * float(serverStats[str(t)][17])) # 18 + except KeyError: + # If fail in getting value, take previous values + CPU.append(CPU[-1]) + RAM.append(RAM[-1]) + loadAvg.append(loadAvg[-1]) + + print("ERROR - Some values missing for CPU and Memory usage [try to load for time=%s]" % t) + + return (time, requestTime, CPU, RAM, reqPerSec, errorRate, loadAvg) + + +def getRequestTimeAndCount(data, time): + reqCount = 0 + errorCount = 0 + totalRequest = 0.0 + + for result in results: + i = 0 + try: + + # Ignore past + while int(result[i][2]) < time: + i += 1 + + # Get infos for present + while int(result[i][2]) == time: + reqCount += 1 + totalRequest += float(result[i][4]) + if result[i][5] != '': + errorCount += 1 + i += 1 + except IndexError: + pass + return (totalRequest / reqCount if reqCount > 0 else 0, reqCount, errorCount) + + +def displayGraph(results, serverStats): + """ + Display all the graph on the same figure + """ + print("Processing data and plot, it may take some time for huge tests") + (time, requestTime, CPU, RAM, reqPerSec, errorCount, loadAvg) = process_data(results, serverStats) + + plt.subplot(221) + plt.plot(time, requestTime, '-', label="Request time (s)") + plt.legend() + plt.subplot(222) + plt.plot(time, CPU, '*', label="CPU usage (%)") + plt.plot(time, loadAvg, '*', label="Load average * 100") + plt.legend() + plt.subplot(223) + plt.plot(time, RAM, '*', label="Used Memory (bytes)") + plt.legend() + plt.subplot(224) + plt.plot(time, reqPerSec, '*', label="Requests/sec") + plt.plot(time, errorCount, '*', label="Errors/sec") + plt.legend() + + +results = get_results() +# process_data(results) +serverStats = get_server_stats() +displayGraph(results, serverStats) +plt.show() diff --git a/tests/DISET_HTTPS_migration/multi-mechanize/plot.py b/tests/DISET_HTTPS_migration/multi-mechanize/plot.py new file mode 100755 index 00000000000..dfba9c2de2b --- /dev/null +++ b/tests/DISET_HTTPS_migration/multi-mechanize/plot.py @@ -0,0 +1,53 @@ +""" + A little script to analyze multi-mechanize tests by ploting every test in a single figure + + Because it can't be automatized you have to define path of the folder who contains results + + testTornado and testDirac must have the same length +""" +import csv +import matplotlib.pyplot as plt + +testTornado = ['ping/results/results_2018.06.22_14.50.53', 'service/results/results_2018.06.22_15.24.18'] +testDirac = ['ping/results/results_2018.06.22_14.52.09', 'service/results/results_2018.06.22_14.55.03'] + + +def read_data(test, groupSize): + with open(test + '/results.csv', 'rb') as csvfile: + reader = csv.reader(csvfile, delimiter=',') + + sumgrouptime = 0 + sumgrouprequestTime = 0 + count = 0 + + time = [] + requestTime = [] + + for row in reader: + count += 1 + sumgrouptime += float(row[1]) + sumgrouprequestTime += float(row[4]) + # We group some points to make graph readable + if(count == groupSize): + time.append(sumgrouptime / groupSize) + requestTime.append(sumgrouprequestTime / groupSize) + sumgrouptime = 0 + sumgrouprequestTime = 0 + count = 0 + return (time, requestTime) + + +def displayGraph(testTornado, testDirac, subplot, groupSize): + plt.subplot(subplot) + plt.ylabel('red = dirac') + + (timeTornado, requestTimeTornado) = read_data(testTornado, groupSize) + (timeDirac, requestTimeDirac) = read_data(testDirac, groupSize) + + plt.plot(timeTornado, requestTimeTornado, 'b-', timeDirac, requestTimeDirac, 'r-') + + +# The "100*len(testTornado)+11+i" can look strange but it define a sublot dynamically +for i in range(len(testTornado)): + displayGraph(testTornado[i], testDirac[i], 100 * len(testTornado) + 11 + i, 42) +plt.show() diff --git a/tests/DISET_HTTPS_migration/multi-mechanize/service/config.cfg b/tests/DISET_HTTPS_migration/multi-mechanize/service/config.cfg new file mode 100644 index 00000000000..b9ba7f702d8 --- /dev/null +++ b/tests/DISET_HTTPS_migration/multi-mechanize/service/config.cfg @@ -0,0 +1,13 @@ + +[global] +run_time = 60 +rampup = 60 +results_ts_interval = 1 +progress_bar = on +console_logging = off +xml_report = off + + +[user_group-1] +threads = 60 +script = v_perf.py diff --git a/tests/DISET_HTTPS_migration/multi-mechanize/service/test_scripts/v_perf.py b/tests/DISET_HTTPS_migration/multi-mechanize/service/test_scripts/v_perf.py new file mode 100644 index 00000000000..4c6a984ac08 --- /dev/null +++ b/tests/DISET_HTTPS_migration/multi-mechanize/service/test_scripts/v_perf.py @@ -0,0 +1,25 @@ +from DIRAC.Core.Tornado.Client.TornadoClient import TornadoClient +from DIRAC.Core.DISET.RPCClient import RPCClient +from time import time +from random import randint +import sys + + +class Transaction(object): + def __init__(self): + # If we want we can force to use dirac + if len(sys.argv) > 2 and sys.argv[2].lower() == 'dirac': + self.client = RPCClient('Framework/UserDirac') + else: + self.client = TornadoClient('Framework/User') + return + + def run(self): + s = "Chaine 1" + s2 = "Chaine %d" % randint(0, 42) + + newUser = self.client.addUser(s) + userID = int(newUser['Value']) + User = self.client.getUserName(userID) + self.client.editUser(userID, s2) + User = self.client.getUserName(userID) diff --git a/tests/Integration/DataManagementSystem/Test_Client_DFC.py b/tests/Integration/DataManagementSystem/Test_Client_DFC.py index dc7cb99e3f7..5c56485cfe8 100644 --- a/tests/Integration/DataManagementSystem/Test_Client_DFC.py +++ b/tests/Integration/DataManagementSystem/Test_Client_DFC.py @@ -143,6 +143,7 @@ def test_fileOperations(self): 'GUID': '1000', 'Checksum': '0'}}) self.assertTrue(result['OK'], "addFile failed when adding new file %s" % result) + self.assertTrue(testFile in result['Value'].get('Successful', {}), result) result = self.dfc.exists(testFile) self.assertTrue(result['OK']) diff --git a/tests/Integration/TornadoServices/ConfigTemplate.cfg b/tests/Integration/TornadoServices/ConfigTemplate.cfg new file mode 100644 index 00000000000..9dd8512a155 --- /dev/null +++ b/tests/Integration/TornadoServices/ConfigTemplate.cfg @@ -0,0 +1,9 @@ +# This file should replace DIRAC/FrameworkSystem/ConfigTemplate when running test in jenkins (after server installation) +# This file is read by dirac-install to install a fake service for tests +Services +{ + User + { + Port = 9000 + } +} diff --git a/tests/Integration/TornadoServices/DB/UserDB.py b/tests/Integration/TornadoServices/DB/UserDB.py new file mode 100644 index 00000000000..79844839f55 --- /dev/null +++ b/tests/Integration/TornadoServices/DB/UserDB.py @@ -0,0 +1,74 @@ +""" A test DB in DIRAC, using MySQL as backend +""" + +from DIRAC.Core.Base.DB import DB + +from DIRAC import gLogger, S_OK, S_ERROR + + +class UserDB(DB): + """ Database system for users """ + + def __init__(self): + """ + Initialize the DB + """ + + super(UserDB, self).__init__('UserDB', 'Framework/UserDB') + retVal = self.__initializeDB() + if not retVal['OK']: + raise Exception("Can't create tables: %s" % retVal['Message']) + + def __initializeDB(self): + """ + Create the tables + """ + retVal = self._query("show tables") + if not retVal['OK']: + return retVal + + tablesInDB = [t[0] for t in retVal['Value']] + tablesD = {} + + if 'user_mytable' not in tablesInDB: + tablesD['user_mytable'] = {'Fields': {'Id': 'INTEGER NOT NULL AUTO_INCREMENT', 'Name': 'VARCHAR(64) NOT NULL'}, + 'PrimaryKey': ['Id'] + } + + return self._createTables(tablesD) + + def addUser(self, userName): + """ + Add a user + :param str userName: The name of the user we want to add + :return: S_OK or S_ERROR + """ + gLogger.verbose("Insert %s in DB" % userName) + return self.insertFields('user_mytable', ['Name'], [userName]) + + def editUser(self, uid, value): + """ + Edit a user + :param int uid: The Id of the user in database + :param str value: New user name + :return: S_OK or S_ERROR + """ + return self.updateFields('user_mytable', updateDict={'Name': value}, condDict={'Id': uid}) + + def getUserName(self, uid): + """ + Get a user + :param int uid: The Id of the user in database + :return: S_OK with S_OK['Value'] = TheUserName or S_ERROR if not found + """ + user = self.getFields('user_mytable', condDict={'Id': uid}) + if len(user['Value']) == 1: + return S_OK(user['Value'][0][1]) + return S_ERROR('USER NOT FOUND') + + def listUsers(self): + """ + List all users + :return: S_OK with S_OK['Value'] list of [UserId, UserName] + """ + return self._query('SELECT * FROM user_mytable') diff --git a/tests/Integration/TornadoServices/DB/UserDB.sql b/tests/Integration/TornadoServices/DB/UserDB.sql new file mode 100644 index 00000000000..ebb64fd28a4 --- /dev/null +++ b/tests/Integration/TornadoServices/DB/UserDB.sql @@ -0,0 +1,3 @@ +# Everything is created by the DB object _checkTable method. +USE UserDB; + diff --git a/tests/Integration/TornadoServices/Services/DummyTornadoHandler.py b/tests/Integration/TornadoServices/Services/DummyTornadoHandler.py new file mode 100644 index 00000000000..bf726c5a784 --- /dev/null +++ b/tests/Integration/TornadoServices/Services/DummyTornadoHandler.py @@ -0,0 +1,22 @@ +""" + This handler is only here to test if Tornado server starts + It can be used for some tests (like performance test on ping) + This file must be copied in FrameworkSystem/Service to run tests +""" + + +from DIRAC.Core.Tornado.Server.TornadoService import TornadoService +from DIRAC import S_OK, S_ERROR + + +class DummyTornadoHandler(TornadoService): + + auth_true = ['all'] + + def export_true(self): + return S_OK() + + auth_false = ['all'] + + def export_false(self): + return S_ERROR() diff --git a/tests/Integration/TornadoServices/Services/UserDiracHandler.py b/tests/Integration/TornadoServices/Services/UserDiracHandler.py new file mode 100644 index 00000000000..41e09a7ee7b --- /dev/null +++ b/tests/Integration/TornadoServices/Services/UserDiracHandler.py @@ -0,0 +1,68 @@ +""" Dummy Service is a service for testing new dirac protocol + This file must be copied in FrameworkSystem/Service to run tests +""" + +__RCSID__ = "$Id$" + +from DIRAC import S_OK +from DIRAC.Core.DISET.RequestHandler import RequestHandler +# You need to copy ../DB/UserDB in DIRAC/FrameworkSystem/DB +from DIRAC.FrameworkSystem.DB.UserDB import UserDB # pylint: disable=no-name-in-module, import-error +from DIRAC import gConfig + + +class UserDiracHandler(RequestHandler): + """ + A handler designed for testing Tornado by implementing a basic access to database + Designed to compare Diset and Tornado + """ + + @classmethod + def initializeHandler(cls, serviceInfo): + """ Handler initialization + """ + cls.userDB = UserDB() + return S_OK() + + auth_addUser = ['all'] + types_addUser = [basestring] + + def export_addUser(self, whom): + """ Add a user and return user id + """ + newUser = self.userDB.addUser(whom) + if newUser['OK']: + return S_OK(newUser['lastRowId']) + return newUser + + auth_editUser = ['all'] + types_editUser = [int, basestring] + + def export_editUser(self, uid, value): + """ Edit a user """ + return self.userDB.editUser(uid, value) + + auth_getUserName = ['all'] + types_getUserName = [int] + + def export_getUserName(self, uid): + """ Get a user """ + return self.userDB.getUserName(uid) + + auth_listUsers = ['all'] + types_listUsers = [] + + def export_listUsers(self): + return self.userDB.listUsers() + + auth_unauthorized = ['nobody'] + types_unauthorized = [] + + def export_unauthorized(self): + return S_OK() + + auth_getTestValue = ['all'] + types_getTestValue = [] + + def export_getTestValue(self): + return S_OK(gConfig.getValue('/DIRAC/Configuration/TestUpdateValue')) diff --git a/tests/Integration/TornadoServices/Services/UserHandler.py b/tests/Integration/TornadoServices/Services/UserHandler.py new file mode 100644 index 00000000000..2cec09c7913 --- /dev/null +++ b/tests/Integration/TornadoServices/Services/UserHandler.py @@ -0,0 +1,78 @@ +""" Dummy Service is a service for testing new dirac protocol + This file must be copied in FrameworkSystem/Service to run tests +""" + +from DIRAC.Core.Tornado.Server.TornadoService import TornadoService +from DIRAC import S_OK, gLogger +# You need to copy ../DB/UserDB in DIRAC/FrameworkSystem/DB +from DIRAC.FrameworkSystem.DB.UserDB import UserDB # pylint: disable=no-name-in-module, import-error +from DIRAC import gConfig + + +class UserHandler(TornadoService): + + """ + A handler designed for testing Tornado by implementing a basic access to database + """ + + @classmethod + def initializeHandler(cls, serviceInfoDict): + cls.userDB = UserDB() + return S_OK() + + auth_addUser = ['all'] + + def export_addUser(self, whom): + """ + Add a user + + :param str whom: The name of the user we want to add + :return: S_OK with S_OK['Value'] = The_ID_of_the_user or S_ERROR + """ + newUser = self.userDB.addUser(whom) + if newUser['OK']: + return S_OK(newUser['lastRowId']) + return newUser + + auth_editUser = ['all'] + + def export_editUser(self, uid, value): + """ + Edit a user + + :param int uid: The Id of the user in database + :param str value: New user name + :return: S_OK or S_ERROR + """ + return self.userDB.editUser(uid, value) + + auth_getUserName = ['all'] + + def export_getUserName(self, uid): + """ + Get a user + + :param int uid: The Id of the user in database + :return: S_OK with S_OK['Value'] = TheUserName or S_ERROR if not found + """ + return self.userDB.getUserName(uid) + + auth_listUsers = ['all'] + + def export_listUsers(self): + """ + List all users + + :return: S_OK with S_OK['Value'] list of [UserId, UserName] + """ + return self.userDB.listUsers() + + auth_unauthorized = ['nobody'] + + def export_unauthorized(self): + return S_OK() + + auth_getValue = ['all'] + + def export_getTestValue(self): + return S_OK(gConfig.getValue('/DIRAC/Configuration/TestUpdateValue')) diff --git a/tests/Integration/TornadoServices/Test_TornadoAndDISETmixed.py b/tests/Integration/TornadoServices/Test_TornadoAndDISETmixed.py new file mode 100644 index 00000000000..1dfddd9ef17 --- /dev/null +++ b/tests/Integration/TornadoServices/Test_TornadoAndDISETmixed.py @@ -0,0 +1,190 @@ +""" + (Case 1) This is a integration test with a Service under Tornado(tornado-start-all) + AND a configuration server under DISET (dirac-service) + + (Case 2) We run the same test with Service under DISET and CS under Tornado, + use tornado-start-CS to start the CS. + + [tornado-start-all and tornado-start-CS can be found at DIRAC/TornadoServices/scripts] + + + We test with + - for the CS: Configuration/Server (handlerPath define path to load ConfigurationHandler.py, + or TornadoConfigurationHandler.py in the second case) + - for the service: Framework/User (for the second case handlerPath should define + path to UserDiracHandler.py, see example bellow) + + The CS, service and client (this test) must read different dirac.cfg files and are + not necessary on the same computer + + In the config file distributed by the CS you should see something like:: + ``` + + Systems{ + Framework + { + DevInstance + { + Services + { + URLs + { + #User = https://MrBoincHost/Framework/User # Case 1 + User = dips://MrBoincHost/Framework/User # Case 2 + } + #for case 1 + #User + #{ + # Protocol = https + #} + #for case 2 + User + { + HandlerPath = DIRAC/FrameworkSystem/Service/UserDiracHandler.py + Port = 1234 + Protocol = dips # Not necessary defined, it's the default value, but good for human reading + } + } + + } + } + } + Registry {...} + ``` + For the service you need to define the database (case 1 and 2) and Tornado (case 1), + so in the config file loaded by service you should see:: + ``` + DIRAC + { + Setup = DeveloperSetup + Configuration + { + Servers = dips://localhost:9135/Configuration/Server # Case 1 + #Servers = https://localhost:444/Configuration/Server # Case 2 + } + } + Systems { + # Only in case 1 (Service run under tornado) + # Tornado{ + # DevInstance + # { + # # 443 is already the default, next line is technically useless + # Port = 443 + # } + # } + Framework + { + DevInstance + { + Databases + { + UserDB + { + Host = localhost + User = root + Password = + DBName = dirac + } + } + } + } + } + ``` + + The client should have a very minimal config file, and read only this file:: + ``` + DIRAC + { + Setup = DeveloperSetup + Configuration + { + Servers = dips://localhost:9135/Configuration/Server + #Servers = https://localhost:444/Configuration/Server + } + } + ``` + + + This test can be very long depending on refresh time (5minutes by default) + You can set following values in configuration (before starting service/CS) + They define refresh time in seconds, 300 by default (600 for SlavesGraceTime) + - DIRAC/Configuration/RefreshTime + - DIRAC/Configuration/PropagationTime + - DIRAC/Configuration/SlavesGraceTime +""" + +from DIRAC.Core.Base import Script +Script.parseCommandLine() + +import time + +from string import printable + +from hypothesis import given, settings, unlimited +from hypothesis.strategies import text + +from DIRAC.Core.Tornado.Client.ClientSelector import RPCClientSelector as RPCClient +from DIRAC.Core.Utilities.DErrno import ENOAUTH +from DIRAC import S_ERROR +from DIRAC.ConfigurationSystem.Client.CSAPI import CSAPI +from DIRAC.ConfigurationSystem.Client.ConfigurationData import gConfigurationData + + +from pytest import mark +parametrize = mark.parametrize + + +def test_authorization(): + service = RPCClient("Framework/User") + + authorisation = service.unauthorized() # In the handler this method have no allowed properties + assert authorisation['OK'] is False + assert authorisation['Message'] == S_ERROR(ENOAUTH, "Unauthorized query")['Message'] + + +def test_unknown_method(): + service = RPCClient("Framework/User") + + unknownmethod = service.ThisMethodMayNotExist() + assert unknownmethod['OK'] is False + assert unknownmethod['Message'] == "Unknown method ThisMethodMayNotExist" + + +def test_ping(): + service = RPCClient("Framework/User") + + assert service.ping()['OK'] + + +@settings(deadline=None, max_examples=4) +@given(data=text(printable, max_size=64)) +def test_echo(data): + service = RPCClient("Framework/User") + + assert service.echo(data)['Value'] == data + + +@settings(deadline=None, max_examples=1, timeout=unlimited) +@given(value1=text(printable, max_size=64), value2=text(printable, max_size=64)) +def test_configurationAutoUpdate(value1, value2): + """ + Test if service refresh his configuration. It sent a random value to the CS + and check if Service can return it. + + """ + csapi = CSAPI() + + # SETTING FIRST VALUE + csapi.modifyValue("/DIRAC/Configuration/TestUpdateValue", value1) + csapi.commitChanges() + + # Wait for automatic refresh (+1 to be sure that request is done) + time.sleep(gConfigurationData.getPropagationTime() + 1) + RPCClient("Framework/User").getTestValue() + assert RPCClient("Framework/User").getTestValue()['Value'] == value1 + + # SETTING SECOND VALUE + csapi.modifyValue("/DIRAC/Configuration/TestUpdateValue", value2) + csapi.commitChanges() + time.sleep(gConfigurationData.getPropagationTime() + 1) + assert RPCClient("Framework/User").getTestValue()['Value'] == value2 diff --git a/tests/Integration/TornadoServices/getCPUInfos b/tests/Integration/TornadoServices/getCPUInfos new file mode 100755 index 00000000000..177613ee8cb --- /dev/null +++ b/tests/Integration/TornadoServices/getCPUInfos @@ -0,0 +1,37 @@ +#!/bin/bash + +## This script write infos about server, you may redirect the output into a file then read it with plot-DistributedTest.py +## It takes output from system commands, replace space/tab by ; +## (In fact it transform output in CSV) + + +# The output contains: +# epoch; vamstat -a [17 values]; /proc/loadavg [3 values] +# +# Example: +# 1533628489;0;0;0;1549064;1109052;939168;0;0;18;2;28;62;0;0;100;0;0;0.00;0.00;0.00 +# +# $ vmstat -a +# procs -----------memory-------------- ---swap-- -----io---- --system-- -----cpu----- +# r b swpd free inact active si so bi bo in cs us sy id wa st +# 0 0 0 1549064 1109052 939168 0 0 18 2 28 62 0 0 100 0 0 <-- we keep these infos +# +# $ cat /proc/loadavg +# 0.00 0.00 0.00 1/369 11332 + + +# ---returned--- --ignored-- +vmstat -a -n 1 | while read line; +do + # If line is not a header + if [ "$(echo $line | grep -c 0)" -ne "0" ]; + then + # print timestamp + echo -n "$(date +%s)"; + # print vmstat info + echo -n ";$line" | sed -E 's/ +/;/g'; + # add load info + echo ";$(cat /proc/loadavg | awk {'print $1,$2,$3'}|tr ' ' ';')"; + + fi; +done \ No newline at end of file diff --git a/tests/Integration/TornadoServices/perf-test-DB/config.cfg b/tests/Integration/TornadoServices/perf-test-DB/config.cfg new file mode 100644 index 00000000000..768fb51aa11 --- /dev/null +++ b/tests/Integration/TornadoServices/perf-test-DB/config.cfg @@ -0,0 +1,19 @@ + +[global] +run_time = 4 +rampup = 2 +results_ts_interval = 1 +progress_bar = on +console_logging = off +xml_report = off + + +[user_group-1] +threads = 60 +script = v_perf.py + +#[user_group-2] +#threads = 60 +#script = v_perf2.py + + diff --git a/tests/Integration/TornadoServices/perf-test-DB/test_scripts/v_perf.py b/tests/Integration/TornadoServices/perf-test-DB/test_scripts/v_perf.py new file mode 100644 index 00000000000..39d136895f7 --- /dev/null +++ b/tests/Integration/TornadoServices/perf-test-DB/test_scripts/v_perf.py @@ -0,0 +1,37 @@ +""" + This test is designed to check protocol/server performances with a database + + Configuration and how to run it are explained in the documentation +""" + +from DIRAC.Core.Tornado.Client.ClientSelector import RPCClientSelector +from time import time +from random import random +import sys +import os + + +class Transaction(object): + def __init__(self): + self.client = RPCClientSelector('Framework/User', timeout=30) + return + + def run(self): + # Generate random name + s = str(int(random() * 100)) + s2 = str(int(random() * 100)) + service = self.client + + # Create a user + newUser = service.addUser(s) + userID = int(newUser['Value']) + + # Check if user exist and name is correct + User = service.getUserName(userID) + assert (User['OK']), 'Error in getting user' + assert (User['Value'] == s), 'Error on insertion' + + # Check if update work + service.editUser(userID, s2) + User = service.getUserName(userID) + assert (User['Value'] == s2), 'Error on update' diff --git a/tests/Integration/TornadoServices/perf-test-ping/config.cfg b/tests/Integration/TornadoServices/perf-test-ping/config.cfg new file mode 100644 index 00000000000..e6b3b5f1513 --- /dev/null +++ b/tests/Integration/TornadoServices/perf-test-ping/config.cfg @@ -0,0 +1,19 @@ + +[global] +run_time = 300 +rampup = 200 +results_ts_interval = 1 +progress_bar = on +console_logging = off +xml_report = off + + +[user_group-1] +threads = 60 +script = v_perf.py + +#[user_group-2] +#threads = 30 +#script = v_perf.py Framework/User2 + + diff --git a/tests/Integration/TornadoServices/perf-test-ping/test_scripts/v_perf.py b/tests/Integration/TornadoServices/perf-test-ping/test_scripts/v_perf.py new file mode 100644 index 00000000000..8abec97aaca --- /dev/null +++ b/tests/Integration/TornadoServices/perf-test-ping/test_scripts/v_perf.py @@ -0,0 +1,24 @@ +""" + This test is designed to check protocol/server performances + + Configuration and how to run it are explained in the documentation +""" + +from DIRAC.Core.Tornado.Client.ClientSelector import RPCClientSelector +from time import time +from random import randint +import sys +import os + + +class Transaction(object): + def __init__(self): + # If we want we can force to use another service name (testing multiple services for example) + if len(sys.argv) > 2: + self.client = RPCClientSelector(sys.argv[2]) + else: + self.client = RPCClientSelector('Framework/User') + return + + def run(self): + assert (self.client.ping()['OK']), 'error' diff --git a/tests/Integration/all_integration_server_tests.sh b/tests/Integration/all_integration_server_tests.sh index 3f70c1fc6e1..8127d89a928 100644 --- a/tests/Integration/all_integration_server_tests.sh +++ b/tests/Integration/all_integration_server_tests.sh @@ -60,6 +60,7 @@ diracDFCDB |& tee -a "${SERVER_TEST_OUTPUT}" echo -e "*** $(date -u) Restart the DFC service\n" |& tee -a "${SERVER_TEST_OUTPUT}" dirac-restart-component DataManagement FileCatalog "${DEBUG}" |& tee -a "${SERVER_TEST_OUTPUT}" +dirac-restart-component Tornado Tornado "${DEBUG}" |& tee -a "${SERVER_TEST_OUTPUT}" echo -e "*** $(date -u) Run it with the admin privileges" |& tee -a "${SERVER_TEST_OUTPUT}" echo -e "*** $(date -u) getting the prod role again\n" |& tee -a "${SERVER_TEST_OUTPUT}" diff --git a/tests/Jenkins/install.cfg b/tests/Jenkins/install.cfg index ef6a1f11122..26fd8a35ddd 100644 --- a/tests/Jenkins/install.cfg +++ b/tests/Jenkins/install.cfg @@ -41,6 +41,7 @@ LocalInstallation Systems += Production Systems += Transformation Systems += WorkloadManagement + Systems += Tornado # List of DataBases to be installed - minimal list for a running base server Databases = InstalledComponentsDB Databases += ResourceStatusDB diff --git a/tests/Jenkins/utilities.sh b/tests/Jenkins/utilities.sh index 9ebe6316e76..432d49bd140 100644 --- a/tests/Jenkins/utilities.sh +++ b/tests/Jenkins/utilities.sh @@ -178,11 +178,12 @@ findDatabases() { # # We are avoiding, FileCatalogDB FileCatalogWithFkAndPsDB that is installed in other ways # and InstalledComponentsDB which is installed at the beginning + # We also ignore all the DBs in tests directory # if [[ -n "${DBstoExclude}" ]]; then - find ./*DIRAC/ -name "*DB.sql" | grep -vE '(FileCatalogDB|FileCatalogWithFkAndPsDB|InstalledComponentsDB)' | awk -F "/" '{print $3,$5}' | grep -v "${DBstoExclude}" | grep -v 'DIRAC' | sort -u > databases + find ./*DIRAC/ -path "*/tests/*" -prune -o -name "*DB.sql" -print | grep -vE '(FileCatalogDB|FileCatalogWithFkAndPsDB|InstalledComponentsDB)' | awk -F "/" '{print $3,$5}' | grep -v "${DBstoExclude}" | grep -v 'DIRAC' | sort | uniq > databases else - find ./*DIRAC/ -name "*DB.sql" | grep -vE '(FileCatalogDB|FileCatalogWithFkAndPsDB|InstalledComponentsDB)' | awk -F "/" '{print $3,$5}' | grep "${DBstoSearch}" | grep -v 'DIRAC' | sort -u > databases + find ./*DIRAC/ -path "*/tests/*" -prune -o -name "*DB.sql" -print | grep -vE '(FileCatalogDB|FileCatalogWithFkAndPsDB|InstalledComponentsDB)' | awk -F "/" '{print $3,$5}' | grep "${DBstoSearch}" | grep -v 'DIRAC' | sort | uniq > databases fi echo "found $(wc -l databases)" @@ -751,7 +752,8 @@ diracAddSite() { diracServices(){ echo '==> [diracServices]' - local services=$(cut -d '.' -f 1 < services | grep -v PilotsLogging | grep -v StorageElementHandler | grep -v ^ConfigurationSystem | grep -v Plotting | grep -v RAWIntegrity | grep -v RunDBInterface | grep -v ComponentMonitoring | sed 's/System / /g' | sed 's/Handler//g' | sed 's/ /\//g') + # Ignore tornado services + local services=$(cut -d '.' -f 1 < services | grep -v Tornado | grep -v PilotsLogging | grep -v StorageElementHandler | grep -v ^ConfigurationSystem | grep -v Plotting | grep -v RAWIntegrity | grep -v RunDBInterface | grep -v ComponentMonitoring | sed 's/System / /g' | sed 's/Handler//g' | sed 's/ /\//g') # group proxy, will be uploaded explicitly # echo '==> getting/uploading proxy for prod' @@ -801,7 +803,8 @@ diracUninstallServices(){ findServices - local services=$(cut -d '.' -f 1 getting/uploading proxy for prod' diff --git a/tests/py3CheckDirs.txt b/tests/py3CheckDirs.txt index e69de29bb2d..f1fd71bdd22 100644 --- a/tests/py3CheckDirs.txt +++ b/tests/py3CheckDirs.txt @@ -0,0 +1,20 @@ +ConfigurationSystem/Client/ConfigurationClient.py +ConfigurationSystem/Client/CSAPI.py +ConfigurationSystem/private/Refresher.py +ConfigurationSystem/private/ServiceInterfaceBase.py +ConfigurationSystem/private/ServiceInterface.py +ConfigurationSystem/private/ServiceInterfaceTornado.py +ConfigurationSystem/Service/TornadoConfigurationHandler.py +Core/Base/Client.py +Core/DISET/private/BaseClient.py +Core/DISET/private/Service.py +Core/DISET/RequestHandler.py +Core/Security/m2crypto/X509Chain.py +Core/Tornado +DataManagementSystem/Service/TornadoFileCatalogHandler.py +FrameworkSystem/Client/ComponentInstaller.py +FrameworkSystem/Client/MonitoringClientIOLoop.py +FrameworkSystem/Client/MonitoringClient.py +FrameworkSystem/scripts/dirac-install-tornado-service.py +Resources/Catalog/FileCatalogClient.py +Resources/Storage/GFAL2_StorageBase.py