From 41fc0d6255c2995f34da49dc57f6ac10a4c64892 Mon Sep 17 00:00:00 2001 From: fstagni Date: Fri, 12 Jun 2020 12:34:42 +0200 Subject: [PATCH 1/7] sanitize DNs --- ConfigurationSystem/Client/Helpers/Registry.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/ConfigurationSystem/Client/Helpers/Registry.py b/ConfigurationSystem/Client/Helpers/Registry.py index eaa337add26..13eeef17da8 100644 --- a/ConfigurationSystem/Client/Helpers/Registry.py +++ b/ConfigurationSystem/Client/Helpers/Registry.py @@ -23,6 +23,7 @@ def getUsernameForDN(dn, usersList=None): :return: S_OK(str)/S_ERROR() """ + dn = dn.strip() if not usersList: result = gConfig.getSections("%s/Users" % gBaseRegistrySection) if not result['OK']: @@ -63,6 +64,7 @@ def getGroupsForDN(dn): :return: S_OK(list)/S_ERROR() -- contain list of groups """ + dn = dn.strip() result = getUsernameForDN(dn) if not result['OK']: return result @@ -128,6 +130,7 @@ def getHostnameForDN(dn): :return: S_OK()/S_ERROR() """ + dn = dn.strip() result = gConfig.getSections("%s/Hosts" % gBaseRegistrySection) if not result['OK']: return result @@ -153,6 +156,7 @@ def findDefaultGroupForDN(dn): :return: S_OK()/S_ERROR() """ + dn = dn.strip() result = getUsernameForDN(dn) if not result['OK']: return result From 816e5d66aeb4a9b9c7282ac11d4fa3ba70159bf7 Mon Sep 17 00:00:00 2001 From: fstagni Date: Mon, 15 Jun 2020 10:47:14 +0200 Subject: [PATCH 2/7] minor/py3 fixes --- .../Client/BundleDeliveryClient.py | 2 + FrameworkSystem/Client/ProxyManagerClient.py | 90 +++++++++---------- .../Service/ProxyManagerHandler.py | 74 +++++++-------- 3 files changed, 84 insertions(+), 82 deletions(-) diff --git a/FrameworkSystem/Client/BundleDeliveryClient.py b/FrameworkSystem/Client/BundleDeliveryClient.py index eeb72c8c24e..f73b404b69b 100644 --- a/FrameworkSystem/Client/BundleDeliveryClient.py +++ b/FrameworkSystem/Client/BundleDeliveryClient.py @@ -10,6 +10,8 @@ from DIRAC.Core.DISET.TransferClient import TransferClient from DIRAC.Core.Security import Locations, Utilities from DIRAC.ConfigurationSystem.Client.Helpers.CSGlobals import skipCACheck + + __RCSID__ = "$Id$" diff --git a/FrameworkSystem/Client/ProxyManagerClient.py b/FrameworkSystem/Client/ProxyManagerClient.py index c8a42db8c13..d1df5a017ba 100755 --- a/FrameworkSystem/Client/ProxyManagerClient.py +++ b/FrameworkSystem/Client/ProxyManagerClient.py @@ -1,4 +1,4 @@ -""" ProxyManagementAPI has the functions to "talk" to the ProxyManagement service +""" ProxyManagemerClient has the function to "talk" to the ProxyManagemer service """ import six import os @@ -35,11 +35,11 @@ def __init__(self): def __deleteTemporalFile(self, filename): """ Delete temporal file - :param basestring filename: path to file + :param str filename: path to file """ try: - os.unlink(filename) - except BaseException: + os.remove(filename) + except Exception: pass def clearCaches(self): @@ -89,8 +89,8 @@ def userHasProxy(self, userDN, userGroup, validSeconds=0): """ Check if a user(DN-group) has a proxy in the proxy management Updates internal cache if needed to minimize queries to the service - :param basestring userDN: user DN - :param basestring userGroup: user group + :param str userDN: user DN + :param str userGroup: user group :param int validSeconds: proxy valid time in a seconds :return: S_OK()/S_ERROR() @@ -110,8 +110,8 @@ def getUserPersistence(self, userDN, userGroup, validSeconds=0): """ Check if a user(DN-group) has a proxy in the proxy management Updates internal cache if needed to minimize queries to the service - :param basestring userDN: user DN - :param basestring userGroup: user group + :param str userDN: user DN + :param str userGroup: user group :param int validSeconds: proxy valid time in a seconds :return: S_OK()/S_ERROR() @@ -134,8 +134,8 @@ def getUserPersistence(self, userDN, userGroup, validSeconds=0): def setPersistency(self, userDN, userGroup, persistent): """ Set the persistency for user/group - :param basestring userDN: user DN - :param basestring userGroup: user group + :param str userDN: user DN + :param str userGroup: user group :param boolean persistent: presistent flag :return: S_OK()/S_ERROR() @@ -219,13 +219,13 @@ def downloadProxy(self, userDN, userGroup, limited=False, requiredTimeLeft=1200, cacheTime=14400, proxyToConnect=None, token=None): """ Get a proxy Chain from the proxy management - :param basestring userDN: user DN - :param basestring userGroup: user group + :param str userDN: user DN + :param str userGroup: user group :param boolean limited: if need limited proxy :param int requiredTimeLeft: required proxy live time in a seconds :param int cacheTime: store in a cache time in a seconds :param X509Chain proxyToConnect: proxy as a chain - :param basestring token: valid token to get a proxy + :param str token: valid token to get a proxy :return: S_OK(X509Chain)/S_ERROR() """ @@ -257,14 +257,14 @@ def downloadProxyToFile(self, userDN, userGroup, limited=False, requiredTimeLeft cacheTime=14400, filePath=None, proxyToConnect=None, token=None): """ Get a proxy Chain from the proxy management and write it to file - :param basestring userDN: user DN - :param basestring userGroup: user group + :param str userDN: user DN + :param str userGroup: user group :param boolean limited: if need limited proxy :param int requiredTimeLeft: required proxy live time in a seconds :param int cacheTime: store in a cache time in a seconds - :param basestring filePath: path to save proxy + :param str filePath: path to save proxy :param X509Chain proxyToConnect: proxy as a chain - :param basestring token: valid token to get a proxy + :param str token: valid token to get a proxy :return: S_OK(X509Chain)/S_ERROR() """ @@ -284,14 +284,14 @@ def downloadVOMSProxy(self, userDN, userGroup, limited=False, requiredTimeLeft=1 proxyToConnect=None, token=None): """ Download a proxy if needed and transform it into a VOMS one - :param basestring userDN: user DN - :param basestring userGroup: user group + :param str userDN: user DN + :param str userGroup: user group :param boolean limited: if need limited proxy :param int requiredTimeLeft: required proxy live time in a seconds :param int cacheTime: store in a cache time in a seconds - :param basestring requiredVOMSAttribute: VOMS attr to add to the proxy + :param str requiredVOMSAttribute: VOMS attr to add to the proxy :param X509Chain proxyToConnect: proxy as a chain - :param basestring token: valid token to get a proxy + :param str token: valid token to get a proxy :return: S_OK(X509Chain)/S_ERROR() """ @@ -325,15 +325,15 @@ def downloadVOMSProxyToFile(self, userDN, userGroup, limited=False, requiredTime proxyToConnect=None, token=None): """ Download a proxy if needed, transform it into a VOMS one and write it to file - :param basestring userDN: user DN - :param basestring userGroup: user group + :param str userDN: user DN + :param str userGroup: user group :param boolean limited: if need limited proxy :param int requiredTimeLeft: required proxy live time in a seconds :param int cacheTime: store in a cache time in a seconds - :param basestring requiredVOMSAttribute: VOMS attr to add to the proxy - :param basestring filePath: path to save proxy + :param str requiredVOMSAttribute: VOMS attr to add to the proxy + :param str filePath: path to save proxy :param X509Chain proxyToConnect: proxy as a chain - :param basestring token: valid token to get a proxy + :param str token: valid token to get a proxy :return: S_OK(X509Chain)/S_ERROR() """ @@ -351,9 +351,9 @@ def downloadVOMSProxyToFile(self, userDN, userGroup, limited=False, requiredTime def getPilotProxyFromDIRACGroup(self, userDN, userGroup, requiredTimeLeft=43200, proxyToConnect=None): """ Download a pilot proxy with VOMS extensions depending on the group - :param basestring userDN: user DN - :param basestring userGroup: user group - :param int requiredTimeLeft: required proxy live time in a seconds + :param str userDN: user DN + :param str userGroup: user group + :param int requiredTimeLeft: required proxy live time in seconds :param X509Chain proxyToConnect: proxy as a chain :return: S_OK(X509Chain)/S_ERROR() @@ -361,7 +361,7 @@ def getPilotProxyFromDIRACGroup(self, userDN, userGroup, requiredTimeLeft=43200, # Assign VOMS attribute vomsAttr = Registry.getVOMSAttributeForGroup(userGroup) if not vomsAttr: - gLogger.verbose("No voms attribute assigned to group %s when requested pilot proxy" % userGroup) + gLogger.warn("No voms attribute assigned to group %s when requested pilot proxy" % userGroup) return self.downloadProxy(userDN, userGroup, limited=False, requiredTimeLeft=requiredTimeLeft, proxyToConnect=proxyToConnect) else: @@ -371,8 +371,8 @@ def getPilotProxyFromDIRACGroup(self, userDN, userGroup, requiredTimeLeft=43200, def getPilotProxyFromVOMSGroup(self, userDN, vomsAttr, requiredTimeLeft=43200, proxyToConnect=None): """ Download a pilot proxy with VOMS extensions depending on the group - :param basestring userDN: user DN - :param basestring vomsAttr: VOMS attribute + :param str userDN: user DN + :param str vomsAttr: VOMS attribute :param int requiredTimeLeft: required proxy live time in a seconds :param X509Chain proxyToConnect: proxy as a chain @@ -395,10 +395,10 @@ def getPilotProxyFromVOMSGroup(self, userDN, vomsAttr, requiredTimeLeft=43200, p def getPayloadProxyFromDIRACGroup(self, userDN, userGroup, requiredTimeLeft, token=None, proxyToConnect=None): """ Download a payload proxy with VOMS extensions depending on the group - :param basestring userDN: user DN - :param basestring userGroup: user group + :param str userDN: user DN + :param str userGroup: user group :param int requiredTimeLeft: required proxy live time in a seconds - :param basestring token: valid token to get a proxy + :param str token: valid token to get a proxy :param X509Chain proxyToConnect: proxy as a chain :return: S_OK(X509Chain)/S_ERROR() @@ -417,9 +417,9 @@ def getPayloadProxyFromDIRACGroup(self, userDN, userGroup, requiredTimeLeft, tok def getPayloadProxyFromVOMSGroup(self, userDN, vomsAttr, token, requiredTimeLeft, proxyToConnect=None): """ Download a payload proxy with VOMS extensions depending on the VOMS attr - :param basestring userDN: user DN - :param basestring vomsAttr: VOMS attribute - :param basestring token: valid token to get a proxy + :param str userDN: user DN + :param str vomsAttr: VOMS attribute + :param str token: valid token to get a proxy :param int requiredTimeLeft: required proxy live time in a seconds :param X509Chain proxyToConnect: proxy as a chain @@ -442,10 +442,10 @@ def dumpProxyToFile(self, chain, destinationFile=None, requiredTimeLeft=600): """ Dump a proxy to a file. It's cached so multiple calls won't generate extra files :param X509Chain chain: proxy as a chain - :param basestring destinationFile: path to store proxy + :param str destinationFile: path to store proxy :param int requiredTimeLeft: required proxy live time in a seconds - :return: S_OK(basestring)/S_ERROR() + :return: S_OK(str)/S_ERROR() """ result = chain.hash() if not result['OK']: @@ -477,8 +477,8 @@ def requestToken(self, requesterDN, requesterGroup, numUses=1): """ Request a number of tokens. usesList must be a list of integers and each integer is the number of uses a token must have - :param basestring requesterDN: user DN - :param basestring requesterGroup: user group + :param str requesterDN: user DN + :param str requesterGroup: user group :param int numUses: number of uses :return: S_OK(tuple)/S_ERROR() -- tuple contain token, number uses @@ -573,15 +573,15 @@ def getVOMSAttributes(self, chain): :param X509Chain chain: proxy as a chain - :return: S_OK(basestring)/S_ERROR() + :return: S_OK(str)/S_ERROR() """ return VOMS().getVOMSAttributes(chain) def getUploadedProxyLifeTime(self, DN, group): """ Get the remaining seconds for an uploaded proxy - :param basestring DN: user DN - :param basestring group: group + :param str DN: user DN + :param str group: group :return: S_OK(int)/S_ERROR() """ diff --git a/FrameworkSystem/Service/ProxyManagerHandler.py b/FrameworkSystem/Service/ProxyManagerHandler.py index 8f0b3c0b92e..2aabb0d17a7 100644 --- a/FrameworkSystem/Service/ProxyManagerHandler.py +++ b/FrameworkSystem/Service/ProxyManagerHandler.py @@ -4,7 +4,6 @@ __RCSID__ = "$Id$" -from past.builtins import long import six from DIRAC import gLogger, S_OK, S_ERROR from DIRAC.Core.DISET.RequestHandler import RequestHandler @@ -37,7 +36,8 @@ def initializeHandler(cls, serviceInfoDict): gThreadScheduler.addPeriodicTask(900, cls.__proxyDB.purgeExpiredRequests, elapsedTime=900) gThreadScheduler.addPeriodicTask(21600, cls.__proxyDB.purgeLogs) gThreadScheduler.addPeriodicTask(3600, cls.__proxyDB.purgeExpiredProxies) - gLogger.info("MyProxy: %s\n MyProxy Server: %s" % (useMyProxy, cls.__proxyDB.getMyProxyServer())) + if useMyProxy: + gLogger.info("MyProxy: %s\n MyProxy Server: %s" % (useMyProxy, cls.__proxyDB.getMyProxyServer())) return S_OK() def __generateUserProxiesInfo(self): @@ -86,12 +86,12 @@ def export_getUserProxiesInfo(self): # WARN: Since v7r1 requestDelegationUpload method use only first argument! # WARN: Second argument for compatibility with older versions - types_requestDelegationUpload = [[int, long], [basestring, bool, type(None)]] + types_requestDelegationUpload = [six.integer_types, [six.string_types, bool, type(None)]] def export_requestDelegationUpload(self, requestedUploadTime, diracGroup=None): """ Request a delegation. Send a delegation request to client - :param int,long requestedUploadTime: requested live time + :param int requestedUploadTime: requested live time :return: S_OK(dict)/S_ERROR() -- dict contain id and proxy as string of the request """ @@ -120,13 +120,13 @@ def export_requestDelegationUpload(self, requestedUploadTime, diracGroup=None): gLogger.error("Upload request failed", "by %s:%s : %s" % (userName, userGroup, result['Message'])) return result - types_completeDelegationUpload = [six.integer_types, basestring] + types_completeDelegationUpload = [six.integer_types, six.string_types] def export_completeDelegationUpload(self, requestId, pemChain): """ Upload result of delegation :param int,long requestId: identity number - :param basestring pemChain: certificate as string + :param str pemChain: certificate as string :return: S_OK(dict)/S_ERROR() -- dict contain proxies """ @@ -157,8 +157,8 @@ def export_getRegisteredUsers(self, validSecondsRequired=0): def __checkProperties(self, requestedUserDN, requestedUserGroup): """ Check the properties and return if they can only download limited proxies if authorized - :param basestring requestedUserDN: user DN - :param basestring requestedUserGroup: DIRAC group + :param str requestedUserDN: user DN + :param str requestedUserGroup: DIRAC group :return: S_OK(boolean)/S_ERROR() """ @@ -176,7 +176,7 @@ def __checkProperties(self, requestedUserDN, requestedUserGroup): # Not authorized! return S_ERROR("You can't get proxies!") - types_getProxy = [basestring, basestring, basestring, six.integer_types] + types_getProxy = [six.string_types, six.string_types, six.string_types, six.integer_types] def export_getProxy(self, userDN, userGroup, requestPem, requiredLifetime): """ Get a proxy for a userDN/userGroup @@ -202,13 +202,13 @@ def export_getProxy(self, userDN, userGroup, requestPem, requiredLifetime): def __getProxy(self, userDN, userGroup, requestPem, requiredLifetime, forceLimited): """ Internal to get a proxy - :param basestring userDN: user DN - :param basestring userGroup: DIRAC group - :param basestring requestPem: dump of request certificate - :param int,long requiredLifetime: requested live time of proxy + :param str userDN: user DN + :param str userGroup: DIRAC group + :param str requestPem: dump of request certificate + :param int requiredLifetime: requested live time of proxy :param boolean forceLimited: limited proxy - :return: S_OK(basestring)/S_ERROR() + :return: S_OK(str)/S_ERROR() """ retVal = self.__proxyDB.getProxy(userDN, userGroup, requiredLifeTime=requiredLifetime) if not retVal['OK']: @@ -223,7 +223,9 @@ def __getProxy(self, userDN, userGroup, requestPem, requiredLifetime, forceLimit return retVal return S_OK(retVal['Value']) - types_getVOMSProxy = [basestring, basestring, basestring, six.integer_types, [basestring, type(None), bool]] + types_getVOMSProxy = [six.string_types, six.string_types, + six.string_types, six.integer_types, + [six.string_types, type(None), bool]] def export_getVOMSProxy(self, userDN, userGroup, requestPem, requiredLifetime, vomsAttribute=None): """ Get a proxy for a userDN/userGroup @@ -256,21 +258,17 @@ def __getVOMSProxy(self, userDN, userGroup, requestPem, requiredLifetime, vomsAt chain, secsLeft = retVal['Value'] # If possible we return a proxy 1.5 longer than requested requiredLifetime = int(min(secsLeft, requiredLifetime * self.__maxExtraLifeFactor)) - retVal = chain.generateChainFromRequestString(requestPem, - lifetime=requiredLifetime, - requireLimited=forceLimited) - if not retVal['OK']: - return retVal - _credDict = self.getRemoteCredentials() - return S_OK(retVal['Value']) + return chain.generateChainFromRequestString(requestPem, + lifetime=requiredLifetime, + requireLimited=forceLimited) - types_setPersistency = [basestring, basestring, bool] + types_setPersistency = [six.string_types, six.string_types, bool] def export_setPersistency(self, userDN, userGroup, persistentFlag): """ Set the persistency for a given dn/group - :param basestring userDN: user DN - :param basestring userGroup: DIRAC group + :param str userDN: user DN + :param str userGroup: DIRAC group :param boolean persistentFlag: if proxy persistent :return: S_OK()/S_ERROR() @@ -311,8 +309,8 @@ def export_deleteProxyBundle(self, idList): def export_deleteProxy(self, userDN, userGroup): """ Delete a proxy from the DB - :param basestring userDN: user DN - :param basestring userGroup: DIRAC group + :param str userDN: user DN + :param str userGroup: DIRAC group :return: S_OK()/S_ERROR() """ @@ -333,8 +331,8 @@ def export_getContents(self, selDict, sortDict, start, limit): :param dict selDict: selection fields :param list,tuple sortDict: sorting fields - :param int,long start: search limit start - :param int,long start: search limit amount + :param int start: search limit start + :param int start: search limit amount :return: S_OK(dict)/S_ERROR() -- dict contain fields, record list, total records """ @@ -350,21 +348,21 @@ def export_getLogContents(self, selDict, sortDict, start, limit): :param dict selDict: selection fields :param list,tuple sortDict: search filter - :param int,long start: search limit start - :param int,long start: search limit amount + :param int start: search limit start + :param int start: search limit amount :return: S_OK(dict)/S_ERROR() -- dict contain fields, record list, total records """ return self.__proxyDB.getLogsContent(selDict, sortDict, start, limit) - types_generateToken = [basestring, basestring, six.integer_types] + types_generateToken = [six.string_types, six.string_types, six.integer_types] def export_generateToken(self, requesterDN, requesterGroup, tokenUses): """ Generate tokens for proxy retrieval - :param basestring requesterDN: user DN - :param basestring requesterGroup: DIRAC group - :param int,long tokenUses: number of uses + :param str requesterDN: user DN + :param str requesterGroup: DIRAC group + :param int tokenUses: number of uses :return: S_OK(tuple)/S_ERROR() -- tuple contain token, number uses """ @@ -372,7 +370,7 @@ def export_generateToken(self, requesterDN, requesterGroup, tokenUses): self.__proxyDB.logAction("generate tokens", credDict['DN'], credDict['group'], requesterDN, requesterGroup) return self.__proxyDB.generateToken(requesterDN, requesterGroup, numUses=tokenUses) - types_getProxyWithToken = [basestring, basestring, basestring, six.integer_types, basestring] + types_getProxyWithToken = [six.string_types, six.string_types, six.string_types, six.integer_types, six.string_types] def export_getProxyWithToken(self, userDN, userGroup, requestPem, requiredLifetime, token): """ Get a proxy for a userDN/userGroup @@ -401,7 +399,9 @@ def export_getProxyWithToken(self, userDN, userGroup, requestPem, requiredLifeti self.__proxyDB.logAction("download proxy with token", credDict['DN'], credDict['group'], userDN, userGroup) return self.__getProxy(userDN, userGroup, requestPem, requiredLifetime, True) - types_getVOMSProxyWithToken = [basestring, basestring, basestring, six.integer_types, [basestring, type(None)]] + types_getVOMSProxyWithToken = [six.string_types, six.string_types, + six.string_types, six.integer_types, + [six.string_types, type(None)]] def export_getVOMSProxyWithToken(self, userDN, userGroup, requestPem, requiredLifetime, token, vomsAttribute=None): """ Get a proxy for a userDN/userGroup From 9de4febbac5ff229d6436903127c673be5005d34 Mon Sep 17 00:00:00 2001 From: fstagni Date: Mon, 15 Jun 2020 11:01:34 +0200 Subject: [PATCH 3/7] inheriting from Client --- FrameworkSystem/Client/BundleDeliveryClient.py | 18 +++++++----------- 1 file changed, 7 insertions(+), 11 deletions(-) diff --git a/FrameworkSystem/Client/BundleDeliveryClient.py b/FrameworkSystem/Client/BundleDeliveryClient.py index f73b404b69b..364ae701bcf 100644 --- a/FrameworkSystem/Client/BundleDeliveryClient.py +++ b/FrameworkSystem/Client/BundleDeliveryClient.py @@ -6,28 +6,24 @@ import cStringIO from DIRAC import S_OK, gLogger -from DIRAC.Core.DISET.RPCClient import RPCClient +from DIRAC.Core.Base.Client import Client, createClient from DIRAC.Core.DISET.TransferClient import TransferClient from DIRAC.Core.Security import Locations, Utilities +from DIRAC.Core.Utilities.File import mkDir from DIRAC.ConfigurationSystem.Client.Helpers.CSGlobals import skipCACheck __RCSID__ = "$Id$" -class BundleDeliveryClient(object): +@createClient('Framework/BundleDelivery') +class BundleDeliveryClient(Client): - def __init__(self, rpcClient=False, transferClient=False): - self.rpcClient = rpcClient + def __init__(self, transferClient=False, **kwargs): + Client.__init__(self, **kwargs) self.transferClient = transferClient self.log = gLogger.getSubLogger("BundleDelivery") - def __getRPCClient(self): - if self.rpcClient: - return self.rpcClient - return RPCClient("Framework/BundleDelivery", - skipCACheck=skipCACheck()) - def __getTransferClient(self): if self.transferClient: return self.transferClient @@ -54,7 +50,7 @@ def syncDir(self, bundleID, dirToSyncTo): dirCreated = False if not os.path.isdir(dirToSyncTo): self.log.info("Creating dir %s" % dirToSyncTo) - os.makedirs(dirToSyncTo) + mkDir(dirToSyncTo) dirCreated = True currentHash = self.__getHash(bundleID, dirToSyncTo) self.log.info("Current hash for bundle %s in dir %s is '%s'" % (bundleID, dirToSyncTo, currentHash)) From 846b85160067b258b7b5c2f28ce7c63b4dc2692f Mon Sep 17 00:00:00 2001 From: fstagni Date: Mon, 15 Jun 2020 11:18:01 +0200 Subject: [PATCH 4/7] some py2 to 3 fixes --- .../Client/BundleDeliveryClient.py | 9 +- FrameworkSystem/Client/ComponentInstaller.py | 88 +++++++++---------- 2 files changed, 46 insertions(+), 51 deletions(-) diff --git a/FrameworkSystem/Client/BundleDeliveryClient.py b/FrameworkSystem/Client/BundleDeliveryClient.py index 364ae701bcf..fd37907b5b5 100644 --- a/FrameworkSystem/Client/BundleDeliveryClient.py +++ b/FrameworkSystem/Client/BundleDeliveryClient.py @@ -2,6 +2,7 @@ """ import os +import io import tarfile import cStringIO @@ -32,7 +33,7 @@ def __getTransferClient(self): def __getHash(self, bundleID, dirToSyncTo): try: - with open(os.path.join(dirToSyncTo, ".dab.%s" % bundleID), "rb") as fd: + with io.open(os.path.join(dirToSyncTo, ".dab.%s" % bundleID), "rb") as fd: bdHash = fd.read().strip() return bdHash except BaseException: @@ -41,7 +42,7 @@ def __getHash(self, bundleID, dirToSyncTo): def __setHash(self, bundleID, dirToSyncTo, bdHash): try: fileName = os.path.join(dirToSyncTo, ".dab.%s" % bundleID) - with open(fileName, "wb") as fd: + with io.open(fileName, "wb") as fd: fd.write(bdHash) except Exception as e: self.log.error("Could not save hash after synchronization", "%s: %s" % (fileName, str(e))) @@ -111,7 +112,7 @@ def getCAs(self): # if we can not found the file, we return the directory, where the file should be transferClient = self.__getTransferClient() casFile = os.path.join(os.path.dirname(retVal['Message']), "cas.pem") - with open(casFile, "w") as fd: + with io.open(casFile, "w") as fd: result = transferClient.receiveFile(fd, 'CAs') if not result['OK']: return result @@ -129,7 +130,7 @@ def getCLRs(self): # if we can not found the file, we return the directory, where the file should be transferClient = self.__getTransferClient() casFile = os.path.join(os.path.dirname(retVal['Message']), "crls.pem") - with open(casFile, "w") as fd: + with io.open(casFile, "w") as fd: result = transferClient.receiveFile(fd, 'CRLs') if not result['OK']: return result diff --git a/FrameworkSystem/Client/ComponentInstaller.py b/FrameworkSystem/Client/ComponentInstaller.py index cf60348151e..6382d9ec459 100644 --- a/FrameworkSystem/Client/ComponentInstaller.py +++ b/FrameworkSystem/Client/ComponentInstaller.py @@ -58,6 +58,7 @@ from __future__ import print_function, absolute_import import os +import io import re import glob import stat @@ -1014,7 +1015,7 @@ def getSoftwareComponents(self, extensions): for agent in agentList: if os.path.splitext(agent)[1] == ".py": agentFile = os.path.join(agentDir, agent) - with open(agentFile, 'r') as afile: + with io.open(agentFile, 'rt') as afile: body = afile.read() if body.find('AgentModule') != -1 or body.find('OptimizerModule') != -1: if system not in agents: @@ -1040,7 +1041,7 @@ def getSoftwareComponents(self, extensions): for executor in executorList: if os.path.splitext(executor)[1] == ".py": executorFile = os.path.join(executorDir, executor) - with open(executorFile, 'r') as afile: + with io.open(executorFile, 'rt') as afile: body = afile.read() if body.find('OptimizerExecutor') != -1: if system not in executors: @@ -1091,9 +1092,8 @@ def getInstalledComponents(self): for component in components: try: runFile = os.path.join(systemDir, component, 'run') - rfile = open(runFile, 'r') - body = rfile.read() - rfile.close() + with io.open(runFile, 'rt') as rFile: + body = rFile.read() for cType in self.componentTypes: if body.find('dirac-%s' % (cType)) != -1: @@ -1126,9 +1126,8 @@ def getSetupComponents(self): for component in componentList: try: runFile = os.path.join(self.startDir, component, 'run') - rfile = open(runFile, 'r') - body = rfile.read() - rfile.close() + with io.open(runFile, 'rt') as rfile: + body = rfile.read() for cType in self.componentTypes: if body.find('dirac-%s' % (cType)) != -1: @@ -1463,9 +1462,8 @@ def getLogTail(self, system, component, length=100): if not os.path.exists(logFileName): retDict[compName] = 'No log file found' else: - logFile = open(logFileName, 'r') - lines = [line.strip() for line in logFile.readlines()] - logFile.close() + with io.open(logFileName, 'rt') as logFile: + lines = [line.strip() for line in logFile.readlines()] if len(lines) < length: retDict[compName] = '\n'.join(lines) @@ -1601,7 +1599,7 @@ def setupSite(self, scriptCfg, cfg=None): if not cmdFound: gLogger.notice('Starting runsvdir ...') - with open(os.devnull, 'w') as devnull: + with io.open(os.devnull, 'w') as devnull: subprocess.Popen(['nohup', 'runsvdir', self.startDir, 'log: DIRAC runsv'], stdout=devnull, stderr=devnull, universal_newlines=True) @@ -1815,16 +1813,16 @@ def _createRunitLog(self, runitCompDir): mkDir(logDir) logConfigFile = os.path.join(logDir, 'config') - with open(logConfigFile, 'w') as fd: + with io.open(logConfigFile, 'w') as fd: fd.write( - """s10000000 + u"""s10000000 n20 """) logRunFile = os.path.join(logDir, 'run') - with open(logRunFile, 'w') as fd: + with io.open(logRunFile, 'w') as fd: fd.write( - """#!/bin/bash + u"""#!/bin/bash # rcfile=%(bashrc)s [ -e $rcfile ] && source $rcfile @@ -1896,31 +1894,29 @@ def installComponent(self, componentType, system, component, extensions, compone try: componentCfg = os.path.join(self.linkedRootPath, 'etc', '%s_%s.cfg' % (system, component)) if not os.path.exists(componentCfg): - fd = open(componentCfg, 'w') - fd.close() + io.open(componentCfg, 'w').close() self._createRunitLog(runitCompDir) runFile = os.path.join(runitCompDir, 'run') - fd = open(runFile, 'w') - fd.write( - """#!/bin/bash - rcfile=%(bashrc)s - [ -e $rcfile ] && source $rcfile - # - exec 2>&1 - # - [ "%(componentType)s" = "agent" ] && renice 20 -p $$ - #%(bashVariables)s - # - exec python $DIRAC/DIRAC/Core/scripts/dirac-%(componentType)s.py %(system)s/%(component)s %(componentCfg)s < /dev/null - """ % {'bashrc': os.path.join(self.instancePath, 'bashrc'), - 'bashVariables': bashVars, - 'componentType': componentType, - 'system': system, - 'component': component, - 'componentCfg': componentCfg}) - fd.close() + with io.open(runFile, 'w') as fd: + fd.write( + u"""#!/bin/bash + rcfile=%(bashrc)s + [ -e $rcfile ] && source $rcfile + # + exec 2>&1 + # + [ "%(componentType)s" = "agent" ] && renice 20 -p $$ + #%(bashVariables)s + # + exec python $DIRAC/DIRAC/Core/scripts/dirac-%(componentType)s.py %(system)s/%(component)s %(componentCfg)s < /dev/null + """ % {'bashrc': os.path.join(self.instancePath, 'bashrc'), + 'bashVariables': bashVars, + 'componentType': componentType, + 'system': system, + 'component': component, + 'componentCfg': componentCfg}) os.chmod(runFile, self.gDefaultPerms) @@ -1930,8 +1926,8 @@ def installComponent(self, componentType, system, component, extensions, compone stopFile = os.path.join(runitCompDir, 'control', 't') # This is, e.g., /opt/dirac/control/WorkfloadManagementSystem/Matcher/ controlDir = self.runitDir.replace('runit', 'control') - with open(stopFile, 'w') as fd: - fd.write("""#!/bin/bash + with io.open(stopFile, 'w') as fd: + fd.write(u"""#!/bin/bash echo %(controlDir)s/%(system)s/%(component)s/stop_%(type)s touch %(controlDir)s/%(system)s/%(component)s/stop_%(type)s """ % {'controlDir': controlDir, @@ -2093,9 +2089,9 @@ def installPortal(self): try: self._createRunitLog(runitWebAppDir) runFile = os.path.join(runitWebAppDir, 'run') - with open(runFile, 'w') as fd: + with io.open(runFile, 'w') as fd: fd.write( - """#!/bin/bash + u"""#!/bin/bash rcfile=%(bashrc)s [ -e $rcfile ] && source $rcfile # @@ -2405,9 +2401,8 @@ def _createMySQLCMDLines(self, dbFile): cmdLines = [] - fd = open(dbFile) - dbLines = fd.readlines() - fd.close() + with io.open(dbFile, 'rt') as fd: + dbLines = fd.readlines() for line in dbLines: # Should we first source an SQL file (is this sql file an extension)? @@ -2415,9 +2410,8 @@ def _createMySQLCMDLines(self, dbFile): sourcedDBbFileName = line.split(' ')[1].replace('\n', '') gLogger.info("Found file to source: %s" % sourcedDBbFileName) sourcedDBbFile = os.path.join(rootPath, sourcedDBbFileName) - fdSourced = open(sourcedDBbFile) - dbLinesSourced = fdSourced.readlines() - fdSourced.close() + with io.open(sourcedDBbFile, 'rt') as fdSourced: + dbLinesSourced = fdSourced.readlines() for lineSourced in dbLinesSourced: if lineSourced.strip(): cmdLines.append(lineSourced.strip()) From d5175ac120632d6516f52234481267d05980bd2e Mon Sep 17 00:00:00 2001 From: fstagni Date: Mon, 15 Jun 2020 14:51:00 +0200 Subject: [PATCH 5/7] fix sh --- FrameworkSystem/Client/ComponentInstaller.py | 48 ++++++++++---------- 1 file changed, 25 insertions(+), 23 deletions(-) diff --git a/FrameworkSystem/Client/ComponentInstaller.py b/FrameworkSystem/Client/ComponentInstaller.py index 6382d9ec459..609dba57537 100644 --- a/FrameworkSystem/Client/ComponentInstaller.py +++ b/FrameworkSystem/Client/ComponentInstaller.py @@ -1823,12 +1823,11 @@ def _createRunitLog(self, runitCompDir): with io.open(logRunFile, 'w') as fd: fd.write( u"""#!/bin/bash - # - rcfile=%(bashrc)s - [ -e $rcfile ] && source $rcfile - # - exec svlogd . +rcfile=%(bashrc)s +[[ -e $rcfile ]] && source ${rcfile} +# +exec svlogd . """ % {'bashrc': os.path.join(self.instancePath, 'bashrc')}) os.chmod(logRunFile, self.gDefaultPerms) @@ -1902,15 +1901,16 @@ def installComponent(self, componentType, system, component, extensions, compone with io.open(runFile, 'w') as fd: fd.write( u"""#!/bin/bash - rcfile=%(bashrc)s - [ -e $rcfile ] && source $rcfile - # - exec 2>&1 - # - [ "%(componentType)s" = "agent" ] && renice 20 -p $$ - #%(bashVariables)s - # - exec python $DIRAC/DIRAC/Core/scripts/dirac-%(componentType)s.py %(system)s/%(component)s %(componentCfg)s < /dev/null + +rcfile=%(bashrc)s +[[ -e $rcfile ]] && source ${rcfile} +# +exec 2>&1 +# +[[ "%(componentType)s" = "agent" ]] && renice 20 -p $$ +#%(bashVariables)s +# +exec python $DIRAC/DIRAC/Core/scripts/dirac-%(componentType)s.py %(system)s/%(component)s %(componentCfg)s < /dev/null """ % {'bashrc': os.path.join(self.instancePath, 'bashrc'), 'bashVariables': bashVars, 'componentType': componentType, @@ -1925,9 +1925,10 @@ def installComponent(self, componentType, system, component, extensions, compone # This is, e.g., /opt/dirac/runit/WorkfloadManagementSystem/Matcher/control/t stopFile = os.path.join(runitCompDir, 'control', 't') # This is, e.g., /opt/dirac/control/WorkfloadManagementSystem/Matcher/ - controlDir = self.runitDir.replace('runit', 'control') + controlDir = self.runitDir.replace('runit', 'control') with io.open(stopFile, 'w') as fd: fd.write(u"""#!/bin/bash + echo %(controlDir)s/%(system)s/%(component)s/stop_%(type)s touch %(controlDir)s/%(system)s/%(component)s/stop_%(type)s """ % {'controlDir': controlDir, @@ -2090,16 +2091,17 @@ def installPortal(self): self._createRunitLog(runitWebAppDir) runFile = os.path.join(runitWebAppDir, 'run') with io.open(runFile, 'w') as fd: - fd.write( + fd.write( u"""#!/bin/bash - rcfile=%(bashrc)s - [ -e $rcfile ] && source $rcfile - # - exec 2>&1 - # - exec python %(DIRAC)s/WebAppDIRAC/scripts/dirac-webapp-run.py -p < /dev/null + +rcfile=%(bashrc)s +[[ -e $rcfile ]] && source $rcfile +# +exec 2>&1 +# +exec python %(DIRAC)s/WebAppDIRAC/scripts/dirac-webapp-run.py -p < /dev/null """ % {'bashrc': os.path.join(self.instancePath, 'bashrc'), - 'DIRAC': self.linkedRootPath}) + 'DIRAC': self.linkedRootPath}) os.chmod(runFile, self.gDefaultPerms) except Exception: From 92e679136f27c768b7db36a85472207d02efc052 Mon Sep 17 00:00:00 2001 From: fstagni Date: Mon, 29 Jun 2020 14:07:43 +0200 Subject: [PATCH 6/7] .keys -> list() --- WorkloadManagementSystem/Agent/SiteDirector.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/WorkloadManagementSystem/Agent/SiteDirector.py b/WorkloadManagementSystem/Agent/SiteDirector.py index 6546b2e26d6..10a31b0aeb6 100644 --- a/WorkloadManagementSystem/Agent/SiteDirector.py +++ b/WorkloadManagementSystem/Agent/SiteDirector.py @@ -536,7 +536,7 @@ def submitPilots(self): totalWaitingPilots = 0 manyWaitingPilotsFlag = False if self.pilotWaitingFlag: - tqIDList = additionalInfo.keys() + tqIDList = list(additionalInfo) result = pilotAgentsDB.countPilots({'TaskQueueID': tqIDList, 'Status': WAITING_PILOT_STATUS}, None) @@ -659,7 +659,7 @@ def _ifAndWhereToSubmit(self): def monitorJobsQueuesPilots(self, matchingTQs): """ Just printout of jobs queues and pilots status in TQ """ - tqIDList = matchingTQs.keys() + tqIDList = list(matchingTQs) result = pilotAgentsDB.countPilots({'TaskQueueID': tqIDList, 'Status': WAITING_PILOT_STATUS}, None) From 2079340afbd457a23ca2cc6a4090b7cb5f1d2cba Mon Sep 17 00:00:00 2001 From: fstagni Date: Mon, 29 Jun 2020 14:22:19 +0200 Subject: [PATCH 7/7] use super for Clients --- DataManagementSystem/Client/FTS3Client.py | 2 +- .../Client/S3GatewayClient.py | 2 +- .../Client/BundleDeliveryClient.py | 3 +- FrameworkSystem/Client/ComponentInstaller.py | 56 ++++++------- FrameworkSystem/Client/ProxyManagerClient.py | 82 +++++++++---------- .../Client/SystemAdministratorClient.py | 2 +- .../Service/ProxyManagerHandler.py | 52 ++++++------ ProductionSystem/Client/ProductionClient.py | 5 +- .../Client/StorageManagerClient.py | 2 +- .../Client/TransformationClient.py | 2 +- .../Agent/SiteDirector.py | 2 +- .../Client/PilotsLoggingClient.py | 2 +- 12 files changed, 107 insertions(+), 105 deletions(-) diff --git a/DataManagementSystem/Client/FTS3Client.py b/DataManagementSystem/Client/FTS3Client.py index 29f653e1016..79fdb5f4ba2 100644 --- a/DataManagementSystem/Client/FTS3Client.py +++ b/DataManagementSystem/Client/FTS3Client.py @@ -11,7 +11,7 @@ class FTS3Client(Client): def __init__(self, url=None, **kwargs): """ Constructor function. """ - Client.__init__(self, **kwargs) + super(FTS3Client, self).__init__(**kwargs) self.setServer('DataManagement/FTS3Manager') if url: self.setServer(url) diff --git a/DataManagementSystem/Client/S3GatewayClient.py b/DataManagementSystem/Client/S3GatewayClient.py index 04bd31396a3..263b77b49bb 100644 --- a/DataManagementSystem/Client/S3GatewayClient.py +++ b/DataManagementSystem/Client/S3GatewayClient.py @@ -11,7 +11,7 @@ class S3GatewayClient(Client): def __init__(self, url=None, **kwargs): """ Constructor function. """ - Client.__init__(self, **kwargs) + super(S3GatewayClient, self).__init__(**kwargs) self.setServer('DataManagement/S3Gateway') if url: self.setServer(url) diff --git a/FrameworkSystem/Client/BundleDeliveryClient.py b/FrameworkSystem/Client/BundleDeliveryClient.py index fd37907b5b5..5667199a351 100644 --- a/FrameworkSystem/Client/BundleDeliveryClient.py +++ b/FrameworkSystem/Client/BundleDeliveryClient.py @@ -21,7 +21,8 @@ class BundleDeliveryClient(Client): def __init__(self, transferClient=False, **kwargs): - Client.__init__(self, **kwargs) + super(BundleDeliveryClient, self).__init__(**kwargs) + self.setServer('Framework/BundleDelivery') self.transferClient = transferClient self.log = gLogger.getSubLogger("BundleDelivery") diff --git a/FrameworkSystem/Client/ComponentInstaller.py b/FrameworkSystem/Client/ComponentInstaller.py index 609dba57537..764bb0a4d72 100644 --- a/FrameworkSystem/Client/ComponentInstaller.py +++ b/FrameworkSystem/Client/ComponentInstaller.py @@ -1015,7 +1015,7 @@ def getSoftwareComponents(self, extensions): for agent in agentList: if os.path.splitext(agent)[1] == ".py": agentFile = os.path.join(agentDir, agent) - with io.open(agentFile, 'rt') as afile: + with io.open(agentFile, 'rt') as afile: body = afile.read() if body.find('AgentModule') != -1 or body.find('OptimizerModule') != -1: if system not in agents: @@ -1041,7 +1041,7 @@ def getSoftwareComponents(self, extensions): for executor in executorList: if os.path.splitext(executor)[1] == ".py": executorFile = os.path.join(executorDir, executor) - with io.open(executorFile, 'rt') as afile: + with io.open(executorFile, 'rt') as afile: body = afile.read() if body.find('OptimizerExecutor') != -1: if system not in executors: @@ -1092,8 +1092,8 @@ def getInstalledComponents(self): for component in components: try: runFile = os.path.join(systemDir, component, 'run') - with io.open(runFile, 'rt') as rFile: - body = rFile.read() + with io.open(runFile, 'rt') as rFile: + body = rFile.read() for cType in self.componentTypes: if body.find('dirac-%s' % (cType)) != -1: @@ -1126,8 +1126,8 @@ def getSetupComponents(self): for component in componentList: try: runFile = os.path.join(self.startDir, component, 'run') - with io.open(runFile, 'rt') as rfile: - body = rfile.read() + with io.open(runFile, 'rt') as rfile: + body = rfile.read() for cType in self.componentTypes: if body.find('dirac-%s' % (cType)) != -1: @@ -1462,8 +1462,8 @@ def getLogTail(self, system, component, length=100): if not os.path.exists(logFileName): retDict[compName] = 'No log file found' else: - with io.open(logFileName, 'rt') as logFile: - lines = [line.strip() for line in logFile.readlines()] + with io.open(logFileName, 'rt') as logFile: + lines = [line.strip() for line in logFile.readlines()] if len(lines) < length: retDict[compName] = '\n'.join(lines) @@ -1599,7 +1599,7 @@ def setupSite(self, scriptCfg, cfg=None): if not cmdFound: gLogger.notice('Starting runsvdir ...') - with io.open(os.devnull, 'w') as devnull: + with io.open(os.devnull, 'w') as devnull: subprocess.Popen(['nohup', 'runsvdir', self.startDir, 'log: DIRAC runsv'], stdout=devnull, stderr=devnull, universal_newlines=True) @@ -1815,14 +1815,14 @@ def _createRunitLog(self, runitCompDir): logConfigFile = os.path.join(logDir, 'config') with io.open(logConfigFile, 'w') as fd: fd.write( - u"""s10000000 + u"""s10000000 n20 """) logRunFile = os.path.join(logDir, 'run') with io.open(logRunFile, 'w') as fd: fd.write( - u"""#!/bin/bash + u"""#!/bin/bash rcfile=%(bashrc)s [[ -e $rcfile ]] && source ${rcfile} @@ -1893,14 +1893,14 @@ def installComponent(self, componentType, system, component, extensions, compone try: componentCfg = os.path.join(self.linkedRootPath, 'etc', '%s_%s.cfg' % (system, component)) if not os.path.exists(componentCfg): - io.open(componentCfg, 'w').close() + io.open(componentCfg, 'w').close() self._createRunitLog(runitCompDir) runFile = os.path.join(runitCompDir, 'run') with io.open(runFile, 'w') as fd: - fd.write( - u"""#!/bin/bash + fd.write( + u"""#!/bin/bash rcfile=%(bashrc)s [[ -e $rcfile ]] && source ${rcfile} @@ -1912,11 +1912,11 @@ def installComponent(self, componentType, system, component, extensions, compone # exec python $DIRAC/DIRAC/Core/scripts/dirac-%(componentType)s.py %(system)s/%(component)s %(componentCfg)s < /dev/null """ % {'bashrc': os.path.join(self.instancePath, 'bashrc'), - 'bashVariables': bashVars, - 'componentType': componentType, - 'system': system, - 'component': component, - 'componentCfg': componentCfg}) + 'bashVariables': bashVars, + 'componentType': componentType, + 'system': system, + 'component': component, + 'componentCfg': componentCfg}) os.chmod(runFile, self.gDefaultPerms) @@ -1925,9 +1925,9 @@ def installComponent(self, componentType, system, component, extensions, compone # This is, e.g., /opt/dirac/runit/WorkfloadManagementSystem/Matcher/control/t stopFile = os.path.join(runitCompDir, 'control', 't') # This is, e.g., /opt/dirac/control/WorkfloadManagementSystem/Matcher/ - controlDir = self.runitDir.replace('runit', 'control') - with io.open(stopFile, 'w') as fd: - fd.write(u"""#!/bin/bash + controlDir = self.runitDir.replace('runit', 'control') + with io.open(stopFile, 'w') as fd: + fd.write(u"""#!/bin/bash echo %(controlDir)s/%(system)s/%(component)s/stop_%(type)s touch %(controlDir)s/%(system)s/%(component)s/stop_%(type)s @@ -2090,9 +2090,9 @@ def installPortal(self): try: self._createRunitLog(runitWebAppDir) runFile = os.path.join(runitWebAppDir, 'run') - with io.open(runFile, 'w') as fd: - fd.write( - u"""#!/bin/bash + with io.open(runFile, 'w') as fd: + fd.write( + u"""#!/bin/bash rcfile=%(bashrc)s [[ -e $rcfile ]] && source $rcfile @@ -2101,7 +2101,7 @@ def installPortal(self): # exec python %(DIRAC)s/WebAppDIRAC/scripts/dirac-webapp-run.py -p < /dev/null """ % {'bashrc': os.path.join(self.instancePath, 'bashrc'), - 'DIRAC': self.linkedRootPath}) + 'DIRAC': self.linkedRootPath}) os.chmod(runFile, self.gDefaultPerms) except Exception: @@ -2412,8 +2412,8 @@ def _createMySQLCMDLines(self, dbFile): sourcedDBbFileName = line.split(' ')[1].replace('\n', '') gLogger.info("Found file to source: %s" % sourcedDBbFileName) sourcedDBbFile = os.path.join(rootPath, sourcedDBbFileName) - with io.open(sourcedDBbFile, 'rt') as fdSourced: - dbLinesSourced = fdSourced.readlines() + with io.open(sourcedDBbFile, 'rt') as fdSourced: + dbLinesSourced = fdSourced.readlines() for lineSourced in dbLinesSourced: if lineSourced.strip(): cmdLines.append(lineSourced.strip()) diff --git a/FrameworkSystem/Client/ProxyManagerClient.py b/FrameworkSystem/Client/ProxyManagerClient.py index d1df5a017ba..67a7ddbf71d 100755 --- a/FrameworkSystem/Client/ProxyManagerClient.py +++ b/FrameworkSystem/Client/ProxyManagerClient.py @@ -35,7 +35,7 @@ def __init__(self): def __deleteTemporalFile(self, filename): """ Delete temporal file - :param str filename: path to file + :param str filename: path to file """ try: os.remove(filename) @@ -89,8 +89,8 @@ def userHasProxy(self, userDN, userGroup, validSeconds=0): """ Check if a user(DN-group) has a proxy in the proxy management Updates internal cache if needed to minimize queries to the service - :param str userDN: user DN - :param str userGroup: user group + :param str userDN: user DN + :param str userGroup: user group :param int validSeconds: proxy valid time in a seconds :return: S_OK()/S_ERROR() @@ -110,8 +110,8 @@ def getUserPersistence(self, userDN, userGroup, validSeconds=0): """ Check if a user(DN-group) has a proxy in the proxy management Updates internal cache if needed to minimize queries to the service - :param str userDN: user DN - :param str userGroup: user group + :param str userDN: user DN + :param str userGroup: user group :param int validSeconds: proxy valid time in a seconds :return: S_OK()/S_ERROR() @@ -134,8 +134,8 @@ def getUserPersistence(self, userDN, userGroup, validSeconds=0): def setPersistency(self, userDN, userGroup, persistent): """ Set the persistency for user/group - :param str userDN: user DN - :param str userGroup: user group + :param str userDN: user DN + :param str userGroup: user group :param boolean persistent: presistent flag :return: S_OK()/S_ERROR() @@ -219,13 +219,13 @@ def downloadProxy(self, userDN, userGroup, limited=False, requiredTimeLeft=1200, cacheTime=14400, proxyToConnect=None, token=None): """ Get a proxy Chain from the proxy management - :param str userDN: user DN - :param str userGroup: user group + :param str userDN: user DN + :param str userGroup: user group :param boolean limited: if need limited proxy :param int requiredTimeLeft: required proxy live time in a seconds :param int cacheTime: store in a cache time in a seconds :param X509Chain proxyToConnect: proxy as a chain - :param str token: valid token to get a proxy + :param str token: valid token to get a proxy :return: S_OK(X509Chain)/S_ERROR() """ @@ -257,14 +257,14 @@ def downloadProxyToFile(self, userDN, userGroup, limited=False, requiredTimeLeft cacheTime=14400, filePath=None, proxyToConnect=None, token=None): """ Get a proxy Chain from the proxy management and write it to file - :param str userDN: user DN - :param str userGroup: user group + :param str userDN: user DN + :param str userGroup: user group :param boolean limited: if need limited proxy :param int requiredTimeLeft: required proxy live time in a seconds :param int cacheTime: store in a cache time in a seconds - :param str filePath: path to save proxy + :param str filePath: path to save proxy :param X509Chain proxyToConnect: proxy as a chain - :param str token: valid token to get a proxy + :param str token: valid token to get a proxy :return: S_OK(X509Chain)/S_ERROR() """ @@ -284,14 +284,14 @@ def downloadVOMSProxy(self, userDN, userGroup, limited=False, requiredTimeLeft=1 proxyToConnect=None, token=None): """ Download a proxy if needed and transform it into a VOMS one - :param str userDN: user DN - :param str userGroup: user group + :param str userDN: user DN + :param str userGroup: user group :param boolean limited: if need limited proxy :param int requiredTimeLeft: required proxy live time in a seconds :param int cacheTime: store in a cache time in a seconds - :param str requiredVOMSAttribute: VOMS attr to add to the proxy + :param str requiredVOMSAttribute: VOMS attr to add to the proxy :param X509Chain proxyToConnect: proxy as a chain - :param str token: valid token to get a proxy + :param str token: valid token to get a proxy :return: S_OK(X509Chain)/S_ERROR() """ @@ -325,15 +325,15 @@ def downloadVOMSProxyToFile(self, userDN, userGroup, limited=False, requiredTime proxyToConnect=None, token=None): """ Download a proxy if needed, transform it into a VOMS one and write it to file - :param str userDN: user DN - :param str userGroup: user group + :param str userDN: user DN + :param str userGroup: user group :param boolean limited: if need limited proxy :param int requiredTimeLeft: required proxy live time in a seconds :param int cacheTime: store in a cache time in a seconds - :param str requiredVOMSAttribute: VOMS attr to add to the proxy - :param str filePath: path to save proxy + :param str requiredVOMSAttribute: VOMS attr to add to the proxy + :param str filePath: path to save proxy :param X509Chain proxyToConnect: proxy as a chain - :param str token: valid token to get a proxy + :param str token: valid token to get a proxy :return: S_OK(X509Chain)/S_ERROR() """ @@ -351,9 +351,9 @@ def downloadVOMSProxyToFile(self, userDN, userGroup, limited=False, requiredTime def getPilotProxyFromDIRACGroup(self, userDN, userGroup, requiredTimeLeft=43200, proxyToConnect=None): """ Download a pilot proxy with VOMS extensions depending on the group - :param str userDN: user DN - :param str userGroup: user group - :param int requiredTimeLeft: required proxy live time in seconds + :param str userDN: user DN + :param str userGroup: user group + :param int requiredTimeLeft: required proxy live time in seconds :param X509Chain proxyToConnect: proxy as a chain :return: S_OK(X509Chain)/S_ERROR() @@ -371,8 +371,8 @@ def getPilotProxyFromDIRACGroup(self, userDN, userGroup, requiredTimeLeft=43200, def getPilotProxyFromVOMSGroup(self, userDN, vomsAttr, requiredTimeLeft=43200, proxyToConnect=None): """ Download a pilot proxy with VOMS extensions depending on the group - :param str userDN: user DN - :param str vomsAttr: VOMS attribute + :param str userDN: user DN + :param str vomsAttr: VOMS attribute :param int requiredTimeLeft: required proxy live time in a seconds :param X509Chain proxyToConnect: proxy as a chain @@ -395,10 +395,10 @@ def getPilotProxyFromVOMSGroup(self, userDN, vomsAttr, requiredTimeLeft=43200, p def getPayloadProxyFromDIRACGroup(self, userDN, userGroup, requiredTimeLeft, token=None, proxyToConnect=None): """ Download a payload proxy with VOMS extensions depending on the group - :param str userDN: user DN - :param str userGroup: user group + :param str userDN: user DN + :param str userGroup: user group :param int requiredTimeLeft: required proxy live time in a seconds - :param str token: valid token to get a proxy + :param str token: valid token to get a proxy :param X509Chain proxyToConnect: proxy as a chain :return: S_OK(X509Chain)/S_ERROR() @@ -417,9 +417,9 @@ def getPayloadProxyFromDIRACGroup(self, userDN, userGroup, requiredTimeLeft, tok def getPayloadProxyFromVOMSGroup(self, userDN, vomsAttr, token, requiredTimeLeft, proxyToConnect=None): """ Download a payload proxy with VOMS extensions depending on the VOMS attr - :param str userDN: user DN - :param str vomsAttr: VOMS attribute - :param str token: valid token to get a proxy + :param str userDN: user DN + :param str vomsAttr: VOMS attribute + :param str token: valid token to get a proxy :param int requiredTimeLeft: required proxy live time in a seconds :param X509Chain proxyToConnect: proxy as a chain @@ -442,10 +442,10 @@ def dumpProxyToFile(self, chain, destinationFile=None, requiredTimeLeft=600): """ Dump a proxy to a file. It's cached so multiple calls won't generate extra files :param X509Chain chain: proxy as a chain - :param str destinationFile: path to store proxy + :param str destinationFile: path to store proxy :param int requiredTimeLeft: required proxy live time in a seconds - :return: S_OK(str)/S_ERROR() + :return: S_OK(str)/S_ERROR() """ result = chain.hash() if not result['OK']: @@ -477,8 +477,8 @@ def requestToken(self, requesterDN, requesterGroup, numUses=1): """ Request a number of tokens. usesList must be a list of integers and each integer is the number of uses a token must have - :param str requesterDN: user DN - :param str requesterGroup: user group + :param str requesterDN: user DN + :param str requesterGroup: user group :param int numUses: number of uses :return: S_OK(tuple)/S_ERROR() -- tuple contain token, number uses @@ -573,15 +573,15 @@ def getVOMSAttributes(self, chain): :param X509Chain chain: proxy as a chain - :return: S_OK(str)/S_ERROR() + :return: S_OK(str)/S_ERROR() """ return VOMS().getVOMSAttributes(chain) def getUploadedProxyLifeTime(self, DN, group): """ Get the remaining seconds for an uploaded proxy - :param str DN: user DN - :param str group: group + :param str DN: user DN + :param str group: group :return: S_OK(int)/S_ERROR() """ diff --git a/FrameworkSystem/Client/SystemAdministratorClient.py b/FrameworkSystem/Client/SystemAdministratorClient.py index 784d7b7b824..0e69080403a 100644 --- a/FrameworkSystem/Client/SystemAdministratorClient.py +++ b/FrameworkSystem/Client/SystemAdministratorClient.py @@ -16,7 +16,7 @@ class SystemAdministratorClient(Client): def __init__(self, host, port=None, **kwargs): """ Constructor function. Takes a mandatory host parameter """ - Client.__init__(self, **kwargs) + super(SystemAdministratorClient, self).__init__(**kwargs) if not port: port = SYSADMIN_PORT self.setServer('dips://%s:%s/Framework/SystemAdministrator' % (host, port)) diff --git a/FrameworkSystem/Service/ProxyManagerHandler.py b/FrameworkSystem/Service/ProxyManagerHandler.py index 2aabb0d17a7..eaee0640179 100644 --- a/FrameworkSystem/Service/ProxyManagerHandler.py +++ b/FrameworkSystem/Service/ProxyManagerHandler.py @@ -91,7 +91,7 @@ def export_getUserProxiesInfo(self): def export_requestDelegationUpload(self, requestedUploadTime, diracGroup=None): """ Request a delegation. Send a delegation request to client - :param int requestedUploadTime: requested live time + :param int requestedUploadTime: requested live time :return: S_OK(dict)/S_ERROR() -- dict contain id and proxy as string of the request """ @@ -126,7 +126,7 @@ def export_completeDelegationUpload(self, requestId, pemChain): """ Upload result of delegation :param int,long requestId: identity number - :param str pemChain: certificate as string + :param str pemChain: certificate as string :return: S_OK(dict)/S_ERROR() -- dict contain proxies """ @@ -157,8 +157,8 @@ def export_getRegisteredUsers(self, validSecondsRequired=0): def __checkProperties(self, requestedUserDN, requestedUserGroup): """ Check the properties and return if they can only download limited proxies if authorized - :param str requestedUserDN: user DN - :param str requestedUserGroup: DIRAC group + :param str requestedUserDN: user DN + :param str requestedUserGroup: DIRAC group :return: S_OK(boolean)/S_ERROR() """ @@ -202,13 +202,13 @@ def export_getProxy(self, userDN, userGroup, requestPem, requiredLifetime): def __getProxy(self, userDN, userGroup, requestPem, requiredLifetime, forceLimited): """ Internal to get a proxy - :param str userDN: user DN - :param str userGroup: DIRAC group - :param str requestPem: dump of request certificate - :param int requiredLifetime: requested live time of proxy + :param str userDN: user DN + :param str userGroup: DIRAC group + :param str requestPem: dump of request certificate + :param int requiredLifetime: requested live time of proxy :param boolean forceLimited: limited proxy - :return: S_OK(str)/S_ERROR() + :return: S_OK(str)/S_ERROR() """ retVal = self.__proxyDB.getProxy(userDN, userGroup, requiredLifeTime=requiredLifetime) if not retVal['OK']: @@ -224,8 +224,8 @@ def __getProxy(self, userDN, userGroup, requestPem, requiredLifetime, forceLimit return S_OK(retVal['Value']) types_getVOMSProxy = [six.string_types, six.string_types, - six.string_types, six.integer_types, - [six.string_types, type(None), bool]] + six.string_types, six.integer_types, + [six.string_types, type(None), bool]] def export_getVOMSProxy(self, userDN, userGroup, requestPem, requiredLifetime, vomsAttribute=None): """ Get a proxy for a userDN/userGroup @@ -259,16 +259,16 @@ def __getVOMSProxy(self, userDN, userGroup, requestPem, requiredLifetime, vomsAt # If possible we return a proxy 1.5 longer than requested requiredLifetime = int(min(secsLeft, requiredLifetime * self.__maxExtraLifeFactor)) return chain.generateChainFromRequestString(requestPem, - lifetime=requiredLifetime, - requireLimited=forceLimited) + lifetime=requiredLifetime, + requireLimited=forceLimited) types_setPersistency = [six.string_types, six.string_types, bool] def export_setPersistency(self, userDN, userGroup, persistentFlag): """ Set the persistency for a given dn/group - :param str userDN: user DN - :param str userGroup: DIRAC group + :param str userDN: user DN + :param str userGroup: DIRAC group :param boolean persistentFlag: if proxy persistent :return: S_OK()/S_ERROR() @@ -309,8 +309,8 @@ def export_deleteProxyBundle(self, idList): def export_deleteProxy(self, userDN, userGroup): """ Delete a proxy from the DB - :param str userDN: user DN - :param str userGroup: DIRAC group + :param str userDN: user DN + :param str userGroup: DIRAC group :return: S_OK()/S_ERROR() """ @@ -331,8 +331,8 @@ def export_getContents(self, selDict, sortDict, start, limit): :param dict selDict: selection fields :param list,tuple sortDict: sorting fields - :param int start: search limit start - :param int start: search limit amount + :param int start: search limit start + :param int start: search limit amount :return: S_OK(dict)/S_ERROR() -- dict contain fields, record list, total records """ @@ -348,8 +348,8 @@ def export_getLogContents(self, selDict, sortDict, start, limit): :param dict selDict: selection fields :param list,tuple sortDict: search filter - :param int start: search limit start - :param int start: search limit amount + :param int start: search limit start + :param int start: search limit amount :return: S_OK(dict)/S_ERROR() -- dict contain fields, record list, total records """ @@ -360,9 +360,9 @@ def export_getLogContents(self, selDict, sortDict, start, limit): def export_generateToken(self, requesterDN, requesterGroup, tokenUses): """ Generate tokens for proxy retrieval - :param str requesterDN: user DN - :param str requesterGroup: DIRAC group - :param int tokenUses: number of uses + :param str requesterDN: user DN + :param str requesterGroup: DIRAC group + :param int tokenUses: number of uses :return: S_OK(tuple)/S_ERROR() -- tuple contain token, number uses """ @@ -400,8 +400,8 @@ def export_getProxyWithToken(self, userDN, userGroup, requestPem, requiredLifeti return self.__getProxy(userDN, userGroup, requestPem, requiredLifetime, True) types_getVOMSProxyWithToken = [six.string_types, six.string_types, - six.string_types, six.integer_types, - [six.string_types, type(None)]] + six.string_types, six.integer_types, + [six.string_types, type(None)]] def export_getVOMSProxyWithToken(self, userDN, userGroup, requestPem, requiredLifetime, token, vomsAttribute=None): """ Get a proxy for a userDN/userGroup diff --git a/ProductionSystem/Client/ProductionClient.py b/ProductionSystem/Client/ProductionClient.py index b77bc1ca9d1..09404335569 100644 --- a/ProductionSystem/Client/ProductionClient.py +++ b/ProductionSystem/Client/ProductionClient.py @@ -3,10 +3,11 @@ __RCSID__ = "$Id$" from DIRAC import gLogger, S_OK, S_ERROR -from DIRAC.Core.Base.Client import Client +from DIRAC.Core.Base.Client import Client, createClient from DIRAC.ProductionSystem.Utilities.StateMachine import ProductionsStateMachine +@createClient('Framework/BundleDelivery') class ProductionClient(Client): """ Exposes the functionality available in the ProductionSystem/ProductionManagerHandler @@ -16,7 +17,7 @@ def __init__(self, **kwargs): """ Simple constructor """ - Client.__init__(self, **kwargs) + super(ProductionClient, self).__init__(**kwargs) self.setServer('Production/ProductionManager') self.prodDescription = {} self.stepCounter = 1 diff --git a/StorageManagementSystem/Client/StorageManagerClient.py b/StorageManagementSystem/Client/StorageManagerClient.py index d40f6060b53..fccbc359a23 100644 --- a/StorageManagementSystem/Client/StorageManagerClient.py +++ b/StorageManagementSystem/Client/StorageManagerClient.py @@ -223,5 +223,5 @@ class StorageManagerClient(Client): """ def __init__(self, **kwargs): - Client.__init__(self, **kwargs) + super(StorageManagerClient, self).__init__(**kwargs) self.setServer('StorageManagement/StorageManager') diff --git a/TransformationSystem/Client/TransformationClient.py b/TransformationSystem/Client/TransformationClient.py index 012ab9345c9..1489717ef6d 100644 --- a/TransformationSystem/Client/TransformationClient.py +++ b/TransformationSystem/Client/TransformationClient.py @@ -61,7 +61,7 @@ def __init__(self, **kwargs): """ Simple constructor """ - Client.__init__(self, **kwargs) + super(TransformationClient, self).__init__(**kwargs) opsH = Operations() self.maxResetCounter = opsH.getValue('Transformations/FilesMaxResetCounter', 10) diff --git a/WorkloadManagementSystem/Agent/SiteDirector.py b/WorkloadManagementSystem/Agent/SiteDirector.py index 10a31b0aeb6..922b2cfbbf8 100644 --- a/WorkloadManagementSystem/Agent/SiteDirector.py +++ b/WorkloadManagementSystem/Agent/SiteDirector.py @@ -536,7 +536,7 @@ def submitPilots(self): totalWaitingPilots = 0 manyWaitingPilotsFlag = False if self.pilotWaitingFlag: - tqIDList = list(additionalInfo) + tqIDList = list(additionalInfo) result = pilotAgentsDB.countPilots({'TaskQueueID': tqIDList, 'Status': WAITING_PILOT_STATUS}, None) diff --git a/WorkloadManagementSystem/Client/PilotsLoggingClient.py b/WorkloadManagementSystem/Client/PilotsLoggingClient.py index 87eb9ca17ca..2c3c03a494d 100644 --- a/WorkloadManagementSystem/Client/PilotsLoggingClient.py +++ b/WorkloadManagementSystem/Client/PilotsLoggingClient.py @@ -12,7 +12,7 @@ class PilotsLoggingClient(Client): """ def __init__(self, **kwargs): - Client.__init__(self, **kwargs) + super(PilotsLoggingClient, self).__init__(**kwargs) self.setServer('WorkloadManagement/PilotsLogging') def addPilotsLogging(self, pilotUUID, timestamp, source, phase, status, messageContent):