From 111b871ad728fc0204c689271ff8f19d86b492f1 Mon Sep 17 00:00:00 2001 From: Federico Stagni Date: Thu, 7 Dec 2017 15:51:28 +0100 Subject: [PATCH 01/17] added spaceTokenUsage scripts --- .../Client/SpaceTokenUsage.py | 201 ++++++++++++++++++ .../scripts/dirac-dms-spaceTokens-usage.py | 27 +++ 2 files changed, 228 insertions(+) create mode 100644 DataManagementSystem/Client/SpaceTokenUsage.py create mode 100755 DataManagementSystem/scripts/dirac-dms-spaceTokens-usage.py diff --git a/DataManagementSystem/Client/SpaceTokenUsage.py b/DataManagementSystem/Client/SpaceTokenUsage.py new file mode 100644 index 00000000000..6cd8c13a68a --- /dev/null +++ b/DataManagementSystem/Client/SpaceTokenUsage.py @@ -0,0 +1,201 @@ +""" +This class gets information about storage space tokens from the FC recording, +from SRM and if available from storage dumps +It reports for each site its availability and usage +""" + +import time + +from DIRAC import gLogger +from DIRAC.Core.DISET.RPCClient import RPCClient +from DIRAC.Core.Base import Script + +from DIRAC.DataManagementSystem.Utilities.DMSHelpers import DMSHelpers +from DIRAC.Resources.Storage.StorageElement import StorageElement + + +class SpaceTokenUsage(object): + """ + Class used to get information about Space token occupancy + """ + + def __init__(self): + """ c'tor """ + import lcg_util + self.lcg_util = lcg_util + self.storageusage = RPCClient('DataManagement/StorageUsage') + self.spaceTokenInfo = {} + self.sitesSEs = {} + self.storageElementSet = DMSHelpers().getStorageElements() + self.dmsHelper = DMSHelpers() + self.shortSiteNames = self.dmsHelper.getShortSiteNames(tier=(0, 1)) + self.storageSummary = None + + def execute(self, unit): + """ Parse the request and execute the command """ + + sites = None + for switch in Script.getUnprocessedSwitches(): + if switch[0] in ("u", "Unit"): + unit = switch[1] + if switch[0] in ("S", "Sites"): + sites = switch[1].split(',') + + if not sites: + # Tier0 and all Tier1s + sites = sorted(self.dmsHelper.getTiers(tier=(0, 1))) + else: + # Translate in case it is a short name + allSites = self.dmsHelper.getSites() + sites = [self.shortSiteNames.get(site, site) for site in sites] + badSites = set(sites) - set(allSites) + if badSites: + gLogger.warn("Some sites do not exist", str(sorted(badSites))) + sites = [site for site in sites if site in allSites] + + scaleDict = {'MB': 1000 * 1000.0, + 'GB': 1000 * 1000 * 1000.0, + 'TB': 1000 * 1000 * 1000 * 1000.0, + 'PB': 1000 * 1000 * 1000 * 1000 * 1000.0} + if unit not in scaleDict: + Script.showHelp() + scaleFactor = scaleDict[unit] + + for site in sites: + self.sitesSEs[site] = {} + # Get SEs at site + seList = self.dmsHelper.getSEsAtSite(site).get('Value', []) + for se in seList: + spaceToken = StorageElement(se).getStorageParameters(protocol='srm') + if spaceToken['OK']: + spaceToken = spaceToken['Value'] + st = spaceToken['SpaceToken'] + self.sitesSEs[site].setdefault(st, {}).setdefault('SEs', []).append(se) + # Fill in the endpoints + ep = 'httpg://%s:%s%s' % (spaceToken['Host'], spaceToken['Port'], spaceToken['WSUrl'].split('?')[0]) + self.spaceTokenInfo.setdefault(site.split('.')[1], {}).setdefault(ep, set()).add(st) + + lfcUsage = {} + srmUsage = {} + sdUsage = {} + for site in sites: + # retrieve space usage from LFC + lfcUsage[site] = self.getFCUsage(site) + + # retrieve SRM usage + srmResult = self.getSrmUsage(site) + if srmResult != -1: + srmUsage[site] = srmResult + else: + return 1 + + # retrieve space usage from storage dumps: + sdResult = self.getSDUsage(site) + if sdResult != -1: + sdUsage[site] = sdResult + else: + return 1 + + gLogger.notice("Storage usage summary for site %s - %s " % (site.split('.')[1], time.asctime())) + for st in self.sitesSEs[site]: + gLogger.notice("Space token %s " % st) + gLogger.notice("\tFrom FC: Files: %d, Size: %.2f %s" % + (lfcUsage[site][st]['Files'], + lfcUsage[site][st]['Size'] / scaleFactor, unit)) + if site in srmUsage and st in srmUsage[site]: + gLogger.notice("\tFrom SRM: Total Assigned Space: %.2f %s, Used Space: %.2f %s, Free Space: %.2f %s " % + (srmUsage[site][st]['SRMTotal'] / scaleFactor, unit, + srmUsage[site][st]['SRMUsed'] / scaleFactor, unit, + srmUsage[site][st]['SRMFree'] / scaleFactor, unit)) + else: + gLogger.notice("\tFrom SRM: Information not available") + if site in sdUsage and st in sdUsage[site]: + gLogger.notice("\tFrom storage dumps: Files: %d, Size: %.2f %s - last update %s " % + (sdUsage[site][st]['Files'], + sdUsage[site][st]['Size'] / scaleFactor, unit, + sdUsage[site][st]['LastUpdate'])) + else: + gLogger.notice("\tFrom storage dumps: Information not available") + return 0 + + def getSrmUsage(self, lcgSite): + """Get space usage via SRM interface + """ + try: + site = lcgSite.split('.')[1] + except IndexError: + site = lcgSite + if site not in self.spaceTokenInfo: + gLogger.error("ERROR: information not available for site %s. Space token information from CS: %s " + % (site, sorted(self.spaceTokenInfo))) + return -1 + + result = {} + for ep, stList in self.spaceTokenInfo[site].iteritems(): + for st in stList: + result[st] = {} + srm = self.lcg_util.lcg_stmd(st, ep, True, 0) + if srm[0]: + # This SpaceToken doesn't exist at this endPoint + continue + srmVal = srm[1][0] + srmTotSpace = srmVal['totalsize'] + # correct for the 6% overhead due to castor setup at RAL + if 'gridpp' in ep: + srmTotSpace = (srmVal['totalsize']) * 0.94 + gLogger.warn('WARNING! apply a 0.94 factor to total space for RAL!') + srmFree = srmVal['unusedsize'] + srmUsed = srmTotSpace - srmFree + result[st]['SRMUsed'] = srmUsed + result[st]['SRMFree'] = srmFree + result[st]['SRMTotal'] = srmTotSpace + return result + + # ................................................. + + def getSDUsage(self, lcgSite): + """ get storage usage from storage dumps + """ + try: + site = lcgSite.split('.')[1] + except IndexError: + site = lcgSite + res = self.storageusage.getSTSummary(site) + if not res['OK']: + gLogger.error("ERROR: Cannot get storage dump information for site %s :" % site, res['Message']) + return -1 + if not res['Value']: + gLogger.warn(" No information available for site %s from storage dumps" % site) + sdUsage = {} + for row in res['Value']: + site, spaceTokenWithID, totalSpace, totalFiles, lastUpdate = row + for st in self.sitesSEs[lcgSite]: + sdUsage.setdefault(st, {}) + if st in spaceTokenWithID: + sdUsage[st]['Size'] = totalSpace + sdUsage[st]['Files'] = totalFiles + sdUsage[st]['LastUpdate'] = lastUpdate + break + return sdUsage + + def getFCUsage(self, lcgSite): + """ get storage usage from LFC + """ + if self.storageSummary is None: + res = self.storageusage.getStorageSummary() + if not res['OK']: + gLogger.error('ERROR in getStorageSummary ', res['Message']) + return {} + self.storageSummary = res['Value'] + + usage = {} + for st in self.sitesSEs[lcgSite]: + usage[st] = {'Files': 0, 'Size': 0} + for se in self.sitesSEs[lcgSite][st]['SEs']: + if se in self.storageSummary: + usage[st]['Files'] += self.storageSummary[se]['Files'] + usage[st]['Size'] += self.storageSummary[se]['Size'] + else: + gLogger.error("No FC storage information for SE", se) + + return usage diff --git a/DataManagementSystem/scripts/dirac-dms-spaceTokens-usage.py b/DataManagementSystem/scripts/dirac-dms-spaceTokens-usage.py new file mode 100755 index 00000000000..d1d33818992 --- /dev/null +++ b/DataManagementSystem/scripts/dirac-dms-spaceTokens-usage.py @@ -0,0 +1,27 @@ +#!/usr/bin/env python + +""" Check the space token usage at the site and report the space usage from several sources: + File Catalogue, Storage dumps, SRM interface +""" + +__RCSID__ = "$Id$" + +from DIRAC.Core.Base import Script +import DIRAC + +if __name__ == "__main__": + + unit = 'TB' + Script.registerSwitch("u:", "Unit=", " Unit to use [%s] (MB,GB,TB,PB)" % unit) + Script.registerSwitch("S:", "Sites=", " Sites to consider [ALL] (space or comma separated list, e.g. LCG.CNAF.it") + # Script.registerSwitch( "l:", "Site=", " LCG Site list to check [%s] (e.g. LCG.CERN.cern, LCG.CNAF.it, ... )" %sites ) + + Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], + 'Usage:', + ' %s [option|cfgfile] ...' % + Script.scriptName, ])) + + Script.parseCommandLine(ignoreErrors=False) + + from LHCbDIRAC.DataManagementSystem.Client.SpaceTokenUsage import SpaceTokenUsage + DIRAC.exit(SpaceTokenUsage().execute(unit)) From ede48931a585ff39c64f918d9026942a8e592372 Mon Sep 17 00:00:00 2001 From: Federico Stagni Date: Thu, 7 Dec 2017 18:59:38 +0100 Subject: [PATCH 02/17] moved to functions --- .../Client/SpaceTokenUsage.py | 347 +++++++++--------- 1 file changed, 166 insertions(+), 181 deletions(-) diff --git a/DataManagementSystem/Client/SpaceTokenUsage.py b/DataManagementSystem/Client/SpaceTokenUsage.py index 6cd8c13a68a..e1907d7c3d2 100644 --- a/DataManagementSystem/Client/SpaceTokenUsage.py +++ b/DataManagementSystem/Client/SpaceTokenUsage.py @@ -5,197 +5,182 @@ """ import time +import lcg_util -from DIRAC import gLogger +from DIRAC import gLogger, S_OK, S_ERROR from DIRAC.Core.DISET.RPCClient import RPCClient -from DIRAC.Core.Base import Script from DIRAC.DataManagementSystem.Utilities.DMSHelpers import DMSHelpers from DIRAC.Resources.Storage.StorageElement import StorageElement +sitesSEs = {} +spaceTokenInfo = {} +storageSummary = None -class SpaceTokenUsage(object): - """ - Class used to get information about Space token occupancy + +def combinedResult(unit, sites=None): + """ Checks the space token usage at the site and report the space usage from several sources: + File Catalogue, Storage dumps, SRM interface """ - def __init__(self): - """ c'tor """ - import lcg_util - self.lcg_util = lcg_util - self.storageusage = RPCClient('DataManagement/StorageUsage') - self.spaceTokenInfo = {} - self.sitesSEs = {} - self.storageElementSet = DMSHelpers().getStorageElements() - self.dmsHelper = DMSHelpers() - self.shortSiteNames = self.dmsHelper.getShortSiteNames(tier=(0, 1)) - self.storageSummary = None - - def execute(self, unit): - """ Parse the request and execute the command """ - - sites = None - for switch in Script.getUnprocessedSwitches(): - if switch[0] in ("u", "Unit"): - unit = switch[1] - if switch[0] in ("S", "Sites"): - sites = switch[1].split(',') - - if not sites: - # Tier0 and all Tier1s - sites = sorted(self.dmsHelper.getTiers(tier=(0, 1))) + dmsHelper = DMSHelpers() + + if not sites: + # Tier0 and all Tier1s + sites = sorted(dmsHelper.getTiers(tier=(0, 1))) + else: + # Translate in case it is a short name + allSites = dmsHelper.getSites() + sites = [dmsHelper.getShortSiteNames(tier=(0, 1)).get(site, site) for site in sites] + badSites = set(sites) - set(allSites) + if badSites: + gLogger.warn("Some sites do not exist", str(sorted(badSites))) + sites = [site for site in sites if site in allSites] + + scaleDict = {'MB': 1000 * 1000.0, + 'GB': 1000 * 1000 * 1000.0, + 'TB': 1000 * 1000 * 1000 * 1000.0, + 'PB': 1000 * 1000 * 1000 * 1000 * 1000.0} + if unit not in scaleDict: + return S_ERROR("Unit not in %s" % scaleDict.keys()) + scaleFactor = scaleDict[unit] + + for site in sites: + sitesSEs[site] = {} + # Get SEs at site + seList = dmsHelper.getSEsAtSite(site).get('Value', []) + for se in seList: + spaceToken = StorageElement(se).getStorageParameters(protocol='srm') # FIXME: remove this limit + if spaceToken['OK']: + spaceToken = spaceToken['Value'] + st = spaceToken['SpaceToken'] + sitesSEs[site].setdefault(st, {}).setdefault('SEs', []).append(se) + # Fill in the endpoints + # FIXME: should move all this shit in the StorageElement definition + ep = 'httpg://%s:%s%s' % (spaceToken['Host'], spaceToken['Port'], spaceToken['WSUrl'].split('?')[0]) + spaceTokenInfo.setdefault(site.split('.')[1], {}).setdefault(ep, set()).add(st) + + fcUsage = {} + srmUsage = {} + sdUsage = {} + for site in sites: + # retrieve space usage from FC + fcUsage[site] = getFCUsage(site) + + # retrieve SRM usage + srmResult = getSrmUsage(site) + if srmResult != -1: + srmUsage[site] = srmResult else: - # Translate in case it is a short name - allSites = self.dmsHelper.getSites() - sites = [self.shortSiteNames.get(site, site) for site in sites] - badSites = set(sites) - set(allSites) - if badSites: - gLogger.warn("Some sites do not exist", str(sorted(badSites))) - sites = [site for site in sites if site in allSites] - - scaleDict = {'MB': 1000 * 1000.0, - 'GB': 1000 * 1000 * 1000.0, - 'TB': 1000 * 1000 * 1000 * 1000.0, - 'PB': 1000 * 1000 * 1000 * 1000 * 1000.0} - if unit not in scaleDict: - Script.showHelp() - scaleFactor = scaleDict[unit] - - for site in sites: - self.sitesSEs[site] = {} - # Get SEs at site - seList = self.dmsHelper.getSEsAtSite(site).get('Value', []) - for se in seList: - spaceToken = StorageElement(se).getStorageParameters(protocol='srm') - if spaceToken['OK']: - spaceToken = spaceToken['Value'] - st = spaceToken['SpaceToken'] - self.sitesSEs[site].setdefault(st, {}).setdefault('SEs', []).append(se) - # Fill in the endpoints - ep = 'httpg://%s:%s%s' % (spaceToken['Host'], spaceToken['Port'], spaceToken['WSUrl'].split('?')[0]) - self.spaceTokenInfo.setdefault(site.split('.')[1], {}).setdefault(ep, set()).add(st) - - lfcUsage = {} - srmUsage = {} - sdUsage = {} - for site in sites: - # retrieve space usage from LFC - lfcUsage[site] = self.getFCUsage(site) - - # retrieve SRM usage - srmResult = self.getSrmUsage(site) - if srmResult != -1: - srmUsage[site] = srmResult - else: - return 1 + return 1 - # retrieve space usage from storage dumps: - sdResult = self.getSDUsage(site) - if sdResult != -1: - sdUsage[site] = sdResult + # retrieve space usage from storage dumps: + sdResult = getSDUsage(site) + if sdResult != -1: + sdUsage[site] = sdResult + else: + return 1 + + gLogger.notice("Storage usage summary for site %s - %s " % (site.split('.')[1], time.asctime())) + for st in sitesSEs[site]: + gLogger.notice("Space token %s " % st) + gLogger.notice("\tFrom FC: Files: %d, Size: %.2f %s" % + (fcUsage[site][st]['Files'], + fcUsage[site][st]['Size'] / scaleFactor, unit)) + if site in srmUsage and st in srmUsage[site]: + gLogger.notice("\tFrom SRM: Total Assigned Space: %.2f %s, Used Space: %.2f %s, Free Space: %.2f %s " % + (srmUsage[site][st]['SRMTotal'] / scaleFactor, unit, + srmUsage[site][st]['SRMUsed'] / scaleFactor, unit, + srmUsage[site][st]['SRMFree'] / scaleFactor, unit)) + else: + gLogger.notice("\tFrom SRM: Information not available") + if site in sdUsage and st in sdUsage[site]: + gLogger.notice("\tFrom storage dumps: Files: %d, Size: %.2f %s - last update %s " % + (sdUsage[site][st]['Files'], + sdUsage[site][st]['Size'] / scaleFactor, unit, + sdUsage[site][st]['LastUpdate'])) else: - return 1 - - gLogger.notice("Storage usage summary for site %s - %s " % (site.split('.')[1], time.asctime())) - for st in self.sitesSEs[site]: - gLogger.notice("Space token %s " % st) - gLogger.notice("\tFrom FC: Files: %d, Size: %.2f %s" % - (lfcUsage[site][st]['Files'], - lfcUsage[site][st]['Size'] / scaleFactor, unit)) - if site in srmUsage and st in srmUsage[site]: - gLogger.notice("\tFrom SRM: Total Assigned Space: %.2f %s, Used Space: %.2f %s, Free Space: %.2f %s " % - (srmUsage[site][st]['SRMTotal'] / scaleFactor, unit, - srmUsage[site][st]['SRMUsed'] / scaleFactor, unit, - srmUsage[site][st]['SRMFree'] / scaleFactor, unit)) - else: - gLogger.notice("\tFrom SRM: Information not available") - if site in sdUsage and st in sdUsage[site]: - gLogger.notice("\tFrom storage dumps: Files: %d, Size: %.2f %s - last update %s " % - (sdUsage[site][st]['Files'], - sdUsage[site][st]['Size'] / scaleFactor, unit, - sdUsage[site][st]['LastUpdate'])) - else: - gLogger.notice("\tFrom storage dumps: Information not available") - return 0 - - def getSrmUsage(self, lcgSite): - """Get space usage via SRM interface - """ - try: - site = lcgSite.split('.')[1] - except IndexError: - site = lcgSite - if site not in self.spaceTokenInfo: - gLogger.error("ERROR: information not available for site %s. Space token information from CS: %s " - % (site, sorted(self.spaceTokenInfo))) - return -1 - - result = {} - for ep, stList in self.spaceTokenInfo[site].iteritems(): - for st in stList: - result[st] = {} - srm = self.lcg_util.lcg_stmd(st, ep, True, 0) - if srm[0]: - # This SpaceToken doesn't exist at this endPoint - continue - srmVal = srm[1][0] - srmTotSpace = srmVal['totalsize'] - # correct for the 6% overhead due to castor setup at RAL - if 'gridpp' in ep: - srmTotSpace = (srmVal['totalsize']) * 0.94 - gLogger.warn('WARNING! apply a 0.94 factor to total space for RAL!') - srmFree = srmVal['unusedsize'] - srmUsed = srmTotSpace - srmFree - result[st]['SRMUsed'] = srmUsed - result[st]['SRMFree'] = srmFree - result[st]['SRMTotal'] = srmTotSpace - return result - - # ................................................. - - def getSDUsage(self, lcgSite): - """ get storage usage from storage dumps - """ - try: - site = lcgSite.split('.')[1] - except IndexError: - site = lcgSite - res = self.storageusage.getSTSummary(site) + gLogger.notice("\tFrom storage dumps: Information not available") + return 0 + + +def getSrmUsage(lcgSite): # FIXME: should move this shit in the StorageElement definition (SRM2Storage or GFAL2_SRM2Storage?) + """Get space usage via SRM interface + """ + try: + site = lcgSite.split('.')[1] + except IndexError: + site = lcgSite + if site not in spaceTokenInfo: + gLogger.error("ERROR: information not available for site %s. Space token information from CS: %s " + % (site, sorted(spaceTokenInfo))) + return -1 + + result = {} + for ep, stList in spaceTokenInfo[site].iteritems(): + for st in stList: + result[st] = {} + srm = lcg_util.lcg_stmd(st, ep, True, 0) + if srm[0]: + # This SpaceToken doesn't exist at this endPoint + continue + srmVal = srm[1][0] + srmTotSpace = srmVal['totalsize'] + srmFree = srmVal['unusedsize'] + srmUsed = srmTotSpace - srmFree + result[st]['SRMUsed'] = srmUsed + result[st]['SRMFree'] = srmFree + result[st]['SRMTotal'] = srmTotSpace + return result + +# ................................................. + + +def getSDUsage(lcgSite): + """ get storage usage from storage dumps + """ + try: + site = lcgSite.split('.')[1] + except IndexError: + site = lcgSite + res = RPCClient('DataManagement/StorageUsage').getSTSummary(site) + if not res['OK']: + gLogger.error("ERROR: Cannot get storage dump information for site %s :" % site, res['Message']) + return -1 + if not res['Value']: + gLogger.warn(" No information available for site %s from storage dumps" % site) + sdUsage = {} + for row in res['Value']: + site, spaceTokenWithID, totalSpace, totalFiles, lastUpdate = row + for st in sitesSEs[lcgSite]: + sdUsage.setdefault(st, {}) + if st in spaceTokenWithID: + sdUsage[st]['Size'] = totalSpace + sdUsage[st]['Files'] = totalFiles + sdUsage[st]['LastUpdate'] = lastUpdate + break + return sdUsage + + +def getFCUsage(lcgSite): + """ get storage usage from LFC + """ + if storageSummary is None: + # FIXME: this is in LHCbDIRAC. So it should be moved + res = RPCClient('DataManagement/StorageUsage').getStorageSummary() if not res['OK']: - gLogger.error("ERROR: Cannot get storage dump information for site %s :" % site, res['Message']) - return -1 - if not res['Value']: - gLogger.warn(" No information available for site %s from storage dumps" % site) - sdUsage = {} - for row in res['Value']: - site, spaceTokenWithID, totalSpace, totalFiles, lastUpdate = row - for st in self.sitesSEs[lcgSite]: - sdUsage.setdefault(st, {}) - if st in spaceTokenWithID: - sdUsage[st]['Size'] = totalSpace - sdUsage[st]['Files'] = totalFiles - sdUsage[st]['LastUpdate'] = lastUpdate - break - return sdUsage - - def getFCUsage(self, lcgSite): - """ get storage usage from LFC - """ - if self.storageSummary is None: - res = self.storageusage.getStorageSummary() - if not res['OK']: - gLogger.error('ERROR in getStorageSummary ', res['Message']) - return {} - self.storageSummary = res['Value'] - - usage = {} - for st in self.sitesSEs[lcgSite]: - usage[st] = {'Files': 0, 'Size': 0} - for se in self.sitesSEs[lcgSite][st]['SEs']: - if se in self.storageSummary: - usage[st]['Files'] += self.storageSummary[se]['Files'] - usage[st]['Size'] += self.storageSummary[se]['Size'] - else: - gLogger.error("No FC storage information for SE", se) - - return usage + gLogger.error('ERROR in getStorageSummary ', res['Message']) + return {} + storageSummary = res['Value'] + + usage = {} + for st in sitesSEs[lcgSite]: + usage[st] = {'Files': 0, 'Size': 0} + for se in sitesSEs[lcgSite][st]['SEs']: + if se in storageSummary: + usage[st]['Files'] += storageSummary[se]['Files'] + usage[st]['Size'] += storageSummary[se]['Size'] + else: + gLogger.error("No FC storage information for SE", se) + + return usage From 30c75137d5d116e3a3cc93aa77ed591c09cc39be Mon Sep 17 00:00:00 2001 From: Federico Stagni Date: Fri, 8 Dec 2017 17:07:42 +0100 Subject: [PATCH 03/17] scaffolding for getOccupancy calls --- .../Command/FreeDiskSpaceCommand.py | 3 ++ .../Command/SpaceTokenOccupancyCommand.py | 4 +-- Resources/Storage/DIPStorage.py | 13 +++++++ Resources/Storage/GFAL2_SRM2Storage.py | 36 ++++++++++++++++++- Resources/Storage/GFAL2_StorageBase.py | 1 - Resources/Storage/StorageBase.py | 19 ++++++++-- Resources/Storage/StorageElement.py | 6 +++- 7 files changed, 74 insertions(+), 8 deletions(-) diff --git a/ResourceStatusSystem/Command/FreeDiskSpaceCommand.py b/ResourceStatusSystem/Command/FreeDiskSpaceCommand.py index 384bec6811b..9bb13bdcdad 100644 --- a/ResourceStatusSystem/Command/FreeDiskSpaceCommand.py +++ b/ResourceStatusSystem/Command/FreeDiskSpaceCommand.py @@ -14,6 +14,9 @@ __RCSID__ = '$Id: $' +# FIXME: this command may not be needed anymore, as we can get the space occupancy for DIP SE (and not only) by simply +# se = StorageElement().getOccupancy() + class FreeDiskSpaceCommand( Command ): ''' diff --git a/ResourceStatusSystem/Command/SpaceTokenOccupancyCommand.py b/ResourceStatusSystem/Command/SpaceTokenOccupancyCommand.py index 4adedb1ab1c..0cc81558dcf 100644 --- a/ResourceStatusSystem/Command/SpaceTokenOccupancyCommand.py +++ b/ResourceStatusSystem/Command/SpaceTokenOccupancyCommand.py @@ -97,6 +97,7 @@ def doNew( self, masterParams = None ): return params spaceTokenEndpoint, spaceToken = params[ 'Value' ] + #FIXME: should just use StorageElement('SE').getOccupancy() # 10 secs of timeout. If it works, the reply is immediate. occupancyResult = pythonCall( 10, lcg_util.lcg_stmd, spaceToken, spaceTokenEndpoint, True, 0 ) if not occupancyResult[ 'OK' ]: @@ -106,9 +107,6 @@ def doNew( self, masterParams = None ): else: occupancy = occupancyResult[ 'Value' ] - # Timeout does not work here... - # occupancy = lcg_util.lcg_stmd( spaceToken, spaceTokenEndpoint, True, 0 ) - if occupancy[ 0 ] != 0: return S_ERROR( occupancy ) output = occupancy[ 1 ][ 0 ] diff --git a/Resources/Storage/DIPStorage.py b/Resources/Storage/DIPStorage.py index 98b9ae45765..66cb262447e 100755 --- a/Resources/Storage/DIPStorage.py +++ b/Resources/Storage/DIPStorage.py @@ -500,3 +500,16 @@ def getDirectory( self, path, localPath = False ): resDict = {'Failed':failed, 'Successful':successful} return S_OK( resDict ) + def getOccupancy(self): + """ Gets the DIPStorage occupancy info + """ + rpc = RPCClient( self.getURLBase(), timeout=120 ) + free = rpc.getFreeDiskSpace("/") + if not free[ 'OK' ]: + return free + + total = rpc.getTotalDiskSpace("/") + if not total[ 'OK' ]: + return total + + return S_OK({'Free': free['Value'], 'Total': total['Value']}) diff --git a/Resources/Storage/GFAL2_SRM2Storage.py b/Resources/Storage/GFAL2_SRM2Storage.py index 83c5b4106ec..fd7f9bfb815 100644 --- a/Resources/Storage/GFAL2_SRM2Storage.py +++ b/Resources/Storage/GFAL2_SRM2Storage.py @@ -9,9 +9,12 @@ # pylint: disable=invalid-name +import lcg_util #pylint: disable=import-error + # from DIRAC -from DIRAC.Resources.Storage.GFAL2_StorageBase import GFAL2_StorageBase from DIRAC import gLogger, gConfig, S_OK, S_ERROR +from DIRAC.Core.Utilities.Subprocess import pythonCall +from DIRAC.Resources.Storage.GFAL2_StorageBase import GFAL2_StorageBase from DIRAC.Resources.Storage.Utilities import checkArgumentFormat @@ -205,3 +208,34 @@ def __getProtocols( self ): return S_ERROR( "GFAL2_SRM2Storage.__getProtocols: No local protocols defined and no defaults found." ) return S_OK( protocolsList ) + + + def getOccupancy(self): + """ Gets the GFAL2_SRM2Storage occupancy info. + + It queries the srm interface, and hopefully it will not crash. Out of the + results, we keep totalsize, guaranteedsize, and unusedsize. + """ + # FIXME: not working still! + spaceToken, spaceTokenEndpoint = 'FILL_ME' #FIXME + + occupancyResult = pythonCall( 10, lcg_util.lcg_stmd, spaceToken, spaceTokenEndpoint, True, 0 ) + if not occupancyResult[ 'OK' ]: + self.log.error( "Could not get spaceToken occupancy", "from endPoint/spaceToken %s/%s : %s" % \ + ( spaceTokenEndpoint, spaceToken, occupancyResult['Message'] ) ) + return occupancyResult + else: + occupancy = occupancyResult[ 'Value' ] + + if occupancy[ 0 ] != 0: + return S_ERROR( occupancy ) + output = occupancy[ 1 ][ 0 ] + + sTokenDict = {} + sTokenDict[ 'Endpoint' ] = spaceTokenEndpoint + sTokenDict[ 'Token' ] = spaceToken + sTokenDict[ 'Total' ] = float( output.get( 'totalsize', '0' ) ) / 1e12 # Bytes to Terabytes #FIXME: have to harmonize with other SE types + sTokenDict[ 'Guaranteed' ] = float( output.get( 'guaranteedsize', '0' ) ) / 1e12 + sTokenDict[ 'Free' ] = float( output.get( 'unusedsize', '0' ) ) / 1e12 + + return S_OK(sTokenDict) diff --git a/Resources/Storage/GFAL2_StorageBase.py b/Resources/Storage/GFAL2_StorageBase.py index 4db7be974e0..281338d8c62 100644 --- a/Resources/Storage/GFAL2_StorageBase.py +++ b/Resources/Storage/GFAL2_StorageBase.py @@ -11,7 +11,6 @@ # # imports import os -import sys import datetime import errno from stat import S_ISREG, S_ISDIR, S_IXUSR, S_IRUSR, S_IWUSR, \ diff --git a/Resources/Storage/StorageBase.py b/Resources/Storage/StorageBase.py index bd936e4fc4a..008aa968889 100755 --- a/Resources/Storage/StorageBase.py +++ b/Resources/Storage/StorageBase.py @@ -28,6 +28,10 @@ getName() getParameters() getCurrentURL() + +These are the methods for getting information about the Storage: + getOccupancy() + """ __RCSID__ = "$Id$" @@ -35,8 +39,6 @@ from DIRAC.Core.Utilities.Pfn import pfnparse, pfnunparse from DIRAC.Resources.Storage.Utilities import checkArgumentFormat -import os - class StorageBase( object ): """ .. class:: StorageBase @@ -415,3 +417,16 @@ def _isInputURL( self, url ): return S_OK( True ) return S_OK( urlDict['Protocol'] == self.protocolParameters['Protocol'] ) + + + ############################################################# + # + # These are the methods for getting information about the Storage element: + # + + def getOccupancy(self, *parms, **kws): + """ Get the StorageElement occupancy info. + :returns: S_OK/S_ERROR dictionary + """ + #FIXME: put an implementation that just gets a file and check its content + return S_ERROR( "Storage.occupancy: implement me!" ) diff --git a/Resources/Storage/StorageElement.py b/Resources/Storage/StorageElement.py index 6d66b37f362..6ca7610e709 100755 --- a/Resources/Storage/StorageElement.py +++ b/Resources/Storage/StorageElement.py @@ -94,6 +94,7 @@ class StorageElementItem( object ): createDirectory( lfn ) putDirectory( lfn ) getDirectory( lfn, localPath = False ) + getOccupancy() """ @@ -123,6 +124,7 @@ class StorageElementItem( object ): "createDirectory" : "createDirectory", "putDirectory" : "putDirectory", "getDirectory" : "getDirectory", + "getOccupancy" : "getOccupancy" } # We can set default argument in the __executeFunction which impacts all plugins @@ -224,7 +226,8 @@ def __init__( self, name, plugins = None, vo = None, hideExceptions = False ): 'createDirectory', 'putDirectory' ] - self.removeMethods = [ 'removeFile', 'removeDirectory' ] + self.removeMethods = [ 'removeFile', + 'removeDirectory' ] self.checkMethods = [ 'exists', 'getDirectoryMetadata', @@ -234,6 +237,7 @@ def __init__( self, name, plugins = None, vo = None, hideExceptions = False ): 'listDirectory', 'isDirectory', 'isFile', + 'getOccupancy' ] self.okMethods = [ 'getLocalProtocols', From 23c044da7c6a9ad0ac29900e70ecd6e4df1e3fd3 Mon Sep 17 00:00:00 2001 From: Federico Stagni Date: Fri, 8 Dec 2017 17:18:12 +0100 Subject: [PATCH 04/17] few fixes --- Resources/Storage/GFAL2_SRM2Storage.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/Resources/Storage/GFAL2_SRM2Storage.py b/Resources/Storage/GFAL2_SRM2Storage.py index fd7f9bfb815..0a4aee8e1d0 100644 --- a/Resources/Storage/GFAL2_SRM2Storage.py +++ b/Resources/Storage/GFAL2_SRM2Storage.py @@ -216,8 +216,11 @@ def getOccupancy(self): It queries the srm interface, and hopefully it will not crash. Out of the results, we keep totalsize, guaranteedsize, and unusedsize. """ - # FIXME: not working still! - spaceToken, spaceTokenEndpoint = 'FILL_ME' #FIXME + # FIXME: Untested! + spaceToken = self.protocolParameters['SpaceToken'] + spaceTokenEndpoint = 'httpg://%s:%s%s' % (self.protocolParameters['Host'], + self.protocolParameters['Port'], + self.protocolParameters['WSUrl'].split('?')[0]) occupancyResult = pythonCall( 10, lcg_util.lcg_stmd, spaceToken, spaceTokenEndpoint, True, 0 ) if not occupancyResult[ 'OK' ]: From 7146c26c81cc2036adb23a5ed6461816e80ebdc5 Mon Sep 17 00:00:00 2001 From: Federico Stagni Date: Fri, 8 Dec 2017 17:18:45 +0100 Subject: [PATCH 05/17] style --- Resources/Storage/DIPStorage.py | 6 ++--- Resources/Storage/GFAL2_SRM2Storage.py | 32 +++++++++++++------------- Resources/Storage/StorageBase.py | 3 +-- Resources/Storage/StorageElement.py | 6 ++--- 4 files changed, 23 insertions(+), 24 deletions(-) diff --git a/Resources/Storage/DIPStorage.py b/Resources/Storage/DIPStorage.py index 66cb262447e..73653c92546 100755 --- a/Resources/Storage/DIPStorage.py +++ b/Resources/Storage/DIPStorage.py @@ -503,13 +503,13 @@ def getDirectory( self, path, localPath = False ): def getOccupancy(self): """ Gets the DIPStorage occupancy info """ - rpc = RPCClient( self.getURLBase(), timeout=120 ) + rpc = RPCClient(self.getURLBase(), timeout=120) free = rpc.getFreeDiskSpace("/") - if not free[ 'OK' ]: + if not free['OK']: return free total = rpc.getTotalDiskSpace("/") - if not total[ 'OK' ]: + if not total['OK']: return total return S_OK({'Free': free['Value'], 'Total': total['Value']}) diff --git a/Resources/Storage/GFAL2_SRM2Storage.py b/Resources/Storage/GFAL2_SRM2Storage.py index 0a4aee8e1d0..a6ae588ed2e 100644 --- a/Resources/Storage/GFAL2_SRM2Storage.py +++ b/Resources/Storage/GFAL2_SRM2Storage.py @@ -9,11 +9,11 @@ # pylint: disable=invalid-name -import lcg_util #pylint: disable=import-error +import lcg_util # pylint: disable=import-error # from DIRAC from DIRAC import gLogger, gConfig, S_OK, S_ERROR -from DIRAC.Core.Utilities.Subprocess import pythonCall +from DIRAC.Core.Utilities.Subprocess import pythonCall from DIRAC.Resources.Storage.GFAL2_StorageBase import GFAL2_StorageBase from DIRAC.Resources.Storage.Utilities import checkArgumentFormat @@ -209,7 +209,6 @@ def __getProtocols( self ): return S_OK( protocolsList ) - def getOccupancy(self): """ Gets the GFAL2_SRM2Storage occupancy info. @@ -222,23 +221,24 @@ def getOccupancy(self): self.protocolParameters['Port'], self.protocolParameters['WSUrl'].split('?')[0]) - occupancyResult = pythonCall( 10, lcg_util.lcg_stmd, spaceToken, spaceTokenEndpoint, True, 0 ) - if not occupancyResult[ 'OK' ]: - self.log.error( "Could not get spaceToken occupancy", "from endPoint/spaceToken %s/%s : %s" % \ - ( spaceTokenEndpoint, spaceToken, occupancyResult['Message'] ) ) + occupancyResult = pythonCall(10, lcg_util.lcg_stmd, spaceToken, spaceTokenEndpoint, True, 0) + if not occupancyResult['OK']: + self.log.error("Could not get spaceToken occupancy", "from endPoint/spaceToken %s/%s : %s" % + (spaceTokenEndpoint, spaceToken, occupancyResult['Message'])) return occupancyResult else: - occupancy = occupancyResult[ 'Value' ] + occupancy = occupancyResult['Value'] - if occupancy[ 0 ] != 0: - return S_ERROR( occupancy ) - output = occupancy[ 1 ][ 0 ] + if occupancy[0] != 0: + return S_ERROR(occupancy) + output = occupancy[1][0] sTokenDict = {} - sTokenDict[ 'Endpoint' ] = spaceTokenEndpoint - sTokenDict[ 'Token' ] = spaceToken - sTokenDict[ 'Total' ] = float( output.get( 'totalsize', '0' ) ) / 1e12 # Bytes to Terabytes #FIXME: have to harmonize with other SE types - sTokenDict[ 'Guaranteed' ] = float( output.get( 'guaranteedsize', '0' ) ) / 1e12 - sTokenDict[ 'Free' ] = float( output.get( 'unusedsize', '0' ) ) / 1e12 + sTokenDict['Endpoint'] = spaceTokenEndpoint + sTokenDict['Token'] = spaceToken + # Bytes to Terabytes #FIXME: have to harmonize with other SE types + sTokenDict['Total'] = float(output.get('totalsize', '0')) / 1e12 + sTokenDict['Guaranteed'] = float(output.get('guaranteedsize', '0')) / 1e12 + sTokenDict['Free'] = float(output.get('unusedsize', '0')) / 1e12 return S_OK(sTokenDict) diff --git a/Resources/Storage/StorageBase.py b/Resources/Storage/StorageBase.py index 008aa968889..9768f555ab5 100755 --- a/Resources/Storage/StorageBase.py +++ b/Resources/Storage/StorageBase.py @@ -418,7 +418,6 @@ def _isInputURL( self, url ): return S_OK( urlDict['Protocol'] == self.protocolParameters['Protocol'] ) - ############################################################# # # These are the methods for getting information about the Storage element: @@ -429,4 +428,4 @@ def getOccupancy(self, *parms, **kws): :returns: S_OK/S_ERROR dictionary """ #FIXME: put an implementation that just gets a file and check its content - return S_ERROR( "Storage.occupancy: implement me!" ) + return S_ERROR("Storage.occupancy: implement me!") diff --git a/Resources/Storage/StorageElement.py b/Resources/Storage/StorageElement.py index 6ca7610e709..b409711589b 100755 --- a/Resources/Storage/StorageElement.py +++ b/Resources/Storage/StorageElement.py @@ -124,7 +124,7 @@ class StorageElementItem( object ): "createDirectory" : "createDirectory", "putDirectory" : "putDirectory", "getDirectory" : "getDirectory", - "getOccupancy" : "getOccupancy" + "getOccupancy": "getOccupancy" } # We can set default argument in the __executeFunction which impacts all plugins @@ -226,8 +226,8 @@ def __init__( self, name, plugins = None, vo = None, hideExceptions = False ): 'createDirectory', 'putDirectory' ] - self.removeMethods = [ 'removeFile', - 'removeDirectory' ] + self.removeMethods = ['removeFile', + 'removeDirectory'] self.checkMethods = [ 'exists', 'getDirectoryMetadata', From a2fdd92b314823fb81a8d116723d929c126b41f7 Mon Sep 17 00:00:00 2001 From: Federico Stagni Date: Fri, 8 Dec 2017 17:23:01 +0100 Subject: [PATCH 06/17] re-migrated --- .../Client/SpaceTokenUsage.py | 186 ------------------ .../scripts/dirac-dms-spaceTokens-usage.py | 27 --- 2 files changed, 213 deletions(-) delete mode 100644 DataManagementSystem/Client/SpaceTokenUsage.py delete mode 100755 DataManagementSystem/scripts/dirac-dms-spaceTokens-usage.py diff --git a/DataManagementSystem/Client/SpaceTokenUsage.py b/DataManagementSystem/Client/SpaceTokenUsage.py deleted file mode 100644 index e1907d7c3d2..00000000000 --- a/DataManagementSystem/Client/SpaceTokenUsage.py +++ /dev/null @@ -1,186 +0,0 @@ -""" -This class gets information about storage space tokens from the FC recording, -from SRM and if available from storage dumps -It reports for each site its availability and usage -""" - -import time -import lcg_util - -from DIRAC import gLogger, S_OK, S_ERROR -from DIRAC.Core.DISET.RPCClient import RPCClient - -from DIRAC.DataManagementSystem.Utilities.DMSHelpers import DMSHelpers -from DIRAC.Resources.Storage.StorageElement import StorageElement - -sitesSEs = {} -spaceTokenInfo = {} -storageSummary = None - - -def combinedResult(unit, sites=None): - """ Checks the space token usage at the site and report the space usage from several sources: - File Catalogue, Storage dumps, SRM interface - """ - - dmsHelper = DMSHelpers() - - if not sites: - # Tier0 and all Tier1s - sites = sorted(dmsHelper.getTiers(tier=(0, 1))) - else: - # Translate in case it is a short name - allSites = dmsHelper.getSites() - sites = [dmsHelper.getShortSiteNames(tier=(0, 1)).get(site, site) for site in sites] - badSites = set(sites) - set(allSites) - if badSites: - gLogger.warn("Some sites do not exist", str(sorted(badSites))) - sites = [site for site in sites if site in allSites] - - scaleDict = {'MB': 1000 * 1000.0, - 'GB': 1000 * 1000 * 1000.0, - 'TB': 1000 * 1000 * 1000 * 1000.0, - 'PB': 1000 * 1000 * 1000 * 1000 * 1000.0} - if unit not in scaleDict: - return S_ERROR("Unit not in %s" % scaleDict.keys()) - scaleFactor = scaleDict[unit] - - for site in sites: - sitesSEs[site] = {} - # Get SEs at site - seList = dmsHelper.getSEsAtSite(site).get('Value', []) - for se in seList: - spaceToken = StorageElement(se).getStorageParameters(protocol='srm') # FIXME: remove this limit - if spaceToken['OK']: - spaceToken = spaceToken['Value'] - st = spaceToken['SpaceToken'] - sitesSEs[site].setdefault(st, {}).setdefault('SEs', []).append(se) - # Fill in the endpoints - # FIXME: should move all this shit in the StorageElement definition - ep = 'httpg://%s:%s%s' % (spaceToken['Host'], spaceToken['Port'], spaceToken['WSUrl'].split('?')[0]) - spaceTokenInfo.setdefault(site.split('.')[1], {}).setdefault(ep, set()).add(st) - - fcUsage = {} - srmUsage = {} - sdUsage = {} - for site in sites: - # retrieve space usage from FC - fcUsage[site] = getFCUsage(site) - - # retrieve SRM usage - srmResult = getSrmUsage(site) - if srmResult != -1: - srmUsage[site] = srmResult - else: - return 1 - - # retrieve space usage from storage dumps: - sdResult = getSDUsage(site) - if sdResult != -1: - sdUsage[site] = sdResult - else: - return 1 - - gLogger.notice("Storage usage summary for site %s - %s " % (site.split('.')[1], time.asctime())) - for st in sitesSEs[site]: - gLogger.notice("Space token %s " % st) - gLogger.notice("\tFrom FC: Files: %d, Size: %.2f %s" % - (fcUsage[site][st]['Files'], - fcUsage[site][st]['Size'] / scaleFactor, unit)) - if site in srmUsage and st in srmUsage[site]: - gLogger.notice("\tFrom SRM: Total Assigned Space: %.2f %s, Used Space: %.2f %s, Free Space: %.2f %s " % - (srmUsage[site][st]['SRMTotal'] / scaleFactor, unit, - srmUsage[site][st]['SRMUsed'] / scaleFactor, unit, - srmUsage[site][st]['SRMFree'] / scaleFactor, unit)) - else: - gLogger.notice("\tFrom SRM: Information not available") - if site in sdUsage and st in sdUsage[site]: - gLogger.notice("\tFrom storage dumps: Files: %d, Size: %.2f %s - last update %s " % - (sdUsage[site][st]['Files'], - sdUsage[site][st]['Size'] / scaleFactor, unit, - sdUsage[site][st]['LastUpdate'])) - else: - gLogger.notice("\tFrom storage dumps: Information not available") - return 0 - - -def getSrmUsage(lcgSite): # FIXME: should move this shit in the StorageElement definition (SRM2Storage or GFAL2_SRM2Storage?) - """Get space usage via SRM interface - """ - try: - site = lcgSite.split('.')[1] - except IndexError: - site = lcgSite - if site not in spaceTokenInfo: - gLogger.error("ERROR: information not available for site %s. Space token information from CS: %s " - % (site, sorted(spaceTokenInfo))) - return -1 - - result = {} - for ep, stList in spaceTokenInfo[site].iteritems(): - for st in stList: - result[st] = {} - srm = lcg_util.lcg_stmd(st, ep, True, 0) - if srm[0]: - # This SpaceToken doesn't exist at this endPoint - continue - srmVal = srm[1][0] - srmTotSpace = srmVal['totalsize'] - srmFree = srmVal['unusedsize'] - srmUsed = srmTotSpace - srmFree - result[st]['SRMUsed'] = srmUsed - result[st]['SRMFree'] = srmFree - result[st]['SRMTotal'] = srmTotSpace - return result - -# ................................................. - - -def getSDUsage(lcgSite): - """ get storage usage from storage dumps - """ - try: - site = lcgSite.split('.')[1] - except IndexError: - site = lcgSite - res = RPCClient('DataManagement/StorageUsage').getSTSummary(site) - if not res['OK']: - gLogger.error("ERROR: Cannot get storage dump information for site %s :" % site, res['Message']) - return -1 - if not res['Value']: - gLogger.warn(" No information available for site %s from storage dumps" % site) - sdUsage = {} - for row in res['Value']: - site, spaceTokenWithID, totalSpace, totalFiles, lastUpdate = row - for st in sitesSEs[lcgSite]: - sdUsage.setdefault(st, {}) - if st in spaceTokenWithID: - sdUsage[st]['Size'] = totalSpace - sdUsage[st]['Files'] = totalFiles - sdUsage[st]['LastUpdate'] = lastUpdate - break - return sdUsage - - -def getFCUsage(lcgSite): - """ get storage usage from LFC - """ - if storageSummary is None: - # FIXME: this is in LHCbDIRAC. So it should be moved - res = RPCClient('DataManagement/StorageUsage').getStorageSummary() - if not res['OK']: - gLogger.error('ERROR in getStorageSummary ', res['Message']) - return {} - storageSummary = res['Value'] - - usage = {} - for st in sitesSEs[lcgSite]: - usage[st] = {'Files': 0, 'Size': 0} - for se in sitesSEs[lcgSite][st]['SEs']: - if se in storageSummary: - usage[st]['Files'] += storageSummary[se]['Files'] - usage[st]['Size'] += storageSummary[se]['Size'] - else: - gLogger.error("No FC storage information for SE", se) - - return usage diff --git a/DataManagementSystem/scripts/dirac-dms-spaceTokens-usage.py b/DataManagementSystem/scripts/dirac-dms-spaceTokens-usage.py deleted file mode 100755 index d1d33818992..00000000000 --- a/DataManagementSystem/scripts/dirac-dms-spaceTokens-usage.py +++ /dev/null @@ -1,27 +0,0 @@ -#!/usr/bin/env python - -""" Check the space token usage at the site and report the space usage from several sources: - File Catalogue, Storage dumps, SRM interface -""" - -__RCSID__ = "$Id$" - -from DIRAC.Core.Base import Script -import DIRAC - -if __name__ == "__main__": - - unit = 'TB' - Script.registerSwitch("u:", "Unit=", " Unit to use [%s] (MB,GB,TB,PB)" % unit) - Script.registerSwitch("S:", "Sites=", " Sites to consider [ALL] (space or comma separated list, e.g. LCG.CNAF.it") - # Script.registerSwitch( "l:", "Site=", " LCG Site list to check [%s] (e.g. LCG.CERN.cern, LCG.CNAF.it, ... )" %sites ) - - Script.setUsageMessage('\n'.join([__doc__.split('\n')[1], - 'Usage:', - ' %s [option|cfgfile] ...' % - Script.scriptName, ])) - - Script.parseCommandLine(ignoreErrors=False) - - from LHCbDIRAC.DataManagementSystem.Client.SpaceTokenUsage import SpaceTokenUsage - DIRAC.exit(SpaceTokenUsage().execute(unit)) From 884b9f9f9ed5e712b9cd35adfeddbdade4e91682 Mon Sep 17 00:00:00 2001 From: Federico Stagni Date: Mon, 11 Dec 2017 11:11:15 +0100 Subject: [PATCH 07/17] just style --- .../Command/SpaceTokenOccupancyCommand.py | 150 +++++++++--------- 1 file changed, 72 insertions(+), 78 deletions(-) diff --git a/ResourceStatusSystem/Command/SpaceTokenOccupancyCommand.py b/ResourceStatusSystem/Command/SpaceTokenOccupancyCommand.py index 0cc81558dcf..3aa028e7f71 100644 --- a/ResourceStatusSystem/Command/SpaceTokenOccupancyCommand.py +++ b/ResourceStatusSystem/Command/SpaceTokenOccupancyCommand.py @@ -5,54 +5,51 @@ ''' -import lcg_util #pylint: disable=import-error +import lcg_util # pylint: disable=import-error -from DIRAC import S_OK, S_ERROR -from DIRAC.Core.Utilities.Subprocess import pythonCall -from DIRAC.ResourceStatusSystem.Command.Command import Command +from DIRAC import S_OK, S_ERROR +from DIRAC.Core.Utilities.Subprocess import pythonCall +from DIRAC.ResourceStatusSystem.Command.Command import Command from DIRAC.ResourceStatusSystem.Client.ResourceManagementClient import ResourceManagementClient -from DIRAC.ResourceStatusSystem.Utilities import CSHelpers +from DIRAC.ResourceStatusSystem.Utilities import CSHelpers __RCSID__ = '$Id: $' -class SpaceTokenOccupancyCommand( Command ): +class SpaceTokenOccupancyCommand(Command): ''' Uses lcg_util to query status of endpoint for a given token. ''' + def __init__(self, args=None, clients=None): - def __init__( self, args = None, clients = None ): - - super( SpaceTokenOccupancyCommand, self ).__init__( args, clients ) + super(SpaceTokenOccupancyCommand, self).__init__(args, clients) if 'ResourceManagementClient' in self.apis: - self.rmClient = self.apis[ 'ResourceManagementClient' ] + self.rmClient = self.apis['ResourceManagementClient'] else: self.rmClient = ResourceManagementClient() - - def _storeCommand( self, results ): + def _storeCommand(self, results): ''' Stores the results of doNew method on the database. ''' for result in results: - resQuery = self.rmClient.addOrModifySpaceTokenOccupancyCache( result[ 'Endpoint' ], - result[ 'Token' ], - result[ 'Total' ], - result[ 'Guaranteed' ], - result[ 'Free' ] ) - if not resQuery[ 'OK' ]: + resQuery = self.rmClient.addOrModifySpaceTokenOccupancyCache(result['Endpoint'], + result['Token'], + result['Total'], + result['Guaranteed'], + result['Free']) + if not resQuery['OK']: return resQuery return S_OK() - - def _prepareCommand( self ): + def _prepareCommand(self): ''' SpaceTokenOccupancy requires one argument: - elementName : @@ -62,23 +59,22 @@ def _prepareCommand( self ): ''' if 'name' not in self.args: - return S_ERROR( '"name" not found in self.args' ) - elementName = self.args[ 'name' ] + return S_ERROR('"name" not found in self.args') + elementName = self.args['name'] - endpoint = CSHelpers.getStorageElementEndpoint( elementName ) - if not endpoint[ 'OK' ]: + endpoint = CSHelpers.getStorageElementEndpoint(elementName) + if not endpoint['OK']: return endpoint - endpoint = endpoint[ 'Value' ] + endpoint = endpoint['Value'] - spaceToken = CSHelpers.getSEToken( elementName ) - if not spaceToken[ 'OK' ]: + spaceToken = CSHelpers.getSEToken(elementName) + if not spaceToken['OK']: return spaceToken - spaceToken = spaceToken[ 'Value'] + spaceToken = spaceToken['Value'] - return S_OK( ( endpoint, spaceToken ) ) + return S_OK((endpoint, spaceToken)) - - def doNew( self, masterParams = None ): + def doNew(self, masterParams=None): ''' Gets the parameters to run, either from the master method or from its own arguments. @@ -93,57 +89,55 @@ def doNew( self, masterParams = None ): spaceTokenEndpoint, spaceToken = masterParams else: params = self._prepareCommand() - if not params[ 'OK' ]: + if not params['OK']: return params - spaceTokenEndpoint, spaceToken = params[ 'Value' ] + spaceTokenEndpoint, spaceToken = params['Value'] - #FIXME: should just use StorageElement('SE').getOccupancy() + # FIXME: should just use StorageElement('SE').getOccupancy() # 10 secs of timeout. If it works, the reply is immediate. - occupancyResult = pythonCall( 10, lcg_util.lcg_stmd, spaceToken, spaceTokenEndpoint, True, 0 ) - if not occupancyResult[ 'OK' ]: - self.log.error( "Could not get spaceToken occupancy", "from endPoint/spaceToken %s/%s : %s" % \ - ( spaceTokenEndpoint, spaceToken, occupancyResult['Message'] ) ) + occupancyResult = pythonCall(10, lcg_util.lcg_stmd, spaceToken, spaceTokenEndpoint, True, 0) + if not occupancyResult['OK']: + self.log.error("Could not get spaceToken occupancy", "from endPoint/spaceToken %s/%s : %s" % + (spaceTokenEndpoint, spaceToken, occupancyResult['Message'])) return occupancyResult else: - occupancy = occupancyResult[ 'Value' ] + occupancy = occupancyResult['Value'] - if occupancy[ 0 ] != 0: - return S_ERROR( occupancy ) - output = occupancy[ 1 ][ 0 ] + if occupancy[0] != 0: + return S_ERROR(occupancy) + output = occupancy[1][0] sTokenDict = {} - sTokenDict[ 'Endpoint' ] = spaceTokenEndpoint - sTokenDict[ 'Token' ] = spaceToken - sTokenDict[ 'Total' ] = float( output.get( 'totalsize', '0' ) ) / 1e12 # Bytes to Terabytes - sTokenDict[ 'Guaranteed' ] = float( output.get( 'guaranteedsize', '0' ) ) / 1e12 - sTokenDict[ 'Free' ] = float( output.get( 'unusedsize', '0' ) ) / 1e12 - - storeRes = self._storeCommand( [ sTokenDict ] ) - if not storeRes[ 'OK' ]: + sTokenDict['Endpoint'] = spaceTokenEndpoint + sTokenDict['Token'] = spaceToken + sTokenDict['Total'] = float(output.get('totalsize', '0')) / 1e12 # Bytes to Terabytes + sTokenDict['Guaranteed'] = float(output.get('guaranteedsize', '0')) / 1e12 + sTokenDict['Free'] = float(output.get('unusedsize', '0')) / 1e12 + + storeRes = self._storeCommand([sTokenDict]) + if not storeRes['OK']: return storeRes - return S_OK( [ sTokenDict ] ) - + return S_OK([sTokenDict]) - def doCache( self ): + def doCache(self): ''' Method that reads the cache table and tries to read from it. It will return a list of dictionaries if there are results. ''' params = self._prepareCommand() - if not params[ 'OK' ]: + if not params['OK']: return params - spaceTokenEndpoint, spaceToken = params[ 'Value' ] + spaceTokenEndpoint, spaceToken = params['Value'] - result = self.rmClient.selectSpaceTokenOccupancyCache( spaceTokenEndpoint, spaceToken ) - if result[ 'OK' ]: - result = S_OK( [ dict( zip( result[ 'Columns' ], res ) ) for res in result[ 'Value' ] ] ) + result = self.rmClient.selectSpaceTokenOccupancyCache(spaceTokenEndpoint, spaceToken) + if result['OK']: + result = S_OK([dict(zip(result['Columns'], res)) for res in result['Value']]) return result - - def doMaster( self ): + def doMaster(self): ''' Master method. Gets all endpoints from the storage elements and all the spaceTokens. Could have taken from Shares/Disk as well. @@ -151,40 +145,40 @@ def doMaster( self ): in the database for those combinations, which then are not queried. ''' - self.log.verbose( "Getting all SEs defined in the CS" ) + self.log.verbose("Getting all SEs defined in the CS") storageElementNames = CSHelpers.getStorageElements() - if not storageElementNames[ 'OK' ]: - self.log.warn( storageElementNames['Message'] ) + if not storageElementNames['OK']: + self.log.warn(storageElementNames['Message']) return storageElementNames - storageElementNames = storageElementNames[ 'Value' ] + storageElementNames = storageElementNames['Value'] endpointTokenSet = set() for storageElementName in storageElementNames: - endpoint = CSHelpers.getStorageElementEndpoint( storageElementName ) - if not endpoint[ 'OK' ]: - self.log.warn( endpoint['Message'] ) + endpoint = CSHelpers.getStorageElementEndpoint(storageElementName) + if not endpoint['OK']: + self.log.warn(endpoint['Message']) continue - endpoint = endpoint[ 'Value' ] + endpoint = endpoint['Value'] - spaceToken = CSHelpers.getSEToken( storageElementName ) - if not spaceToken[ 'OK' ]: - self.log.warn( spaceToken['Message'] ) + spaceToken = CSHelpers.getSEToken(storageElementName) + if not spaceToken['OK']: + self.log.warn(spaceToken['Message']) continue - spaceToken = spaceToken[ 'Value' ] + spaceToken = spaceToken['Value'] - endpointTokenSet.add( ( endpoint, spaceToken ) ) + endpointTokenSet.add((endpoint, spaceToken)) - self.log.verbose( 'Processing %s' % endpointTokenSet ) + self.log.verbose('Processing %s' % endpointTokenSet) for elementToQuery in endpointTokenSet: - result = self.doNew( elementToQuery ) - if not result[ 'OK' ]: - self.metrics[ 'failed' ].append( result ) + result = self.doNew(elementToQuery) + if not result['OK']: + self.metrics['failed'].append(result) - return S_OK( self.metrics ) + return S_OK(self.metrics) # ............................................................................... From 3acc8871fbd515d4b8ba6c6af5f31b82345d5705 Mon Sep 17 00:00:00 2001 From: Federico Stagni Date: Mon, 11 Dec 2017 11:21:07 +0100 Subject: [PATCH 08/17] using StorageElement.getOccupancy() --- .../Command/FreeDiskSpaceCommand.py | 1 + .../Command/SpaceTokenOccupancyCommand.py | 27 +++++++------------ 2 files changed, 11 insertions(+), 17 deletions(-) diff --git a/ResourceStatusSystem/Command/FreeDiskSpaceCommand.py b/ResourceStatusSystem/Command/FreeDiskSpaceCommand.py index 9bb13bdcdad..041b8ff8d5a 100644 --- a/ResourceStatusSystem/Command/FreeDiskSpaceCommand.py +++ b/ResourceStatusSystem/Command/FreeDiskSpaceCommand.py @@ -16,6 +16,7 @@ # FIXME: this command may not be needed anymore, as we can get the space occupancy for DIP SE (and not only) by simply # se = StorageElement().getOccupancy() +# Alternatively, this may survive and we remove SpaceTokenOccupancyCommand class FreeDiskSpaceCommand( Command ): diff --git a/ResourceStatusSystem/Command/SpaceTokenOccupancyCommand.py b/ResourceStatusSystem/Command/SpaceTokenOccupancyCommand.py index 3aa028e7f71..edbfbabee1a 100644 --- a/ResourceStatusSystem/Command/SpaceTokenOccupancyCommand.py +++ b/ResourceStatusSystem/Command/SpaceTokenOccupancyCommand.py @@ -4,18 +4,18 @@ ''' - -import lcg_util # pylint: disable=import-error +__RCSID__ = '$Id:$' from DIRAC import S_OK, S_ERROR -from DIRAC.Core.Utilities.Subprocess import pythonCall from DIRAC.ResourceStatusSystem.Command.Command import Command from DIRAC.ResourceStatusSystem.Client.ResourceManagementClient import ResourceManagementClient from DIRAC.ResourceStatusSystem.Utilities import CSHelpers +from DIRAC.Resources.Storage.StorageElement import StorageElement - -__RCSID__ = '$Id: $' +# FIXME: this command by calling StorageElement(se).getOccupancy() becomes a duplicate of the FreeDiskSpaceCommand +# so only one should survive. +# Also, the concept of "space token" should remain hidden in StorageElement interface. class SpaceTokenOccupancyCommand(Command): @@ -93,26 +93,19 @@ def doNew(self, masterParams=None): return params spaceTokenEndpoint, spaceToken = params['Value'] - # FIXME: should just use StorageElement('SE').getOccupancy() - # 10 secs of timeout. If it works, the reply is immediate. - occupancyResult = pythonCall(10, lcg_util.lcg_stmd, spaceToken, spaceTokenEndpoint, True, 0) + se = StorageElement(self.args['name']) + occupancyResult = se.getOccupancy if not occupancyResult['OK']: - self.log.error("Could not get spaceToken occupancy", "from endPoint/spaceToken %s/%s : %s" % - (spaceTokenEndpoint, spaceToken, occupancyResult['Message'])) return occupancyResult else: occupancy = occupancyResult['Value'] - if occupancy[0] != 0: - return S_ERROR(occupancy) - output = occupancy[1][0] - sTokenDict = {} sTokenDict['Endpoint'] = spaceTokenEndpoint sTokenDict['Token'] = spaceToken - sTokenDict['Total'] = float(output.get('totalsize', '0')) / 1e12 # Bytes to Terabytes - sTokenDict['Guaranteed'] = float(output.get('guaranteedsize', '0')) / 1e12 - sTokenDict['Free'] = float(output.get('unusedsize', '0')) / 1e12 + sTokenDict['Total'] = float(occupancy.get('totalsize', '0')) / 1e12 # Bytes to Terabytes + sTokenDict['Guaranteed'] = float(occupancy.get('guaranteedsize', '0')) / 1e12 + sTokenDict['Free'] = float(occupancy.get('unusedsize', '0')) / 1e12 storeRes = self._storeCommand([sTokenDict]) if not storeRes['OK']: From 596503608219bfeb31f225db8110c0aa8d721979 Mon Sep 17 00:00:00 2001 From: Federico Stagni Date: Mon, 11 Dec 2017 11:22:55 +0100 Subject: [PATCH 09/17] just style --- .../Command/FreeDiskSpaceCommand.py | 70 +++++++++---------- 1 file changed, 35 insertions(+), 35 deletions(-) diff --git a/ResourceStatusSystem/Command/FreeDiskSpaceCommand.py b/ResourceStatusSystem/Command/FreeDiskSpaceCommand.py index 041b8ff8d5a..ed361269ca6 100644 --- a/ResourceStatusSystem/Command/FreeDiskSpaceCommand.py +++ b/ResourceStatusSystem/Command/FreeDiskSpaceCommand.py @@ -4,12 +4,12 @@ ''' -from datetime import datetime -from DIRAC import S_OK, S_ERROR, gLogger -from DIRAC.ResourceStatusSystem.Command.Command import Command -from DIRAC.Core.DISET.RPCClient import RPCClient -from DIRAC.ResourceStatusSystem.Utilities import CSHelpers -from DIRAC.Resources.Storage.StorageElement import StorageElement +from datetime import datetime +from DIRAC import S_OK, S_ERROR, gLogger +from DIRAC.ResourceStatusSystem.Command.Command import Command +from DIRAC.Core.DISET.RPCClient import RPCClient +from DIRAC.ResourceStatusSystem.Utilities import CSHelpers +from DIRAC.Resources.Storage.StorageElement import StorageElement from DIRAC.ResourceStatusSystem.Client.ResourceManagementClient import ResourceManagementClient __RCSID__ = '$Id: $' @@ -19,31 +19,31 @@ # Alternatively, this may survive and we remove SpaceTokenOccupancyCommand -class FreeDiskSpaceCommand( Command ): +class FreeDiskSpaceCommand(Command): ''' Uses diskSpace method to get the free space ''' - def __init__( self, args = None, clients = None ): + def __init__(self, args=None, clients=None): - super( FreeDiskSpaceCommand, self ).__init__( args, clients = clients ) + super(FreeDiskSpaceCommand, self).__init__(args, clients=clients) self.rpc = None self.rsClient = ResourceManagementClient() - def _prepareCommand( self ): + def _prepareCommand(self): ''' FreeDiskSpaceCommand requires one argument: - name : ''' if 'name' not in self.args: - return S_ERROR( '"name" not found in self.args' ) - elementName = self.args[ 'name' ] + return S_ERROR('"name" not found in self.args') + elementName = self.args['name'] - return S_OK( elementName ) + return S_OK(elementName) - def doNew( self, masterParams = None ): + def doNew(self, masterParams=None): """ Gets the total and the free disk space of a DIPS storage element that is found in the CS and inserts the results in the SpaceTokenOccupancyCache table @@ -54,30 +54,30 @@ def doNew( self, masterParams = None ): elementName = masterParams else: elementName = self._prepareCommand() - if not elementName[ 'OK' ]: + if not elementName['OK']: return elementName se = StorageElement(elementName) - elementURL = se.getStorageParameters(protocol = "dips") + elementURL = se.getStorageParameters(protocol="dips") if elementURL['OK']: - elementURL = se.getStorageParameters(protocol = "dips")['Value']['URLBase'] + elementURL = se.getStorageParameters(protocol="dips")['Value']['URLBase'] else: - gLogger.verbose( "Not a DIPS storage element, skipping..." ) + gLogger.verbose("Not a DIPS storage element, skipping...") return S_OK() - self.rpc = RPCClient( elementURL, timeout=120 ) + self.rpc = RPCClient(elementURL, timeout=120) free = self.rpc.getFreeDiskSpace("/") - if not free[ 'OK' ]: + if not free['OK']: return free free = free['Value'] total = self.rpc.getTotalDiskSpace("/") - if not total[ 'OK' ]: + if not total['OK']: return total total = total['Value'] @@ -86,32 +86,32 @@ def doNew( self, masterParams = None ): if total and total < 1: total = 1 - result = self.rsClient.addOrModifySpaceTokenOccupancyCache( endpoint = elementURL, - lastCheckTime = datetime.utcnow(), - free = free, total = total, - token = elementName ) - if not result[ 'OK' ]: + result = self.rsClient.addOrModifySpaceTokenOccupancyCache(endpoint=elementURL, + lastCheckTime=datetime.utcnow(), + free=free, total=total, + token=elementName) + if not result['OK']: return result return S_OK() - def doCache( self ): + def doCache(self): """ This is a method that gets the element's details from the spaceTokenOccupancy cache. """ elementName = self._prepareCommand() - if not elementName[ 'OK' ]: + if not elementName['OK']: return elementName - result = self.rsClient.selectSpaceTokenOccupancyCache(token = elementName) + result = self.rsClient.selectSpaceTokenOccupancyCache(token=elementName) - if not result[ 'OK' ]: + if not result['OK']: return result - return S_OK( result ) + return S_OK(result) - def doMaster( self ): + def doMaster(self): """ This method calls the doNew method for each storage element that exists in the CS. @@ -120,9 +120,9 @@ def doMaster( self ): elements = CSHelpers.getStorageElements() for name in elements['Value']: - diskSpace = self.doNew( name ) - if not diskSpace[ 'OK' ]: - gLogger.error( "Unable to calculate free disk space", "name: %s" % name ) + diskSpace = self.doNew(name) + if not diskSpace['OK']: + gLogger.error("Unable to calculate free disk space", "name: %s" % name) continue return S_OK() From bd97a88560206ce71226935a7f51b16027d0f6bb Mon Sep 17 00:00:00 2001 From: Federico Stagni Date: Mon, 11 Dec 2017 16:33:05 +0100 Subject: [PATCH 10/17] bug fix --- ResourceStatusSystem/Command/SpaceTokenOccupancyCommand.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ResourceStatusSystem/Command/SpaceTokenOccupancyCommand.py b/ResourceStatusSystem/Command/SpaceTokenOccupancyCommand.py index edbfbabee1a..323111df019 100644 --- a/ResourceStatusSystem/Command/SpaceTokenOccupancyCommand.py +++ b/ResourceStatusSystem/Command/SpaceTokenOccupancyCommand.py @@ -94,7 +94,7 @@ def doNew(self, masterParams=None): spaceTokenEndpoint, spaceToken = params['Value'] se = StorageElement(self.args['name']) - occupancyResult = se.getOccupancy + occupancyResult = se.getOccupancy() if not occupancyResult['OK']: return occupancyResult else: From fbd0783081fedf3ff5f2ca80dd0b210353ba6e4c Mon Sep 17 00:00:00 2001 From: Federico Stagni Date: Tue, 12 Dec 2017 10:39:24 +0100 Subject: [PATCH 11/17] autopep8 style --- Resources/Storage/DIPStorage.py | 363 ++++++------- Resources/Storage/StorageBase.py | 234 +++++---- Resources/Storage/StorageElement.py | 772 ++++++++++++++-------------- 3 files changed, 685 insertions(+), 684 deletions(-) diff --git a/Resources/Storage/DIPStorage.py b/Resources/Storage/DIPStorage.py index 73653c92546..1c49caf8013 100755 --- a/Resources/Storage/DIPStorage.py +++ b/Resources/Storage/DIPStorage.py @@ -16,48 +16,47 @@ import os import random -from DIRAC import gLogger, S_OK, S_ERROR -from DIRAC.Resources.Storage.Utilities import checkArgumentFormat -from DIRAC.Resources.Storage.StorageBase import StorageBase -from DIRAC.Core.Utilities.Pfn import pfnparse, pfnunparse -from DIRAC.Core.DISET.TransferClient import TransferClient -from DIRAC.Core.DISET.RPCClient import RPCClient -from DIRAC.Core.Utilities.File import getSize +from DIRAC import gLogger, S_OK, S_ERROR +from DIRAC.Resources.Storage.Utilities import checkArgumentFormat +from DIRAC.Resources.Storage.StorageBase import StorageBase +from DIRAC.Core.Utilities.Pfn import pfnparse, pfnunparse +from DIRAC.Core.DISET.TransferClient import TransferClient +from DIRAC.Core.DISET.RPCClient import RPCClient +from DIRAC.Core.Utilities.File import getSize -class DIPStorage( StorageBase ): +class DIPStorage(StorageBase): _INPUT_PROTOCOLS = ['file', 'dip', 'dips'] _OUTPUT_PROTOCOLS = ['dip', 'dips'] - - def __init__( self, storageName, parameters ): + def __init__(self, storageName, parameters): """ """ - StorageBase.__init__( self, storageName, parameters ) + StorageBase.__init__(self, storageName, parameters) self.pluginName = 'DIP' - self.log = gLogger.getSubLogger( "DIPStorage", True ) + self.log = gLogger.getSubLogger("DIPStorage", True) # Several ports can be specified as comma separated list, choose # randomly one of those ports - ports = self.protocolParameters['Port'].split( ',' ) - random.shuffle( ports ) + ports = self.protocolParameters['Port'].split(',') + random.shuffle(ports) self.protocolParameters['Port'] = ports[0] - pathDict = dict( self.protocolParameters ) + pathDict = dict(self.protocolParameters) pathDict['Path'] = self.basePath - result = pfnunparse( pathDict ) + result = pfnunparse(pathDict) if result['OK']: self.url = result['Value'] self.checkSum = "CheckSum" self.isok = True - def setParameters( self, parameters ): + def setParameters(self, parameters): """ Applying extra storage parameters """ - StorageBase.setParameters( self, parameters ) + StorageBase.setParameters(self, parameters) if "CheckSum" in parameters and parameters['CheckSum'].lower() in ['0', 'no', 'false', 'off']: self.checkSum = "NoCheckSum" return S_OK() @@ -67,187 +66,187 @@ def setParameters( self, parameters ): # These are the methods for file manipulation # - def exists( self, path ): + def exists(self, path): """ Check if the given path exists. The 'path' variable can be a string or a list of strings. """ - res = checkArgumentFormat( path ) + res = checkArgumentFormat(path) if not res['OK']: return res urls = res['Value'] successful = {} failed = {} - serviceClient = RPCClient( self.url ) + serviceClient = RPCClient(self.url) for url in urls: - gLogger.debug( "DIPStorage.exists: Determining existence of %s." % url ) - res = serviceClient.exists( url ) + gLogger.debug("DIPStorage.exists: Determining existence of %s." % url) + res = serviceClient.exists(url) if res['OK']: successful[url] = res['Value'] else: failed[url] = res['Message'] - resDict = {'Failed':failed, 'Successful':successful} - return S_OK( resDict ) + resDict = {'Failed': failed, 'Successful': successful} + return S_OK(resDict) - def putFile( self, path, sourceSize = 0 ): + def putFile(self, path, sourceSize=0): """Put a file to the physical storage """ - res = checkArgumentFormat( path ) + res = checkArgumentFormat(path) if not res['OK']: return res urls = res['Value'] successful = {} failed = {} for dest_url, src_file in urls.items(): - gLogger.debug( "DIPStorage.putFile: Executing transfer of %s to %s" % ( src_file, dest_url ) ) - res = self.__putFile( src_file, dest_url ) + gLogger.debug("DIPStorage.putFile: Executing transfer of %s to %s" % (src_file, dest_url)) + res = self.__putFile(src_file, dest_url) if res['OK']: successful[dest_url] = res['Value'] else: failed[dest_url] = res['Message'] - resDict = {'Failed':failed, 'Successful':successful} - return S_OK( resDict ) + resDict = {'Failed': failed, 'Successful': successful} + return S_OK(resDict) - def __putFile( self, src_file, dest_url ): - res = pfnparse( src_file ) + def __putFile(self, src_file, dest_url): + res = pfnparse(src_file) if not res['OK']: return res localCache = False srcDict = res['Value'] if srcDict['Protocol'] in ['dips', 'dip']: # Make the service URL from the file URL by stripping off the file part - serviceDict = dict( srcDict ) - serviceDict['Path'] = '/'.join( srcDict['Path'].split('/')[:3] ) + serviceDict = dict(srcDict) + serviceDict['Path'] = '/'.join(srcDict['Path'].split('/')[:3]) serviceDict['FileName'] = '' - res = pfnunparse( serviceDict ) + res = pfnunparse(serviceDict) if not res['OK']: return res srcSEURL = res['Value'] localCache = True - transferClient = TransferClient( srcSEURL ) - res = transferClient.receiveFile( srcDict['FileName'], os.path.join( srcDict['Path'], srcDict['FileName'] ) ) + transferClient = TransferClient(srcSEURL) + res = transferClient.receiveFile(srcDict['FileName'], os.path.join(srcDict['Path'], srcDict['FileName'])) if not res['OK']: return res src_file = srcDict['FileName'] - if not os.path.exists( src_file ): + if not os.path.exists(src_file): errStr = "DIPStorage.__putFile: The source local file does not exist." - gLogger.error( errStr, src_file ) - return S_ERROR( errStr ) - sourceSize = getSize( src_file ) + gLogger.error(errStr, src_file) + return S_ERROR(errStr) + sourceSize = getSize(src_file) if sourceSize == -1: errStr = "DIPStorage.__putFile: Failed to get file size." - gLogger.error( errStr, src_file ) - return S_ERROR( errStr ) - transferClient = TransferClient( self.url ) - res = transferClient.sendFile( src_file, dest_url, token = self.checkSum ) + gLogger.error(errStr, src_file) + return S_ERROR(errStr) + transferClient = TransferClient(self.url) + res = transferClient.sendFile(src_file, dest_url, token=self.checkSum) if localCache: - os.unlink( src_file ) + os.unlink(src_file) if res['OK']: - return S_OK( sourceSize ) + return S_OK(sourceSize) else: return res - def getFile( self, path, localPath = False ): + def getFile(self, path, localPath=False): """Get a local copy in the current directory of a physical file specified by its path """ - res = checkArgumentFormat( path ) + res = checkArgumentFormat(path) if not res['OK']: return res urls = res['Value'] successful = {} failed = {} for src_url in urls: - fileName = os.path.basename( src_url ) + fileName = os.path.basename(src_url) if localPath: - dest_file = "%s/%s" % ( localPath, fileName ) + dest_file = "%s/%s" % (localPath, fileName) else: - dest_file = "%s/%s" % ( os.getcwd(), fileName ) - gLogger.debug( "DIPStorage.getFile: Executing transfer of %s to %s" % ( src_url, dest_file ) ) - res = self.__getFile( src_url, dest_file ) + dest_file = "%s/%s" % (os.getcwd(), fileName) + gLogger.debug("DIPStorage.getFile: Executing transfer of %s to %s" % (src_url, dest_file)) + res = self.__getFile(src_url, dest_file) if res['OK']: successful[src_url] = res['Value'] else: failed[src_url] = res['Message'] - resDict = {'Failed':failed, 'Successful':successful} - return S_OK( resDict ) + resDict = {'Failed': failed, 'Successful': successful} + return S_OK(resDict) - def __getFile( self, src_url, dest_file ): - transferClient = TransferClient( self.url ) - res = transferClient.receiveFile( dest_file, src_url, token = self.checkSum ) + def __getFile(self, src_url, dest_file): + transferClient = TransferClient(self.url) + res = transferClient.receiveFile(dest_file, src_url, token=self.checkSum) if not res['OK']: return res - if not os.path.exists( dest_file ): + if not os.path.exists(dest_file): errStr = "DIPStorage.__getFile: The destination local file does not exist." - gLogger.error( errStr, dest_file ) - return S_ERROR( errStr ) - destSize = getSize( dest_file ) + gLogger.error(errStr, dest_file) + return S_ERROR(errStr) + destSize = getSize(dest_file) if destSize == -1: errStr = "DIPStorage.__getFile: Failed to get the local file size." - gLogger.error( errStr, dest_file ) - return S_ERROR( errStr ) - return S_OK( destSize ) + gLogger.error(errStr, dest_file) + return S_ERROR(errStr) + return S_OK(destSize) - def removeFile( self, path ): + def removeFile(self, path): """Remove physically the file specified by its path """ - res = checkArgumentFormat( path ) + res = checkArgumentFormat(path) if not res['OK']: return res urls = res['Value'] - if not len( urls ) > 0: - return S_ERROR( "DIPStorage.removeFile: No surls supplied." ) + if not len(urls) > 0: + return S_ERROR("DIPStorage.removeFile: No surls supplied.") successful = {} failed = {} - serviceClient = RPCClient( self.url ) + serviceClient = RPCClient(self.url) for url in urls: - gLogger.debug( "DIPStorage.removeFile: Attempting to remove %s." % url ) - res = serviceClient.remove( url, '' ) + gLogger.debug("DIPStorage.removeFile: Attempting to remove %s." % url) + res = serviceClient.remove(url, '') if res['OK']: successful[url] = True else: failed[url] = res['Message'] - resDict = {'Failed':failed, 'Successful':successful} - return S_OK( resDict ) + resDict = {'Failed': failed, 'Successful': successful} + return S_OK(resDict) - def isFile( self, path ): + def isFile(self, path): """ Determine whether the path is a directory """ - res = checkArgumentFormat( path ) + res = checkArgumentFormat(path) if not res['OK']: return res urls = res['Value'] successful = {} failed = {} - gLogger.debug( "DIPStorage.isFile: Attempting to determine whether %s paths are files." % len( urls ) ) - serviceClient = RPCClient( self.url ) + gLogger.debug("DIPStorage.isFile: Attempting to determine whether %s paths are files." % len(urls)) + serviceClient = RPCClient(self.url) for url in urls: - res = serviceClient.getMetadata( url ) + res = serviceClient.getMetadata(url) if res['OK']: if res['Value']['Exists']: if res['Value']['Type'] == 'File': - gLogger.debug( "DIPStorage.isFile: Successfully obtained metadata for %s." % url ) + gLogger.debug("DIPStorage.isFile: Successfully obtained metadata for %s." % url) successful[url] = True else: successful[url] = False else: failed[url] = 'File does not exist' else: - gLogger.error( "DIPStorage.isFile: Failed to get metadata for url", - "%s: %s" % ( url, res['Message'] ) ) + gLogger.error("DIPStorage.isFile: Failed to get metadata for url", + "%s: %s" % (url, res['Message'])) failed[url] = res['Message'] - resDict = {'Failed':failed, 'Successful':successful} - return S_OK( resDict ) + resDict = {'Failed': failed, 'Successful': successful} + return S_OK(resDict) - def getFileSize( self, path ): + def getFileSize(self, path): """ Get size of supplied files """ - res = checkArgumentFormat( path ) + res = checkArgumentFormat(path) if not res['OK']: return res urls = res['Value'] successful = {} failed = {} - gLogger.debug( "DIPStorage.getFileSize: Attempting to obtain size for %s files." % len( urls ) ) - res = self.getFileMetadata( urls ) + gLogger.debug("DIPStorage.getFileSize: Attempting to obtain size for %s files." % len(urls)) + res = self.getFileMetadata(urls) if not res['OK']: return res for url, urlDict in res['Value']['Successful'].items(): @@ -257,59 +256,59 @@ def getFileSize( self, path ): failed[url] = 'File does not exist' for url, error in res['Value']['Failed'].items(): failed[url] = error - resDict = {'Failed':failed, 'Successful':successful} - return S_OK( resDict ) + resDict = {'Failed': failed, 'Successful': successful} + return S_OK(resDict) - def getFileMetadata( self, path ): + def getFileMetadata(self, path): """ Get metadata associated to the file """ - res = checkArgumentFormat( path ) + res = checkArgumentFormat(path) if not res['OK']: return res urls = res['Value'] successful = {} failed = {} - gLogger.debug( "DIPStorage.getFileMetadata: Attempting to obtain metadata for %s files." % len( urls ) ) - serviceClient = RPCClient( self.url ) + gLogger.debug("DIPStorage.getFileMetadata: Attempting to obtain metadata for %s files." % len(urls)) + serviceClient = RPCClient(self.url) for url in urls: pfn = url - if url.find( self.url ) == 0: - pfn = url[ ( len( self.url ) ):] - res = serviceClient.getMetadata( pfn ) + if url.find(self.url) == 0: + pfn = url[(len(self.url)):] + res = serviceClient.getMetadata(pfn) if res['OK']: if res['Value']['Exists']: if res['Value']['Type'] == 'File': - gLogger.debug( "DIPStorage.getFileMetadata: Successfully obtained metadata for %s." % url ) + gLogger.debug("DIPStorage.getFileMetadata: Successfully obtained metadata for %s." % url) successful[url] = res['Value'] else: failed[url] = 'Supplied path is not a file' else: failed[url] = 'File does not exist' else: - gLogger.error( "DIPStorage.getFileMetadata: Failed to get metadata for url", - "%s: %s" % ( url, res['Message'] ) ) + gLogger.error("DIPStorage.getFileMetadata: Failed to get metadata for url", + "%s: %s" % (url, res['Message'])) failed[url] = res['Message'] - resDict = {'Failed':failed, 'Successful':successful} - return S_OK( resDict ) + resDict = {'Failed': failed, 'Successful': successful} + return S_OK(resDict) ############################################################# # # These are the methods for directory manipulation # - def listDirectory( self, path ): + def listDirectory(self, path): """ List the contents of the directory """ - res = checkArgumentFormat( path ) + res = checkArgumentFormat(path) if not res['OK']: return res urls = res['Value'] successful = {} failed = {} - gLogger.debug( "DIPStorage.listDirectory: Attempting to list %s directories." % len( urls ) ) - serviceClient = RPCClient( self.url ) + gLogger.debug("DIPStorage.listDirectory: Attempting to list %s directories." % len(urls)) + serviceClient = RPCClient(self.url) for url in urls: - res = serviceClient.listDirectory( url, 'l' ) + res = serviceClient.listDirectory(url, 'l') if not res['OK']: failed[url] = res['Message'] else: @@ -323,182 +322,186 @@ def listDirectory( self, path ): successful[url] = {} successful[url]['SubDirs'] = subDirs successful[url]['Files'] = files - resDict = {'Failed':failed, 'Successful':successful} - return S_OK( resDict ) + resDict = {'Failed': failed, 'Successful': successful} + return S_OK(resDict) - def isDirectory( self, path ): + def isDirectory(self, path): """ Determine whether the path is a directory """ - res = checkArgumentFormat( path ) + res = checkArgumentFormat(path) if not res['OK']: return res urls = res['Value'] successful = {} failed = {} - gLogger.debug( "DIPStorage.isDirectory: Attempting to determine whether %s paths are directories." % len( urls ) ) - serviceClient = RPCClient( self.url ) + gLogger.debug("DIPStorage.isDirectory: Attempting to determine whether %s paths are directories." % len(urls)) + serviceClient = RPCClient(self.url) for url in urls: - res = serviceClient.getMetadata( url ) + res = serviceClient.getMetadata(url) if res['OK']: if res['Value']['Exists']: if res['Value']['Type'] == 'Directory': - gLogger.debug( "DIPStorage.isDirectory: Successfully obtained metadata for %s." % url ) + gLogger.debug("DIPStorage.isDirectory: Successfully obtained metadata for %s." % url) successful[url] = True else: successful[url] = False else: failed[url] = 'Path does not exist' else: - gLogger.error( "DIPStorage.isDirectory: Failed to get metadata for url", - "%s: %s" % ( url, res['Message'] ) ) + gLogger.error("DIPStorage.isDirectory: Failed to get metadata for url", + "%s: %s" % (url, res['Message'])) failed[url] = res['Message'] - resDict = {'Failed':failed, 'Successful':successful} - return S_OK( resDict ) + resDict = {'Failed': failed, 'Successful': successful} + return S_OK(resDict) - def getDirectorySize( self, path ): + def getDirectorySize(self, path): """ Get the size of the contents of the directory """ - res = checkArgumentFormat( path ) + res = checkArgumentFormat(path) if not res['OK']: return res urls = res['Value'] successful = {} failed = {} - gLogger.debug( "DIPStorage.isDirectory: Attempting to determine whether %s paths are directories." % len( urls ) ) - serviceClient = RPCClient( self.url ) + gLogger.debug("DIPStorage.isDirectory: Attempting to determine whether %s paths are directories." % len(urls)) + serviceClient = RPCClient(self.url) for url in urls: - res = serviceClient.getDirectorySize( url ) + res = serviceClient.getDirectorySize(url) if not res['OK']: failed[url] = res['Message'] else: - successful[url] = {'Files':0, 'Size':res['Value'], 'SubDirs':0} - resDict = {'Failed':failed, 'Successful':successful} - return S_OK( resDict ) + successful[url] = {'Files': 0, 'Size': res['Value'], 'SubDirs': 0} + resDict = {'Failed': failed, 'Successful': successful} + return S_OK(resDict) - def getDirectoryMetadata( self, path ): + def getDirectoryMetadata(self, path): """ Get metadata associated to the directory """ - res = checkArgumentFormat( path ) + res = checkArgumentFormat(path) if not res['OK']: return res urls = res['Value'] successful = {} failed = {} - gLogger.debug( "DIPStorage.getFileMetadata: Attempting to obtain metadata for %s directories." % len( urls ) ) - serviceClient = RPCClient( self.url ) + gLogger.debug("DIPStorage.getFileMetadata: Attempting to obtain metadata for %s directories." % len(urls)) + serviceClient = RPCClient(self.url) for url in urls: - res = serviceClient.getMetadata( url ) + res = serviceClient.getMetadata(url) if res['OK']: if res['Value']['Exists']: if res['Value']['Type'] == 'Directory': res['Value']['Directory'] = True - gLogger.debug( "DIPStorage.getFileMetadata: Successfully obtained metadata for %s." % url ) + gLogger.debug("DIPStorage.getFileMetadata: Successfully obtained metadata for %s." % url) successful[url] = res['Value'] else: failed[url] = 'Supplied path is not a directory' else: failed[url] = 'Directory does not exist' else: - gLogger.error( "DIPStorage.getFileMetadata: Failed to get metadata for url", - "%s: %s" % ( url, res['Message'] ) ) + gLogger.error("DIPStorage.getFileMetadata: Failed to get metadata for url", + "%s: %s" % (url, res['Message'])) failed[url] = res['Message'] - resDict = {'Failed':failed, 'Successful':successful} - return S_OK( resDict ) + resDict = {'Failed': failed, 'Successful': successful} + return S_OK(resDict) - def createDirectory( self, path ): + def createDirectory(self, path): """ Create the remote directory """ - res = checkArgumentFormat( path ) + res = checkArgumentFormat(path) if not res['OK']: return res urls = res['Value'] successful = {} failed = {} - gLogger.debug( "DIPStorage.createDirectory: Attempting to create %s directories." % len( urls ) ) - serviceClient = RPCClient( self.url ) + gLogger.debug("DIPStorage.createDirectory: Attempting to create %s directories." % len(urls)) + serviceClient = RPCClient(self.url) for url in urls: - res = serviceClient.createDirectory( url ) + res = serviceClient.createDirectory(url) if res['OK']: - gLogger.debug( "DIPStorage.createDirectory: Successfully created directory on storage: %s" % url ) + gLogger.debug("DIPStorage.createDirectory: Successfully created directory on storage: %s" % url) successful[url] = True else: - gLogger.error( "DIPStorage.createDirectory: Failed to create directory on storage.", "%s: %s" % ( url, res['Message'] ) ) + gLogger.error( + "DIPStorage.createDirectory: Failed to create directory on storage.", "%s: %s" % + (url, res['Message'])) failed[url] = res['Message'] - resDict = {'Failed':failed, 'Successful':successful} - return S_OK( resDict ) + resDict = {'Failed': failed, 'Successful': successful} + return S_OK(resDict) - def putDirectory( self, path ): + def putDirectory(self, path): """ Put a local directory to the physical storage together with all its files and subdirectories. """ - res = checkArgumentFormat( path ) + res = checkArgumentFormat(path) if not res['OK']: return res urls = res['Value'] successful = {} failed = {} - gLogger.debug( "DIPStorage.putDirectory: Attemping to put %s directories to remote storage." % len( urls ) ) - transferClient = TransferClient( self.url ) + gLogger.debug("DIPStorage.putDirectory: Attemping to put %s directories to remote storage." % len(urls)) + transferClient = TransferClient(self.url) for destDir, sourceDir in urls.items(): - tmpList = os.listdir( sourceDir ) - sourceFiles = [ "%s/%s" % ( sourceDir, x ) for x in tmpList ] - res = transferClient.sendBulk( sourceFiles, destDir ) + tmpList = os.listdir(sourceDir) + sourceFiles = ["%s/%s" % (sourceDir, x) for x in tmpList] + res = transferClient.sendBulk(sourceFiles, destDir) if res['OK']: - successful[destDir] = {'Files':0, 'Size':0} + successful[destDir] = {'Files': 0, 'Size': 0} else: failed[destDir] = res['Message'] - resDict = {'Failed':failed, 'Successful':successful} - return S_OK( resDict ) + resDict = {'Failed': failed, 'Successful': successful} + return S_OK(resDict) - def removeDirectory( self, path, recursive = False ): + def removeDirectory(self, path, recursive=False): """ Remove a directory from the storage together with all its files and subdirectories. """ - res = checkArgumentFormat( path ) + res = checkArgumentFormat(path) if not res['OK']: return res urls = res['Value'] successful = {} failed = {} - gLogger.debug( "DIPStorage.removeDirectory: Attempting to remove %s directories." % len( urls ) ) - serviceClient = RPCClient( self.url ) + gLogger.debug("DIPStorage.removeDirectory: Attempting to remove %s directories." % len(urls)) + serviceClient = RPCClient(self.url) for url in urls: - res = serviceClient.removeDirectory( url, '' ) + res = serviceClient.removeDirectory(url, '') if res['OK']: - gLogger.debug( "DIPStorage.removeDirectory: Successfully removed directory on storage: %s" % url ) - successful[url] = {'FilesRemoved':0, 'SizeRemoved':0} + gLogger.debug("DIPStorage.removeDirectory: Successfully removed directory on storage: %s" % url) + successful[url] = {'FilesRemoved': 0, 'SizeRemoved': 0} else: - gLogger.error( "DIPStorage.removeDirectory: Failed to remove directory from storage.", "%s: %s" % ( url, res['Message'] ) ) + gLogger.error( + "DIPStorage.removeDirectory: Failed to remove directory from storage.", "%s: %s" % + (url, res['Message'])) failed[url] = res['Message'] - resDict = {'Failed':failed, 'Successful':successful} - return S_OK( resDict ) + resDict = {'Failed': failed, 'Successful': successful} + return S_OK(resDict) - def getDirectory( self, path, localPath = False ): + def getDirectory(self, path, localPath=False): """ Get a local copy in the current directory of a physical file specified by its path """ - res = checkArgumentFormat( path ) + res = checkArgumentFormat(path) if not res['OK']: return res urls = res['Value'] failed = {} successful = {} - gLogger.debug( "DIPStorage.getDirectory: Attempting to get local copies of %s directories." % len( urls ) ) - transferClient = TransferClient( self.url ) + gLogger.debug("DIPStorage.getDirectory: Attempting to get local copies of %s directories." % len(urls)) + transferClient = TransferClient(self.url) for src_dir in urls: if localPath: dest_dir = localPath else: dest_dir = os.getcwd() - if not os.path.exists( dest_dir ): - os.mkdir( dest_dir ) - res = transferClient.receiveBulk( dest_dir, src_dir ) + if not os.path.exists(dest_dir): + os.mkdir(dest_dir) + res = transferClient.receiveBulk(dest_dir, src_dir) if res['OK']: - gLogger.debug( "DIPStorage.getDirectory: Successfully got local copy of %s" % src_dir ) - successful[src_dir] = {'Files':0, 'Size':0} + gLogger.debug("DIPStorage.getDirectory: Successfully got local copy of %s" % src_dir) + successful[src_dir] = {'Files': 0, 'Size': 0} else: - gLogger.error( "DIPStorage.getDirectory: Failed to get entire directory.", src_dir ) + gLogger.error("DIPStorage.getDirectory: Failed to get entire directory.", src_dir) failed[src_dir] = res['Message'] - resDict = {'Failed':failed, 'Successful':successful} - return S_OK( resDict ) + resDict = {'Failed': failed, 'Successful': successful} + return S_OK(resDict) def getOccupancy(self): """ Gets the DIPStorage occupancy info diff --git a/Resources/Storage/StorageBase.py b/Resources/Storage/StorageBase.py index 9768f555ab5..772d4b13b34 100755 --- a/Resources/Storage/StorageBase.py +++ b/Resources/Storage/StorageBase.py @@ -39,36 +39,36 @@ from DIRAC.Core.Utilities.Pfn import pfnparse, pfnunparse from DIRAC.Resources.Storage.Utilities import checkArgumentFormat -class StorageBase( object ): + +class StorageBase(object): """ .. class:: StorageBase """ - PROTOCOL_PARAMETERS = [ "Protocol", "Host", "Path", "Port", "SpaceToken", "WSUrl" ] + PROTOCOL_PARAMETERS = ["Protocol", "Host", "Path", "Port", "SpaceToken", "WSUrl"] # Options to be prepended in the URL # keys are the name of the parameters in the CS # values are the name of the options as they appear in the URL DYNAMIC_OPTIONS = {} - def __init__( self, name, parameterDict ): + def __init__(self, name, parameterDict): self.name = name self.pluginName = '' self.protocolParameters = {} - self.__updateParameters( parameterDict ) - + self.__updateParameters(parameterDict) - if hasattr( self, '_INPUT_PROTOCOLS' ): - self.protocolParameters['InputProtocols'] = getattr( self, '_INPUT_PROTOCOLS' ) + if hasattr(self, '_INPUT_PROTOCOLS'): + self.protocolParameters['InputProtocols'] = getattr(self, '_INPUT_PROTOCOLS') else: - self.protocolParameters['InputProtocols'] = [ self.protocolParameters['Protocol'], 'file'] + self.protocolParameters['InputProtocols'] = [self.protocolParameters['Protocol'], 'file'] - if hasattr( self, '_OUTPUT_PROTOCOLS' ): - self.protocolParameters['OutputProtocols'] = getattr( self, '_OUTPUT_PROTOCOLS' ) + if hasattr(self, '_OUTPUT_PROTOCOLS'): + self.protocolParameters['OutputProtocols'] = getattr(self, '_OUTPUT_PROTOCOLS') else: - self.protocolParameters['OutputProtocols'] = [ self.protocolParameters['Protocol']] + self.protocolParameters['OutputProtocols'] = [self.protocolParameters['Protocol']] self.basePath = parameterDict['Path'] self.cwd = self.basePath @@ -78,223 +78,222 @@ def __init__( self, name, parameterDict ): # use True for backward compatibility self.srmSpecificParse = True - def setStorageElement( self, se ): + def setStorageElement(self, se): self.se = se - def setParameters( self, parameterDict ): + def setParameters(self, parameterDict): """ Set standard parameters, method can be overriden in subclasses to process specific parameters """ - self.__updateParameters( parameterDict ) + self.__updateParameters(parameterDict) - def __updateParameters( self, parameterDict ): + def __updateParameters(self, parameterDict): """ setParameters implementation method """ for item in self.PROTOCOL_PARAMETERS: - self.protocolParameters[item] = parameterDict.get( item, '' ) + self.protocolParameters[item] = parameterDict.get(item, '') - def getParameters( self ): + def getParameters(self): """ Get the parameters with which the storage was instantiated """ - parameterDict = dict( self.protocolParameters ) + parameterDict = dict(self.protocolParameters) parameterDict["StorageName"] = self.name parameterDict["PluginName"] = self.pluginName - parameterDict['URLBase'] = self.getURLBase().get( 'Value', '' ) + parameterDict['URLBase'] = self.getURLBase().get('Value', '') return parameterDict - def exists( self, *parms, **kws ): + def exists(self, *parms, **kws): """Check if the given path exists """ - return S_ERROR( "Storage.exists: implement me!" ) + return S_ERROR("Storage.exists: implement me!") ############################################################# # # These are the methods for file manipulation # - def isFile( self, *parms, **kws ): + def isFile(self, *parms, **kws): """Check if the given path exists and it is a file """ - return S_ERROR( "Storage.isFile: implement me!" ) + return S_ERROR("Storage.isFile: implement me!") - def getFile( self, *parms, **kws ): + def getFile(self, *parms, **kws): """Get a local copy of the file specified by its path """ - return S_ERROR( "Storage.getFile: implement me!" ) + return S_ERROR("Storage.getFile: implement me!") - def putFile( self, *parms, **kws ): + def putFile(self, *parms, **kws): """Put a copy of the local file to the current directory on the physical storage """ - return S_ERROR( "Storage.putFile: implement me!" ) + return S_ERROR("Storage.putFile: implement me!") - def removeFile( self, *parms, **kws ): + def removeFile(self, *parms, **kws): """Remove physically the file specified by its path """ - return S_ERROR( "Storage.removeFile: implement me!" ) + return S_ERROR("Storage.removeFile: implement me!") - def getFileMetadata( self, *parms, **kws ): + def getFileMetadata(self, *parms, **kws): """ Get metadata associated to the file """ - return S_ERROR( "Storage.getFileMetadata: implement me!" ) + return S_ERROR("Storage.getFileMetadata: implement me!") - def getFileSize( self, *parms, **kws ): + def getFileSize(self, *parms, **kws): """Get the physical size of the given file """ - return S_ERROR( "Storage.getFileSize: implement me!" ) + return S_ERROR("Storage.getFileSize: implement me!") - def prestageFile( self, *parms, **kws ): + def prestageFile(self, *parms, **kws): """ Issue prestage request for file """ - return S_ERROR( "Storage.prestageFile: implement me!" ) + return S_ERROR("Storage.prestageFile: implement me!") - def prestageFileStatus( self, *parms, **kws ): + def prestageFileStatus(self, *parms, **kws): """ Obtain the status of the prestage request """ - return S_ERROR( "Storage.prestageFileStatus: implement me!" ) + return S_ERROR("Storage.prestageFileStatus: implement me!") - def pinFile( self, *parms, **kws ): + def pinFile(self, *parms, **kws): """ Pin the file on the destination storage element """ - return S_ERROR( "Storage.pinFile: implement me!" ) + return S_ERROR("Storage.pinFile: implement me!") - def releaseFile( self, *parms, **kws ): + def releaseFile(self, *parms, **kws): """ Release the file on the destination storage element """ - return S_ERROR( "Storage.releaseFile: implement me!" ) + return S_ERROR("Storage.releaseFile: implement me!") ############################################################# # # These are the methods for directory manipulation # - def isDirectory( self, *parms, **kws ): + def isDirectory(self, *parms, **kws): """Check if the given path exists and it is a directory """ - return S_ERROR( "Storage.isDirectory: implement me!" ) + return S_ERROR("Storage.isDirectory: implement me!") - def getDirectory( self, *parms, **kws ): + def getDirectory(self, *parms, **kws): """Get locally a directory from the physical storage together with all its files and subdirectories. """ - return S_ERROR( "Storage.getDirectory: implement me!" ) + return S_ERROR("Storage.getDirectory: implement me!") - def putDirectory( self, *parms, **kws ): + def putDirectory(self, *parms, **kws): """Put a local directory to the physical storage together with all its files and subdirectories. """ - return S_ERROR( "Storage.putDirectory: implement me!" ) + return S_ERROR("Storage.putDirectory: implement me!") - def createDirectory( self, *parms, **kws ): + def createDirectory(self, *parms, **kws): """ Make a new directory on the physical storage """ - return S_ERROR( "Storage.createDirectory: implement me!" ) + return S_ERROR("Storage.createDirectory: implement me!") - def removeDirectory( self, *parms, **kws ): + def removeDirectory(self, *parms, **kws): """Remove a directory on the physical storage together with all its files and subdirectories. """ - return S_ERROR( "Storage.removeDirectory: implement me!" ) + return S_ERROR("Storage.removeDirectory: implement me!") - def listDirectory( self, *parms, **kws ): + def listDirectory(self, *parms, **kws): """ List the supplied path """ - return S_ERROR( "Storage.listDirectory: implement me!" ) + return S_ERROR("Storage.listDirectory: implement me!") - def getDirectoryMetadata( self, *parms, **kws ): + def getDirectoryMetadata(self, *parms, **kws): """ Get the metadata for the directory """ - return S_ERROR( "Storage.getDirectoryMetadata: implement me!" ) + return S_ERROR("Storage.getDirectoryMetadata: implement me!") - def getDirectorySize( self, *parms, **kws ): + def getDirectorySize(self, *parms, **kws): """ Get the size of the directory on the storage """ - return S_ERROR( "Storage.getDirectorySize: implement me!" ) - + return S_ERROR("Storage.getDirectorySize: implement me!") ############################################################# # # These are the methods for manipulating the client # - def isOK( self ): + def isOK(self): return self.isok - def resetCurrentDirectory( self ): + def resetCurrentDirectory(self): """ Reset the working directory to the base dir """ self.cwd = self.basePath - def changeDirectory( self, directory ): + def changeDirectory(self, directory): """ Change the directory to the supplied directory """ - if directory.startswith( '/' ): - self.cwd = "%s/%s" % ( self.basePath, directory ) + if directory.startswith('/'): + self.cwd = "%s/%s" % (self.basePath, directory) else: - self.cwd = '%s/%s' % ( self.cwd, directory ) + self.cwd = '%s/%s' % (self.cwd, directory) - def getCurrentDirectory( self ): + def getCurrentDirectory(self): """ Get the current directory """ return self.cwd - def getCurrentURL( self, fileName ): + def getCurrentURL(self, fileName): """ Obtain the current file URL from the current working directory and the filename :param self: self reference :param str fileName: path on storage """ - urlDict = dict( self.protocolParameters ) - if not fileName.startswith( '/' ): + urlDict = dict(self.protocolParameters) + if not fileName.startswith('/'): # Relative path is given urlDict['Path'] = self.cwd - result = pfnunparse( urlDict, srmSpecific = self.srmSpecificParse ) + result = pfnunparse(urlDict, srmSpecific=self.srmSpecificParse) if not result['OK']: return result cwdUrl = result['Value'] - fullUrl = '%s%s' % ( cwdUrl, fileName ) - return S_OK( fullUrl ) + fullUrl = '%s%s' % (cwdUrl, fileName) + return S_OK(fullUrl) - def getName( self ): + def getName(self): """ The name with which the storage was instantiated """ return self.name - def getURLBase( self, withWSUrl = False ): + def getURLBase(self, withWSUrl=False): """ This will get the URL base. This is then appended with the LFN in DIRAC convention. :param self: self reference :param bool withWSUrl: flag to include Web Service part of the url :returns: URL """ - urlDict = dict( self.protocolParameters ) + urlDict = dict(self.protocolParameters) if not withWSUrl: urlDict['WSUrl'] = '' - return pfnunparse( urlDict, srmSpecific = self.srmSpecificParse ) + return pfnunparse(urlDict, srmSpecific=self.srmSpecificParse) - def isURL( self, path ): + def isURL(self, path): """ Guess if the path looks like a URL :param self: self reference :param string path: input file LFN or URL :returns boolean: True if URL, False otherwise """ - if self.basePath and path.startswith( self.basePath ): - return S_OK( True ) + if self.basePath and path.startswith(self.basePath): + return S_OK(True) - result = pfnparse( path, srmSpecific = self.srmSpecificParse ) + result = pfnparse(path, srmSpecific=self.srmSpecificParse) if not result['OK']: return result - if len( result['Value']['Protocol'] ) != 0: - return S_OK( True ) + if len(result['Value']['Protocol']) != 0: + return S_OK(True) - if result['Value']['Path'].startswith( self.basePath ): - return S_OK( True ) + if result['Value']['Path'].startswith(self.basePath): + return S_OK(True) - return S_OK( False ) + return S_OK(False) - def getTransportURL( self, pathDict, protocols ): + def getTransportURL(self, pathDict, protocols): """ Get a transport URL for a given URL. For a simple storage plugin it is just returning input URL if the plugin protocol is one of the requested protocols @@ -304,7 +303,7 @@ def getTransportURL( self, pathDict, protocols ): :param protocols: a list of acceptable transport protocols in priority order :type protocols: `python:list` """ - res = checkArgumentFormat( pathDict ) + res = checkArgumentFormat(pathDict) if not res['OK']: return res urls = res['Value'] @@ -312,15 +311,15 @@ def getTransportURL( self, pathDict, protocols ): failed = {} if protocols and not self.protocolParameters['Protocol'] in protocols: - return S_ERROR( 'No native protocol requested' ) + return S_ERROR('No native protocol requested') for url in urls: successful[url] = url - resDict = {'Failed':failed, 'Successful':successful} - return S_OK( resDict ) + resDict = {'Failed': failed, 'Successful': successful} + return S_OK(resDict) - def constructURLFromLFN( self, lfn, withWSUrl = False ): + def constructURLFromLFN(self, lfn, withWSUrl=False): """ Construct URL from the given LFN according to the VO convention for the primary protocol of the storage plagin @@ -332,27 +331,27 @@ def constructURLFromLFN( self, lfn, withWSUrl = False ): # Check the LFN convention: # 1. LFN must start with the VO name as the top level directory # 2. VO name must not appear as any subdirectory or file name - lfnSplitList = lfn.split( '/' ) + lfnSplitList = lfn.split('/') voLFN = lfnSplitList[1] # TODO comparison to Sandbox below is for backward compatibility, should be removed in the next release if voLFN != self.se.vo and voLFN != "SandBox" and voLFN != "Sandbox": - return S_ERROR( 'LFN does not follow the DIRAC naming convention %s' % lfn ) + return S_ERROR('LFN does not follow the DIRAC naming convention %s' % lfn) - urlDict = dict( self.protocolParameters ) - urlDict['Options'] = '&'.join( "%s=%s" % ( optionName, urlDict[paramName] ) - for paramName, optionName in self.DYNAMIC_OPTIONS.iteritems() - if urlDict.get( paramName ) ) + urlDict = dict(self.protocolParameters) + urlDict['Options'] = '&'.join("%s=%s" % (optionName, urlDict[paramName]) + for paramName, optionName in self.DYNAMIC_OPTIONS.iteritems() + if urlDict.get(paramName)) if not withWSUrl: urlDict['WSUrl'] = '' - urlDict['FileName'] = lfn.lstrip( '/' ) + urlDict['FileName'] = lfn.lstrip('/') - return pfnunparse( urlDict, srmSpecific = self.srmSpecificParse ) + return pfnunparse(urlDict, srmSpecific=self.srmSpecificParse) - def updateURL( self, url, withWSUrl = False ): + def updateURL(self, url, withWSUrl=False): """ Update the URL according to the current SE parameters """ - result = pfnparse( url, srmSpecific = self.srmSpecificParse ) + result = pfnparse(url, srmSpecific=self.srmSpecificParse) if not result['OK']: return result urlDict = result['Value'] @@ -364,22 +363,22 @@ def updateURL( self, url, withWSUrl = False ): if withWSUrl: urlDict['WSUrl'] = self.protocolParameters['WSUrl'] - return pfnunparse( urlDict, srmSpecific = self.srmSpecificParse ) + return pfnunparse(urlDict, srmSpecific=self.srmSpecificParse) - def isNativeURL( self, url ): + def isNativeURL(self, url): """ Check if URL :url: is valid for :self.protocol: :param self: self reference :param str url: URL """ - res = pfnparse( url, srmSpecific = self.srmSpecificParse ) + res = pfnparse(url, srmSpecific=self.srmSpecificParse) if not res['OK']: return res urlDict = res['Value'] - return S_OK( urlDict['Protocol'] == self.protocolParameters['Protocol'] ) + return S_OK(urlDict['Protocol'] == self.protocolParameters['Protocol']) @staticmethod - def _addCommonMetadata( metadataDict ): + def _addCommonMetadata(metadataDict): """ To make the output of getFileMetadata uniform throughout the protocols this returns a minimum set of metadata with default value, that are then complemented with the protocol specific metadata @@ -388,35 +387,34 @@ def _addCommonMetadata( metadataDict ): :returns: dictionnary with all the metadata (specific and basic) """ - commonMetadata = { 'Checksum' : '', - 'Directory' : False, - 'File' : False, - 'Mode' : 0o000, - 'Size' : 0, - 'Accessible' : True, + commonMetadata = {'Checksum': '', + 'Directory': False, + 'File': False, + 'Mode': 0o000, + 'Size': 0, + 'Accessible': True, } - commonMetadata.update( metadataDict ) + commonMetadata.update(metadataDict) return commonMetadata - - def _isInputURL( self, url ): + def _isInputURL(self, url): """ Check if the given url can be taken as input :param self: self reference :param str url: URL """ - res = pfnparse( url ) + res = pfnparse(url) if not res['OK']: return res urlDict = res['Value'] # Special case of 'file' protocol which can be just a URL if not urlDict['Protocol'] and 'file' in self.protocolParameters['InputProtocols']: - return S_OK( True ) + return S_OK(True) - return S_OK( urlDict['Protocol'] == self.protocolParameters['Protocol'] ) + return S_OK(urlDict['Protocol'] == self.protocolParameters['Protocol']) ############################################################# # @@ -427,5 +425,5 @@ def getOccupancy(self, *parms, **kws): """ Get the StorageElement occupancy info. :returns: S_OK/S_ERROR dictionary """ - #FIXME: put an implementation that just gets a file and check its content + # FIXME: put an implementation that just gets a file and check its content return S_ERROR("Storage.occupancy: implement me!") diff --git a/Resources/Storage/StorageElement.py b/Resources/Storage/StorageElement.py index b409711589b..856b4418567 100755 --- a/Resources/Storage/StorageElement.py +++ b/Resources/Storage/StorageElement.py @@ -26,17 +26,18 @@ from DIRAC.AccountingSystem.Client.Types.DataOperation import DataOperation from DIRAC.AccountingSystem.Client.DataStoreClient import gDataStoreClient from DIRAC.DataManagementSystem.Utilities.DMSHelpers import DMSHelpers +from functools import reduce __RCSID__ = "$Id$" -class StorageElementCache( object ): +class StorageElementCache(object): - def __init__( self ): + def __init__(self): self.seCache = DictCache() - def __call__( self, name, plugins = None, vo = None, hideExceptions = False ): - self.seCache.purgeExpired( expiredInSeconds = 60 ) + def __call__(self, name, plugins=None, vo=None, hideExceptions=False): + self.seCache.purgeExpired(expiredInSeconds=60) tId = threading.current_thread().ident if not vo: @@ -45,17 +46,18 @@ def __call__( self, name, plugins = None, vo = None, hideExceptions = False ): return vo = result['Value'] - argTuple = ( tId, name, plugins, vo ) - seObj = self.seCache.get( argTuple ) + argTuple = (tId, name, plugins, vo) + seObj = self.seCache.get(argTuple) if not seObj: - seObj = StorageElementItem( name, plugins, vo, hideExceptions = hideExceptions ) + seObj = StorageElementItem(name, plugins, vo, hideExceptions=hideExceptions) # Add the StorageElement to the cache for 1/2 hour - self.seCache.add( argTuple, 1800, seObj ) + self.seCache.add(argTuple, 1800, seObj) return seObj -class StorageElementItem( object ): + +class StorageElementItem(object): """ .. class:: StorageElement @@ -104,39 +106,39 @@ class StorageElementItem( object ): # Some methods have a different name in the StorageElement and the plugins... # We could avoid this static list in the __getattr__ by checking the storage plugin and so on # but fine... let's not be too smart, otherwise it becomes unreadable :-) - __equivalentMethodNames = { "exists" : "exists", - "isFile" : "isFile", - "getFile" : "getFile", - "putFile" : "putFile", - "replicateFile" : "putFile", - "getFileMetadata" : "getFileMetadata", - "getFileSize" : "getFileSize", - "removeFile" : "removeFile", - "prestageFile" : "prestageFile", - "prestageFileStatus" : "prestageFileStatus", - "pinFile" : "pinFile", - "releaseFile" : "releaseFile", - "isDirectory" : "isDirectory", - "getDirectoryMetadata" : "getDirectoryMetadata", - "getDirectorySize" : "getDirectorySize", - "listDirectory" : "listDirectory", - "removeDirectory" : "removeDirectory", - "createDirectory" : "createDirectory", - "putDirectory" : "putDirectory", - "getDirectory" : "getDirectory", - "getOccupancy": "getOccupancy" - } + __equivalentMethodNames = {"exists": "exists", + "isFile": "isFile", + "getFile": "getFile", + "putFile": "putFile", + "replicateFile": "putFile", + "getFileMetadata": "getFileMetadata", + "getFileSize": "getFileSize", + "removeFile": "removeFile", + "prestageFile": "prestageFile", + "prestageFileStatus": "prestageFileStatus", + "pinFile": "pinFile", + "releaseFile": "releaseFile", + "isDirectory": "isDirectory", + "getDirectoryMetadata": "getDirectoryMetadata", + "getDirectorySize": "getDirectorySize", + "listDirectory": "listDirectory", + "removeDirectory": "removeDirectory", + "createDirectory": "createDirectory", + "putDirectory": "putDirectory", + "getDirectory": "getDirectory", + "getOccupancy": "getOccupancy" + } # We can set default argument in the __executeFunction which impacts all plugins - __defaultsArguments = {"putFile" : {"sourceSize" : 0 }, - "getFile": { "localPath": False }, - "prestageFile" : { "lifetime" : 86400 }, - "pinFile" : { "lifetime" : 60 * 60 * 24 }, - "removeDirectory" : { "recursive" : False }, - "getDirectory" : { "localPath" : False }, - } - - def __init__( self, name, plugins = None, vo = None, hideExceptions = False ): + __defaultsArguments = {"putFile": {"sourceSize": 0}, + "getFile": {"localPath": False}, + "prestageFile": {"lifetime": 86400}, + "pinFile": {"lifetime": 60 * 60 * 24}, + "removeDirectory": {"recursive": False}, + "getDirectory": {"localPath": False}, + } + + def __init__(self, name, plugins=None, vo=None, hideExceptions=False): """ c'tor :param str name: SE name @@ -153,24 +155,35 @@ def __init__( self, name, plugins = None, vo = None, hideExceptions = False ): if not result['OK']: return self.vo = result['Value'] - self.opHelper = Operations( vo = self.vo ) + self.opHelper = Operations(vo=self.vo) # These things will soon have to go as well. 'AccessProtocol.1' is all but flexible. - proxiedProtocols = gConfig.getValue( '/LocalSite/StorageElements/ProxyProtocols', "" ).split( ',' ) - self.useProxy = ( gConfig.getValue( "/Resources/StorageElements/%s/AccessProtocol.1/Protocol" % name, "UnknownProtocol" ) - in proxiedProtocols ) + proxiedProtocols = gConfig.getValue('/LocalSite/StorageElements/ProxyProtocols', "").split(',') + self.useProxy = ( + gConfig.getValue( + "/Resources/StorageElements/%s/AccessProtocol.1/Protocol" % + name, "UnknownProtocol") in proxiedProtocols) if not self.useProxy: - self.useProxy = gConfig.getValue( '/LocalSite/StorageElements/%s/UseProxy' % name, False ) + self.useProxy = gConfig.getValue('/LocalSite/StorageElements/%s/UseProxy' % name, False) if not self.useProxy: - self.useProxy = self.opHelper.getValue( '/Services/StorageElements/%s/UseProxy' % name, False ) + self.useProxy = self.opHelper.getValue('/Services/StorageElements/%s/UseProxy' % name, False) self.valid = True - if plugins == None: - res = StorageFactory( useProxy = self.useProxy, vo = self.vo ).getStorages( name, pluginList = [], hideExceptions = hideExceptions ) + if plugins is None: + res = StorageFactory( + useProxy=self.useProxy, + vo=self.vo).getStorages( + name, + pluginList=[], + hideExceptions=hideExceptions) else: - res = StorageFactory( useProxy = self.useProxy, vo = self.vo ).getStorages( name, pluginList = plugins, hideExceptions = hideExceptions ) - + res = StorageFactory( + useProxy=self.useProxy, + vo=self.vo).getStorages( + name, + pluginList=plugins, + hideExceptions=hideExceptions) if not res['OK']: self.valid = False @@ -188,115 +201,111 @@ def __init__( self, name, plugins = None, vo = None, hideExceptions = False ): for storage in self.storages: - storage.setStorageElement( self ) + storage.setStorageElement(self) - - self.log = gLogger.getSubLogger( "SE[%s]" % self.name ) + self.log = gLogger.getSubLogger("SE[%s]" % self.name) if self.valid: - self.useCatalogURL = gConfig.getValue( '/Resources/StorageElements/%s/UseCatalogURL' % self.name, False ) - self.log.debug( "useCatalogURL: %s" % self.useCatalogURL ) + self.useCatalogURL = gConfig.getValue('/Resources/StorageElements/%s/UseCatalogURL' % self.name, False) + self.log.debug("useCatalogURL: %s" % self.useCatalogURL) - self.__dmsHelper = DMSHelpers( vo = vo ) + self.__dmsHelper = DMSHelpers(vo=vo) # Allow SE to overwrite general operation config - accessProto = self.options.get( 'AccessProtocols' ) + accessProto = self.options.get('AccessProtocols') self.localAccessProtocolList = accessProto if accessProto else self.__dmsHelper.getAccessProtocols() - self.log.debug( "localAccessProtocolList %s" % self.localAccessProtocolList ) + self.log.debug("localAccessProtocolList %s" % self.localAccessProtocolList) - writeProto = self.options.get( 'WriteProtocols' ) + writeProto = self.options.get('WriteProtocols') self.localWriteProtocolList = writeProto if writeProto else self.__dmsHelper.getWriteProtocols() - self.log.debug( "localWriteProtocolList %s" % self.localWriteProtocolList ) - - - + self.log.debug("localWriteProtocolList %s" % self.localWriteProtocolList) # 'getTransportURL', - self.readMethods = [ 'getFile', - 'prestageFile', - 'prestageFileStatus', - 'getDirectory'] - - self.writeMethods = [ 'retransferOnlineFile', - 'putFile', - 'replicateFile', - 'pinFile', - 'releaseFile', - 'createDirectory', - 'putDirectory' ] + self.readMethods = ['getFile', + 'prestageFile', + 'prestageFileStatus', + 'getDirectory'] + + self.writeMethods = ['retransferOnlineFile', + 'putFile', + 'replicateFile', + 'pinFile', + 'releaseFile', + 'createDirectory', + 'putDirectory'] self.removeMethods = ['removeFile', 'removeDirectory'] - self.checkMethods = [ 'exists', - 'getDirectoryMetadata', - 'getDirectorySize', - 'getFileSize', - 'getFileMetadata', - 'listDirectory', - 'isDirectory', - 'isFile', - 'getOccupancy' - ] - - self.okMethods = [ 'getLocalProtocols', - 'getProtocols', - 'getRemoteProtocols', - 'storageElementName', - 'getStorageParameters', - 'getTransportURL', - 'isLocalSE' ] + self.checkMethods = ['exists', + 'getDirectoryMetadata', + 'getDirectorySize', + 'getFileSize', + 'getFileMetadata', + 'listDirectory', + 'isDirectory', + 'isFile', + 'getOccupancy' + ] + + self.okMethods = ['getLocalProtocols', + 'getProtocols', + 'getRemoteProtocols', + 'storageElementName', + 'getStorageParameters', + 'getTransportURL', + 'isLocalSE'] self.__fileCatalog = None - def dump( self ): + def dump(self): """ Dump to the logger a summary of the StorageElement items. """ - log = self.log.getSubLogger( 'dump', True ) - log.verbose( "Preparing dump for StorageElement %s." % self.name ) + log = self.log.getSubLogger('dump', True) + log.verbose("Preparing dump for StorageElement %s." % self.name) if not self.valid: - log.debug( "Failed to create StorageElement plugins.", self.errorReason ) + log.debug("Failed to create StorageElement plugins.", self.errorReason) return i = 1 outStr = "\n\n============ Options ============\n" - for key in sorted( self.options ): - outStr = "%s%s: %s\n" % ( outStr, key.ljust( 15 ), self.options[key] ) + for key in sorted(self.options): + outStr = "%s%s: %s\n" % (outStr, key.ljust(15), self.options[key]) for storage in self.storages: - outStr = "%s============Protocol %s ============\n" % ( outStr, i ) + outStr = "%s============Protocol %s ============\n" % (outStr, i) storageParameters = storage.getParameters() - for key in sorted( storageParameters ): - outStr = "%s%s: %s\n" % ( outStr, key.ljust( 15 ), storageParameters[key] ) + for key in sorted(storageParameters): + outStr = "%s%s: %s\n" % (outStr, key.ljust(15), storageParameters[key]) i = i + 1 - log.verbose( outStr ) + log.verbose(outStr) ################################################################################################# # # These are the basic get functions for storage configuration # - def getStorageElementName( self ): + def getStorageElementName(self): """ SE name getter for backward compatibility """ - return S_OK( self.storageElementName() ) + return S_OK(self.storageElementName()) - def storageElementName( self ): + def storageElementName(self): """ SE name getter """ - self.log.getSubLogger( 'storageElementName' ).verbose( "The Storage Element name is %s." % self.name ) + self.log.getSubLogger('storageElementName').verbose("The Storage Element name is %s." % self.name) return self.name - def getChecksumType( self ): + def getChecksumType(self): """ Checksum type getter for backward compatibility """ - return S_OK( self.checksumType() ) + return S_OK(self.checksumType()) - def checksumType( self ): + def checksumType(self): """ get specific /Resources/StorageElements//ChecksumType option if defined, otherwise global /Resources/StorageElements/ChecksumType """ - self.log.getSubLogger( 'checksumType' ).verbose( "get checksum type for %s." % self.name ) - return self.options["ChecksumType"].upper() \ - if "ChecksumType" in self.options else gConfig.getValue( "/Resources/StorageElements/ChecksumType", "ADLER32" ).upper() + self.log.getSubLogger('checksumType').verbose("get checksum type for %s." % self.name) + return self.options["ChecksumType"].upper() if "ChecksumType" in self.options else gConfig.getValue( + "/Resources/StorageElements/ChecksumType", "ADLER32").upper() - def getStatus( self ): + def getStatus(self): """ Return Status of the SE only if the SE is valid It returns an S_OK/S_ERROR structure @@ -304,9 +313,9 @@ def getStatus( self ): valid = self.isValid() if not valid['OK']: return valid - return S_OK( self.status() ) + return S_OK(self.status()) - def status( self ): + def status(self): """ Return Status of the SE, a dictionary with: @@ -324,7 +333,7 @@ def status( self ): It returns directly the dictionary """ - self.log.getSubLogger( 'getStatus' ).verbose( "determining status of %s." % self.name ) + self.log.getSubLogger('getStatus').verbose("determining status of %s." % self.name) retDict = {} if not self.valid: @@ -340,126 +349,129 @@ def status( self ): # If nothing is defined in the CS Access is allowed # If something is defined, then it must be set to Active - retDict['Read'] = not ( 'ReadAccess' in self.options and self.options['ReadAccess'] not in ( 'Active', 'Degraded' ) ) - retDict['Write'] = not ( 'WriteAccess' in self.options and self.options['WriteAccess'] not in ( 'Active', 'Degraded' ) ) - retDict['Remove'] = not ( 'RemoveAccess' in self.options and self.options['RemoveAccess'] not in ( 'Active', 'Degraded' ) ) + retDict['Read'] = not ('ReadAccess' in self.options and self.options['ReadAccess'] not in ('Active', 'Degraded')) + retDict['Write'] = not ('WriteAccess' in self.options and self.options['WriteAccess'] not in ('Active', 'Degraded')) + retDict['Remove'] = not ( + 'RemoveAccess' in self.options and self.options['RemoveAccess'] not in ( + 'Active', 'Degraded')) if retDict['Read']: retDict['Check'] = True else: - retDict['Check'] = not ( 'CheckAccess' in self.options and self.options['CheckAccess'] not in ( 'Active', 'Degraded' ) ) + retDict['Check'] = not ( + 'CheckAccess' in self.options and self.options['CheckAccess'] not in ( + 'Active', 'Degraded')) diskSE = True tapeSE = False if 'SEType' in self.options: # Type should follow the convention TXDY seType = self.options['SEType'] - diskSE = re.search( 'D[1-9]', seType ) != None - tapeSE = re.search( 'T[1-9]', seType ) != None + diskSE = re.search('D[1-9]', seType) is not None + tapeSE = re.search('T[1-9]', seType) is not None retDict['DiskSE'] = diskSE retDict['TapeSE'] = tapeSE try: - retDict['TotalCapacityTB'] = float( self.options['TotalCapacityTB'] ) + retDict['TotalCapacityTB'] = float(self.options['TotalCapacityTB']) except Exception: retDict['TotalCapacityTB'] = -1 try: - retDict['DiskCacheTB'] = float( self.options['DiskCacheTB'] ) + retDict['DiskCacheTB'] = float(self.options['DiskCacheTB']) except Exception: retDict['DiskCacheTB'] = -1 return retDict - def isValid( self, operation = None ): + def isValid(self, operation=None): """ check CS/RSS statuses for :operation: :param str operation: operation name """ - log = self.log.getSubLogger( 'isValid', True ) - log.verbose( "Determining if the StorageElement %s is valid for VO %s" % ( self.name, self.vo ) ) + log = self.log.getSubLogger('isValid', True) + log.verbose("Determining if the StorageElement %s is valid for VO %s" % (self.name, self.vo)) if not self.valid: - log.debug( "Failed to create StorageElement plugins.", self.errorReason ) - return S_ERROR( "SE.isValid: Failed to create StorageElement plugins: %s" % self.errorReason ) + log.debug("Failed to create StorageElement plugins.", self.errorReason) + return S_ERROR("SE.isValid: Failed to create StorageElement plugins: %s" % self.errorReason) # Check if the Storage Element is eligible for the user's VO - if 'VO' in self.options and not self.vo in self.options['VO']: - log.debug( "StorageElement is not allowed for VO", self.vo ) - return S_ERROR( errno.EACCES, "StorageElement.isValid: StorageElement is not allowed for VO" ) - log.verbose( "Determining if the StorageElement %s is valid for operation '%s'" % ( self.name, operation ) ) - if ( not operation ) or ( operation in self.okMethods ): + if 'VO' in self.options and self.vo not in self.options['VO']: + log.debug("StorageElement is not allowed for VO", self.vo) + return S_ERROR(errno.EACCES, "StorageElement.isValid: StorageElement is not allowed for VO") + log.verbose("Determining if the StorageElement %s is valid for operation '%s'" % (self.name, operation)) + if (not operation) or (operation in self.okMethods): return S_OK() # Determine whether the StorageElement is valid for checking, reading, writing status = self.status() - checking = status[ 'Check' ] - reading = status[ 'Read' ] - writing = status[ 'Write' ] - removing = status[ 'Remove' ] + checking = status['Check'] + reading = status['Read'] + writing = status['Write'] + removing = status['Remove'] # Determine whether the requested operation can be fulfilled - if ( not operation ) and ( not reading ) and ( not writing ) and ( not checking ): - log.debug( "Read, write and check access not permitted." ) - return S_ERROR( errno.EACCES, "SE.isValid: Read, write and check access not permitted." ) - + if (not operation) and (not reading) and (not writing) and (not checking): + log.debug("Read, write and check access not permitted.") + return S_ERROR(errno.EACCES, "SE.isValid: Read, write and check access not permitted.") # The supplied operation can be 'Read','Write' or any of the possible StorageElement methods. - if ( operation in self.readMethods ) or ( operation.lower() in ( 'read', 'readaccess' ) ): + if (operation in self.readMethods) or (operation.lower() in ('read', 'readaccess')): operation = 'ReadAccess' - elif operation in self.writeMethods or ( operation.lower() in ( 'write', 'writeaccess' ) ): + elif operation in self.writeMethods or (operation.lower() in ('write', 'writeaccess')): operation = 'WriteAccess' - elif operation in self.removeMethods or ( operation.lower() in ( 'remove', 'removeaccess' ) ): + elif operation in self.removeMethods or (operation.lower() in ('remove', 'removeaccess')): operation = 'RemoveAccess' - elif operation in self.checkMethods or ( operation.lower() in ( 'check', 'checkaccess' ) ): + elif operation in self.checkMethods or (operation.lower() in ('check', 'checkaccess')): operation = 'CheckAccess' else: - log.debug( "The supplied operation is not known.", operation ) - return S_ERROR( DErrno.ENOMETH , "SE.isValid: The supplied operation is not known." ) - log.debug( "check the operation: %s " % operation ) + log.debug("The supplied operation is not known.", operation) + return S_ERROR(DErrno.ENOMETH, "SE.isValid: The supplied operation is not known.") + log.debug("check the operation: %s " % operation) # Check if the operation is valid if operation == 'CheckAccess': if not reading: if not checking: - log.debug( "Check access not currently permitted." ) - return S_ERROR( errno.EACCES, "SE.isValid: Check access not currently permitted." ) + log.debug("Check access not currently permitted.") + return S_ERROR(errno.EACCES, "SE.isValid: Check access not currently permitted.") if operation == 'ReadAccess': if not reading: - log.debug( "Read access not currently permitted." ) - return S_ERROR( errno.EACCES, "SE.isValid: Read access not currently permitted." ) + log.debug("Read access not currently permitted.") + return S_ERROR(errno.EACCES, "SE.isValid: Read access not currently permitted.") if operation == 'WriteAccess': if not writing: - log.debug( "Write access not currently permitted." ) - return S_ERROR( errno.EACCES, "SE.isValid: Write access not currently permitted." ) + log.debug("Write access not currently permitted.") + return S_ERROR(errno.EACCES, "SE.isValid: Write access not currently permitted.") if operation == 'RemoveAccess': if not removing: - log.debug( "Remove access not currently permitted." ) - return S_ERROR( errno.EACCES, "SE.isValid: Remove access not currently permitted." ) + log.debug("Remove access not currently permitted.") + return S_ERROR(errno.EACCES, "SE.isValid: Remove access not currently permitted.") return S_OK() - def getPlugins( self ): + def getPlugins(self): """ Get the list of all the plugins defined for this Storage Element """ - self.log.getSubLogger( 'getPlugins' ).verbose( "Obtaining all plugins of %s." % self.name ) + self.log.getSubLogger('getPlugins').verbose("Obtaining all plugins of %s." % self.name) if not self.valid: - return S_ERROR( self.errorReason ) + return S_ERROR(self.errorReason) allPlugins = self.localPlugins + self.remotePlugins - return S_OK( allPlugins ) + return S_OK(allPlugins) - def getRemotePlugins( self ): + def getRemotePlugins(self): """ Get the list of all the remote access protocols defined for this Storage Element """ - self.log.getSubLogger( 'getRemotePlugins' ).verbose( "Obtaining remote protocols for %s." % self.name ) + self.log.getSubLogger('getRemotePlugins').verbose("Obtaining remote protocols for %s." % self.name) if not self.valid: - return S_ERROR( self.errorReason ) - return S_OK( self.remotePlugins ) + return S_ERROR(self.errorReason) + return S_OK(self.remotePlugins) - def getLocalPlugins( self ): + def getLocalPlugins(self): """ Get the list of all the local access protocols defined for this Storage Element """ - self.log.getSubLogger( 'getLocalPlugins' ).verbose( "Obtaining local protocols for %s." % self.name ) + self.log.getSubLogger('getLocalPlugins').verbose("Obtaining local protocols for %s." % self.name) if not self.valid: - return S_ERROR( self.errorReason ) - return S_OK( self.localPlugins ) + return S_ERROR(self.errorReason) + return S_OK(self.localPlugins) - def getStorageParameters( self, plugin = None, protocol = None ): + def getStorageParameters(self, plugin=None, protocol=None): """ Get plugin specific options :param plugin : plugin we are interested in :param protocol: protocol we are interested in @@ -469,49 +481,46 @@ def getStorageParameters( self, plugin = None, protocol = None ): # both set if plugin and protocol: - return S_ERROR( errno.EINVAL, "plugin and protocol cannot be set together." ) + return S_ERROR(errno.EINVAL, "plugin and protocol cannot be set together.") # both None - elif not ( plugin or protocol ): - return S_ERROR( errno.EINVAL, "plugin and protocol cannot be None together." ) + elif not (plugin or protocol): + return S_ERROR(errno.EINVAL, "plugin and protocol cannot be None together.") - log = self.log.getSubLogger( 'getStorageParameters' ) + log = self.log.getSubLogger('getStorageParameters') reqStr = "plugin %s" % plugin if plugin else "protocol %s" % protocol - log.verbose( "Obtaining storage parameters for %s for %s." % ( self.name, - reqStr ) ) - + log.verbose("Obtaining storage parameters for %s for %s." % (self.name, + reqStr)) for storage in self.storages: storageParameters = storage.getParameters() if plugin and storageParameters['PluginName'] == plugin: - return S_OK( storageParameters ) + return S_OK(storageParameters) elif protocol and storageParameters['Protocol'] == protocol: - return S_OK( storageParameters ) + return S_OK(storageParameters) errStr = "Requested plugin or protocol not available." - log.debug( errStr, "%s for %s" % ( reqStr, self.name ) ) - return S_ERROR( errStr ) + log.debug(errStr, "%s for %s" % (reqStr, self.name)) + return S_ERROR(errStr) - def __getAllProtocols( self, protoType ): + def __getAllProtocols(self, protoType): """ Returns the list of all protocols for Input or Output :param proto = InputProtocols or OutputProtocols """ - return set( reduce( lambda x, y:x + y, [plugin.protocolParameters[protoType] for plugin in self.storages ] ) ) + return set(reduce(lambda x, y: x + y, [plugin.protocolParameters[protoType] for plugin in self.storages])) - - def _getAllInputProtocols( self ): + def _getAllInputProtocols(self): """ Returns all the protocols supported by the SE for Input """ - return self.__getAllProtocols( 'InputProtocols' ) + return self.__getAllProtocols('InputProtocols') - def _getAllOutputProtocols( self ): + def _getAllOutputProtocols(self): """ Returns all the protocols supported by the SE for Output """ - return self.__getAllProtocols( 'OutputProtocols' ) - + return self.__getAllProtocols('OutputProtocols') - def generateTransferURLsBetweenSEs(self, lfns, sourceSE, protocols = None): + def generateTransferURLsBetweenSEs(self, lfns, sourceSE, protocols=None): """ This negociate the URLs to be used for third party copy. This is mostly useful for FTS. If protocols is given, it restricts the list of plugins to use @@ -522,18 +531,18 @@ def generateTransferURLsBetweenSEs(self, lfns, sourceSE, protocols = None): :return:dictionnary Successful/Failed with pair (src, dest) urls """ - log = self.log.getSubLogger( 'generateTransferURLsBetweenSEs' ) + log = self.log.getSubLogger('generateTransferURLsBetweenSEs') - result = checkArgumentFormat( lfns ) + result = checkArgumentFormat(lfns) if result['OK']: lfns = result['Value'] else: errStr = "Supplied urls must be string, list of strings or a dictionary." - log.debug( errStr ) - return S_ERROR( errno.EINVAL, errStr ) + log.debug(errStr) + return S_ERROR(errno.EINVAL, errStr) # First, find common protocols to use - res = self.negociateProtocolWithOtherSE(sourceSE, protocols = protocols) + res = self.negociateProtocolWithOtherSE(sourceSE, protocols=protocols) if not res['OK']: return res @@ -545,7 +554,7 @@ def generateTransferURLsBetweenSEs(self, lfns, sourceSE, protocols = None): srcPlugin = None destPlugin = None - log.debug("Trying to find plugins for protocol %s"%proto) + log.debug("Trying to find plugins for protocol %s" % proto) # Finding the source storage plugin for storagePlugin in sourceSE.storages: @@ -553,15 +562,15 @@ def generateTransferURLsBetweenSEs(self, lfns, sourceSE, protocols = None): storageParameters = storagePlugin.getParameters() nativeProtocol = storageParameters['Protocol'] # If the native protocol of the plugin is allowed for read - if nativeProtocol in sourceSE.localAccessProtocolList: + if nativeProtocol in sourceSE.localAccessProtocolList: # If the plugin can generate the protocol we are interested in if proto in storageParameters['OutputProtocols']: log.debug("Selecting it") srcPlugin = storagePlugin break # If we did not find a source plugin, continue - if srcPlugin is None : - log.debug("Could not find a source plugin for protocol %s"%proto) + if srcPlugin is None: + log.debug("Could not find a source plugin for protocol %s" % proto) continue # Finding the destination storage plugin @@ -571,7 +580,7 @@ def generateTransferURLsBetweenSEs(self, lfns, sourceSE, protocols = None): storageParameters = storagePlugin.getParameters() nativeProtocol = storageParameters['Protocol'] # If the native protocol of the plugin is allowed for write - if nativeProtocol in self.localWriteProtocolList: + if nativeProtocol in self.localWriteProtocolList: # If the plugin can accept the protocol we are interested in if proto in storageParameters['InputProtocols']: log.debug("Selecting it") @@ -581,42 +590,40 @@ def generateTransferURLsBetweenSEs(self, lfns, sourceSE, protocols = None): # If we found both a source and destination plugin, we are happy, # otherwise we continue with the next protocol if destPlugin is None: - log.debug("Could not find a destination plugin for protocol %s"%proto) + log.debug("Could not find a destination plugin for protocol %s" % proto) srcPlugin = None continue - failed = {} successful = {} # Generate the URLs for lfn in lfns: # Source URL first - res = srcPlugin.constructURLFromLFN( lfn, withWSUrl = True ) + res = srcPlugin.constructURLFromLFN(lfn, withWSUrl=True) if not res['OK']: - errMsg = "Error generating source url: %s"%res['Message'] - gLogger.debug("Error generating source url",errMsg) + errMsg = "Error generating source url: %s" % res['Message'] + gLogger.debug("Error generating source url", errMsg) failed[lfn] = errMsg continue srcURL = res['Value'] # Destination URL - res = destPlugin.constructURLFromLFN( lfn, withWSUrl = True ) + res = destPlugin.constructURLFromLFN(lfn, withWSUrl=True) if not res['OK']: - errMsg = "Error generating destination url: %s"%res['Message'] - gLogger.debug("Error generating destination url",errMsg) + errMsg = "Error generating destination url: %s" % res['Message'] + gLogger.debug("Error generating destination url", errMsg) failed[lfn] = errMsg continue destURL = res['Value'] successful[lfn] = (srcURL, destURL) - return S_OK({'Successful':successful, 'Failed':failed}) + return S_OK({'Successful': successful, 'Failed': failed}) return S_ERROR(errno.ENOPROTOOPT, "Could not find a protocol ") - - def negociateProtocolWithOtherSE( self, sourceSE, protocols = None ): + def negociateProtocolWithOtherSE(self, sourceSE, protocols=None): """ Negotiate what protocol could be used for a third party transfer between the sourceSE and ourselves. If protocols is given, the chosen protocol has to be among those @@ -630,22 +637,21 @@ def negociateProtocolWithOtherSE( self, sourceSE, protocols = None ): # No common protocols if this is a proxy storage if self.useProxy: - return S_OK( [] ) + return S_OK([]) - log = self.log.getSubLogger( 'negociateProtocolWithOtherSE', child = True ) - - log.debug( "Negociating protocols between %s and %s (protocols %s)" % ( sourceSE.name, self.name, protocols ) ) + log = self.log.getSubLogger('negociateProtocolWithOtherSE', child=True) + log.debug("Negociating protocols between %s and %s (protocols %s)" % (sourceSE.name, self.name, protocols)) # Take all the protocols the destination can accept as input destProtocols = self._getAllInputProtocols() - log.debug( "Destination input protocols %s" % destProtocols ) + log.debug("Destination input protocols %s" % destProtocols) # Take all the protocols the source can provide sourceProtocols = sourceSE._getAllOutputProtocols() - log.debug( "Source output protocols %s" % sourceProtocols ) + log.debug("Source output protocols %s" % sourceProtocols) commonProtocols = destProtocols & sourceProtocols @@ -653,35 +659,39 @@ def negociateProtocolWithOtherSE( self, sourceSE, protocols = None ): # take the intersection, and sort the commonProtocols # based on the protocolList order if protocols: - protocolList = list( protocols ) - commonProtocols = sorted( commonProtocols & set( protocolList ), key = lambda x : self.__getIndexInList( x, protocolList ) ) + protocolList = list(protocols) + commonProtocols = sorted( + commonProtocols & set(protocolList), + key=lambda x: self.__getIndexInList( + x, + protocolList)) - log.debug( "Common protocols %s" % commonProtocols ) + log.debug("Common protocols %s" % commonProtocols) - return S_OK( list( commonProtocols ) ) + return S_OK(list(commonProtocols)) ################################################################################################# # # These are the basic get functions for lfn manipulation # - def __getURLPath( self, url ): + def __getURLPath(self, url): """ Get the part of the URL path below the basic storage path. This path must coincide with the LFN of the file in order to be compliant with the DIRAC conventions. """ - log = self.log.getSubLogger( '__getURLPath' ) - log.verbose( "Getting path from url in %s." % self.name ) + log = self.log.getSubLogger('__getURLPath') + log.verbose("Getting path from url in %s." % self.name) if not self.valid: - return S_ERROR( self.errorReason ) - res = pfnparse( url ) + return S_ERROR(self.errorReason) + res = pfnparse(url) if not res['OK']: return res - fullURLPath = '%s/%s' % ( res['Value']['Path'], res['Value']['FileName'] ) + fullURLPath = '%s/%s' % (res['Value']['Path'], res['Value']['FileName']) # Check all available storages and check whether the url is for that protocol urlPath = '' for storage in self.storages: - res = storage.isNativeURL( url ) + res = storage.isNativeURL(url) if res['OK']: if res['Value']: parameters = storage.getParameters() @@ -690,91 +700,92 @@ def __getURLPath( self, url ): # If the sa path doesn't exist then the url path is the entire string urlPath = fullURLPath else: - if re.search( saPath, fullURLPath ): + if re.search(saPath, fullURLPath): # Remove the sa path from the fullURLPath - urlPath = fullURLPath.replace( saPath, '' ) + urlPath = fullURLPath.replace(saPath, '') if urlPath: - return S_OK( urlPath ) + return S_OK(urlPath) # This should never happen. DANGER!! errStr = "Failed to get the url path for any of the protocols!!" - log.debug( errStr ) - return S_ERROR( errStr ) + log.debug(errStr) + return S_ERROR(errStr) - def getLFNFromURL( self, urls ): + def getLFNFromURL(self, urls): """ Get the LFN from the PFNS . :param lfn : input lfn or lfns (list/dict) """ - result = checkArgumentFormat( urls ) + result = checkArgumentFormat(urls) if result['OK']: urlDict = result['Value'] else: errStr = "Supplied urls must be string, list of strings or a dictionary." - self.log.getSubLogger( 'getLFNFromURL' ).debug( errStr ) - return S_ERROR( errno.EINVAL, errStr ) + self.log.getSubLogger('getLFNFromURL').debug(errStr) + return S_ERROR(errno.EINVAL, errStr) - retDict = { "Successful" : {}, "Failed" : {} } + retDict = {"Successful": {}, "Failed": {}} for url in urlDict: - res = self.__getURLPath( url ) + res = self.__getURLPath(url) if res["OK"]: retDict["Successful"][url] = res["Value"] else: retDict["Failed"][url] = res["Message"] - return S_OK( retDict ) + return S_OK(retDict) ########################################################################################### # # This is the generic wrapper for file operations # - def getURL( self, lfn, protocol = False, replicaDict = None ): + def getURL(self, lfn, protocol=False, replicaDict=None): """ execute 'getTransportURL' operation. :param str lfn: string, list or dictionary of lfns :param protocol: if no protocol is specified, we will request self.turlProtocols :param replicaDict: optional results from the File Catalog replica query """ - self.log.getSubLogger( 'getURL' ).verbose( "Getting accessUrl %s for lfn in %s." % ( "(%s)" % protocol if protocol else "", self.name ) ) + self.log.getSubLogger('getURL').verbose("Getting accessUrl %s for lfn in %s." % + ("(%s)" % protocol if protocol else "", self.name)) if not protocol: # This turlProtocols seems totally useless. # Get ride of it when gfal2 is totally ready # and replace it with the localAccessProtocol list protocols = self.turlProtocols - elif isinstance( protocol, list ): + elif isinstance(protocol, list): protocols = protocol - elif isinstance( protocol, basestring ): + elif isinstance(protocol, basestring): protocols = [protocol] self.methodName = "getTransportURL" - result = self.__executeMethod( lfn, protocols = protocols ) + result = self.__executeMethod(lfn, protocols=protocols) return result - def __isLocalSE( self ): + def __isLocalSE(self): """ Test if the Storage Element is local in the current context """ - self.log.getSubLogger( 'LocalSE' ).verbose( "Determining whether %s is a local SE." % self.name ) + self.log.getSubLogger('LocalSE').verbose("Determining whether %s is a local SE." % self.name) import DIRAC - localSEs = getSEsForSite( DIRAC.siteName() )['Value'] + localSEs = getSEsForSite(DIRAC.siteName())['Value'] if self.name in localSEs: - return S_OK( True ) + return S_OK(True) else: - return S_OK( False ) + return S_OK(False) - def __getFileCatalog( self ): + def __getFileCatalog(self): if not self.__fileCatalog: - self.__fileCatalog = FileCatalog( vo = self.vo ) + self.__fileCatalog = FileCatalog(vo=self.vo) return self.__fileCatalog - def __generateURLDict( self, lfns, storage, replicaDict = None ): + def __generateURLDict(self, lfns, storage, replicaDict=None): """ Generates a dictionary (url : lfn ), where the url are constructed from the lfn using the constructURLFromLFN method of the storage plugins. :param: lfns : dictionary {lfn:whatever} :returns dictionary {constructed url : lfn} """ - log = self.log.getSubLogger( "__generateURLDict" ) - log.verbose( "generating url dict for %s lfn in %s." % ( len( lfns ), self.name ) ) + log = self.log.getSubLogger("__generateURLDict") + log.verbose("generating url dict for %s lfn in %s." % (len(lfns), self.name)) if not replicaDict: replicaDict = {} @@ -784,7 +795,7 @@ def __generateURLDict( self, lfns, storage, replicaDict = None ): for lfn in lfns: if self.useCatalogURL: # Is this self.name alias proof? - url = replicaDict.get( lfn, {} ).get( self.name, '' ) + url = replicaDict.get(lfn, {}).get(self.name, '') if url: urlDict[url] = lfn continue @@ -793,32 +804,32 @@ def __generateURLDict( self, lfns, storage, replicaDict = None ): result = fc.getReplicas() if not result['OK']: failed[lfn] = result['Message'] - url = result['Value']['Successful'].get( lfn, {} ).get( self.name, '' ) + url = result['Value']['Successful'].get(lfn, {}).get(self.name, '') if not url: failed[lfn] = 'Failed to get catalog replica' else: # Update the URL according to the current SE description - result = returnSingleResult( storage.updateURL( url ) ) + result = returnSingleResult(storage.updateURL(url)) if not result['OK']: failed[lfn] = result['Message'] else: urlDict[result['Value']] = lfn else: - result = storage.constructURLFromLFN( lfn, withWSUrl = True ) + result = storage.constructURLFromLFN(lfn, withWSUrl=True) if not result['OK']: errStr = result['Message'] - log.debug( errStr, 'for %s' % ( lfn ) ) - failed[lfn] = "%s %s" % ( failed[lfn], errStr ) if lfn in failed else errStr + log.debug(errStr, 'for %s' % (lfn)) + failed[lfn] = "%s %s" % (failed[lfn], errStr) if lfn in failed else errStr else: urlDict[result['Value']] = lfn - res = S_OK( {'Successful': urlDict, 'Failed' : failed} ) + res = S_OK({'Successful': urlDict, 'Failed': failed}) # res['Failed'] = failed return res @staticmethod - def __getIndexInList( x, l ): + def __getIndexInList(x, l): """ Return the index of the element x in the list l or sys.maxint if it does not exist @@ -828,11 +839,11 @@ def __getIndexInList( x, l ): :return: the index or sys.maxint """ try: - return l.index( x ) + return l.index(x) except ValueError: - return sys.maxint + return sys.maxsize - def __filterPlugins( self, methodName, protocols = None, inputProtocol = None ): + def __filterPlugins(self, methodName, protocols=None, inputProtocol=None): """ Determine the list of plugins that can be used for a particular action @@ -846,11 +857,11 @@ def __filterPlugins( self, methodName, protocols = None, inputProtocol = None ): list: list of storage plugins """ - log = self.log.getSubLogger( '__filterPlugins', child = True ) + log = self.log.getSubLogger('__filterPlugins', child=True) - log.debug( "Filtering plugins for %s (protocol = %s ; inputProtocol = %s)" % ( methodName, protocols, inputProtocol ) ) + log.debug("Filtering plugins for %s (protocol = %s ; inputProtocol = %s)" % (methodName, protocols, inputProtocol)) - if isinstance( protocols, basestring ): + if isinstance(protocols, basestring): protocols = [protocols] pluginsToUse = [] @@ -860,7 +871,7 @@ def __filterPlugins( self, methodName, protocols = None, inputProtocol = None ): if methodName in self.readMethods + self.checkMethods: allowedProtocols = self.localAccessProtocolList - elif methodName in self.removeMethods + self.writeMethods : + elif methodName in self.removeMethods + self.writeMethods: allowedProtocols = self.localWriteProtocolList else: # OK methods @@ -868,66 +879,68 @@ def __filterPlugins( self, methodName, protocols = None, inputProtocol = None ): # can generate such protocol # otherwise we return them all if protocols: - setProtocol = set( protocols ) + setProtocol = set(protocols) for plugin in self.storages: - if set( plugin.protocolParameters.get( "OutputProtocols", [] ) ) & setProtocol: - log.debug( "Plugin %s can generate compatible protocol" % plugin.pluginName ) - pluginsToUse.append( plugin ) + if set(plugin.protocolParameters.get("OutputProtocols", [])) & setProtocol: + log.debug("Plugin %s can generate compatible protocol" % plugin.pluginName) + pluginsToUse.append(plugin) else: pluginsToUse = self.storages # The closest list for "OK" methods is the AccessProtocol preference, so we sort based on that - pluginsToUse.sort( key = lambda x: self.__getIndexInList( x.protocolParameters['Protocol'] , self.localAccessProtocolList ) ) - log.debug( "Plugins to be used for %s: %s" % ( methodName, [p.pluginName for p in pluginsToUse] ) ) + pluginsToUse.sort( + key=lambda x: self.__getIndexInList( + x.protocolParameters['Protocol'], + self.localAccessProtocolList)) + log.debug("Plugins to be used for %s: %s" % (methodName, [p.pluginName for p in pluginsToUse])) return pluginsToUse - log.debug( "Allowed protocol: %s" % allowedProtocols ) + log.debug("Allowed protocol: %s" % allowedProtocols) # if a list of protocol is specified, take it into account if protocols: - potentialProtocols = list( set( allowedProtocols ) & set( protocols ) ) + potentialProtocols = list(set(allowedProtocols) & set(protocols)) else: potentialProtocols = allowedProtocols - log.debug( 'Potential protocols %s' % potentialProtocols ) + log.debug('Potential protocols %s' % potentialProtocols) localSE = self.__isLocalSE()['Value'] for plugin in self.storages: # Determine whether to use this storage object pluginParameters = plugin.getParameters() - pluginName = pluginParameters.get( 'PluginName' ) + pluginName = pluginParameters.get('PluginName') if not pluginParameters: - log.debug( "Failed to get storage parameters.", "%s %s" % ( self.name, pluginName ) ) + log.debug("Failed to get storage parameters.", "%s %s" % (self.name, pluginName)) continue - if not ( pluginName in self.remotePlugins ) and not localSE and not pluginName == "Proxy": + if not (pluginName in self.remotePlugins) and not localSE and not pluginName == "Proxy": # If the SE is not local then we can't use local protocols - log.debug( "Local protocol not appropriate for remote use: %s." % pluginName ) + log.debug("Local protocol not appropriate for remote use: %s." % pluginName) continue if pluginParameters['Protocol'] not in potentialProtocols: - log.debug( "Plugin %s not allowed for %s." % ( pluginName, methodName ) ) + log.debug("Plugin %s not allowed for %s." % (pluginName, methodName)) continue # If we are attempting a putFile and we know the inputProtocol if methodName == 'putFile' and inputProtocol: if inputProtocol not in pluginParameters['InputProtocols']: - log.debug( "Plugin %s not appropriate for %s protocol as input." % ( pluginName, inputProtocol ) ) + log.debug("Plugin %s not appropriate for %s protocol as input." % (pluginName, inputProtocol)) continue - pluginsToUse.append( plugin ) + pluginsToUse.append(plugin) # sort the plugins according to the lists in the CS - pluginsToUse.sort( key = lambda x: self.__getIndexInList( x.protocolParameters['Protocol'] , allowedProtocols ) ) + pluginsToUse.sort(key=lambda x: self.__getIndexInList(x.protocolParameters['Protocol'], allowedProtocols)) - log.debug( "Plugins to be used for %s: %s" % ( methodName, [p.pluginName for p in pluginsToUse] ) ) + log.debug("Plugins to be used for %s: %s" % (methodName, [p.pluginName for p in pluginsToUse])) return pluginsToUse - - def __executeMethod( self, lfn, *args, **kwargs ): + def __executeMethod(self, lfn, *args, **kwargs): """ Forward the call to each storage in turn until one works. The method to be executed is stored in self.methodName :param lfn : string, list or dictionary @@ -941,68 +954,63 @@ def __executeMethod( self, lfn, *args, **kwargs ): the protocol used as source protocol, since there is in principle only one. """ - removedArgs = {} - log = self.log.getSubLogger( '__executeMethod' ) - log.verbose( "preparing the execution of %s" % ( self.methodName ) ) + log = self.log.getSubLogger('__executeMethod') + log.verbose("preparing the execution of %s" % (self.methodName)) # args should normaly be empty to avoid problem... - if len( args ): - log.verbose( "args should be empty!%s" % args ) + if len(args): + log.verbose("args should be empty!%s" % args) # because there is normally only one kw argument, I can move it from args to kwargs - methDefaultArgs = StorageElementItem.__defaultsArguments.get( self.methodName, {} ).keys() - if len( methDefaultArgs ): - kwargs[methDefaultArgs[0] ] = args[0] + methDefaultArgs = StorageElementItem.__defaultsArguments.get(self.methodName, {}).keys() + if len(methDefaultArgs): + kwargs[methDefaultArgs[0]] = args[0] args = args[1:] - log.verbose( "put it in kwargs, but dirty and might be dangerous!args %s kwargs %s" % ( args, kwargs ) ) - + log.verbose("put it in kwargs, but dirty and might be dangerous!args %s kwargs %s" % (args, kwargs)) # We check the deprecated arguments for depArg in StorageElementItem.__deprecatedArguments: if depArg in kwargs: - log.verbose( "%s is not an allowed argument anymore. Please change your code!" % depArg ) + log.verbose("%s is not an allowed argument anymore. Please change your code!" % depArg) removedArgs[depArg] = kwargs[depArg] del kwargs[depArg] - - # Set default argument if any - methDefaultArgs = StorageElementItem.__defaultsArguments.get( self.methodName, {} ) + methDefaultArgs = StorageElementItem.__defaultsArguments.get(self.methodName, {}) for argName in methDefaultArgs: if argName not in kwargs: - log.debug( "default argument %s for %s not present.\ - Setting value %s." % ( argName, self.methodName, methDefaultArgs[argName] ) ) + log.debug("default argument %s for %s not present.\ + Setting value %s." % (argName, self.methodName, methDefaultArgs[argName])) kwargs[argName] = methDefaultArgs[argName] - res = checkArgumentFormat( lfn ) + res = checkArgumentFormat(lfn) if not res['OK']: errStr = "Supplied lfns must be string, list of strings or a dictionary." - log.debug( errStr ) + log.debug(errStr) return res lfnDict = res['Value'] - log.verbose( "Attempting to perform '%s' operation with %s lfns." % ( self.methodName, len( lfnDict ) ) ) + log.verbose("Attempting to perform '%s' operation with %s lfns." % (self.methodName, len(lfnDict))) - res = self.isValid( operation = self.methodName ) + res = self.isValid(operation=self.methodName) if not res['OK']: return res else: if not self.valid: - return S_ERROR( self.errorReason ) + return S_ERROR(self.errorReason) # In case executing putFile, we can assume that all the source urls # are from the same protocol. This optional parameter, if defined # can be used to ignore some storage plugins and thus save time # and avoid fake failures showing in the accounting - inputProtocol = kwargs.pop( 'inputProtocol', None ) - + inputProtocol = kwargs.pop('inputProtocol', None) successful = {} failed = {} - filteredPlugins = self.__filterPlugins( self.methodName, kwargs.get( 'protocols' ), inputProtocol ) + filteredPlugins = self.__filterPlugins(self.methodName, kwargs.get('protocols'), inputProtocol) if not filteredPlugins: - return S_ERROR( errno.EPROTONOSUPPORT, "No storage plugins matching the requirements\ - (operation %s protocols %s inputProtocol %s)"\ - % ( self.methodName, kwargs.get( 'protocols' ), inputProtocol ) ) + return S_ERROR(errno.EPROTONOSUPPORT, "No storage plugins matching the requirements\ + (operation %s protocols %s inputProtocol %s)" + % (self.methodName, kwargs.get('protocols'), inputProtocol)) # Try all of the storages one by one for storage in filteredPlugins: # Determine whether to use this storage object @@ -1010,45 +1018,46 @@ def __executeMethod( self, lfn, *args, **kwargs ): pluginName = storageParameters['PluginName'] if not lfnDict: - log.debug( "No lfns to be attempted for %s protocol." % pluginName ) + log.debug("No lfns to be attempted for %s protocol." % pluginName) continue - log.verbose( "Generating %s protocol URLs for %s." % ( len( lfnDict ), pluginName ) ) - replicaDict = kwargs.pop( 'replicaDict', {} ) + log.verbose("Generating %s protocol URLs for %s." % (len(lfnDict), pluginName)) + replicaDict = kwargs.pop('replicaDict', {}) if storage.pluginName != "Proxy": - res = self.__generateURLDict( lfnDict, storage, replicaDict = replicaDict ) + res = self.__generateURLDict(lfnDict, storage, replicaDict=replicaDict) urlDict = res['Value']['Successful'] # url : lfn - failed.update( res['Value']['Failed'] ) + failed.update(res['Value']['Failed']) else: - urlDict = dict( [ ( lfn, lfn ) for lfn in lfnDict ] ) - if not len( urlDict ): - log.verbose( "__executeMethod No urls generated for protocol %s." % pluginName ) + urlDict = dict([(lfn, lfn) for lfn in lfnDict]) + if not len(urlDict): + log.verbose("__executeMethod No urls generated for protocol %s." % pluginName) else: - log.verbose( "Attempting to perform '%s' for %s physical files" % ( self.methodName, len( urlDict ) ) ) + log.verbose("Attempting to perform '%s' for %s physical files" % (self.methodName, len(urlDict))) fcn = None - if hasattr( storage, self.methodName ) and callable( getattr( storage, self.methodName ) ): - fcn = getattr( storage, self.methodName ) + if hasattr(storage, self.methodName) and callable(getattr(storage, self.methodName)): + fcn = getattr(storage, self.methodName) if not fcn: - return S_ERROR( DErrno.ENOMETH, "SE.__executeMethod: unable to invoke %s, it isn't a member function of storage" ) + return S_ERROR( + DErrno.ENOMETH, + "SE.__executeMethod: unable to invoke %s, it isn't a member function of storage") urlsToUse = {} # url : the value of the lfn dictionary for the lfn of this url for url in urlDict: urlsToUse[url] = lfnDict[urlDict[url]] startDate = datetime.datetime.utcnow() startTime = time.time() - res = fcn( urlsToUse, *args, **kwargs ) + res = fcn(urlsToUse, *args, **kwargs) elapsedTime = time.time() - startTime - - self.addAccountingOperation( urlsToUse, startDate, elapsedTime, storageParameters, res ) + self.addAccountingOperation(urlsToUse, startDate, elapsedTime, storageParameters, res) if not res['OK']: errStr = "Completely failed to perform %s." % self.methodName - log.debug( errStr, 'with plugin %s: %s' % ( pluginName, res['Message'] ) ) + log.debug(errStr, 'with plugin %s: %s' % (pluginName, res['Message'])) for lfn in urlDict.values(): if lfn not in failed: failed[lfn] = '' - failed[lfn] = "%s %s" % ( failed[lfn], res['Message'] ) if failed[lfn] else res['Message'] + failed[lfn] = "%s %s" % (failed[lfn], res['Message']) if failed[lfn] else res['Message'] else: for url, lfn in urlDict.items(): @@ -1056,40 +1065,33 @@ def __executeMethod( self, lfn, *args, **kwargs ): if lfn not in failed: failed[lfn] = '' if url in res['Value']['Failed']: - self.log.debug( res['Value']['Failed'][url] ) - failed[lfn] = "%s %s" % ( failed[lfn], res['Value']['Failed'][url] ) if failed[lfn] else res['Value']['Failed'][url] + self.log.debug(res['Value']['Failed'][url]) + failed[lfn] = "%s %s" % (failed[lfn], res['Value']['Failed'][url] + ) if failed[lfn] else res['Value']['Failed'][url] else: errStr = 'No error returned from plug-in' - failed[lfn] = "%s %s" % ( failed[lfn], errStr ) if failed[lfn] else errStr + failed[lfn] = "%s %s" % (failed[lfn], errStr) if failed[lfn] else errStr else: successful[lfn] = res['Value']['Successful'][url] if lfn in failed: - failed.pop( lfn ) - lfnDict.pop( lfn ) - - + failed.pop(lfn) + lfnDict.pop(lfn) gDataStoreClient.commit() + return S_OK({'Failed': failed, 'Successful': successful}) - - return S_OK( { 'Failed': failed, 'Successful': successful } ) - - - def __getattr__( self, name ): + def __getattr__(self, name): """ Forwards the equivalent Storage calls to __executeMethod""" # We take either the equivalent name, or the name itself - self.methodName = StorageElementItem.__equivalentMethodNames.get( name, None ) + self.methodName = StorageElementItem.__equivalentMethodNames.get(name, None) if self.methodName: return self.__executeMethod - raise AttributeError( "StorageElement does not have a method '%s'" % name ) - - + raise AttributeError("StorageElement does not have a method '%s'" % name) - - def addAccountingOperation( self, lfns, startDate, elapsedTime, storageParameters, callRes ): + def addAccountingOperation(self, lfns, startDate, elapsedTime, storageParameters, callRes): """ Generates a DataOperation accounting if needs to be, and adds it to the DataStore client cache @@ -1107,12 +1109,12 @@ def addAccountingOperation( self, lfns, startDate, elapsedTime, storageParameter """ - if self.methodName not in ( self.readMethods + self.writeMethods + self.removeMethods ): + if self.methodName not in (self.readMethods + self.writeMethods + self.removeMethods): return baseAccountingDict = {} baseAccountingDict['OperationType'] = 'se.%s' % self.methodName - baseAccountingDict['User'] = getProxyInfo().get( 'Value', {} ).get( 'username', 'unknown' ) + baseAccountingDict['User'] = getProxyInfo().get('Value', {}).get('username', 'unknown') baseAccountingDict['RegistrationTime'] = 0.0 baseAccountingDict['RegistrationOK'] = 0 baseAccountingDict['RegistrationTotal'] = 0 @@ -1120,7 +1122,7 @@ def addAccountingOperation( self, lfns, startDate, elapsedTime, storageParameter # if it is a get method, then source and destination of the transfer should be inverted if self.methodName == 'getFile': baseAccountingDict['Destination'] = siteName() - baseAccountingDict[ 'Source'] = self.name + baseAccountingDict['Source'] = self.name else: baseAccountingDict['Destination'] = self.name baseAccountingDict['Source'] = siteName() @@ -1132,58 +1134,56 @@ def addAccountingOperation( self, lfns, startDate, elapsedTime, storageParameter baseAccountingDict['FinalStatus'] = 'Successful' oDataOperation = DataOperation() - oDataOperation.setValuesFromDict( baseAccountingDict ) - oDataOperation.setStartTime( startDate ) - oDataOperation.setEndTime( startDate + datetime.timedelta( seconds = elapsedTime ) ) - oDataOperation.setValueByKey( 'TransferTime', elapsedTime ) - oDataOperation.setValueByKey( 'Protocol', storageParameters.get( 'Protocol', 'unknown' ) ) + oDataOperation.setValuesFromDict(baseAccountingDict) + oDataOperation.setStartTime(startDate) + oDataOperation.setEndTime(startDate + datetime.timedelta(seconds=elapsedTime)) + oDataOperation.setValueByKey('TransferTime', elapsedTime) + oDataOperation.setValueByKey('Protocol', storageParameters.get('Protocol', 'unknown')) if not callRes['OK']: # Everything failed - oDataOperation.setValueByKey( 'TransferTotal', len( lfns ) ) - oDataOperation.setValueByKey( 'FinalStatus', 'Failed' ) + oDataOperation.setValueByKey('TransferTotal', len(lfns)) + oDataOperation.setValueByKey('FinalStatus', 'Failed') else: - succ = callRes.get( 'Value', {} ).get( 'Successful', {} ) - failed = callRes.get( 'Value', {} ).get( 'Failed', {} ) + succ = callRes.get('Value', {}).get('Successful', {}) + failed = callRes.get('Value', {}).get('Failed', {}) totalSize = 0 # We don't take len(lfns) in order to make two # separate entries in case of few failures - totalSucc = len( succ ) + totalSucc = len(succ) - if self.methodName in ( 'putFile', 'getFile' ): + if self.methodName in ('putFile', 'getFile'): # putFile and getFile return for each entry # in the successful dir the size of the corresponding file - totalSize = sum( succ.values() ) + totalSize = sum(succ.values()) - elif self.methodName in ( 'putDirectory', 'getDirectory' ): + elif self.methodName in ('putDirectory', 'getDirectory'): # putDirectory and getDirectory return for each dir name # a dictionnary with the keys 'Files' and 'Size' - totalSize = sum( val.get( 'Size', 0 ) for val in succ.values() if isinstance( val, dict ) ) - totalSucc = sum( val.get( 'Files', 0 ) for val in succ.values() if isinstance( val, dict ) ) - oDataOperation.setValueByKey( 'TransferOK', len( succ ) ) + totalSize = sum(val.get('Size', 0) for val in succ.values() if isinstance(val, dict)) + totalSucc = sum(val.get('Files', 0) for val in succ.values() if isinstance(val, dict)) + oDataOperation.setValueByKey('TransferOK', len(succ)) - oDataOperation.setValueByKey( 'TransferSize', totalSize ) - oDataOperation.setValueByKey( 'TransferTotal', totalSucc ) - oDataOperation.setValueByKey( 'TransferOK', totalSucc ) + oDataOperation.setValueByKey('TransferSize', totalSize) + oDataOperation.setValueByKey('TransferTotal', totalSucc) + oDataOperation.setValueByKey('TransferOK', totalSucc) if callRes['Value']['Failed']: - oDataOperationFailed = copy.deepcopy( oDataOperation ) - oDataOperationFailed.setValueByKey( 'TransferTotal', len( failed ) ) - oDataOperationFailed.setValueByKey( 'TransferOK', 0 ) - oDataOperationFailed.setValueByKey( 'TransferSize', 0 ) - oDataOperationFailed.setValueByKey( 'FinalStatus', 'Failed' ) + oDataOperationFailed = copy.deepcopy(oDataOperation) + oDataOperationFailed.setValueByKey('TransferTotal', len(failed)) + oDataOperationFailed.setValueByKey('TransferOK', 0) + oDataOperationFailed.setValueByKey('TransferSize', 0) + oDataOperationFailed.setValueByKey('FinalStatus', 'Failed') - accRes = gDataStoreClient.addRegister( oDataOperationFailed ) + accRes = gDataStoreClient.addRegister(oDataOperationFailed) if not accRes['OK']: - self.log.error( "Could not send failed accounting report", accRes['Message'] ) - + self.log.error("Could not send failed accounting report", accRes['Message']) - accRes = gDataStoreClient.addRegister( oDataOperation ) + accRes = gDataStoreClient.addRegister(oDataOperation) if not accRes['OK']: - self.log.error( "Could not send accounting report", accRes['Message'] ) - + self.log.error("Could not send accounting report", accRes['Message']) StorageElement = StorageElementCache() From 87f55a090854d7a658a1f13ac2dc059bcb7d0fe3 Mon Sep 17 00:00:00 2001 From: Federico Stagni Date: Tue, 19 Dec 2017 16:16:49 +0100 Subject: [PATCH 12/17] FreeDiskSpace replaces SpaceTokenOccupancy --- .../Agent/CacheFeederAgent.py | 1 - .../Command/DowntimeCommand.py | 2 +- .../Command/FreeDiskSpaceCommand.py | 77 ++++---- .../Command/SpaceTokenOccupancyCommand.py | 178 ------------------ ResourceStatusSystem/Policy/Configurations.py | 32 +++- ...upancyPolicy.py => FreeDiskSpacePolicy.py} | 27 +-- .../Test_RSS_Policy_FreeDiskSpacePolicy.py | 103 ++++++++++ ...st_RSS_Policy_SpaceTokenOccupancyPolicy.py | 16 +- Resources/Storage/DIPStorage.py | 9 +- Resources/Storage/GFAL2_SRM2Storage.py | 16 +- 10 files changed, 196 insertions(+), 265 deletions(-) delete mode 100644 ResourceStatusSystem/Command/SpaceTokenOccupancyCommand.py rename ResourceStatusSystem/Policy/{SpaceTokenOccupancyPolicy.py => FreeDiskSpacePolicy.py} (67%) create mode 100644 ResourceStatusSystem/Policy/test/Test_RSS_Policy_FreeDiskSpacePolicy.py diff --git a/ResourceStatusSystem/Agent/CacheFeederAgent.py b/ResourceStatusSystem/Agent/CacheFeederAgent.py index aa9b6b44d1a..ef850cb4638 100644 --- a/ResourceStatusSystem/Agent/CacheFeederAgent.py +++ b/ResourceStatusSystem/Agent/CacheFeederAgent.py @@ -50,7 +50,6 @@ def initialize(self): self.rmClient = ResourceManagementClient() self.commands['Downtime'] = [{'Downtime': {}}] - self.commands['SpaceTokenOccupancy'] = [{'SpaceTokenOccupancy': {}}] self.commands['GOCDBSync'] = [{'GOCDBSync': {}}] self.commands['FreeDiskSpace'] = [{'FreeDiskSpace': {}}] diff --git a/ResourceStatusSystem/Command/DowntimeCommand.py b/ResourceStatusSystem/Command/DowntimeCommand.py index da8aff9133e..7ff3aa9037b 100644 --- a/ResourceStatusSystem/Command/DowntimeCommand.py +++ b/ResourceStatusSystem/Command/DowntimeCommand.py @@ -74,7 +74,7 @@ def _cleanCommand(self, element, elementNames): currentDate = datetime.utcnow() - if len(uniformResult) == 0: + if not uniformResult: continue # get the list of all ongoing DTs from GocDB diff --git a/ResourceStatusSystem/Command/FreeDiskSpaceCommand.py b/ResourceStatusSystem/Command/FreeDiskSpaceCommand.py index ed361269ca6..6bcb6395661 100644 --- a/ResourceStatusSystem/Command/FreeDiskSpaceCommand.py +++ b/ResourceStatusSystem/Command/FreeDiskSpaceCommand.py @@ -1,23 +1,20 @@ ''' FreeDiskSpaceCommand - The Command gets the free space that is left in a DIRAC Storage Element + The Command gets the free space that is left in a Storage Element ''' +__RCSID__ = '$Id:$' + from datetime import datetime + from DIRAC import S_OK, S_ERROR, gLogger from DIRAC.ResourceStatusSystem.Command.Command import Command -from DIRAC.Core.DISET.RPCClient import RPCClient from DIRAC.ResourceStatusSystem.Utilities import CSHelpers from DIRAC.Resources.Storage.StorageElement import StorageElement from DIRAC.ResourceStatusSystem.Client.ResourceManagementClient import ResourceManagementClient -__RCSID__ = '$Id: $' - -# FIXME: this command may not be needed anymore, as we can get the space occupancy for DIP SE (and not only) by simply -# se = StorageElement().getOccupancy() -# Alternatively, this may survive and we remove SpaceTokenOccupancyCommand - +#FIXME: use unit! class FreeDiskSpaceCommand(Command): ''' @@ -28,8 +25,7 @@ def __init__(self, args=None, clients=None): super(FreeDiskSpaceCommand, self).__init__(args, clients=clients) - self.rpc = None - self.rsClient = ResourceManagementClient() + self.rmClient = ResourceManagementClient() def _prepareCommand(self): ''' @@ -41,54 +37,44 @@ def _prepareCommand(self): return S_ERROR('"name" not found in self.args') elementName = self.args['name'] - return S_OK(elementName) + unit = 'TB' + if 'unit' in self.args: + unit = self.args['unit'] + + return S_OK((elementName, unit)) def doNew(self, masterParams=None): """ - Gets the total and the free disk space of a DIPS storage element that - is found in the CS and inserts the results in the SpaceTokenOccupancyCache table + Gets the parameters to run, either from the master method or from its + own arguments. + + Gets the total and the free disk space of a storage element + and inserts the results in the SpaceTokenOccupancyCache table of ResourceManagementDB database. """ if masterParams is not None: elementName = masterParams + unit = 'TB' else: - elementName = self._prepareCommand() + elementName, unit = self._prepareCommand() if not elementName['OK']: return elementName - se = StorageElement(elementName) - - elementURL = se.getStorageParameters(protocol="dips") + endpointResult = CSHelpers.getStorageElementEndpoint(elementName) + if not endpointResult['OK']: + return endpointResult - if elementURL['OK']: - elementURL = se.getStorageParameters(protocol="dips")['Value']['URLBase'] - else: - gLogger.verbose("Not a DIPS storage element, skipping...") - return S_OK() - - self.rpc = RPCClient(elementURL, timeout=120) - - free = self.rpc.getFreeDiskSpace("/") - - if not free['OK']: - return free - free = free['Value'] - - total = self.rpc.getTotalDiskSpace("/") - - if not total['OK']: - return total - total = total['Value'] - - if free and free < 1: - free = 1 - if total and total < 1: - total = 1 + se = StorageElement(elementName) + occupancyResult = se.getOccupancy() + if not occupancyResult['OK']: + return occupancyResult + occupancy = occupancyResult['Value'] - result = self.rsClient.addOrModifySpaceTokenOccupancyCache(endpoint=elementURL, + result = self.rmClient.addOrModifySpaceTokenOccupancyCache(endpoint=endpointResult['Value'], lastCheckTime=datetime.utcnow(), - free=free, total=total, + free=occupancy['Free'], + total=occupancy['Total'], token=elementName) if not result['OK']: return result @@ -98,13 +84,14 @@ def doNew(self, masterParams=None): def doCache(self): """ This is a method that gets the element's details from the spaceTokenOccupancy cache. + It will return a list of dictionaries if there are results. """ - elementName = self._prepareCommand() + elementName, unit = self._prepareCommand() if not elementName['OK']: return elementName - result = self.rsClient.selectSpaceTokenOccupancyCache(token=elementName) + result = self.rmClient.selectSpaceTokenOccupancyCache(token=elementName) if not result['OK']: return result diff --git a/ResourceStatusSystem/Command/SpaceTokenOccupancyCommand.py b/ResourceStatusSystem/Command/SpaceTokenOccupancyCommand.py deleted file mode 100644 index 323111df019..00000000000 --- a/ResourceStatusSystem/Command/SpaceTokenOccupancyCommand.py +++ /dev/null @@ -1,178 +0,0 @@ -''' SpaceTokenOccupancyCommand - - The Command gets information of the SpaceTokenOccupancy from the lcg_utils - -''' - -__RCSID__ = '$Id:$' - - -from DIRAC import S_OK, S_ERROR -from DIRAC.ResourceStatusSystem.Command.Command import Command -from DIRAC.ResourceStatusSystem.Client.ResourceManagementClient import ResourceManagementClient -from DIRAC.ResourceStatusSystem.Utilities import CSHelpers -from DIRAC.Resources.Storage.StorageElement import StorageElement - -# FIXME: this command by calling StorageElement(se).getOccupancy() becomes a duplicate of the FreeDiskSpaceCommand -# so only one should survive. -# Also, the concept of "space token" should remain hidden in StorageElement interface. - - -class SpaceTokenOccupancyCommand(Command): - ''' - Uses lcg_util to query status of endpoint for a given token. - ''' - - def __init__(self, args=None, clients=None): - - super(SpaceTokenOccupancyCommand, self).__init__(args, clients) - - if 'ResourceManagementClient' in self.apis: - self.rmClient = self.apis['ResourceManagementClient'] - else: - self.rmClient = ResourceManagementClient() - - def _storeCommand(self, results): - ''' - Stores the results of doNew method on the database. - ''' - - for result in results: - - resQuery = self.rmClient.addOrModifySpaceTokenOccupancyCache(result['Endpoint'], - result['Token'], - result['Total'], - result['Guaranteed'], - result['Free']) - if not resQuery['OK']: - return resQuery - - return S_OK() - - def _prepareCommand(self): - ''' - SpaceTokenOccupancy requires one argument: - - elementName : - - Given a (storage)elementName, we calculate its endpoint and spaceToken, - which are used to query the srm interface. - ''' - - if 'name' not in self.args: - return S_ERROR('"name" not found in self.args') - elementName = self.args['name'] - - endpoint = CSHelpers.getStorageElementEndpoint(elementName) - if not endpoint['OK']: - return endpoint - endpoint = endpoint['Value'] - - spaceToken = CSHelpers.getSEToken(elementName) - if not spaceToken['OK']: - return spaceToken - spaceToken = spaceToken['Value'] - - return S_OK((endpoint, spaceToken)) - - def doNew(self, masterParams=None): - ''' - Gets the parameters to run, either from the master method or from its - own arguments. - - It queries the srm interface, and hopefully it will not crash. Out of the - results, we keep totalsize, guaranteedsuze, and unusedsize. - - Then, they are recorded and returned. - ''' - - if masterParams is not None: - spaceTokenEndpoint, spaceToken = masterParams - else: - params = self._prepareCommand() - if not params['OK']: - return params - spaceTokenEndpoint, spaceToken = params['Value'] - - se = StorageElement(self.args['name']) - occupancyResult = se.getOccupancy() - if not occupancyResult['OK']: - return occupancyResult - else: - occupancy = occupancyResult['Value'] - - sTokenDict = {} - sTokenDict['Endpoint'] = spaceTokenEndpoint - sTokenDict['Token'] = spaceToken - sTokenDict['Total'] = float(occupancy.get('totalsize', '0')) / 1e12 # Bytes to Terabytes - sTokenDict['Guaranteed'] = float(occupancy.get('guaranteedsize', '0')) / 1e12 - sTokenDict['Free'] = float(occupancy.get('unusedsize', '0')) / 1e12 - - storeRes = self._storeCommand([sTokenDict]) - if not storeRes['OK']: - return storeRes - - return S_OK([sTokenDict]) - - def doCache(self): - ''' - Method that reads the cache table and tries to read from it. It will - return a list of dictionaries if there are results. - ''' - - params = self._prepareCommand() - if not params['OK']: - return params - spaceTokenEndpoint, spaceToken = params['Value'] - - result = self.rmClient.selectSpaceTokenOccupancyCache(spaceTokenEndpoint, spaceToken) - if result['OK']: - result = S_OK([dict(zip(result['Columns'], res)) for res in result['Value']]) - - return result - - def doMaster(self): - ''' - Master method. Gets all endpoints from the storage elements and all - the spaceTokens. Could have taken from Shares/Disk as well. - It queries for all their possible combinations, unless there are records - in the database for those combinations, which then are not queried. - ''' - - self.log.verbose("Getting all SEs defined in the CS") - storageElementNames = CSHelpers.getStorageElements() - if not storageElementNames['OK']: - self.log.warn(storageElementNames['Message']) - return storageElementNames - storageElementNames = storageElementNames['Value'] - - endpointTokenSet = set() - - for storageElementName in storageElementNames: - - endpoint = CSHelpers.getStorageElementEndpoint(storageElementName) - if not endpoint['OK']: - self.log.warn(endpoint['Message']) - continue - endpoint = endpoint['Value'] - - spaceToken = CSHelpers.getSEToken(storageElementName) - if not spaceToken['OK']: - self.log.warn(spaceToken['Message']) - continue - spaceToken = spaceToken['Value'] - - endpointTokenSet.add((endpoint, spaceToken)) - - self.log.verbose('Processing %s' % endpointTokenSet) - - for elementToQuery in endpointTokenSet: - - result = self.doNew(elementToQuery) - if not result['OK']: - self.metrics['failed'].append(result) - - return S_OK(self.metrics) - - -# ............................................................................... -# EOF diff --git a/ResourceStatusSystem/Policy/Configurations.py b/ResourceStatusSystem/Policy/Configurations.py index ec4abf3afea..0938ff0103a 100644 --- a/ResourceStatusSystem/Policy/Configurations.py +++ b/ResourceStatusSystem/Policy/Configurations.py @@ -13,7 +13,7 @@ """ -__RCSID__ = '$Id: $' +__RCSID__ = '$Id: $' POLICIESMETA = { # DownTime POLICIES @@ -38,16 +38,32 @@ 'args' : { 'hours' : 12, 'onlyCache' : True }, }, - # Space Token POLICIES........................................................ - 'SpaceTokenOccupancy': { - 'description' : "Space token occupancy", - 'module' : 'SpaceTokenOccupancyPolicy', - 'command' : ( 'SpaceTokenOccupancyCommand', 'SpaceTokenOccupancyCommand' ), - 'args' : { 'onlyCache' : True }, + # Free Disk Space in Terabytes + 'FreeDiskSpaceTB': { + 'description' : "Free disk space", + 'module' : 'FreeDiskSpacePolicy', + 'command' : ( 'FreeDiskSpaceCommand', 'FreeDiskSpaceCommand' ), + 'args' : { 'unit': 'TB', 'onlyCache' : True }, + }, + + # Free Disk Space in Gigabytes + 'FreeDiskSpaceTB': { + 'description' : "Free disk space", + 'module' : 'FreeDiskSpacePolicy', + 'command' : ( 'FreeDiskSpaceCommand', 'FreeDiskSpaceCommand' ), + 'args' : { 'unit': 'GB', 'onlyCache' : True }, + }, + + # Free Disk Space in Megabytes + 'FreeDiskSpaceTB': { + 'description' : "Free disk space", + 'module' : 'FreeDiskSpacePolicy', + 'command' : ( 'FreeDiskSpaceCommand', 'FreeDiskSpaceCommand' ), + 'args' : { 'unit': 'MB', 'onlyCache' : True }, }, - # Job POLICIES.............................................................. + # Job POLICIES 'JobDoneRatio': { 'description' : "done / ( completed + done ) jobs ( 30 min )", 'module' : 'JobDoneRatioPolicy', diff --git a/ResourceStatusSystem/Policy/SpaceTokenOccupancyPolicy.py b/ResourceStatusSystem/Policy/FreeDiskSpacePolicy.py similarity index 67% rename from ResourceStatusSystem/Policy/SpaceTokenOccupancyPolicy.py rename to ResourceStatusSystem/Policy/FreeDiskSpacePolicy.py index 0fceb412150..a8d82d2f6e6 100644 --- a/ResourceStatusSystem/Policy/SpaceTokenOccupancyPolicy.py +++ b/ResourceStatusSystem/Policy/FreeDiskSpacePolicy.py @@ -1,6 +1,6 @@ -""" SpaceTokenOccupancyPolicy +""" FreeDiskSpacePolicy - SpaceTokenOccupancyPolicy.__bases__: + FreeDiskSpacePolicy.__bases__: DIRAC.ResourceStatusSystem.PolicySystem.PolicyBase.PolicyBase """ @@ -11,23 +11,23 @@ from DIRAC.ResourceStatusSystem.PolicySystem.PolicyBase import PolicyBase -class SpaceTokenOccupancyPolicy(PolicyBase): +class FreeDiskSpacePolicy(PolicyBase): """ - The SpaceTokenOccupancyPolicy class is a policy class satisfied when a SE has a + The FreeDiskSpacePolicy class is a policy class satisfied when a SE has a low occupancy. - SpaceTokenOccupancyPolicy, given the space left at the element, proposes a new status. + FreeDiskSpacePolicy, given the space left at the element, proposes a new status. """ @staticmethod def _evaluate(commandResult): """ - Evaluate policy on SE occupancy: Use SpaceTokenOccupancyCommand + Evaluate policy on SE occupancy: Use FreeDiskSpaceCommand :Parameters: **commandResult** - S_OK / S_ERROR result of the command. It is expected ( iff S_OK ) a dictionary like - { 'Total' : .., 'Free' : .., 'Guaranteed': .. } + { 'Total' : .., 'Free' : ..} :return: { @@ -52,25 +52,26 @@ def _evaluate(commandResult): commandResult = commandResult[0] - for key in ['Total', 'Free', 'Guaranteed']: + for key in ['Total', 'Free']: - if key not in commandResult.keys(): + if key not in commandResult: result['Status'] = 'Error' result['Reason'] = 'Key %s missing' % key.lower() return S_OK(result) free = float(commandResult['Free']) - # Units are TB ! ( 0.01 == 10 GB ) + # Units (TB, GB, MB) may change, + # depending on the configuration of the command in Configurations.py if free < 0.1: result['Status'] = 'Banned' - result['Reason'] = 'Free space < 100GB' + result['Reason'] = 'Too little free space' elif free < 5: result['Status'] = 'Degraded' - result['Reason'] = 'Free space < 5TB' + result['Reason'] = 'Little free space' else: result['Status'] = 'Active' - result['Reason'] = 'Free space > 5TB' + result['Reason'] = 'Enough free space' return S_OK(result) diff --git a/ResourceStatusSystem/Policy/test/Test_RSS_Policy_FreeDiskSpacePolicy.py b/ResourceStatusSystem/Policy/test/Test_RSS_Policy_FreeDiskSpacePolicy.py new file mode 100644 index 00000000000..403929b4de4 --- /dev/null +++ b/ResourceStatusSystem/Policy/test/Test_RSS_Policy_FreeDiskSpacePolicy.py @@ -0,0 +1,103 @@ +''' Test_RSS_Policy_FreeDiskSpacePolicy +''' + +import unittest + +import DIRAC.ResourceStatusSystem.Policy.FreeDiskSpacePolicy as moduleTested + +################################################################################ + +class FreeDiskSpacePolicy_TestCase( unittest.TestCase ): + + def setUp( self ): + ''' + Setup + ''' + + self.moduleTested = moduleTested + self.testClass = self.moduleTested.FreeDiskSpacePolicy + + def tearDown( self ): + ''' + Tear down + ''' + + del self.moduleTested + del self.testClass + +################################################################################ + +class FreeDiskSpacePolicy_Success( FreeDiskSpacePolicy_TestCase ): + + def test_instantiate( self ): + ''' tests that we can instantiate one object of the tested class + ''' + + module = self.testClass() + self.assertEqual( 'FreeDiskSpacePolicy', module.__class__.__name__ ) + + def test_evaluate( self ): + ''' tests the method _evaluate + ''' + + module = self.testClass() + + res = module._evaluate( { 'OK' : False, 'Message' : 'Bo!' } ) + self.assertTrue(res['OK']) + self.assertEqual( 'Error', res[ 'Value' ][ 'Status' ] ) + self.assertEqual( 'Bo!', res[ 'Value' ][ 'Reason' ] ) + + res = module._evaluate( { 'OK' : True, 'Value' : None } ) + self.assertTrue(res['OK']) + self.assertEqual( 'Unknown', res[ 'Value' ][ 'Status' ] ) + self.assertEqual( 'No values to take a decision', res[ 'Value' ][ 'Reason' ] ) + + res = module._evaluate( { 'OK' : True, 'Value' : [] } ) + self.assertTrue(res['OK']) + self.assertEqual( 'Unknown', res[ 'Value' ][ 'Status' ] ) + self.assertEqual( 'No values to take a decision', res[ 'Value' ][ 'Reason' ] ) + + res = module._evaluate( { 'OK' : True, 'Value' : [{ 'A' : 1 }] } ) + self.assertTrue(res['OK']) + self.assertEqual( 'Error', res[ 'Value' ][ 'Status' ] ) + self.assertEqual( 'Key total missing', res[ 'Value' ][ 'Reason' ] ) + + res = module._evaluate( { 'OK' : True, 'Value' : [{ 'Total' : 1 }] } ) + self.assertTrue(res['OK']) + self.assertEqual( 'Error', res[ 'Value' ][ 'Status' ] ) + self.assertEqual( 'Key free missing', res[ 'Value' ][ 'Reason' ] ) + + res = module._evaluate( { 'OK' : True, 'Value' : [{'Total' : 100, 'Free' : 0.0 }] } ) + self.assertTrue(res['OK']) + self.assertEqual( 'Error', res[ 'Value' ][ 'Status' ] ) + self.assertEqual( 'Key guaranteed missing', res[ 'Value' ][ 'Reason' ] ) + + res = module._evaluate( { 'OK' : True, 'Value' : [{'Total' : 100, 'Free' : 0.0, + 'Guaranteed' : 1 }] } ) + self.assertTrue(res['OK']) + self.assertEqual( 'Banned', res[ 'Value' ][ 'Status' ] ) + self.assertEqual( 'Free space < 100GB', res[ 'Value' ][ 'Reason' ] ) + + res = module._evaluate( { 'OK' : True, 'Value' : [{'Total' : 100, 'Free' : 4.0, + 'Guaranteed' : 1 }] } ) + self.assertTrue(res['OK']) + self.assertEqual( 'Degraded', res[ 'Value' ][ 'Status' ] ) + self.assertEqual( 'Free space < 5TB', + res[ 'Value' ][ 'Reason' ] ) + + res = module._evaluate( { 'OK' : True, 'Value' : [{'Total' : 100, 'Free' : 100, + 'Guaranteed' : 1 }] } ) + self.assertTrue(res['OK']) + self.assertEqual( 'Active', res[ 'Value' ][ 'Status' ] ) + self.assertEqual( 'Free space > 5TB', + res[ 'Value' ][ 'Reason' ] ) + + +################################################################################ + +if __name__ == '__main__': + suite = unittest.defaultTestLoader.loadTestsFromTestCase( FreeDiskSpacePolicy_TestCase ) + suite.addTest( unittest.defaultTestLoader.loadTestsFromTestCase( FreeDiskSpacePolicy_Success ) ) + testResult = unittest.TextTestRunner( verbosity = 2 ).run( suite ) + +#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF \ No newline at end of file diff --git a/ResourceStatusSystem/Policy/test/Test_RSS_Policy_SpaceTokenOccupancyPolicy.py b/ResourceStatusSystem/Policy/test/Test_RSS_Policy_SpaceTokenOccupancyPolicy.py index 2ae74a68468..403929b4de4 100644 --- a/ResourceStatusSystem/Policy/test/Test_RSS_Policy_SpaceTokenOccupancyPolicy.py +++ b/ResourceStatusSystem/Policy/test/Test_RSS_Policy_SpaceTokenOccupancyPolicy.py @@ -1,13 +1,13 @@ -''' Test_RSS_Policy_SpaceTokenOccupancyPolicy +''' Test_RSS_Policy_FreeDiskSpacePolicy ''' import unittest -import DIRAC.ResourceStatusSystem.Policy.SpaceTokenOccupancyPolicy as moduleTested +import DIRAC.ResourceStatusSystem.Policy.FreeDiskSpacePolicy as moduleTested ################################################################################ -class SpaceTokenOccupancyPolicy_TestCase( unittest.TestCase ): +class FreeDiskSpacePolicy_TestCase( unittest.TestCase ): def setUp( self ): ''' @@ -15,7 +15,7 @@ def setUp( self ): ''' self.moduleTested = moduleTested - self.testClass = self.moduleTested.SpaceTokenOccupancyPolicy + self.testClass = self.moduleTested.FreeDiskSpacePolicy def tearDown( self ): ''' @@ -27,14 +27,14 @@ def tearDown( self ): ################################################################################ -class SpaceTokenOccupancyPolicy_Success( SpaceTokenOccupancyPolicy_TestCase ): +class FreeDiskSpacePolicy_Success( FreeDiskSpacePolicy_TestCase ): def test_instantiate( self ): ''' tests that we can instantiate one object of the tested class ''' module = self.testClass() - self.assertEqual( 'SpaceTokenOccupancyPolicy', module.__class__.__name__ ) + self.assertEqual( 'FreeDiskSpacePolicy', module.__class__.__name__ ) def test_evaluate( self ): ''' tests the method _evaluate @@ -96,8 +96,8 @@ def test_evaluate( self ): ################################################################################ if __name__ == '__main__': - suite = unittest.defaultTestLoader.loadTestsFromTestCase( SpaceTokenOccupancyPolicy_TestCase ) - suite.addTest( unittest.defaultTestLoader.loadTestsFromTestCase( SpaceTokenOccupancyPolicy_Success ) ) + suite = unittest.defaultTestLoader.loadTestsFromTestCase( FreeDiskSpacePolicy_TestCase ) + suite.addTest( unittest.defaultTestLoader.loadTestsFromTestCase( FreeDiskSpacePolicy_Success ) ) testResult = unittest.TextTestRunner( verbosity = 2 ).run( suite ) #EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF \ No newline at end of file diff --git a/Resources/Storage/DIPStorage.py b/Resources/Storage/DIPStorage.py index 1c49caf8013..dd1734d4128 100755 --- a/Resources/Storage/DIPStorage.py +++ b/Resources/Storage/DIPStorage.py @@ -503,15 +503,18 @@ def getDirectory(self, path, localPath=False): resDict = {'Failed': failed, 'Successful': successful} return S_OK(resDict) - def getOccupancy(self): + def getOccupancy(self, *parms, **kws): """ Gets the DIPStorage occupancy info + + :return: S_OK/S_ERROR (free and total space, in MB) """ rpc = RPCClient(self.getURLBase(), timeout=120) - free = rpc.getFreeDiskSpace("/") + + free = rpc.getFreeDiskSpace() if not free['OK']: return free - total = rpc.getTotalDiskSpace("/") + total = rpc.getTotalDiskSpace() if not total['OK']: return total diff --git a/Resources/Storage/GFAL2_SRM2Storage.py b/Resources/Storage/GFAL2_SRM2Storage.py index 65ce24c03d9..de80ec88ae4 100644 --- a/Resources/Storage/GFAL2_SRM2Storage.py +++ b/Resources/Storage/GFAL2_SRM2Storage.py @@ -9,8 +9,6 @@ # pylint: disable=invalid-name -import lcg_util # pylint: disable=import-error - # from DIRAC from DIRAC import gLogger, gConfig, S_OK, S_ERROR from DIRAC.Core.Utilities.Subprocess import pythonCall @@ -170,12 +168,12 @@ def __getSingleTransportURL( self, path, protocols = False ): if res['OK']: return S_OK( res['Value']['user.replicas'] ) - else: - errStr = 'GFAL2_SRM2Storage.__getSingleTransportURL: Extended attribute tURL is not set.' - self.log.debug( errStr, res['Message'] ) - return res - def getOccupancy(self): + errStr = 'GFAL2_SRM2Storage.__getSingleTransportURL: Extended attribute tURL is not set.' + self.log.debug( errStr, res['Message'] ) + return res + + def getOccupancy(self, *parms, **kws): """ Gets the GFAL2_SRM2Storage occupancy info. It queries the srm interface, and hopefully it will not crash. Out of the @@ -187,6 +185,8 @@ def getOccupancy(self): self.protocolParameters['Port'], self.protocolParameters['WSUrl'].split('?')[0]) + # FIXME: re-do without lcg_util (gfal1) + import lcg_util # pylint: disable=import-error occupancyResult = pythonCall(10, lcg_util.lcg_stmd, spaceToken, spaceTokenEndpoint, True, 0) if not occupancyResult['OK']: self.log.error("Could not get spaceToken occupancy", "from endPoint/spaceToken %s/%s : %s" % @@ -202,7 +202,7 @@ def getOccupancy(self): sTokenDict = {} sTokenDict['Endpoint'] = spaceTokenEndpoint sTokenDict['Token'] = spaceToken - # Bytes to Terabytes #FIXME: have to harmonize with other SE types + # Bytes to Terabytes #FIXME: have to harmonize with other SE types ---> use MB? sTokenDict['Total'] = float(output.get('totalsize', '0')) / 1e12 sTokenDict['Guaranteed'] = float(output.get('guaranteedsize', '0')) / 1e12 sTokenDict['Free'] = float(output.get('unusedsize', '0')) / 1e12 From 739710a372a4bd376fc5f7c0a743a1b2745f3b53 Mon Sep 17 00:00:00 2001 From: Federico Stagni Date: Tue, 19 Dec 2017 17:33:35 +0100 Subject: [PATCH 13/17] styling --- .../Command/FreeDiskSpaceCommand.py | 3 +- ResourceStatusSystem/Policy/Configurations.py | 138 +++++++++--------- .../Test_RSS_Policy_FreeDiskSpacePolicy.py | 92 ++++++------ ...st_RSS_Policy_SpaceTokenOccupancyPolicy.py | 103 ------------- Resources/Storage/GFAL2_SRM2Storage.py | 90 +++++------- 5 files changed, 158 insertions(+), 268 deletions(-) delete mode 100644 ResourceStatusSystem/Policy/test/Test_RSS_Policy_SpaceTokenOccupancyPolicy.py diff --git a/ResourceStatusSystem/Command/FreeDiskSpaceCommand.py b/ResourceStatusSystem/Command/FreeDiskSpaceCommand.py index 6bcb6395661..a972d7a98ad 100644 --- a/ResourceStatusSystem/Command/FreeDiskSpaceCommand.py +++ b/ResourceStatusSystem/Command/FreeDiskSpaceCommand.py @@ -14,7 +14,8 @@ from DIRAC.Resources.Storage.StorageElement import StorageElement from DIRAC.ResourceStatusSystem.Client.ResourceManagementClient import ResourceManagementClient -#FIXME: use unit! +# FIXME: use unit! + class FreeDiskSpaceCommand(Command): ''' diff --git a/ResourceStatusSystem/Policy/Configurations.py b/ResourceStatusSystem/Policy/Configurations.py index 0938ff0103a..75fb8d8b9e6 100644 --- a/ResourceStatusSystem/Policy/Configurations.py +++ b/ResourceStatusSystem/Policy/Configurations.py @@ -16,126 +16,126 @@ __RCSID__ = '$Id: $' -POLICIESMETA = { # DownTime POLICIES - 'DTOngoing': {'description' : "Ongoing and scheduled down-times", - 'module' : 'DowntimePolicy', - 'command' : ( 'DowntimeCommand', 'DowntimeCommand' ), - 'args' : { 'hours' : 0, 'onlyCache' : True }, }, +POLICIESMETA = { # DownTime POLICIES + 'DTOngoing': {'description': "Ongoing and scheduled down-times", + 'module': 'DowntimePolicy', + 'command': ('DowntimeCommand', 'DowntimeCommand'), + 'args': {'hours': 0, 'onlyCache': True}, }, - 'DTScheduled1': {'description' : "Ongoing and scheduled down-times", - 'module' : 'DowntimePolicy', - 'command' : ( 'DowntimeCommand', 'DowntimeCommand' ), - 'args' : { 'hours' : 1, 'onlyCache' : True }, }, + 'DTScheduled1': {'description': "Ongoing and scheduled down-times", + 'module': 'DowntimePolicy', + 'command': ('DowntimeCommand', 'DowntimeCommand'), + 'args': {'hours': 1, 'onlyCache': True}, }, - 'DTScheduled3': {'description' : "Ongoing and scheduled down-times", - 'module' : 'DowntimePolicy', - 'command' : ( 'DowntimeCommand', 'DowntimeCommand' ), - 'args' : { 'hours' : 3, 'onlyCache' : True }, }, + 'DTScheduled3': {'description': "Ongoing and scheduled down-times", + 'module': 'DowntimePolicy', + 'command': ('DowntimeCommand', 'DowntimeCommand'), + 'args': {'hours': 3, 'onlyCache': True}, }, - 'DTScheduled': {'description' : "Scheduled down-times, starting in ", - 'module' : 'DowntimePolicy', - 'command' : ( 'DowntimeCommand', 'DowntimeCommand' ), - 'args' : { 'hours' : 12, 'onlyCache' : True }, }, + 'DTScheduled': {'description': "Scheduled down-times, starting in ", + 'module': 'DowntimePolicy', + 'command': ('DowntimeCommand', 'DowntimeCommand'), + 'args': {'hours': 12, 'onlyCache': True}, }, # Free Disk Space in Terabytes 'FreeDiskSpaceTB': { - 'description' : "Free disk space", - 'module' : 'FreeDiskSpacePolicy', - 'command' : ( 'FreeDiskSpaceCommand', 'FreeDiskSpaceCommand' ), - 'args' : { 'unit': 'TB', 'onlyCache' : True }, + 'description': "Free disk space", + 'module': 'FreeDiskSpacePolicy', + 'command': ('FreeDiskSpaceCommand', 'FreeDiskSpaceCommand'), + 'args': {'unit': 'TB', 'onlyCache': True}, }, # Free Disk Space in Gigabytes 'FreeDiskSpaceTB': { - 'description' : "Free disk space", - 'module' : 'FreeDiskSpacePolicy', - 'command' : ( 'FreeDiskSpaceCommand', 'FreeDiskSpaceCommand' ), - 'args' : { 'unit': 'GB', 'onlyCache' : True }, + 'description': "Free disk space", + 'module': 'FreeDiskSpacePolicy', + 'command': ('FreeDiskSpaceCommand', 'FreeDiskSpaceCommand'), + 'args': {'unit': 'GB', 'onlyCache': True}, }, # Free Disk Space in Megabytes 'FreeDiskSpaceTB': { - 'description' : "Free disk space", - 'module' : 'FreeDiskSpacePolicy', - 'command' : ( 'FreeDiskSpaceCommand', 'FreeDiskSpaceCommand' ), - 'args' : { 'unit': 'MB', 'onlyCache' : True }, + 'description': "Free disk space", + 'module': 'FreeDiskSpacePolicy', + 'command': ('FreeDiskSpaceCommand', 'FreeDiskSpaceCommand'), + 'args': {'unit': 'MB', 'onlyCache': True}, }, # Job POLICIES 'JobDoneRatio': { - 'description' : "done / ( completed + done ) jobs ( 30 min )", - 'module' : 'JobDoneRatioPolicy', - 'command' : ( 'JobCommand', 'JobCommand' ), - 'args' : { 'onlyCache' : True, 'timespan' : 1800 }, + 'description': "done / ( completed + done ) jobs ( 30 min )", + 'module': 'JobDoneRatioPolicy', + 'command': ('JobCommand', 'JobCommand'), + 'args': {'onlyCache': True, 'timespan': 1800}, }, 'JobEfficiency': { - 'description' : "( completed + done ) / ( completed + done + failed ) jobs ( 30 min )", - 'module' : 'JobEfficiencyPolicy', - 'command' : ( 'JobCommand', 'JobCommand' ), - 'args' : { 'onlyCache' : True, 'timespan' : 1800 }, + 'description': "( completed + done ) / ( completed + done + failed ) jobs ( 30 min )", + 'module': 'JobEfficiencyPolicy', + 'command': ('JobCommand', 'JobCommand'), + 'args': {'onlyCache': True, 'timespan': 1800}, }, 'JobRunningMatchedRatio': { - 'description' : "running / ( running + matched + received + checking ) jobs ( 30 min )", - 'module' : 'JobRunningMatchedRatioPolicy', - 'command' : ( 'JobCommand', 'JobCommand' ), - 'args' : { 'onlyCache' : True, 'timespan' : 1800 }, + 'description': "running / ( running + matched + received + checking ) jobs ( 30 min )", + 'module': 'JobRunningMatchedRatioPolicy', + 'command': ('JobCommand', 'JobCommand'), + 'args': {'onlyCache': True, 'timespan': 1800}, }, 'JobRunningWaitingRatio': { - 'description' : "running / ( running + waiting + staging ) jobs ( 30 min )", - 'module' : 'JobRunningWaitingRatioPolicy', - 'command' : ( 'JobCommand', 'JobCommand' ), - 'args' : { 'onlyCache' : True, 'timespan' : 1800 }, + 'description': "running / ( running + waiting + staging ) jobs ( 30 min )", + 'module': 'JobRunningWaitingRatioPolicy', + 'command': ('JobCommand', 'JobCommand'), + 'args': {'onlyCache': True, 'timespan': 1800}, }, # Pilot POLICIES.............................................................. 'PilotInstantEfficiency': { - 'description' : "Pilots Instant Efficiency ( 30 min )", - 'module' : 'PilotEfficiencyPolicy', - 'command' : ( 'PilotCommand', 'PilotCommand' ), - 'args' : { 'onlyCache' : True, 'timespan' : 1800 } + 'description': "Pilots Instant Efficiency ( 30 min )", + 'module': 'PilotEfficiencyPolicy', + 'command': ('PilotCommand', 'PilotCommand'), + 'args': {'onlyCache': True, 'timespan': 1800} }, # Site status propagation POLICIES.............................................................. 'PropagationPolicy': { - 'description' : "Site status propagation", - 'module' : 'PropagationPolicy', - 'command' : ( 'PropagationCommand', 'PropagationCommand' ), - 'args' : { 'onlyCache' : True, 'timespan' : 1800 } + 'description': "Site status propagation", + 'module': 'PropagationPolicy', + 'command': ('PropagationCommand', 'PropagationCommand'), + 'args': {'onlyCache': True, 'timespan': 1800} }, # ALWAYS SOMETHING POLICIES................................................... 'AlwaysActive': { - 'description' : "A Policy that always returns Active", - 'module' : 'AlwaysActivePolicy', - 'command' : None, - 'args' : None + 'description': "A Policy that always returns Active", + 'module': 'AlwaysActivePolicy', + 'command': None, + 'args': None }, 'AlwaysDegraded': { - 'description' : "A Policy that always returns Degraded", - 'module' : 'AlwaysDegradedPolicy', - 'command' : None, - 'args' : None + 'description': "A Policy that always returns Degraded", + 'module': 'AlwaysDegradedPolicy', + 'command': None, + 'args': None }, 'AlwaysProbing': { - 'description' : "A Policy that always returns Probing", - 'module' : 'AlwaysProbingPolicy', - 'command' : None, - 'args' : None + 'description': "A Policy that always returns Probing", + 'module': 'AlwaysProbingPolicy', + 'command': None, + 'args': None }, 'AlwaysBanned': { - 'description' : "A Policy that always returns Banned", - 'module' : 'AlwaysBannedPolicy', - 'command' : None, - 'args' : None + 'description': "A Policy that always returns Banned", + 'module': 'AlwaysBannedPolicy', + 'command': None, + 'args': None } } diff --git a/ResourceStatusSystem/Policy/test/Test_RSS_Policy_FreeDiskSpacePolicy.py b/ResourceStatusSystem/Policy/test/Test_RSS_Policy_FreeDiskSpacePolicy.py index 403929b4de4..c023b884193 100644 --- a/ResourceStatusSystem/Policy/test/Test_RSS_Policy_FreeDiskSpacePolicy.py +++ b/ResourceStatusSystem/Policy/test/Test_RSS_Policy_FreeDiskSpacePolicy.py @@ -7,17 +7,18 @@ ################################################################################ -class FreeDiskSpacePolicy_TestCase( unittest.TestCase ): - - def setUp( self ): + +class FreeDiskSpacePolicy_TestCase(unittest.TestCase): + + def setUp(self): ''' Setup ''' self.moduleTested = moduleTested - self.testClass = self.moduleTested.FreeDiskSpacePolicy + self.testClass = self.moduleTested.FreeDiskSpacePolicy - def tearDown( self ): + def tearDown(self): ''' Tear down ''' @@ -27,77 +28,78 @@ def tearDown( self ): ################################################################################ -class FreeDiskSpacePolicy_Success( FreeDiskSpacePolicy_TestCase ): - def test_instantiate( self ): +class FreeDiskSpacePolicy_Success(FreeDiskSpacePolicy_TestCase): + + def test_instantiate(self): ''' tests that we can instantiate one object of the tested class ''' module = self.testClass() - self.assertEqual( 'FreeDiskSpacePolicy', module.__class__.__name__ ) + self.assertEqual('FreeDiskSpacePolicy', module.__class__.__name__) - def test_evaluate( self ): + def test_evaluate(self): ''' tests the method _evaluate ''' module = self.testClass() - res = module._evaluate( { 'OK' : False, 'Message' : 'Bo!' } ) + res = module._evaluate({'OK': False, 'Message': 'Bo!'}) self.assertTrue(res['OK']) - self.assertEqual( 'Error', res[ 'Value' ][ 'Status' ] ) - self.assertEqual( 'Bo!', res[ 'Value' ][ 'Reason' ] ) + self.assertEqual('Error', res['Value']['Status']) + self.assertEqual('Bo!', res['Value']['Reason']) - res = module._evaluate( { 'OK' : True, 'Value' : None } ) + res = module._evaluate({'OK': True, 'Value': None}) self.assertTrue(res['OK']) - self.assertEqual( 'Unknown', res[ 'Value' ][ 'Status' ] ) - self.assertEqual( 'No values to take a decision', res[ 'Value' ][ 'Reason' ] ) + self.assertEqual('Unknown', res['Value']['Status']) + self.assertEqual('No values to take a decision', res['Value']['Reason']) - res = module._evaluate( { 'OK' : True, 'Value' : [] } ) + res = module._evaluate({'OK': True, 'Value': []}) self.assertTrue(res['OK']) - self.assertEqual( 'Unknown', res[ 'Value' ][ 'Status' ] ) - self.assertEqual( 'No values to take a decision', res[ 'Value' ][ 'Reason' ] ) + self.assertEqual('Unknown', res['Value']['Status']) + self.assertEqual('No values to take a decision', res['Value']['Reason']) - res = module._evaluate( { 'OK' : True, 'Value' : [{ 'A' : 1 }] } ) + res = module._evaluate({'OK': True, 'Value': [{'A': 1}]}) self.assertTrue(res['OK']) - self.assertEqual( 'Error', res[ 'Value' ][ 'Status' ] ) - self.assertEqual( 'Key total missing', res[ 'Value' ][ 'Reason' ] ) + self.assertEqual('Error', res['Value']['Status']) + self.assertEqual('Key total missing', res['Value']['Reason']) - res = module._evaluate( { 'OK' : True, 'Value' : [{ 'Total' : 1 }] } ) + res = module._evaluate({'OK': True, 'Value': [{'Total': 1}]}) self.assertTrue(res['OK']) - self.assertEqual( 'Error', res[ 'Value' ][ 'Status' ] ) - self.assertEqual( 'Key free missing', res[ 'Value' ][ 'Reason' ] ) + self.assertEqual('Error', res['Value']['Status']) + self.assertEqual('Key free missing', res['Value']['Reason']) - res = module._evaluate( { 'OK' : True, 'Value' : [{'Total' : 100, 'Free' : 0.0 }] } ) + res = module._evaluate({'OK': True, 'Value': [{'Total': 100, 'Free': 0.0}]}) self.assertTrue(res['OK']) - self.assertEqual( 'Error', res[ 'Value' ][ 'Status' ] ) - self.assertEqual( 'Key guaranteed missing', res[ 'Value' ][ 'Reason' ] ) + self.assertEqual('Error', res['Value']['Status']) + self.assertEqual('Key guaranteed missing', res['Value']['Reason']) - res = module._evaluate( { 'OK' : True, 'Value' : [{'Total' : 100, 'Free' : 0.0, - 'Guaranteed' : 1 }] } ) + res = module._evaluate({'OK': True, 'Value': [{'Total': 100, 'Free': 0.0, + 'Guaranteed': 1}]}) self.assertTrue(res['OK']) - self.assertEqual( 'Banned', res[ 'Value' ][ 'Status' ] ) - self.assertEqual( 'Free space < 100GB', res[ 'Value' ][ 'Reason' ] ) + self.assertEqual('Banned', res['Value']['Status']) + self.assertEqual('Free space < 100GB', res['Value']['Reason']) - res = module._evaluate( { 'OK' : True, 'Value' : [{'Total' : 100, 'Free' : 4.0, - 'Guaranteed' : 1 }] } ) + res = module._evaluate({'OK': True, 'Value': [{'Total': 100, 'Free': 4.0, + 'Guaranteed': 1}]}) self.assertTrue(res['OK']) - self.assertEqual( 'Degraded', res[ 'Value' ][ 'Status' ] ) - self.assertEqual( 'Free space < 5TB', - res[ 'Value' ][ 'Reason' ] ) + self.assertEqual('Degraded', res['Value']['Status']) + self.assertEqual('Free space < 5TB', + res['Value']['Reason']) - res = module._evaluate( { 'OK' : True, 'Value' : [{'Total' : 100, 'Free' : 100, - 'Guaranteed' : 1 }] } ) + res = module._evaluate({'OK': True, 'Value': [{'Total': 100, 'Free': 100, + 'Guaranteed': 1}]}) self.assertTrue(res['OK']) - self.assertEqual( 'Active', res[ 'Value' ][ 'Status' ] ) - self.assertEqual( 'Free space > 5TB', - res[ 'Value' ][ 'Reason' ] ) + self.assertEqual('Active', res['Value']['Status']) + self.assertEqual('Free space > 5TB', + res['Value']['Reason']) ################################################################################ if __name__ == '__main__': - suite = unittest.defaultTestLoader.loadTestsFromTestCase( FreeDiskSpacePolicy_TestCase ) - suite.addTest( unittest.defaultTestLoader.loadTestsFromTestCase( FreeDiskSpacePolicy_Success ) ) - testResult = unittest.TextTestRunner( verbosity = 2 ).run( suite ) + suite = unittest.defaultTestLoader.loadTestsFromTestCase(FreeDiskSpacePolicy_TestCase) + suite.addTest(unittest.defaultTestLoader.loadTestsFromTestCase(FreeDiskSpacePolicy_Success)) + testResult = unittest.TextTestRunner(verbosity=2).run(suite) -#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF \ No newline at end of file +# EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF diff --git a/ResourceStatusSystem/Policy/test/Test_RSS_Policy_SpaceTokenOccupancyPolicy.py b/ResourceStatusSystem/Policy/test/Test_RSS_Policy_SpaceTokenOccupancyPolicy.py deleted file mode 100644 index 403929b4de4..00000000000 --- a/ResourceStatusSystem/Policy/test/Test_RSS_Policy_SpaceTokenOccupancyPolicy.py +++ /dev/null @@ -1,103 +0,0 @@ -''' Test_RSS_Policy_FreeDiskSpacePolicy -''' - -import unittest - -import DIRAC.ResourceStatusSystem.Policy.FreeDiskSpacePolicy as moduleTested - -################################################################################ - -class FreeDiskSpacePolicy_TestCase( unittest.TestCase ): - - def setUp( self ): - ''' - Setup - ''' - - self.moduleTested = moduleTested - self.testClass = self.moduleTested.FreeDiskSpacePolicy - - def tearDown( self ): - ''' - Tear down - ''' - - del self.moduleTested - del self.testClass - -################################################################################ - -class FreeDiskSpacePolicy_Success( FreeDiskSpacePolicy_TestCase ): - - def test_instantiate( self ): - ''' tests that we can instantiate one object of the tested class - ''' - - module = self.testClass() - self.assertEqual( 'FreeDiskSpacePolicy', module.__class__.__name__ ) - - def test_evaluate( self ): - ''' tests the method _evaluate - ''' - - module = self.testClass() - - res = module._evaluate( { 'OK' : False, 'Message' : 'Bo!' } ) - self.assertTrue(res['OK']) - self.assertEqual( 'Error', res[ 'Value' ][ 'Status' ] ) - self.assertEqual( 'Bo!', res[ 'Value' ][ 'Reason' ] ) - - res = module._evaluate( { 'OK' : True, 'Value' : None } ) - self.assertTrue(res['OK']) - self.assertEqual( 'Unknown', res[ 'Value' ][ 'Status' ] ) - self.assertEqual( 'No values to take a decision', res[ 'Value' ][ 'Reason' ] ) - - res = module._evaluate( { 'OK' : True, 'Value' : [] } ) - self.assertTrue(res['OK']) - self.assertEqual( 'Unknown', res[ 'Value' ][ 'Status' ] ) - self.assertEqual( 'No values to take a decision', res[ 'Value' ][ 'Reason' ] ) - - res = module._evaluate( { 'OK' : True, 'Value' : [{ 'A' : 1 }] } ) - self.assertTrue(res['OK']) - self.assertEqual( 'Error', res[ 'Value' ][ 'Status' ] ) - self.assertEqual( 'Key total missing', res[ 'Value' ][ 'Reason' ] ) - - res = module._evaluate( { 'OK' : True, 'Value' : [{ 'Total' : 1 }] } ) - self.assertTrue(res['OK']) - self.assertEqual( 'Error', res[ 'Value' ][ 'Status' ] ) - self.assertEqual( 'Key free missing', res[ 'Value' ][ 'Reason' ] ) - - res = module._evaluate( { 'OK' : True, 'Value' : [{'Total' : 100, 'Free' : 0.0 }] } ) - self.assertTrue(res['OK']) - self.assertEqual( 'Error', res[ 'Value' ][ 'Status' ] ) - self.assertEqual( 'Key guaranteed missing', res[ 'Value' ][ 'Reason' ] ) - - res = module._evaluate( { 'OK' : True, 'Value' : [{'Total' : 100, 'Free' : 0.0, - 'Guaranteed' : 1 }] } ) - self.assertTrue(res['OK']) - self.assertEqual( 'Banned', res[ 'Value' ][ 'Status' ] ) - self.assertEqual( 'Free space < 100GB', res[ 'Value' ][ 'Reason' ] ) - - res = module._evaluate( { 'OK' : True, 'Value' : [{'Total' : 100, 'Free' : 4.0, - 'Guaranteed' : 1 }] } ) - self.assertTrue(res['OK']) - self.assertEqual( 'Degraded', res[ 'Value' ][ 'Status' ] ) - self.assertEqual( 'Free space < 5TB', - res[ 'Value' ][ 'Reason' ] ) - - res = module._evaluate( { 'OK' : True, 'Value' : [{'Total' : 100, 'Free' : 100, - 'Guaranteed' : 1 }] } ) - self.assertTrue(res['OK']) - self.assertEqual( 'Active', res[ 'Value' ][ 'Status' ] ) - self.assertEqual( 'Free space > 5TB', - res[ 'Value' ][ 'Reason' ] ) - - -################################################################################ - -if __name__ == '__main__': - suite = unittest.defaultTestLoader.loadTestsFromTestCase( FreeDiskSpacePolicy_TestCase ) - suite.addTest( unittest.defaultTestLoader.loadTestsFromTestCase( FreeDiskSpacePolicy_Success ) ) - testResult = unittest.TextTestRunner( verbosity = 2 ).run( suite ) - -#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF \ No newline at end of file diff --git a/Resources/Storage/GFAL2_SRM2Storage.py b/Resources/Storage/GFAL2_SRM2Storage.py index de80ec88ae4..c84e5b3b981 100644 --- a/Resources/Storage/GFAL2_SRM2Storage.py +++ b/Resources/Storage/GFAL2_SRM2Storage.py @@ -18,20 +18,19 @@ __RCSID__ = "$Id$" -class GFAL2_SRM2Storage( GFAL2_StorageBase ): + +class GFAL2_SRM2Storage(GFAL2_StorageBase): """ SRM2 SE class that inherits from GFAL2StorageBase """ _INPUT_PROTOCOLS = ['file', 'root', 'srm', 'gsiftp'] _OUTPUT_PROTOCOLS = ['file', 'root', 'dcap', 'gsidcap', 'rfio', 'srm', 'gsiftp'] - - - def __init__( self, storageName, parameters ): + def __init__(self, storageName, parameters): """ """ - super( GFAL2_SRM2Storage, self ).__init__( storageName, parameters ) - self.log = gLogger.getSubLogger( "GFAL2_SRM2Storage", True ) - self.log.debug( "GFAL2_SRM2Storage.__init__: Initializing object" ) + super(GFAL2_SRM2Storage, self).__init__(storageName, parameters) + self.log = gLogger.getSubLogger("GFAL2_SRM2Storage", True) + self.log.debug("GFAL2_SRM2Storage.__init__: Initializing object") self.pluginName = 'GFAL2_SRM2' # This attribute is used to know the file status (OFFLINE,NEARLINE,ONLINE) @@ -43,35 +42,31 @@ def __init__( self, storageName, parameters ): # the proper values and then calling the base class method. # ## - self.gfal2requestLifetime = gConfig.getValue( '/Resources/StorageElements/RequestLifeTime', 100 ) + self.gfal2requestLifetime = gConfig.getValue('/Resources/StorageElements/RequestLifeTime', 100) self.__setSRMOptionsToDefault() - # This lists contains the list of protocols to ask to SRM to get a URL # It can be either defined in the plugin of the SE, or as a global option if 'ProtocolsList' in parameters: self.protocolsList = parameters['ProtocolsList'].split(',') else: - self.log.debug( "GFAL2_SRM2Storage: No protocols provided, using the default protocols." ) + self.log.debug("GFAL2_SRM2Storage: No protocols provided, using the default protocols.") self.protocolsList = self.defaultLocalProtocols - self.log.debug( 'GFAL2_SRM2Storage: protocolsList = %s' % self.protocolsList ) - - + self.log.debug('GFAL2_SRM2Storage: protocolsList = %s' % self.protocolsList) - def __setSRMOptionsToDefault( self ): + def __setSRMOptionsToDefault(self): ''' Resetting the SRM options back to default ''' - self.ctx.set_opt_integer( "SRM PLUGIN", "OPERATION_TIMEOUT", self.gfal2Timeout ) - self.ctx.set_opt_string( "SRM PLUGIN", "SPACETOKENDESC", self.spaceToken ) - self.ctx.set_opt_integer( "SRM PLUGIN", "REQUEST_LIFETIME", self.gfal2requestLifetime ) + self.ctx.set_opt_integer("SRM PLUGIN", "OPERATION_TIMEOUT", self.gfal2Timeout) + self.ctx.set_opt_string("SRM PLUGIN", "SPACETOKENDESC", self.spaceToken) + self.ctx.set_opt_integer("SRM PLUGIN", "REQUEST_LIFETIME", self.gfal2requestLifetime) # Setting the TURL protocol to gsiftp because with other protocols we have authorisation problems # self.ctx.set_opt_string_list( "SRM PLUGIN", "TURL_PROTOCOLS", self.defaultLocalProtocols ) - self.ctx.set_opt_string_list( "SRM PLUGIN", "TURL_PROTOCOLS", ['gsiftp'] ) + self.ctx.set_opt_string_list("SRM PLUGIN", "TURL_PROTOCOLS", ['gsiftp']) - - def _updateMetadataDict( self, metadataDict, attributeDict ): + def _updateMetadataDict(self, metadataDict, attributeDict): """ Updating the metadata dictionary with srm specific attributes :param self: self reference @@ -80,15 +75,14 @@ def _updateMetadataDict( self, metadataDict, attributeDict ): """ # 'user.status' is the extended attribute we are interested in - user_status = attributeDict.get( 'user.status', '' ) - metadataDict['Cached'] = int( 'ONLINE' in user_status ) - metadataDict['Migrated'] = int( 'NEARLINE' in user_status ) - metadataDict['Lost'] = int( user_status == 'LOST' ) - metadataDict['Unavailable'] = int( user_status == 'UNAVAILABLE' ) + user_status = attributeDict.get('user.status', '') + metadataDict['Cached'] = int('ONLINE' in user_status) + metadataDict['Migrated'] = int('NEARLINE' in user_status) + metadataDict['Lost'] = int(user_status == 'LOST') + metadataDict['Unavailable'] = int(user_status == 'UNAVAILABLE') metadataDict['Accessible'] = not metadataDict['Lost'] and metadataDict['Cached'] and not metadataDict['Unavailable'] - - def getTransportURL( self, path, protocols = False ): + def getTransportURL(self, path, protocols=False): """ obtain the tURLs for the supplied path and protocols :param self: self reference @@ -98,60 +92,56 @@ def getTransportURL( self, path, protocols = False ): Successful dict {path : transport url} S_ERROR in case of argument problems """ - res = checkArgumentFormat( path ) + res = checkArgumentFormat(path) if not res['OK']: return res urls = res['Value'] - self.log.debug( 'GFAL2_SRM2Storage.getTransportURL: Attempting to retrieve tURL for %s paths' % len( urls ) ) + self.log.debug('GFAL2_SRM2Storage.getTransportURL: Attempting to retrieve tURL for %s paths' % len(urls)) failed = {} successful = {} if not protocols: listProtocols = self.protocolsList if not listProtocols: - return S_ERROR( "GFAL2_SRM2Storage.getTransportURL: No local protocols defined and no defaults found." ) - elif isinstance( protocols, basestring ): + return S_ERROR("GFAL2_SRM2Storage.getTransportURL: No local protocols defined and no defaults found.") + elif isinstance(protocols, basestring): listProtocols = [protocols] - elif isinstance( protocols, list ): + elif isinstance(protocols, list): listProtocols = protocols else: - return S_ERROR( "getTransportURL: Must supply desired protocols to this plug-in." ) + return S_ERROR("getTransportURL: Must supply desired protocols to this plug-in.") # Compatibility because of castor returning a castor: url if you ask # for a root URL, and a root: url if you ask for a xroot url... if 'root' in listProtocols and 'xroot' not in listProtocols: - listProtocols.insert( listProtocols.index( 'root' ), 'xroot' ) + listProtocols.insert(listProtocols.index('root'), 'xroot') elif 'xroot' in listProtocols and 'root' not in listProtocols: - listProtocols.insert( listProtocols.index( 'xroot' ) + 1, 'root' ) - + listProtocols.insert(listProtocols.index('xroot') + 1, 'root') if self.protocolParameters['Protocol'] in listProtocols: successful = {} failed = {} for url in urls: - if self.isURL( url )['Value']: + if self.isURL(url)['Value']: successful[url] = url else: failed[url] = 'getTransportURL: Failed to obtain turls.' - return S_OK( {'Successful' : successful, 'Failed' : failed} ) + return S_OK({'Successful': successful, 'Failed': failed}) for url in urls: - res = self.__getSingleTransportURL( url, listProtocols ) - self.log.debug( 'res = %s' % res ) + res = self.__getSingleTransportURL(url, listProtocols) + self.log.debug('res = %s' % res) if not res['OK']: failed[url] = res['Message'] else: successful[url] = res['Value'] - return S_OK( { 'Failed' : failed, 'Successful' : successful } ) - - - + return S_OK({'Failed': failed, 'Successful': successful}) - def __getSingleTransportURL( self, path, protocols = False ): + def __getSingleTransportURL(self, path, protocols=False): """ Get the tURL from path with getxattr from gfal2 :param self: self reference @@ -159,18 +149,18 @@ def __getSingleTransportURL( self, path, protocols = False ): :returns: S_OK( Transport_URL ) in case of success S_ERROR( errStr ) in case of a failure """ - self.log.debug( 'GFAL2_SRM2Storage.__getSingleTransportURL: trying to retrieve tURL for %s' % path ) + self.log.debug('GFAL2_SRM2Storage.__getSingleTransportURL: trying to retrieve tURL for %s' % path) if protocols: - self.ctx.set_opt_string_list( "SRM PLUGIN", "TURL_PROTOCOLS", protocols ) + self.ctx.set_opt_string_list("SRM PLUGIN", "TURL_PROTOCOLS", protocols) - res = self._getExtendedAttributes( path, attributes = ['user.replicas'] ) + res = self._getExtendedAttributes(path, attributes=['user.replicas']) self.__setSRMOptionsToDefault() if res['OK']: - return S_OK( res['Value']['user.replicas'] ) + return S_OK(res['Value']['user.replicas']) errStr = 'GFAL2_SRM2Storage.__getSingleTransportURL: Extended attribute tURL is not set.' - self.log.debug( errStr, res['Message'] ) + self.log.debug(errStr, res['Message']) return res def getOccupancy(self, *parms, **kws): From ce2ed75bcd01c881af541ea645c8d0435b43bc41 Mon Sep 17 00:00:00 2001 From: Federico Stagni Date: Tue, 19 Dec 2017 17:50:15 +0100 Subject: [PATCH 14/17] fix test --- .../Policy/FreeDiskSpacePolicy.py | 2 +- .../test/Test_RSS_Policy_FreeDiskSpacePolicy.py | 17 ++++++++--------- 2 files changed, 9 insertions(+), 10 deletions(-) diff --git a/ResourceStatusSystem/Policy/FreeDiskSpacePolicy.py b/ResourceStatusSystem/Policy/FreeDiskSpacePolicy.py index a8d82d2f6e6..ce077e9708a 100644 --- a/ResourceStatusSystem/Policy/FreeDiskSpacePolicy.py +++ b/ResourceStatusSystem/Policy/FreeDiskSpacePolicy.py @@ -56,7 +56,7 @@ def _evaluate(commandResult): if key not in commandResult: result['Status'] = 'Error' - result['Reason'] = 'Key %s missing' % key.lower() + result['Reason'] = 'Key %s missing' % key return S_OK(result) free = float(commandResult['Free']) diff --git a/ResourceStatusSystem/Policy/test/Test_RSS_Policy_FreeDiskSpacePolicy.py b/ResourceStatusSystem/Policy/test/Test_RSS_Policy_FreeDiskSpacePolicy.py index c023b884193..1b86287e65d 100644 --- a/ResourceStatusSystem/Policy/test/Test_RSS_Policy_FreeDiskSpacePolicy.py +++ b/ResourceStatusSystem/Policy/test/Test_RSS_Policy_FreeDiskSpacePolicy.py @@ -62,36 +62,35 @@ def test_evaluate(self): res = module._evaluate({'OK': True, 'Value': [{'A': 1}]}) self.assertTrue(res['OK']) self.assertEqual('Error', res['Value']['Status']) - self.assertEqual('Key total missing', res['Value']['Reason']) + self.assertEqual('Key Total missing', res['Value']['Reason']) res = module._evaluate({'OK': True, 'Value': [{'Total': 1}]}) self.assertTrue(res['OK']) self.assertEqual('Error', res['Value']['Status']) - self.assertEqual('Key free missing', res['Value']['Reason']) + self.assertEqual('Key Free missing', res['Value']['Reason']) - res = module._evaluate({'OK': True, 'Value': [{'Total': 100, 'Free': 0.0}]}) + res = module._evaluate({'OK': True, 'Value': [{'Total': 100, 'Fre': 0.0}]}) self.assertTrue(res['OK']) self.assertEqual('Error', res['Value']['Status']) - self.assertEqual('Key guaranteed missing', res['Value']['Reason']) + self.assertEqual('Key Free missing', res['Value']['Reason']) - res = module._evaluate({'OK': True, 'Value': [{'Total': 100, 'Free': 0.0, - 'Guaranteed': 1}]}) + res = module._evaluate({'OK': True, 'Value': [{'Total': 100, 'Free': 0.0}]}) self.assertTrue(res['OK']) self.assertEqual('Banned', res['Value']['Status']) - self.assertEqual('Free space < 100GB', res['Value']['Reason']) + self.assertEqual('Too little free space', res['Value']['Reason']) res = module._evaluate({'OK': True, 'Value': [{'Total': 100, 'Free': 4.0, 'Guaranteed': 1}]}) self.assertTrue(res['OK']) self.assertEqual('Degraded', res['Value']['Status']) - self.assertEqual('Free space < 5TB', + self.assertEqual('Little free space', res['Value']['Reason']) res = module._evaluate({'OK': True, 'Value': [{'Total': 100, 'Free': 100, 'Guaranteed': 1}]}) self.assertTrue(res['OK']) self.assertEqual('Active', res['Value']['Status']) - self.assertEqual('Free space > 5TB', + self.assertEqual('Enough free space', res['Value']['Reason']) From 2c249d748772fa68bb73be3c26bb912329334cad Mon Sep 17 00:00:00 2001 From: Federico Stagni Date: Wed, 20 Dec 2017 16:04:14 +0100 Subject: [PATCH 15/17] using unit inside the command --- .../Service/StorageElementHandler.py | 23 +++---- .../Command/FreeDiskSpaceCommand.py | 64 +++++++++++++------ ResourceStatusSystem/Policy/Configurations.py | 11 ++-- 3 files changed, 59 insertions(+), 39 deletions(-) diff --git a/DataManagementSystem/Service/StorageElementHandler.py b/DataManagementSystem/Service/StorageElementHandler.py index 51a5cf01e34..3ca1cd844e8 100755 --- a/DataManagementSystem/Service/StorageElementHandler.py +++ b/DataManagementSystem/Service/StorageElementHandler.py @@ -16,6 +16,8 @@ removeDirectory() - remove on directory recursively removeFileList() - remove files in the list getAdminInfo() - get administration information about the SE status +getFreeDiskSpace() - get the free disk space +getTotalDiskSpace() - get the free total space The handler implements also the DISET data transfer calls toClient(), fromClient(), bulkToClient(), bulkFromClient @@ -50,32 +52,25 @@ MAX_STORAGE_SIZE = 0 USE_TOKENS = False -UNIT_CONVERSION = {"KB": 1024, "MB": 1024 * 1024, "GB": 1024 * 1024 * 1024, "TB": 1024 * 1024 * 1024 * 1024} -def getDiskSpace(path, size='TB', total=False): +def getDiskSpace(path, total=False): """ - Returns disk usage of the given path. - If no size is specified, terabytes will be used by default. + Returns disk usage of the given path, in MB. If total is set to true, the total disk space will be returned instead. """ - size_to_convert = size.upper() - if size_to_convert not in UNIT_CONVERSION: - return S_ERROR("No valid size specified") - convert = UNIT_CONVERSION[size_to_convert] - try: st = os.statvfs(path) if total: # return total space - queried_size = st.f_blocks + queriedSize = st.f_blocks else: # return free space - queried_size = st.f_bavail + queriedSize = st.f_bavail - result = float(queried_size * st.f_frsize) / float(convert) + result = float(queriedSize * st.f_frsize) / float(1024*1024) except OSError as e: return S_ERROR(errno.EIO, "Error while getting the available disk space: %s" % repr(e)) @@ -94,7 +89,7 @@ def getTotalDiskSpace(): global BASE_PATH global MAX_STORAGE_SIZE - result = getDiskSpace(BASE_PATH, size='MB', total=True) + result = getDiskSpace(BASE_PATH, total=True) if not result['OK']: return result totalSpace = result['Value'] @@ -112,7 +107,7 @@ def getFreeDiskSpace(ignoreMaxStorageSize=True): global MAX_STORAGE_SIZE global BASE_PATH - result = getDiskSpace(BASE_PATH, 'MB') + result = getDiskSpace(BASE_PATH) if not result['OK']: return result freeSpace = result['Value'] diff --git a/ResourceStatusSystem/Command/FreeDiskSpaceCommand.py b/ResourceStatusSystem/Command/FreeDiskSpaceCommand.py index a972d7a98ad..8ba2431368a 100644 --- a/ResourceStatusSystem/Command/FreeDiskSpaceCommand.py +++ b/ResourceStatusSystem/Command/FreeDiskSpaceCommand.py @@ -1,6 +1,10 @@ ''' FreeDiskSpaceCommand + The Command gets the free space that is left in a Storage Element - The Command gets the free space that is left in a Storage Element + Note: there are, still, many references to "space tokens", + for example ResourceManagementClient().selectSpaceTokenOccupancyCache(token=elementName) + This is for historical reasons, and shoud be fixed one day. + For the moment, when you see "token" or "space token" here, just read "StorageElement". ''' @@ -14,7 +18,8 @@ from DIRAC.Resources.Storage.StorageElement import StorageElement from DIRAC.ResourceStatusSystem.Client.ResourceManagementClient import ResourceManagementClient -# FIXME: use unit! + +UNIT_CONVERSION = {"MB": 1, "GB": 1024, "TB": 1024 * 1024} class FreeDiskSpaceCommand(Command): @@ -38,9 +43,9 @@ def _prepareCommand(self): return S_ERROR('"name" not found in self.args') elementName = self.args['name'] - unit = 'TB' - if 'unit' in self.args: - unit = self.args['unit'] + # We keep TB as default as this is what was used (and will still be used) + # in the policy for "space tokens" ("real", "data" SEs) + unit = self.args.get('unit', 'TB') return S_OK((elementName, unit)) @@ -52,15 +57,19 @@ def doNew(self, masterParams=None): Gets the total and the free disk space of a storage element and inserts the results in the SpaceTokenOccupancyCache table of ResourceManagementDB database. + + The result is also returned to the caller, not only inserted. + What is inserted in the DB will be in MB, + what is returned will be in the specified unit. """ if masterParams is not None: - elementName = masterParams - unit = 'TB' + elementName, unit = masterParams else: - elementName, unit = self._prepareCommand() - if not elementName['OK']: - return elementName + params = self._prepareCommand() + if not params['OK']: + return params + elementName, unit = params['Value'] endpointResult = CSHelpers.getStorageElementEndpoint(elementName) if not endpointResult['OK']: @@ -71,33 +80,48 @@ def doNew(self, masterParams=None): if not occupancyResult['OK']: return occupancyResult occupancy = occupancyResult['Value'] + free = occupancy['Free'] + total = occupancy['Total'] result = self.rmClient.addOrModifySpaceTokenOccupancyCache(endpoint=endpointResult['Value'], lastCheckTime=datetime.utcnow(), - free=occupancy['Free'], - total=occupancy['Total'], + free=free, + total=total, token=elementName) if not result['OK']: return result - return S_OK() + # results are normally in 'MB' + unit = unit.upper() + if unit not in UNIT_CONVERSION: + return S_ERROR("No valid unit specified") + convert = UNIT_CONVERSION[unit] + return S_OK({'Free': float(free)/float(convert), 'Total': float(total)/float(convert)}) def doCache(self): """ - This is a method that gets the element's details from the spaceTokenOccupancy cache. - It will return a list of dictionaries if there are results. + This is a method that gets the element's details from the spaceTokenOccupancyCache DB table. + It will return a dictionary with th results, converted to "correct" unit. """ - elementName, unit = self._prepareCommand() - if not elementName['OK']: - return elementName + params = self._prepareCommand() + if not params['OK']: + return params + elementName, unit = params['Value'] result = self.rmClient.selectSpaceTokenOccupancyCache(token=elementName) if not result['OK']: return result - return S_OK(result) + # results are normally in 'MB' + free = result['Value'][0][3] + total = result['Value'][0][4] + unit = unit.upper() + if unit not in UNIT_CONVERSION: + return S_ERROR("No valid unit specified") + convert = UNIT_CONVERSION[unit] + return S_OK({'Free': float(free)/float(convert), 'Total': float(total)/float(convert)}) def doMaster(self): """ @@ -110,7 +134,7 @@ def doMaster(self): for name in elements['Value']: diskSpace = self.doNew(name) if not diskSpace['OK']: - gLogger.error("Unable to calculate free disk space", "name: %s" % name) + gLogger.error("Unable to calculate free/total disk space", "name: %s" % name) continue return S_OK() diff --git a/ResourceStatusSystem/Policy/Configurations.py b/ResourceStatusSystem/Policy/Configurations.py index 75fb8d8b9e6..9709fd3161d 100644 --- a/ResourceStatusSystem/Policy/Configurations.py +++ b/ResourceStatusSystem/Policy/Configurations.py @@ -1,3 +1,4 @@ + """ Configurations module Configuration to use policies. @@ -40,23 +41,23 @@ # Free Disk Space in Terabytes 'FreeDiskSpaceTB': { - 'description': "Free disk space", + 'description': "Free disk space, in TB", 'module': 'FreeDiskSpacePolicy', 'command': ('FreeDiskSpaceCommand', 'FreeDiskSpaceCommand'), 'args': {'unit': 'TB', 'onlyCache': True}, }, # Free Disk Space in Gigabytes - 'FreeDiskSpaceTB': { - 'description': "Free disk space", + 'FreeDiskSpaceGB': { + 'description': "Free disk space, in GB", 'module': 'FreeDiskSpacePolicy', 'command': ('FreeDiskSpaceCommand', 'FreeDiskSpaceCommand'), 'args': {'unit': 'GB', 'onlyCache': True}, }, # Free Disk Space in Megabytes - 'FreeDiskSpaceTB': { - 'description': "Free disk space", + 'FreeDiskSpaceMB': { + 'description': "Free disk space, in MB", 'module': 'FreeDiskSpacePolicy', 'command': ('FreeDiskSpaceCommand', 'FreeDiskSpaceCommand'), 'args': {'unit': 'MB', 'onlyCache': True}, From f3aa637b8b600f497b4467dafe05b3a393d2c2f5 Mon Sep 17 00:00:00 2001 From: Federico Stagni Date: Wed, 20 Dec 2017 16:20:59 +0100 Subject: [PATCH 16/17] bug fix plus style --- .../Service/StorageElementHandler.py | 3 +- .../Command/FreeDiskSpaceCommand.py | 4 +- .../Service/SandboxStoreHandler.py | 448 +++++++++--------- 3 files changed, 229 insertions(+), 226 deletions(-) diff --git a/DataManagementSystem/Service/StorageElementHandler.py b/DataManagementSystem/Service/StorageElementHandler.py index 3ca1cd844e8..64897fbd700 100755 --- a/DataManagementSystem/Service/StorageElementHandler.py +++ b/DataManagementSystem/Service/StorageElementHandler.py @@ -53,7 +53,6 @@ USE_TOKENS = False - def getDiskSpace(path, total=False): """ Returns disk usage of the given path, in MB. @@ -70,7 +69,7 @@ def getDiskSpace(path, total=False): # return free space queriedSize = st.f_bavail - result = float(queriedSize * st.f_frsize) / float(1024*1024) + result = float(queriedSize * st.f_frsize) / float(1024 * 1024) except OSError as e: return S_ERROR(errno.EIO, "Error while getting the available disk space: %s" % repr(e)) diff --git a/ResourceStatusSystem/Command/FreeDiskSpaceCommand.py b/ResourceStatusSystem/Command/FreeDiskSpaceCommand.py index 8ba2431368a..b7e463830a2 100644 --- a/ResourceStatusSystem/Command/FreeDiskSpaceCommand.py +++ b/ResourceStatusSystem/Command/FreeDiskSpaceCommand.py @@ -96,7 +96,7 @@ def doNew(self, masterParams=None): if unit not in UNIT_CONVERSION: return S_ERROR("No valid unit specified") convert = UNIT_CONVERSION[unit] - return S_OK({'Free': float(free)/float(convert), 'Total': float(total)/float(convert)}) + return S_OK({'Free': float(free) / float(convert), 'Total': float(total) / float(convert)}) def doCache(self): """ @@ -121,7 +121,7 @@ def doCache(self): if unit not in UNIT_CONVERSION: return S_ERROR("No valid unit specified") convert = UNIT_CONVERSION[unit] - return S_OK({'Free': float(free)/float(convert), 'Total': float(total)/float(convert)}) + return S_OK({'Free': float(free) / float(convert), 'Total': float(total) / float(convert)}) def doMaster(self): """ diff --git a/WorkloadManagementSystem/Service/SandboxStoreHandler.py b/WorkloadManagementSystem/Service/SandboxStoreHandler.py index 77f43d4e504..3f019740ed1 100755 --- a/WorkloadManagementSystem/Service/SandboxStoreHandler.py +++ b/WorkloadManagementSystem/Service/SandboxStoreHandler.py @@ -11,7 +11,7 @@ from DIRAC.Core.Utilities.File import mkDir from DIRAC.Core.DISET.RequestHandler import RequestHandler from DIRAC.WorkloadManagementSystem.DB.SandboxMetadataDB import SandboxMetadataDB -from DIRAC.DataManagementSystem.Client.DataManager import DataManager +from DIRAC.DataManagementSystem.Client.DataManager import DataManager from DIRAC.DataManagementSystem.Service.StorageElementHandler import getDiskSpace from DIRAC.RequestManagementSystem.Client.ReqClient import ReqClient from DIRAC.RequestManagementSystem.Client.Request import Request @@ -24,21 +24,23 @@ sandboxDB = False -def initializeSandboxStoreHandler( serviceInfo ): + +def initializeSandboxStoreHandler(serviceInfo): global sandboxDB, gSBDeletionPool sandboxDB = SandboxMetadataDB() return S_OK() -class SandboxStoreHandler( RequestHandler ): + +class SandboxStoreHandler(RequestHandler): __purgeCount = -1 __purgeLock = threading.Lock() __purgeWorking = False - def initialize( self ): - self.__backend = self.getCSOption( "Backend", "local" ) - self.__localSEName = self.getCSOption( "LocalSE", "SandboxSE" ) - self.__maxUploadBytes = self.getCSOption( "MaxSandboxSizeMiB", 10 ) * 1048576 + def initialize(self): + self.__backend = self.getCSOption("Backend", "local") + self.__localSEName = self.getCSOption("LocalSE", "SandboxSE") + self.__maxUploadBytes = self.getCSOption("MaxSandboxSizeMiB", 10) * 1048576 if self.__backend.lower() == "local" or self.__backend == self.__localSEName: self.__useLocalStorage = True self.__seNameToUse = self.__localSEName @@ -48,450 +50,452 @@ def initialize( self ): self.__seNameToUse = self.__backend # Execute the purge once every 100 calls SandboxStoreHandler.__purgeCount += 1 - if SandboxStoreHandler.__purgeCount > self.getCSOption( "QueriesBeforePurge", 1000 ): + if SandboxStoreHandler.__purgeCount > self.getCSOption("QueriesBeforePurge", 1000): SandboxStoreHandler.__purgeCount = 0 if SandboxStoreHandler.__purgeCount == 0: - threading.Thread( target = self.purgeUnusedSandboxes ).start() + threading.Thread(target=self.purgeUnusedSandboxes).start() - def __getSandboxPath( self, md5 ): + def __getSandboxPath(self, md5): """ Generate the sandbox path """ # prefix = self.getCSOption( "SandboxPrefix", "SandBox" ) prefix = "SandBox" credDict = self.getRemoteCredentials() - if Properties.JOB_SHARING in credDict[ 'properties' ]: - idField = credDict[ 'group' ] + if Properties.JOB_SHARING in credDict['properties']: + idField = credDict['group'] else: - idField = "%s.%s" % ( credDict[ 'username' ], credDict[ 'group' ] ) - pathItems = [ "/", prefix, idField[0], idField ] - pathItems.extend( [ md5[0:3], md5[3:6], md5 ] ) - return os.path.join( *pathItems ) - + idField = "%s.%s" % (credDict['username'], credDict['group']) + pathItems = ["/", prefix, idField[0], idField] + pathItems.extend([md5[0:3], md5[3:6], md5]) + return os.path.join(*pathItems) - def transfer_fromClient( self, fileId, token, fileSize, fileHelper ): + def transfer_fromClient(self, fileId, token, fileSize, fileHelper): """ Receive a file as a sandbox """ if self.__maxUploadBytes and fileSize > self.__maxUploadBytes: fileHelper.markAsTransferred() - return S_ERROR( "Sandbox is too big. Please upload it to a grid storage element" ) + return S_ERROR("Sandbox is too big. Please upload it to a grid storage element") - if isinstance( fileId, ( list, tuple ) ): - if len( fileId ) > 1: + if isinstance(fileId, (list, tuple)): + if len(fileId) > 1: assignTo = fileId[1] fileId = fileId[0] else: - return S_ERROR( "File identified tuple has to have length greater than 1" ) + return S_ERROR("File identified tuple has to have length greater than 1") else: assignTo = {} - extPos = fileId.find( ".tar" ) + extPos = fileId.find(".tar") if extPos > -1: - extension = fileId[ extPos + 1: ] - aHash = fileId[ :extPos ] + extension = fileId[extPos + 1:] + aHash = fileId[:extPos] else: extension = "" aHash = fileId - gLogger.info( "Upload requested for %s [%s]" % ( aHash, extension ) ) + gLogger.info("Upload requested for %s [%s]" % (aHash, extension)) credDict = self.getRemoteCredentials() - sbPath = self.__getSandboxPath( "%s.%s" % ( aHash, extension ) ) + sbPath = self.__getSandboxPath("%s.%s" % (aHash, extension)) # Generate the location - result = self.__generateLocation( sbPath ) - if not result[ 'OK' ]: + result = self.__generateLocation(sbPath) + if not result['OK']: return result - seName, sePFN = result[ 'Value' ] + seName, sePFN = result['Value'] - result = sandboxDB.getSandboxId( seName, sePFN, credDict[ 'username' ], credDict[ 'group' ] ) - if result[ 'OK' ]: - gLogger.info( "Sandbox already exists. Skipping upload" ) + result = sandboxDB.getSandboxId(seName, sePFN, credDict['username'], credDict['group']) + if result['OK']: + gLogger.info("Sandbox already exists. Skipping upload") fileHelper.markAsTransferred() - sbURL = "SB:%s|%s" % ( seName, sePFN ) - assignTo = dict( [ ( key, [ ( sbURL, assignTo[ key ] ) ] ) for key in assignTo ] ) - result = self.export_assignSandboxesToEntities( assignTo ) - if not result[ 'OK' ]: + sbURL = "SB:%s|%s" % (seName, sePFN) + assignTo = dict([(key, [(sbURL, assignTo[key])]) for key in assignTo]) + result = self.export_assignSandboxesToEntities(assignTo) + if not result['OK']: return result - return S_OK( sbURL ) + return S_OK(sbURL) if self.__useLocalStorage: - hdPath = self.__sbToHDPath( sbPath ) + hdPath = self.__sbToHDPath(sbPath) else: hdPath = False # Write to local file - result = self.__networkToFile( fileHelper, hdPath ) - if not result[ 'OK' ]: - gLogger.error( "Error while receiving sandbox file", "%s" % result['Message'] ) + result = self.__networkToFile(fileHelper, hdPath) + if not result['OK']: + gLogger.error("Error while receiving sandbox file", "%s" % result['Message']) return result - hdPath = result[ 'Value' ] - gLogger.info( "Wrote sandbox to file %s" % hdPath ) + hdPath = result['Value'] + gLogger.info("Wrote sandbox to file %s" % hdPath) # Check hash! if fileHelper.getHash() != aHash: - self.__secureUnlinkFile( hdPath ) - gLogger.error( "Hashes don't match! Client defined hash is different with received data hash!" ) - return S_ERROR( "Hashes don't match!" ) + self.__secureUnlinkFile(hdPath) + gLogger.error("Hashes don't match! Client defined hash is different with received data hash!") + return S_ERROR("Hashes don't match!") # If using remote storage, copy there! if not self.__useLocalStorage: - gLogger.info( "Uploading sandbox to external storage" ) - result = self.__copyToExternalSE( hdPath, sbPath ) - self.__secureUnlinkFile( hdPath ) - if not result[ 'OK' ]: + gLogger.info("Uploading sandbox to external storage") + result = self.__copyToExternalSE(hdPath, sbPath) + self.__secureUnlinkFile(hdPath) + if not result['OK']: return result - sbPath = result[ 'Value' ][1] + sbPath = result['Value'][1] # Register! - gLogger.info( "Registering sandbox in the DB with", "SB:%s|%s" % ( self.__seNameToUse, sbPath ) ) - result = sandboxDB.registerAndGetSandbox( credDict[ 'username' ], credDict[ 'DN' ], credDict[ 'group' ], - self.__seNameToUse, sbPath, fileHelper.getTransferedBytes() ) - if not result[ 'OK' ]: - self.__secureUnlinkFile( hdPath ) + gLogger.info("Registering sandbox in the DB with", "SB:%s|%s" % (self.__seNameToUse, sbPath)) + result = sandboxDB.registerAndGetSandbox(credDict['username'], credDict['DN'], credDict['group'], + self.__seNameToUse, sbPath, fileHelper.getTransferedBytes()) + if not result['OK']: + self.__secureUnlinkFile(hdPath) return result - sbURL = "SB:%s|%s" % ( self.__seNameToUse, sbPath ) - assignTo = dict( [ ( key, [ ( sbURL, assignTo[ key ] ) ] ) for key in assignTo ] ) - result = self.export_assignSandboxesToEntities( assignTo ) - if not result[ 'OK' ]: + sbURL = "SB:%s|%s" % (self.__seNameToUse, sbPath) + assignTo = dict([(key, [(sbURL, assignTo[key])]) for key in assignTo]) + result = self.export_assignSandboxesToEntities(assignTo) + if not result['OK']: return result - return S_OK( sbURL ) + return S_OK(sbURL) - def transfer_bulkFromClient( self, fileId, token, fileSize, fileHelper ): + def transfer_bulkFromClient(self, fileId, token, fileSize, fileHelper): """ Receive files packed into a tar archive by the fileHelper logic. token is used for access rights confirmation. """ - result = self.__networkToFile( fileHelper ) - if not result[ 'OK' ]: + result = self.__networkToFile(fileHelper) + if not result['OK']: return result - tmpFilePath = result[ 'OK' ] - gLogger.info( "Got Sandbox to local storage", tmpFilePath ) + tmpFilePath = result['OK'] + gLogger.info("Got Sandbox to local storage", tmpFilePath) - extension = fileId[ fileId.find( ".tar" ) + 1: ] - sbPath = "%s.%s" % ( self.__getSandboxPath( fileHelper.getHash() ), extension ) - gLogger.info( "Sandbox path will be", sbPath ) + extension = fileId[fileId.find(".tar") + 1:] + sbPath = "%s.%s" % (self.__getSandboxPath(fileHelper.getHash()), extension) + gLogger.info("Sandbox path will be", sbPath) # Generate the location - result = self.__generateLocation( sbPath ) - if not result[ 'OK' ]: + result = self.__generateLocation(sbPath) + if not result['OK']: return result - seName, sePFN = result[ 'Value' ] + seName, sePFN = result['Value'] # Register in DB credDict = self.getRemoteCredentials() - result = sandboxDB.getSandboxId( seName, sePFN, credDict[ 'username' ], credDict[ 'group' ] ) - if result[ 'OK' ]: - return S_OK( "SB:%s|%s" % ( seName, sePFN ) ) - - result = sandboxDB.registerAndGetSandbox( credDict[ 'username' ], credDict[ 'DN' ], credDict[ 'group' ], - seName, sePFN, fileHelper.getTransferedBytes() ) - if not result[ 'OK' ]: - self.__secureUnlinkFile( tmpFilePath ) + result = sandboxDB.getSandboxId(seName, sePFN, credDict['username'], credDict['group']) + if result['OK']: + return S_OK("SB:%s|%s" % (seName, sePFN)) + + result = sandboxDB.registerAndGetSandbox(credDict['username'], credDict['DN'], credDict['group'], + seName, sePFN, fileHelper.getTransferedBytes()) + if not result['OK']: + self.__secureUnlinkFile(tmpFilePath) return result - sbid, _newSandbox = result[ 'Value' ] - gLogger.info( "Registered in DB", "with SBId %s" % sbid ) + sbid, _newSandbox = result['Value'] + gLogger.info("Registered in DB", "with SBId %s" % sbid) - result = self.__moveToFinalLocation( tmpFilePath, sbPath ) - self.__secureUnlinkFile( tmpFilePath ) - if not result[ 'OK' ]: - gLogger.error( "Could not move sandbox to final destination", result[ 'Message' ] ) + result = self.__moveToFinalLocation(tmpFilePath, sbPath) + self.__secureUnlinkFile(tmpFilePath) + if not result['OK']: + gLogger.error("Could not move sandbox to final destination", result['Message']) return result - gLogger.info( "Moved to final destination" ) + gLogger.info("Moved to final destination") # Unlink temporal file if it's there - self.__secureUnlinkFile( tmpFilePath ) - return S_OK( "SB:%s|%s" % ( seName, sePFN ) ) + self.__secureUnlinkFile(tmpFilePath) + return S_OK("SB:%s|%s" % (seName, sePFN)) - def __generateLocation( self, sbPath ): + def __generateLocation(self, sbPath): """ Generate the location string """ if self.__useLocalStorage: - return S_OK( ( self.__localSEName, sbPath ) ) + return S_OK((self.__localSEName, sbPath)) # It's external storage - storageElement = StorageElement( self.__externalSEName ) + storageElement = StorageElement(self.__externalSEName) res = storageElement.isValid() if not res['OK']: errStr = "Failed to instantiate destination StorageElement" - gLogger.error( errStr, self.__externalSEName ) - return S_ERROR( errStr ) - result = storageElement.getURL( sbPath ) + gLogger.error(errStr, self.__externalSEName) + return S_ERROR(errStr) + result = storageElement.getURL(sbPath) if not result['OK'] or sbPath not in result['Value']['Successful']: errStr = "Failed to generate PFN" - gLogger.error( errStr, self.__externalSEName ) - return S_ERROR( errStr ) + gLogger.error(errStr, self.__externalSEName) + return S_ERROR(errStr) destPfn = result['Value']['Successful'][sbPath] - return S_OK( ( self.__externalSEName, destPfn ) ) + return S_OK((self.__externalSEName, destPfn)) - def __sbToHDPath( self, sbPath ): + def __sbToHDPath(self, sbPath): while sbPath and sbPath[0] == "/": sbPath = sbPath[1:] - basePath = self.getCSOption( "BasePath", "/opt/dirac/storage/sandboxes" ) - return os.path.join( basePath, sbPath ) + basePath = self.getCSOption("BasePath", "/opt/dirac/storage/sandboxes") + return os.path.join(basePath, sbPath) - def __networkToFile( self, fileHelper, destFileName = False ): + def __networkToFile(self, fileHelper, destFileName=False): """ Dump incoming network data to temporal file """ tfd = None if not destFileName: try: - tfd, destFileName = tempfile.mkstemp( prefix = "DSB." ) + tfd, destFileName = tempfile.mkstemp(prefix="DSB.") tfd.close() except Exception as e: - gLogger.error( "%s" % repr( e ).replace( ',)', ')' ) ) - return S_ERROR( "Cannot create temporary file" ) + gLogger.error("%s" % repr(e).replace(',)', ')')) + return S_ERROR("Cannot create temporary file") - destFileName = os.path.realpath( destFileName ) - mkDir( os.path.dirname( destFileName ) ) + destFileName = os.path.realpath(destFileName) + mkDir(os.path.dirname(destFileName)) try: if tfd is not None: fd = tfd else: - fd = open( destFileName, "wb" ) - result = fileHelper.networkToDataSink( fd, maxFileSize = self.__maxUploadBytes ) + fd = open(destFileName, "wb") + result = fileHelper.networkToDataSink(fd, maxFileSize=self.__maxUploadBytes) fd.close() except Exception as e: - gLogger.error( "Cannot open to write destination file", "%s: %s" % ( destFileName, repr( e ).replace( ',)', ')' ) ) ) - return S_ERROR( "Cannot open to write destination file" ) - if not result[ 'OK' ]: + gLogger.error("Cannot open to write destination file", "%s: %s" % (destFileName, repr(e).replace(',)', ')'))) + return S_ERROR("Cannot open to write destination file") + if not result['OK']: return result - return S_OK( destFileName ) + return S_OK(destFileName) - def __secureUnlinkFile( self, filePath ): + def __secureUnlinkFile(self, filePath): try: - os.unlink( filePath ) + os.unlink(filePath) except Exception as e: - gLogger.warn( "Could not unlink file %s: %s" % ( filePath, repr( e ).replace( ',)', ')' ) ) ) + gLogger.warn("Could not unlink file %s: %s" % (filePath, repr(e).replace(',)', ')'))) return False return True - def __moveToFinalLocation( self, localFilePath, sbPath ): + def __moveToFinalLocation(self, localFilePath, sbPath): if self.__useLocalStorage: - hdFilePath = self.__sbToHDPath( sbPath ) - result = S_OK( ( self.__localSEName, sbPath ) ) - if os.path.isfile( hdFilePath ): - gLogger.info( "There was already a sandbox with that name, skipping copy", sbPath ) + hdFilePath = self.__sbToHDPath(sbPath) + result = S_OK((self.__localSEName, sbPath)) + if os.path.isfile(hdFilePath): + gLogger.info("There was already a sandbox with that name, skipping copy", sbPath) else: - hdDirPath = os.path.dirname( hdFilePath ) + hdDirPath = os.path.dirname(hdFilePath) mkDir(hdDirPath) try: - os.rename( localFilePath, hdFilePath ) + os.rename(localFilePath, hdFilePath) except OSError as e: errMsg = "Cannot move temporal file to final path" - gLogger.error( errMsg, repr( e ).replace( ',)', ')' ) ) - result = S_ERROR( errMsg ) + gLogger.error(errMsg, repr(e).replace(',)', ')')) + result = S_ERROR(errMsg) else: - result = self.__copyToExternalSE( localFilePath, sbPath ) + result = self.__copyToExternalSE(localFilePath, sbPath) return result - def __copyToExternalSE( self, localFilePath, sbPath ): + def __copyToExternalSE(self, localFilePath, sbPath): """ Copy uploaded file to external SE """ try: dm = DataManager() - result = dm.put( sbPath, localFilePath, self.__externalSEName ) - if not result[ 'OK' ]: + result = dm.put(sbPath, localFilePath, self.__externalSEName) + if not result['OK']: return result - if 'Successful' not in result[ 'Value' ]: - gLogger.verbose( "Oops, no successful transfers there", str( result ) ) - return S_ERROR( "RM returned OK to the action but no successful transfers were there" ) - okTrans = result[ 'Value' ][ 'Successful' ] + if 'Successful' not in result['Value']: + gLogger.verbose("Oops, no successful transfers there", str(result)) + return S_ERROR("RM returned OK to the action but no successful transfers were there") + okTrans = result['Value']['Successful'] if sbPath not in okTrans: - gLogger.verbose( "Ooops, SB transfer wasn't in the successful ones", str( result ) ) - return S_ERROR( "RM returned OK to the action but SB transfer wasn't in the successful ones" ) - return S_OK( ( self.__externalSEName, okTrans[ sbPath ] ) ) + gLogger.verbose("Ooops, SB transfer wasn't in the successful ones", str(result)) + return S_ERROR("RM returned OK to the action but SB transfer wasn't in the successful ones") + return S_OK((self.__externalSEName, okTrans[sbPath])) except Exception as e: - gLogger.error( "Error while moving sandbox to SE", "%s" % repr( e ).replace( ',)', ')' ) ) - return S_ERROR( "Error while moving sandbox to SE" ) + gLogger.error("Error while moving sandbox to SE", "%s" % repr(e).replace(',)', ')')) + return S_ERROR("Error while moving sandbox to SE") ################## # Assigning sbs to jobs - types_assignSandboxesToEntities = [ dict ] - def export_assignSandboxesToEntities( self, enDict, ownerName = "", ownerGroup = "", entitySetup = False ): + types_assignSandboxesToEntities = [dict] + + def export_assignSandboxesToEntities(self, enDict, ownerName="", ownerGroup="", entitySetup=False): """ Assign sandboxes to jobs. Expects a dict of { entityId : [ ( SB, SBType ), ... ] } """ if not entitySetup: - entitySetup = self.serviceInfoDict[ 'clientSetup' ] + entitySetup = self.serviceInfoDict['clientSetup'] credDict = self.getRemoteCredentials() - return sandboxDB.assignSandboxesToEntities( enDict, credDict[ 'username' ], credDict[ 'group' ], entitySetup, - ownerName, ownerGroup ) + return sandboxDB.assignSandboxesToEntities(enDict, credDict['username'], credDict['group'], entitySetup, + ownerName, ownerGroup) ################## # Unassign sbs to jobs - types_unassignEntities = [ ( list, tuple ) ] - def export_unassignEntities( self, entitiesList, entitiesSetup = False ): + types_unassignEntities = [(list, tuple)] + + def export_unassignEntities(self, entitiesList, entitiesSetup=False): """ Unassign a list of jobs """ if not entitiesSetup: - entitiesSetup = self.serviceInfoDict[ 'clientSetup' ] + entitiesSetup = self.serviceInfoDict['clientSetup'] credDict = self.getRemoteCredentials() - return sandboxDB.unassignEntities( { entitiesSetup : entitiesList }, credDict[ 'username' ], credDict[ 'group' ] ) + return sandboxDB.unassignEntities({entitiesSetup: entitiesList}, credDict['username'], credDict['group']) ################## # Getting assigned sandboxes - types_getSandboxesAssignedToEntity = [ basestring ] - def export_getSandboxesAssignedToEntity( self, entityId, entitySetup = False ): + types_getSandboxesAssignedToEntity = [basestring] + + def export_getSandboxesAssignedToEntity(self, entityId, entitySetup=False): """ Get the sandboxes associated to a job and the association type """ if not entitySetup: - entitySetup = self.serviceInfoDict[ 'clientSetup' ] + entitySetup = self.serviceInfoDict['clientSetup'] credDict = self.getRemoteCredentials() - result = sandboxDB.getSandboxesAssignedToEntity( entityId, entitySetup, - credDict[ 'username' ], credDict[ 'group' ] ) - if not result[ 'OK' ]: + result = sandboxDB.getSandboxesAssignedToEntity(entityId, entitySetup, + credDict['username'], credDict['group']) + if not result['OK']: return result sbDict = {} - for SEName, SEPFN, SBType in result[ 'Value' ]: + for SEName, SEPFN, SBType in result['Value']: if SBType not in sbDict: - sbDict[ SBType ] = [] - sbDict[ SBType ].append( "SB:%s|%s" % ( SEName, SEPFN ) ) - return S_OK( sbDict ) - + sbDict[SBType] = [] + sbDict[SBType].append("SB:%s|%s" % (SEName, SEPFN)) + return S_OK(sbDict) ################## # Disk space left management types_getFreeDiskSpace = [basestring] - def export_getFreeDiskSpace( self, path, size = 'TB' ): + + def export_getFreeDiskSpace(self, path): """ Get the free disk space of the storage element If no size is specified, terabytes will be used by default. """ - return getDiskSpace(path, size) + return getDiskSpace(path) types_getTotalDiskSpace = [basestring] - def export_getTotalDiskSpace( self, path, size = 'TB' ): + + def export_getTotalDiskSpace(self, path): """ Get the total disk space of the storage element If no size is specified, terabytes will be used by default. """ - return getDiskSpace(path, size, total = True) - + return getDiskSpace(path, total=True) ################## # Download sandboxes - def transfer_toClient( self, fileID, token, fileHelper ): + def transfer_toClient(self, fileID, token, fileHelper): """ Method to send files to clients. fileID is the local file name in the SE. token is used for access rights confirmation. """ credDict = self.getRemoteCredentials() serviceURL = self.serviceInfoDict['URL'] - filePath = fileID.replace( serviceURL, '' ) - result = sandboxDB.getSandboxId( self.__localSEName, filePath, credDict[ 'username' ], credDict[ 'group' ] ) - if not result[ 'OK' ]: + filePath = fileID.replace(serviceURL, '') + result = sandboxDB.getSandboxId(self.__localSEName, filePath, credDict['username'], credDict['group']) + if not result['OK']: return result - sbId = result[ 'Value' ] - sandboxDB.accessedSandboxById( sbId ) + sbId = result['Value'] + sandboxDB.accessedSandboxById(sbId) # If it's a local file - hdPath = self.__sbToHDPath( filePath ) - if not os.path.isfile( hdPath ): - return S_ERROR( "Sandbox does not exist" ) - result = fileHelper.getFileDescriptor( hdPath, 'rb' ) - if not result[ 'OK' ]: - return S_ERROR( 'Failed to get file descriptor: %s' % result[ 'Message' ] ) - fd = result[ 'Value' ] - result = fileHelper.FDToNetwork( fd ) + hdPath = self.__sbToHDPath(filePath) + if not os.path.isfile(hdPath): + return S_ERROR("Sandbox does not exist") + result = fileHelper.getFileDescriptor(hdPath, 'rb') + if not result['OK']: + return S_ERROR('Failed to get file descriptor: %s' % result['Message']) + fd = result['Value'] + result = fileHelper.FDToNetwork(fd) fileHelper.oFile.close() return result ################## # Purge sandboxes - def purgeUnusedSandboxes( self ): + def purgeUnusedSandboxes(self): # If a purge is already working skip SandboxStoreHandler.__purgeLock.acquire() try: if SandboxStoreHandler.__purgeWorking: if time.time() - SandboxStoreHandler.__purgeWorking < 86400: - gLogger.info( "Sandbox purge still working" ) + gLogger.info("Sandbox purge still working") return S_OK() SandboxStoreHandler.__purgeWorking = time.time() finally: SandboxStoreHandler.__purgeLock.release() - gLogger.info( "Purging sandboxes" ) + gLogger.info("Purging sandboxes") result = sandboxDB.getUnusedSandboxes() - if not result[ 'OK' ]: - gLogger.error( "Error while retrieving sandboxes to purge", result[ 'Message' ] ) + if not result['OK']: + gLogger.error("Error while retrieving sandboxes to purge", result['Message']) SandboxStoreHandler.__purgeWorking = False return result - sbList = result[ 'Value' ] - gLogger.info( "Got %s sandboxes to purge" % len( sbList ) ) + sbList = result['Value'] + gLogger.info("Got %s sandboxes to purge" % len(sbList)) for sbId, SEName, SEPFN in sbList: - self.__purgeSandbox( sbId, SEName, SEPFN ) + self.__purgeSandbox(sbId, SEName, SEPFN) SandboxStoreHandler.__purgeWorking = False return S_OK() - def __purgeSandbox( self, sbId, SEName, SEPFN ): - result = self.__deleteSandboxFromBackend( SEName, SEPFN ) - if not result[ 'OK' ]: - gLogger.error( "Cannot delete sandbox from backend", result[ 'Message' ] ) + def __purgeSandbox(self, sbId, SEName, SEPFN): + result = self.__deleteSandboxFromBackend(SEName, SEPFN) + if not result['OK']: + gLogger.error("Cannot delete sandbox from backend", result['Message']) return - result = sandboxDB.deleteSandboxes( [ sbId ] ) - if not result[ 'OK' ]: - gLogger.error( "Cannot delete sandbox from DB", result[ 'Message' ] ) + result = sandboxDB.deleteSandboxes([sbId]) + if not result['OK']: + gLogger.error("Cannot delete sandbox from DB", result['Message']) - def __deleteSandboxFromBackend( self, SEName, SEPFN ): - gLogger.info( "Purging sandbox" "SB:%s|%s" % ( SEName, SEPFN ) ) + def __deleteSandboxFromBackend(self, SEName, SEPFN): + gLogger.info("Purging sandbox" "SB:%s|%s" % (SEName, SEPFN)) if SEName != self.__localSEName: - return self.__deleteSandboxFromExternalBackend( SEName, SEPFN ) + return self.__deleteSandboxFromExternalBackend(SEName, SEPFN) else: - hdPath = self.__sbToHDPath( SEPFN ) + hdPath = self.__sbToHDPath(SEPFN) try: - if not os.path.isfile( hdPath ): + if not os.path.isfile(hdPath): return S_OK() except Exception as e: - gLogger.error( "Cannot perform isfile", "%s : %s" % ( hdPath, repr( e ).replace( ',)', ')' ) ) ) - return S_ERROR( "Error checking %s" % hdPath ) + gLogger.error("Cannot perform isfile", "%s : %s" % (hdPath, repr(e).replace(',)', ')'))) + return S_ERROR("Error checking %s" % hdPath) try: - os.unlink( hdPath ) + os.unlink(hdPath) except Exception as e: - gLogger.error( "Cannot delete local sandbox", "%s : %s" % ( hdPath, repr( e ).replace( ',)', ')' ) ) ) + gLogger.error("Cannot delete local sandbox", "%s : %s" % (hdPath, repr(e).replace(',)', ')'))) while hdPath: - hdPath = os.path.dirname( hdPath ) - gLogger.info( "Checking if dir %s is empty" % hdPath ) + hdPath = os.path.dirname(hdPath) + gLogger.info("Checking if dir %s is empty" % hdPath) try: - if not os.path.isdir( hdPath ): + if not os.path.isdir(hdPath): break - if len( os.listdir( hdPath ) ) > 0: + if len(os.listdir(hdPath)) > 0: break - gLogger.info( "Trying to clean dir %s" % hdPath ) + gLogger.info("Trying to clean dir %s" % hdPath) # Empty dir! - os.rmdir( hdPath ) + os.rmdir(hdPath) except Exception as e: - gLogger.error( "Cannot clean directory", "%s : %s" % ( hdPath, repr( e ).replace( ',)', ')' ) ) ) + gLogger.error("Cannot clean directory", "%s : %s" % (hdPath, repr(e).replace(',)', ')'))) break return S_OK() - def __deleteSandboxFromExternalBackend( self, SEName, SEPFN ): - if self.getCSOption( "DelayedExternalDeletion", True ): - gLogger.info( "Setting deletion request" ) + def __deleteSandboxFromExternalBackend(self, SEName, SEPFN): + if self.getCSOption("DelayedExternalDeletion", True): + gLogger.info("Setting deletion request") try: request = Request() - request.RequestName = "RemoteSBDeletion:%s|%s:%s" % ( SEName, SEPFN, time.time() ) + request.RequestName = "RemoteSBDeletion:%s|%s:%s" % (SEName, SEPFN, time.time()) physicalRemoval = Operation() physicalRemoval.Type = "PhysicalRemoval" physicalRemoval.TargetSE = SEName fileToRemove = File() fileToRemove.PFN = SEPFN - physicalRemoval.addFile( fileToRemove ) - request.addOperation( physicalRemoval ) - return ReqClient().putRequest( request ) + physicalRemoval.addFile(fileToRemove) + request.addOperation(physicalRemoval) + return ReqClient().putRequest(request) except Exception as e: - gLogger.exception( "Exception while setting deletion request" ) - return S_ERROR( "Cannot set deletion request: %s" % str( e ) ) + gLogger.exception("Exception while setting deletion request") + return S_ERROR("Cannot set deletion request: %s" % str(e)) else: - gLogger.info( "Deleting external Sandbox" ) + gLogger.info("Deleting external Sandbox") try: - return StorageElement( SEName ).removeFile( SEPFN ) + return StorageElement(SEName).removeFile(SEPFN) except Exception as e: - gLogger.exception( "RM raised an exception while trying to delete a remote sandbox" ) - return S_ERROR( "RM raised an exception while trying to delete a remote sandbox" ) + gLogger.exception("RM raised an exception while trying to delete a remote sandbox") + return S_ERROR("RM raised an exception while trying to delete a remote sandbox") From 397c2e63f760bbf2a13b74d88cb0f78ba21f69e0 Mon Sep 17 00:00:00 2001 From: Federico Stagni Date: Mon, 8 Jan 2018 15:38:33 +0100 Subject: [PATCH 17/17] restored function for extensibility --- .../Command/FreeDiskSpaceCommand.py | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/ResourceStatusSystem/Command/FreeDiskSpaceCommand.py b/ResourceStatusSystem/Command/FreeDiskSpaceCommand.py index b7e463830a2..92e1e5e0eaa 100644 --- a/ResourceStatusSystem/Command/FreeDiskSpaceCommand.py +++ b/ResourceStatusSystem/Command/FreeDiskSpaceCommand.py @@ -83,11 +83,11 @@ def doNew(self, masterParams=None): free = occupancy['Free'] total = occupancy['Total'] - result = self.rmClient.addOrModifySpaceTokenOccupancyCache(endpoint=endpointResult['Value'], - lastCheckTime=datetime.utcnow(), - free=free, - total=total, - token=elementName) + results = {'Endpoint': endpointResult['Value'], + 'Free': free, + 'Total': total, + 'ElementName': elementName} + result = self._storeCommand(results) if not result['OK']: return result @@ -98,6 +98,15 @@ def doNew(self, masterParams=None): convert = UNIT_CONVERSION[unit] return S_OK({'Free': float(free) / float(convert), 'Total': float(total) / float(convert)}) + def _storeCommand(self, results): + """ Here purely for extensibility + """ + return self.rmClient.addOrModifySpaceTokenOccupancyCache(endpoint=results['Endpoint'], + lastCheckTime=datetime.utcnow(), + free=results['Free'], + total=results['Total'], + token=results['ElementName']) + def doCache(self): """ This is a method that gets the element's details from the spaceTokenOccupancyCache DB table.