diff --git a/Core/Utilities/File.py b/Core/Utilities/File.py index 49724ee435d..4daad35240d 100755 --- a/Core/Utilities/File.py +++ b/Core/Utilities/File.py @@ -4,8 +4,8 @@ By default on Error they return None. """ -#pylint: skip-file -## getGlobbedFiles gives "RuntimeError: maximum recursion depth exceeded" in pylint +# pylint: skip-file +# getGlobbedFiles gives "RuntimeError: maximum recursion depth exceeded" in pylint import os import hashlib @@ -17,20 +17,42 @@ __RCSID__ = "$Id$" -def mkDir( path ): +# Translation table of a given unit to Bytes +# I know, it should be kB... +SIZE_UNIT_CONVERSION = { + 'B': 1, + 'KB': 1024, + 'MB': 1024 * + 1024, + 'GB': 1024 * + 1024 * + 1024, + 'TB': 1024 * + 1024 * + 1024 * + 1024, + 'PB': 1024 * + 1024 * + 1024 * + 1024 * + 1024} + + +def mkDir(path): """ Emulate 'mkdir -p path' (if path exists already, don't raise an exception) """ try: if os.path.isdir(path): return - os.makedirs( path ) + os.makedirs(path) except OSError as osError: - if osError.errno == errno.EEXIST and os.path.isdir( path ): + if osError.errno == errno.EEXIST and os.path.isdir(path): pass else: raise -def mkLink( src, dst ): + +def mkLink(src, dst): """ Protected creation of simbolic link """ try: @@ -41,7 +63,8 @@ def mkLink( src, dst ): else: raise -def makeGuid( fileName = None ): + +def makeGuid(fileName=None): """Utility to create GUID. If a filename is provided the GUID will correspond to its content's hexadecimal md5 checksum. Otherwise a random seed is used to create the GUID. @@ -55,18 +78,19 @@ def makeGuid( fileName = None ): myMd5 = hashlib.md5() if fileName: try: - with open( fileName, 'r' ) as fd: - data = fd.read( 10 * 1024 * 1024 ) - myMd5.update( data ) - except: + with open(fileName, 'r') as fd: + data = fd.read(10 * 1024 * 1024) + myMd5.update(data) + except BaseException: return None else: - myMd5.update( str( random.getrandbits( 128 ) ) ) + myMd5.update(str(random.getrandbits(128))) md5HexString = myMd5.hexdigest().upper() - return generateGuid( md5HexString, "MD5" ) + return generateGuid(md5HexString, "MD5") + -def generateGuid( checksum, checksumtype ): +def generateGuid(checksum, checksumtype): """ Generate a GUID based on the file checksum """ @@ -74,31 +98,32 @@ def generateGuid( checksum, checksumtype ): if checksumtype == "MD5": checksumString = checksum elif checksumtype == "Adler32": - checksumString = str( checksum ).zfill( 32 ) + checksumString = str(checksum).zfill(32) else: checksumString = '' if checksumString: - guid = "%s-%s-%s-%s-%s" % ( checksumString[0:8], - checksumString[8:12], - checksumString[12:16], - checksumString[16:20], - checksumString[20:32] ) + guid = "%s-%s-%s-%s-%s" % (checksumString[0:8], + checksumString[8:12], + checksumString[12:16], + checksumString[16:20], + checksumString[20:32]) guid = guid.upper() return guid # Failed to use the check sum, generate a new guid myMd5 = hashlib.md5() - myMd5.update( str( random.getrandbits( 128 ) ) ) + myMd5.update(str(random.getrandbits(128))) md5HexString = myMd5.hexdigest() - guid = "%s-%s-%s-%s-%s" % ( md5HexString[0:8], - md5HexString[8:12], - md5HexString[12:16], - md5HexString[16:20], - md5HexString[20:32] ) + guid = "%s-%s-%s-%s-%s" % (md5HexString[0:8], + md5HexString[8:12], + md5HexString[12:16], + md5HexString[16:20], + md5HexString[20:32]) guid = guid.upper() return guid -def checkGuid( guid ): + +def checkGuid(guid): """Checks whether a supplied GUID is of the correct format. The guid is a string of 36 characters [0-9A-F] long split into 5 parts of length 8-4-4-4-12. @@ -110,16 +135,17 @@ def checkGuid( guid ): :param string guid: string to be checked :return: True (False) if supplied string is (not) a valid GUID. """ - reGUID = re.compile( "^[0-9A-F]{8}(-[0-9A-F]{4}){3}-[0-9A-F]{12}$" ) - if reGUID.match( guid.upper() ): + reGUID = re.compile("^[0-9A-F]{8}(-[0-9A-F]{4}){3}-[0-9A-F]{12}$") + if reGUID.match(guid.upper()): return True else: - guid = [ len( x ) for x in guid.split( "-" ) ] - if ( guid == [ 8, 4, 4, 4, 12 ] ): + guid = [len(x) for x in guid.split("-")] + if (guid == [8, 4, 4, 4, 12]): return True return False -def getSize( fileName ): + +def getSize(fileName): """Get size of a file. :param string fileName: name of file to be checked @@ -132,87 +158,91 @@ def getSize( fileName ): """ try: - return os.stat( fileName )[6] + return os.stat(fileName)[6] except OSError: return - 1 -def getGlobbedTotalSize( files ): + +def getGlobbedTotalSize(files): """Get total size of a list of files or a single file. Globs the parameter to allow regular expressions. :params list files: list or tuple of strings of files """ totalSize = 0 - if isinstance( files, (list, tuple) ): + if isinstance(files, (list, tuple)): for entry in files: - size = getGlobbedTotalSize( entry ) + size = getGlobbedTotalSize(entry) if size == -1: size = 0 totalSize += size else: - for path in glob.glob( files ): - if os.path.isdir( path ): - for content in os.listdir( path ): - totalSize += getGlobbedTotalSize( os.path.join( path, content ) ) - if os.path.isfile( path ): - size = getSize( path ) + for path in glob.glob(files): + if os.path.isdir(path): + for content in os.listdir(path): + totalSize += getGlobbedTotalSize(os.path.join(path, content)) + if os.path.isfile(path): + size = getSize(path) if size == -1: size = 0 totalSize += size return totalSize -def getGlobbedFiles( files ): + +def getGlobbedFiles(files): """Get list of files or a single file. Globs the parameter to allow regular expressions. :params list files: list or tuple of strings of files """ globbedFiles = [] - if isinstance( files, ( list, tuple ) ): + if isinstance(files, (list, tuple)): for entry in files: - globbedFiles += getGlobbedFiles( entry ) + globbedFiles += getGlobbedFiles(entry) else: - for path in glob.glob( files ): - if os.path.isdir( path ): - for content in os.listdir( path ): - globbedFiles += getGlobbedFiles( os.path.join( path, content ) ) - if os.path.isfile( path ): - globbedFiles.append( path ) + for path in glob.glob(files): + if os.path.isdir(path): + for content in os.listdir(path): + globbedFiles += getGlobbedFiles(os.path.join(path, content)) + if os.path.isfile(path): + globbedFiles.append(path) return globbedFiles -def getCommonPath( files ): + +def getCommonPath(files): """Get the common path for all files in the file list. :param files: list of strings with paths :type files: python:list """ - def properSplit( dirPath ): + def properSplit(dirPath): """Splitting of path to drive and path parts for non-Unix file systems. :param string dirPath: path """ - nDrive, nPath = os.path.splitdrive( dirPath ) - return [ nDrive ] + [ d for d in nPath.split( os.sep ) if d.strip() ] + nDrive, nPath = os.path.splitdrive(dirPath) + return [nDrive] + [d for d in nPath.split(os.sep) if d.strip()] if not files: return "" - commonPath = properSplit( files[0] ) + commonPath = properSplit(files[0]) for fileName in files: - if os.path.isdir( fileName ): + if os.path.isdir(fileName): dirPath = fileName else: - dirPath = os.path.dirname( fileName ) - nPath = properSplit( dirPath ) + dirPath = os.path.dirname(fileName) + nPath = properSplit(dirPath) tPath = [] - for i in range( min( len( commonPath ), len( nPath ) ) ): - if commonPath[ i ] != nPath[ i ]: + for i in range(min(len(commonPath), len(nPath))): + if commonPath[i] != nPath[i]: break - tPath .append( commonPath[ i ] ) + tPath .append(commonPath[i]) if not tPath: return "" commonPath = tPath - return tPath[0] + os.sep + os.path.join( *tPath[1:] ) + return tPath[0] + os.sep + os.path.join(*tPath[1:]) + -def getMD5ForFiles( fileList ): +def getMD5ForFiles(fileList): """Calculate md5 for the content of all the files. :param fileList: list of paths @@ -221,15 +251,47 @@ def getMD5ForFiles( fileList ): fileList.sort() hashMD5 = hashlib.md5() for filePath in fileList: - if os.path.isdir( filePath ): + if os.path.isdir(filePath): continue - with open( filePath, "rb" ) as fd: - buf = fd.read( 4096 ) + with open(filePath, "rb") as fd: + buf = fd.read(4096) while buf: - hashMD5.update( buf ) - buf = fd.read( 4096 ) + hashMD5.update(buf) + buf = fd.read(4096) return hashMD5.hexdigest() + +def convertSizeUnits(size, srcUnit, dstUnit): + """ Converts a number from a given source unit to a destination unit. + + Example: + In [1]: convertSizeUnits(1024, 'B', 'kB') + Out[1]: 1 + + In [2]: convertSizeUnits(1024, 'MB', 'kB') + Out[2]: 1048576 + + + :param size: number to convert + :param srcUnit: unit of the number. Any of ( 'B', 'kB', 'MB', 'GB', 'TB', 'PB') + :param dstUnit: unit expected for the return. Any of ( 'B', 'kB', 'MB', 'GB', 'TB', 'PB') + + :returns: the size number converted in the dstUnit. In case of problem -sys.maxint is returned (negative) + """ + + srcUnit = srcUnit.upper() + dstUnit = dstUnit.upper() + + try: + convertedValue = float(size) * SIZE_UNIT_CONVERSION[srcUnit] / SIZE_UNIT_CONVERSION[dstUnit] + return convertedValue + + # TypeError, ValueError: size is not a number + # KeyError: srcUnit or dstUnit are not in the conversion list + except (TypeError, ValueError, KeyError): + return -sys.maxsize + + if __name__ == "__main__": for p in sys.argv[1:]: - print "%s : %s bytes" % ( p, getGlobbedTotalSize( p ) ) + print "%s : %s bytes" % (p, getGlobbedTotalSize(p)) diff --git a/Core/Utilities/test/Test_File.py b/Core/Utilities/test/Test_File.py index 9cea354b30f..9665ee8a1e5 100644 --- a/Core/Utilities/test/Test_File.py +++ b/Core/Utilities/test/Test_File.py @@ -20,13 +20,22 @@ # @date 2011/01/17 14:01:18 # @brief Definition of FileTestCase class. -## imports +# imports import unittest import os import re +import sys +from hypothesis import given +from hypothesis.strategies import floats + +from pytest import mark, approx # sut -from DIRAC.Core.Utilities.File import checkGuid, makeGuid, getSize, getMD5ForFiles, getCommonPath +from DIRAC.Core.Utilities.File import checkGuid, makeGuid, getSize,\ + getMD5ForFiles, getCommonPath, convertSizeUnits, SIZE_UNIT_CONVERSION + +parametrize = mark.parametrize + __RCSID__ = "$Id $" @@ -46,96 +55,133 @@ def setUp(self): .. function:: FileTestCase.setUp(self) :param self: "Me, myself and Irene" """ - self.filesList = [ os.path.abspath(".") + os.sep + x for x in os.listdir( "." ) ] + self.filesList = [os.path.abspath(".") + os.sep + x for x in os.listdir(".")] - def tearDown( self ): + def tearDown(self): """ remove filesList """ self.filesList = None - def testCheckGuid( self ): + def testCheckGuid(self): """ checkGuid tests """ # empty string guid = "" - self.assertEqual( checkGuid( guid ), False, "empty guid" ) + self.assertEqual(checkGuid(guid), False, "empty guid") # wrong length in a 1st field guid = '012345678-0123-0123-0123-0123456789AB' - self.assertEqual( checkGuid( guid ), False, 'wrong length in 1st field' ) + self.assertEqual(checkGuid(guid), False, 'wrong length in 1st field') guid = '0123456-0123-0123-0123-0123456789AB' - self.assertEqual( checkGuid( guid ), False, 'wrong length in 1st field') + self.assertEqual(checkGuid(guid), False, 'wrong length in 1st field') # wrong length in a 2nd field guid = '01234567-01234-0123-0123-0123456789AB' - self.assertEqual( checkGuid( guid ), False, 'wrong length in 2nd field' ) + self.assertEqual(checkGuid(guid), False, 'wrong length in 2nd field') guid = '01234567-012-0123-0123-0123456789AB' - self.assertEqual( checkGuid( guid ), False, 'wrong length in 2nd field' ) + self.assertEqual(checkGuid(guid), False, 'wrong length in 2nd field') # wrong length in a 3rd field guid = '01234567-0123-01234-0123-0123456789AB' - self.assertEqual( checkGuid( guid ), False, 'wrong length in 3rd field' ) + self.assertEqual(checkGuid(guid), False, 'wrong length in 3rd field') guid = '01234567-0123-012-0123-0123456789AB' - self.assertEqual( checkGuid( guid ), False, 'wrong length in 3rd field') + self.assertEqual(checkGuid(guid), False, 'wrong length in 3rd field') # wrong length in a 4th field guid = '01234567-0123-0123-01234-0123456789AB' - self.assertEqual( checkGuid( guid ), False, 'wrong length in 4th field' ) + self.assertEqual(checkGuid(guid), False, 'wrong length in 4th field') guid = '01234567-0123-0123-012-0123456789AB' - self.assertEqual( checkGuid( guid ), False, 'wrong length in 4th field' ) + self.assertEqual(checkGuid(guid), False, 'wrong length in 4th field') # wrong length in a 5th field guid = '01234567-0123-0123-0123-0123-0123456789ABC' - self.assertEqual( checkGuid( guid ), False, 'wrong length in 5th field' ) + self.assertEqual(checkGuid(guid), False, 'wrong length in 5th field') guid = '01234567-0123-0123-0123-0123-0123456789A' - self.assertEqual( checkGuid( guid ), False, 'wrong length in 5th field' ) + self.assertEqual(checkGuid(guid), False, 'wrong length in 5th field') # small caps guid = '01234567-9ABC-0DEF-0123-456789ABCDEF'.lower() - self.assertEqual( checkGuid( guid ), True, "small caps in guid, zut!" ) + self.assertEqual(checkGuid(guid), True, "small caps in guid, zut!") # wrong characters not in [0-9A-F] guid = 'NEEDMORE-SPAM-SPAM-SPAM-SPAMWITHEGGS' - self.assertEqual( checkGuid( guid ), True, "wrong set of characters, zut!" ) + self.assertEqual(checkGuid(guid), True, "wrong set of characters, zut!") # normal operation guid = '01234567-9ABC-0DEF-0123-456789ABCDEF' - self.assertEqual( checkGuid( guid ), True, "proper GUID" ) + self.assertEqual(checkGuid(guid), True, "proper GUID") - def testMakeGuid( self ): + def testMakeGuid(self): """ makeGuid tests """ # no filename - fake guid produced - self.assertEqual( checkGuid( makeGuid() ), True , "fake guid for inexisting file" ) + self.assertEqual(checkGuid(makeGuid()), True, "fake guid for inexisting file") # using this python file - self.assertEqual( checkGuid( makeGuid( os.path.abspath(__file__) ) ), True, "guid for FileTestCase.py file" ) - - - def testGetSize( self ): + self.assertEqual( + checkGuid( + makeGuid( + os.path.abspath(__file__))), + True, + "guid for FileTestCase.py file") + + def testGetSize(self): """ getSize tests """ # non existing file - self.assertEqual( getSize("/spam/eggs/eggs"), -1, "inexisting file" ) + self.assertEqual(getSize("/spam/eggs/eggs"), -1, "inexisting file") # file unreadable - self.assertEqual( getSize('/root/.login'), -1 , "unreadable file") - + self.assertEqual(getSize('/root/.login'), -1, "unreadable file") - def testGetMD5ForFiles( self ): + def testGetMD5ForFiles(self): """ getMD5ForFiles tests """ - md5sum = getMD5ForFiles( self.filesList ) - reMD5 = re.compile( "^[0-9a-fA-F]+$" ) - self.assertTrue( reMD5.match( md5sum) != None ) + md5sum = getMD5ForFiles(self.filesList) + reMD5 = re.compile("^[0-9a-fA-F]+$") + self.assertTrue(reMD5.match(md5sum) is not None) # OK for python 2.7 # self.assertRegexpMatches( md5sum, reMD5, "regexp doesn't match" ) - def testGetCommonPath( self ): + def testGetCommonPath(self): """ getCommonPath tests """ # empty list - self.assertEqual( getCommonPath( [] ), "" ) + self.assertEqual(getCommonPath([]), "") # this folder - filesList = [ os.path.abspath(".") + os.sep + x for x in os.listdir(".") ] + self.filesList - self.assertEqual( getCommonPath( filesList ), os.path.abspath(".") ) + filesList = [os.path.abspath(".") + os.sep + x for x in os.listdir(".")] + self.filesList + self.assertEqual(getCommonPath(filesList), os.path.abspath(".")) + + +@given(nb=floats(allow_nan=False, allow_infinity=False, min_value=1)) +def test_convert_to_bigger_unit_floats(nb): + """ Make sure that converting to bigger unit gets the number smaller . + Also tests that two steps are equal to two consecutive steps + """ + toKB = convertSizeUnits(nb, 'B', 'kB') + toMB = convertSizeUnits(nb, 'B', 'MB') + fromkBtoMB = convertSizeUnits(toKB, 'kB', 'MB') + + assert toKB < nb + assert toMB < toKB + assert toMB == fromkBtoMB + + +def test_convert_error_to_maxint(): + """ Make sure that on error we receive -sys.maxint """ + assert convertSizeUnits('size', 'B', 'kB') == -sys.maxsize + assert convertSizeUnits(0, 'srcUnit', 'kB') == -sys.maxsize + assert convertSizeUnits(0, 'B', 'dstUnit') == -sys.maxsize + + +@given(nb=floats(allow_nan=False, allow_infinity=False, min_value=1)) +@parametrize('srcUnit', SIZE_UNIT_CONVERSION) +@parametrize('dstUnit', SIZE_UNIT_CONVERSION) +def test_convert_loop(nb, srcUnit, dstUnit): + """ Make sure that converting a size back and forth preserves the number """ + + converted = convertSizeUnits(convertSizeUnits(nb, srcUnit, dstUnit), dstUnit, srcUnit) + # We exclude the infinity case + if converted != float('Inf'): + assert converted == nb + # test case execution if __name__ == "__main__": TESTLOADER = unittest.TestLoader() - SUITE = TESTLOADER.loadTestsFromTestCase( FileTestCase ) - unittest.TextTestRunner(verbosity=3).run( SUITE ) + SUITE = TESTLOADER.loadTestsFromTestCase(FileTestCase) + unittest.TextTestRunner(verbosity=3).run(SUITE) diff --git a/DataManagementSystem/Service/StorageElementHandler.py b/DataManagementSystem/Service/StorageElementHandler.py index 51a5cf01e34..64897fbd700 100755 --- a/DataManagementSystem/Service/StorageElementHandler.py +++ b/DataManagementSystem/Service/StorageElementHandler.py @@ -16,6 +16,8 @@ removeDirectory() - remove on directory recursively removeFileList() - remove files in the list getAdminInfo() - get administration information about the SE status +getFreeDiskSpace() - get the free disk space +getTotalDiskSpace() - get the free total space The handler implements also the DISET data transfer calls toClient(), fromClient(), bulkToClient(), bulkFromClient @@ -50,32 +52,24 @@ MAX_STORAGE_SIZE = 0 USE_TOKENS = False -UNIT_CONVERSION = {"KB": 1024, "MB": 1024 * 1024, "GB": 1024 * 1024 * 1024, "TB": 1024 * 1024 * 1024 * 1024} - -def getDiskSpace(path, size='TB', total=False): +def getDiskSpace(path, total=False): """ - Returns disk usage of the given path. - If no size is specified, terabytes will be used by default. + Returns disk usage of the given path, in MB. If total is set to true, the total disk space will be returned instead. """ - size_to_convert = size.upper() - if size_to_convert not in UNIT_CONVERSION: - return S_ERROR("No valid size specified") - convert = UNIT_CONVERSION[size_to_convert] - try: st = os.statvfs(path) if total: # return total space - queried_size = st.f_blocks + queriedSize = st.f_blocks else: # return free space - queried_size = st.f_bavail + queriedSize = st.f_bavail - result = float(queried_size * st.f_frsize) / float(convert) + result = float(queriedSize * st.f_frsize) / float(1024 * 1024) except OSError as e: return S_ERROR(errno.EIO, "Error while getting the available disk space: %s" % repr(e)) @@ -94,7 +88,7 @@ def getTotalDiskSpace(): global BASE_PATH global MAX_STORAGE_SIZE - result = getDiskSpace(BASE_PATH, size='MB', total=True) + result = getDiskSpace(BASE_PATH, total=True) if not result['OK']: return result totalSpace = result['Value'] @@ -112,7 +106,7 @@ def getFreeDiskSpace(ignoreMaxStorageSize=True): global MAX_STORAGE_SIZE global BASE_PATH - result = getDiskSpace(BASE_PATH, 'MB') + result = getDiskSpace(BASE_PATH) if not result['OK']: return result freeSpace = result['Value'] diff --git a/ResourceStatusSystem/Agent/CacheFeederAgent.py b/ResourceStatusSystem/Agent/CacheFeederAgent.py index aa9b6b44d1a..ef850cb4638 100644 --- a/ResourceStatusSystem/Agent/CacheFeederAgent.py +++ b/ResourceStatusSystem/Agent/CacheFeederAgent.py @@ -50,7 +50,6 @@ def initialize(self): self.rmClient = ResourceManagementClient() self.commands['Downtime'] = [{'Downtime': {}}] - self.commands['SpaceTokenOccupancy'] = [{'SpaceTokenOccupancy': {}}] self.commands['GOCDBSync'] = [{'GOCDBSync': {}}] self.commands['FreeDiskSpace'] = [{'FreeDiskSpace': {}}] diff --git a/ResourceStatusSystem/Command/DowntimeCommand.py b/ResourceStatusSystem/Command/DowntimeCommand.py index da8aff9133e..7ff3aa9037b 100644 --- a/ResourceStatusSystem/Command/DowntimeCommand.py +++ b/ResourceStatusSystem/Command/DowntimeCommand.py @@ -74,7 +74,7 @@ def _cleanCommand(self, element, elementNames): currentDate = datetime.utcnow() - if len(uniformResult) == 0: + if not uniformResult: continue # get the list of all ongoing DTs from GocDB diff --git a/ResourceStatusSystem/Command/FreeDiskSpaceCommand.py b/ResourceStatusSystem/Command/FreeDiskSpaceCommand.py index 384bec6811b..a317678a2d7 100644 --- a/ResourceStatusSystem/Command/FreeDiskSpaceCommand.py +++ b/ResourceStatusSystem/Command/FreeDiskSpaceCommand.py @@ -1,113 +1,136 @@ ''' FreeDiskSpaceCommand + The Command gets the free space that is left in a Storage Element - The Command gets the free space that is left in a DIRAC Storage Element + Note: there are, still, many references to "space tokens", + for example ResourceManagementClient().selectSpaceTokenOccupancyCache(token=elementName) + This is for historical reasons, and shoud be fixed one day. + For the moment, when you see "token" or "space token" here, just read "StorageElement". ''' -from datetime import datetime -from DIRAC import S_OK, S_ERROR, gLogger -from DIRAC.ResourceStatusSystem.Command.Command import Command -from DIRAC.Core.DISET.RPCClient import RPCClient -from DIRAC.ResourceStatusSystem.Utilities import CSHelpers -from DIRAC.Resources.Storage.StorageElement import StorageElement -from DIRAC.ResourceStatusSystem.Client.ResourceManagementClient import ResourceManagementClient +__RCSID__ = '$Id:$' + +import sys + +from datetime import datetime -__RCSID__ = '$Id: $' +from DIRAC import S_OK, S_ERROR, gLogger +from DIRAC.Core.Utilities.File import convertSizeUnits +from DIRAC.ResourceStatusSystem.Command.Command import Command +from DIRAC.ResourceStatusSystem.Utilities import CSHelpers +from DIRAC.Resources.Storage.StorageElement import StorageElement +from DIRAC.ResourceStatusSystem.Client.ResourceManagementClient import ResourceManagementClient -class FreeDiskSpaceCommand( Command ): +class FreeDiskSpaceCommand(Command): ''' Uses diskSpace method to get the free space ''' - def __init__( self, args = None, clients = None ): + def __init__(self, args=None, clients=None): - super( FreeDiskSpaceCommand, self ).__init__( args, clients = clients ) + super(FreeDiskSpaceCommand, self).__init__(args, clients=clients) - self.rpc = None - self.rsClient = ResourceManagementClient() + self.rmClient = ResourceManagementClient() - def _prepareCommand( self ): + def _prepareCommand(self): ''' FreeDiskSpaceCommand requires one argument: - name : ''' if 'name' not in self.args: - return S_ERROR( '"name" not found in self.args' ) - elementName = self.args[ 'name' ] + return S_ERROR('"name" not found in self.args') + elementName = self.args['name'] + + # We keep TB as default as this is what was used (and will still be used) + # in the policy for "space tokens" ("real", "data" SEs) + unit = self.args.get('unit', 'TB') - return S_OK( elementName ) + return S_OK((elementName, unit)) - def doNew( self, masterParams = None ): + def doNew(self, masterParams=None): """ - Gets the total and the free disk space of a DIPS storage element that - is found in the CS and inserts the results in the SpaceTokenOccupancyCache table + Gets the parameters to run, either from the master method or from its + own arguments. + + Gets the total and the free disk space of a storage element + and inserts the results in the SpaceTokenOccupancyCache table of ResourceManagementDB database. + + The result is also returned to the caller, not only inserted. + What is inserted in the DB will be in MB, + what is returned will be in the specified unit. """ if masterParams is not None: - elementName = masterParams + elementName, unit = masterParams else: - elementName = self._prepareCommand() - if not elementName[ 'OK' ]: - return elementName - - se = StorageElement(elementName) + params = self._prepareCommand() + if not params['OK']: + return params + elementName, unit = params['Value'] - elementURL = se.getStorageParameters(protocol = "dips") - - if elementURL['OK']: - elementURL = se.getStorageParameters(protocol = "dips")['Value']['URLBase'] - else: - gLogger.verbose( "Not a DIPS storage element, skipping..." ) - return S_OK() + endpointResult = CSHelpers.getStorageElementEndpoint(elementName) + if not endpointResult['OK']: + return endpointResult - self.rpc = RPCClient( elementURL, timeout=120 ) + se = StorageElement(elementName) + occupancyResult = se.getOccupancy(unit=unit) + if not occupancyResult['OK']: + return occupancyResult + occupancy = occupancyResult['Value'] + free = occupancy['Free'] + total = occupancy['Total'] + + results = {'Endpoint': endpointResult['Value'], + 'Free': free, + 'Total': total, + 'ElementName': elementName} + result = self._storeCommand(results) + if not result['OK']: + return result - free = self.rpc.getFreeDiskSpace("/") + return S_OK({'Free': free, 'Total': total}) - if not free[ 'OK' ]: - return free - free = free['Value'] + def _storeCommand(self, results): + """ Here purely for extensibility + """ + return self.rmClient.addOrModifySpaceTokenOccupancyCache(endpoint=results['Endpoint'], + lastCheckTime=datetime.utcnow(), + free=results['Free'], + total=results['Total'], + token=results['ElementName']) - total = self.rpc.getTotalDiskSpace("/") + def doCache(self): + """ + This is a method that gets the element's details from the spaceTokenOccupancyCache DB table. + It will return a dictionary with th results, converted to "correct" unit. + """ - if not total[ 'OK' ]: - return total - total = total['Value'] + params = self._prepareCommand() + if not params['OK']: + return params + elementName, unit = params['Value'] - if free and free < 1: - free = 1 - if total and total < 1: - total = 1 + result = self.rmClient.selectSpaceTokenOccupancyCache(token=elementName) - result = self.rsClient.addOrModifySpaceTokenOccupancyCache( endpoint = elementURL, - lastCheckTime = datetime.utcnow(), - free = free, total = total, - token = elementName ) - if not result[ 'OK' ]: + if not result['OK']: return result - return S_OK() + # results are normally in 'MB' + free = result['Value'][0][3] + total = result['Value'][0][4] - def doCache( self ): - """ - This is a method that gets the element's details from the spaceTokenOccupancy cache. - """ - - elementName = self._prepareCommand() - if not elementName[ 'OK' ]: - return elementName + free = convertSizeUnits(free, 'MB', unit) + total = convertSizeUnits(total, 'MB', unit) - result = self.rsClient.selectSpaceTokenOccupancyCache(token = elementName) - - if not result[ 'OK' ]: - return result + if free == -sys.maxsize or total == -sys.maxsize: + return S_ERROR("No valid unit specified") - return S_OK( result ) + return S_OK({'Free': free, 'Total': total}) - def doMaster( self ): + def doMaster(self): """ This method calls the doNew method for each storage element that exists in the CS. @@ -116,9 +139,9 @@ def doMaster( self ): elements = CSHelpers.getStorageElements() for name in elements['Value']: - diskSpace = self.doNew( name ) - if not diskSpace[ 'OK' ]: - gLogger.error( "Unable to calculate free disk space", "name: %s" % name ) + diskSpace = self.doNew(name) + if not diskSpace['OK']: + gLogger.error("Unable to calculate free/total disk space", "name: %s" % name) continue return S_OK() diff --git a/ResourceStatusSystem/Command/SpaceTokenOccupancyCommand.py b/ResourceStatusSystem/Command/SpaceTokenOccupancyCommand.py deleted file mode 100644 index 4adedb1ab1c..00000000000 --- a/ResourceStatusSystem/Command/SpaceTokenOccupancyCommand.py +++ /dev/null @@ -1,193 +0,0 @@ -''' SpaceTokenOccupancyCommand - - The Command gets information of the SpaceTokenOccupancy from the lcg_utils - -''' - - -import lcg_util #pylint: disable=import-error - - -from DIRAC import S_OK, S_ERROR -from DIRAC.Core.Utilities.Subprocess import pythonCall -from DIRAC.ResourceStatusSystem.Command.Command import Command -from DIRAC.ResourceStatusSystem.Client.ResourceManagementClient import ResourceManagementClient -from DIRAC.ResourceStatusSystem.Utilities import CSHelpers - - -__RCSID__ = '$Id: $' - - -class SpaceTokenOccupancyCommand( Command ): - ''' - Uses lcg_util to query status of endpoint for a given token. - ''' - - - def __init__( self, args = None, clients = None ): - - super( SpaceTokenOccupancyCommand, self ).__init__( args, clients ) - - if 'ResourceManagementClient' in self.apis: - self.rmClient = self.apis[ 'ResourceManagementClient' ] - else: - self.rmClient = ResourceManagementClient() - - - def _storeCommand( self, results ): - ''' - Stores the results of doNew method on the database. - ''' - - for result in results: - - resQuery = self.rmClient.addOrModifySpaceTokenOccupancyCache( result[ 'Endpoint' ], - result[ 'Token' ], - result[ 'Total' ], - result[ 'Guaranteed' ], - result[ 'Free' ] ) - if not resQuery[ 'OK' ]: - return resQuery - - return S_OK() - - - def _prepareCommand( self ): - ''' - SpaceTokenOccupancy requires one argument: - - elementName : - - Given a (storage)elementName, we calculate its endpoint and spaceToken, - which are used to query the srm interface. - ''' - - if 'name' not in self.args: - return S_ERROR( '"name" not found in self.args' ) - elementName = self.args[ 'name' ] - - endpoint = CSHelpers.getStorageElementEndpoint( elementName ) - if not endpoint[ 'OK' ]: - return endpoint - endpoint = endpoint[ 'Value' ] - - spaceToken = CSHelpers.getSEToken( elementName ) - if not spaceToken[ 'OK' ]: - return spaceToken - spaceToken = spaceToken[ 'Value'] - - return S_OK( ( endpoint, spaceToken ) ) - - - def doNew( self, masterParams = None ): - ''' - Gets the parameters to run, either from the master method or from its - own arguments. - - It queries the srm interface, and hopefully it will not crash. Out of the - results, we keep totalsize, guaranteedsuze, and unusedsize. - - Then, they are recorded and returned. - ''' - - if masterParams is not None: - spaceTokenEndpoint, spaceToken = masterParams - else: - params = self._prepareCommand() - if not params[ 'OK' ]: - return params - spaceTokenEndpoint, spaceToken = params[ 'Value' ] - - # 10 secs of timeout. If it works, the reply is immediate. - occupancyResult = pythonCall( 10, lcg_util.lcg_stmd, spaceToken, spaceTokenEndpoint, True, 0 ) - if not occupancyResult[ 'OK' ]: - self.log.error( "Could not get spaceToken occupancy", "from endPoint/spaceToken %s/%s : %s" % \ - ( spaceTokenEndpoint, spaceToken, occupancyResult['Message'] ) ) - return occupancyResult - else: - occupancy = occupancyResult[ 'Value' ] - - # Timeout does not work here... - # occupancy = lcg_util.lcg_stmd( spaceToken, spaceTokenEndpoint, True, 0 ) - - if occupancy[ 0 ] != 0: - return S_ERROR( occupancy ) - output = occupancy[ 1 ][ 0 ] - - sTokenDict = {} - sTokenDict[ 'Endpoint' ] = spaceTokenEndpoint - sTokenDict[ 'Token' ] = spaceToken - sTokenDict[ 'Total' ] = float( output.get( 'totalsize', '0' ) ) / 1e12 # Bytes to Terabytes - sTokenDict[ 'Guaranteed' ] = float( output.get( 'guaranteedsize', '0' ) ) / 1e12 - sTokenDict[ 'Free' ] = float( output.get( 'unusedsize', '0' ) ) / 1e12 - - storeRes = self._storeCommand( [ sTokenDict ] ) - if not storeRes[ 'OK' ]: - return storeRes - - return S_OK( [ sTokenDict ] ) - - - def doCache( self ): - ''' - Method that reads the cache table and tries to read from it. It will - return a list of dictionaries if there are results. - ''' - - params = self._prepareCommand() - if not params[ 'OK' ]: - return params - spaceTokenEndpoint, spaceToken = params[ 'Value' ] - - result = self.rmClient.selectSpaceTokenOccupancyCache( spaceTokenEndpoint, spaceToken ) - if result[ 'OK' ]: - result = S_OK( [ dict( zip( result[ 'Columns' ], res ) ) for res in result[ 'Value' ] ] ) - - return result - - - def doMaster( self ): - ''' - Master method. Gets all endpoints from the storage elements and all - the spaceTokens. Could have taken from Shares/Disk as well. - It queries for all their possible combinations, unless there are records - in the database for those combinations, which then are not queried. - ''' - - self.log.verbose( "Getting all SEs defined in the CS" ) - storageElementNames = CSHelpers.getStorageElements() - if not storageElementNames[ 'OK' ]: - self.log.warn( storageElementNames['Message'] ) - return storageElementNames - storageElementNames = storageElementNames[ 'Value' ] - - endpointTokenSet = set() - - for storageElementName in storageElementNames: - - endpoint = CSHelpers.getStorageElementEndpoint( storageElementName ) - if not endpoint[ 'OK' ]: - self.log.warn( endpoint['Message'] ) - continue - endpoint = endpoint[ 'Value' ] - - spaceToken = CSHelpers.getSEToken( storageElementName ) - if not spaceToken[ 'OK' ]: - self.log.warn( spaceToken['Message'] ) - continue - spaceToken = spaceToken[ 'Value' ] - - endpointTokenSet.add( ( endpoint, spaceToken ) ) - - self.log.verbose( 'Processing %s' % endpointTokenSet ) - - for elementToQuery in endpointTokenSet: - - result = self.doNew( elementToQuery ) - if not result[ 'OK' ]: - self.metrics[ 'failed' ].append( result ) - - return S_OK( self.metrics ) - - -# ............................................................................... -# EOF diff --git a/ResourceStatusSystem/Policy/Configurations.py b/ResourceStatusSystem/Policy/Configurations.py index ec4abf3afea..9709fd3161d 100644 --- a/ResourceStatusSystem/Policy/Configurations.py +++ b/ResourceStatusSystem/Policy/Configurations.py @@ -1,3 +1,4 @@ + """ Configurations module Configuration to use policies. @@ -13,113 +14,129 @@ """ -__RCSID__ = '$Id: $' +__RCSID__ = '$Id: $' + +POLICIESMETA = { # DownTime POLICIES + 'DTOngoing': {'description': "Ongoing and scheduled down-times", + 'module': 'DowntimePolicy', + 'command': ('DowntimeCommand', 'DowntimeCommand'), + 'args': {'hours': 0, 'onlyCache': True}, }, -POLICIESMETA = { # DownTime POLICIES - 'DTOngoing': {'description' : "Ongoing and scheduled down-times", - 'module' : 'DowntimePolicy', - 'command' : ( 'DowntimeCommand', 'DowntimeCommand' ), - 'args' : { 'hours' : 0, 'onlyCache' : True }, }, + 'DTScheduled1': {'description': "Ongoing and scheduled down-times", + 'module': 'DowntimePolicy', + 'command': ('DowntimeCommand', 'DowntimeCommand'), + 'args': {'hours': 1, 'onlyCache': True}, }, - 'DTScheduled1': {'description' : "Ongoing and scheduled down-times", - 'module' : 'DowntimePolicy', - 'command' : ( 'DowntimeCommand', 'DowntimeCommand' ), - 'args' : { 'hours' : 1, 'onlyCache' : True }, }, + 'DTScheduled3': {'description': "Ongoing and scheduled down-times", + 'module': 'DowntimePolicy', + 'command': ('DowntimeCommand', 'DowntimeCommand'), + 'args': {'hours': 3, 'onlyCache': True}, }, - 'DTScheduled3': {'description' : "Ongoing and scheduled down-times", - 'module' : 'DowntimePolicy', - 'command' : ( 'DowntimeCommand', 'DowntimeCommand' ), - 'args' : { 'hours' : 3, 'onlyCache' : True }, }, + 'DTScheduled': {'description': "Scheduled down-times, starting in ", + 'module': 'DowntimePolicy', + 'command': ('DowntimeCommand', 'DowntimeCommand'), + 'args': {'hours': 12, 'onlyCache': True}, }, - 'DTScheduled': {'description' : "Scheduled down-times, starting in ", - 'module' : 'DowntimePolicy', - 'command' : ( 'DowntimeCommand', 'DowntimeCommand' ), - 'args' : { 'hours' : 12, 'onlyCache' : True }, }, + # Free Disk Space in Terabytes + 'FreeDiskSpaceTB': { + 'description': "Free disk space, in TB", + 'module': 'FreeDiskSpacePolicy', + 'command': ('FreeDiskSpaceCommand', 'FreeDiskSpaceCommand'), + 'args': {'unit': 'TB', 'onlyCache': True}, + }, + + # Free Disk Space in Gigabytes + 'FreeDiskSpaceGB': { + 'description': "Free disk space, in GB", + 'module': 'FreeDiskSpacePolicy', + 'command': ('FreeDiskSpaceCommand', 'FreeDiskSpaceCommand'), + 'args': {'unit': 'GB', 'onlyCache': True}, + }, - # Space Token POLICIES........................................................ - 'SpaceTokenOccupancy': { - 'description' : "Space token occupancy", - 'module' : 'SpaceTokenOccupancyPolicy', - 'command' : ( 'SpaceTokenOccupancyCommand', 'SpaceTokenOccupancyCommand' ), - 'args' : { 'onlyCache' : True }, + # Free Disk Space in Megabytes + 'FreeDiskSpaceMB': { + 'description': "Free disk space, in MB", + 'module': 'FreeDiskSpacePolicy', + 'command': ('FreeDiskSpaceCommand', 'FreeDiskSpaceCommand'), + 'args': {'unit': 'MB', 'onlyCache': True}, }, - # Job POLICIES.............................................................. + # Job POLICIES 'JobDoneRatio': { - 'description' : "done / ( completed + done ) jobs ( 30 min )", - 'module' : 'JobDoneRatioPolicy', - 'command' : ( 'JobCommand', 'JobCommand' ), - 'args' : { 'onlyCache' : True, 'timespan' : 1800 }, + 'description': "done / ( completed + done ) jobs ( 30 min )", + 'module': 'JobDoneRatioPolicy', + 'command': ('JobCommand', 'JobCommand'), + 'args': {'onlyCache': True, 'timespan': 1800}, }, 'JobEfficiency': { - 'description' : "( completed + done ) / ( completed + done + failed ) jobs ( 30 min )", - 'module' : 'JobEfficiencyPolicy', - 'command' : ( 'JobCommand', 'JobCommand' ), - 'args' : { 'onlyCache' : True, 'timespan' : 1800 }, + 'description': "( completed + done ) / ( completed + done + failed ) jobs ( 30 min )", + 'module': 'JobEfficiencyPolicy', + 'command': ('JobCommand', 'JobCommand'), + 'args': {'onlyCache': True, 'timespan': 1800}, }, 'JobRunningMatchedRatio': { - 'description' : "running / ( running + matched + received + checking ) jobs ( 30 min )", - 'module' : 'JobRunningMatchedRatioPolicy', - 'command' : ( 'JobCommand', 'JobCommand' ), - 'args' : { 'onlyCache' : True, 'timespan' : 1800 }, + 'description': "running / ( running + matched + received + checking ) jobs ( 30 min )", + 'module': 'JobRunningMatchedRatioPolicy', + 'command': ('JobCommand', 'JobCommand'), + 'args': {'onlyCache': True, 'timespan': 1800}, }, 'JobRunningWaitingRatio': { - 'description' : "running / ( running + waiting + staging ) jobs ( 30 min )", - 'module' : 'JobRunningWaitingRatioPolicy', - 'command' : ( 'JobCommand', 'JobCommand' ), - 'args' : { 'onlyCache' : True, 'timespan' : 1800 }, + 'description': "running / ( running + waiting + staging ) jobs ( 30 min )", + 'module': 'JobRunningWaitingRatioPolicy', + 'command': ('JobCommand', 'JobCommand'), + 'args': {'onlyCache': True, 'timespan': 1800}, }, # Pilot POLICIES.............................................................. 'PilotInstantEfficiency': { - 'description' : "Pilots Instant Efficiency ( 30 min )", - 'module' : 'PilotEfficiencyPolicy', - 'command' : ( 'PilotCommand', 'PilotCommand' ), - 'args' : { 'onlyCache' : True, 'timespan' : 1800 } + 'description': "Pilots Instant Efficiency ( 30 min )", + 'module': 'PilotEfficiencyPolicy', + 'command': ('PilotCommand', 'PilotCommand'), + 'args': {'onlyCache': True, 'timespan': 1800} }, # Site status propagation POLICIES.............................................................. 'PropagationPolicy': { - 'description' : "Site status propagation", - 'module' : 'PropagationPolicy', - 'command' : ( 'PropagationCommand', 'PropagationCommand' ), - 'args' : { 'onlyCache' : True, 'timespan' : 1800 } + 'description': "Site status propagation", + 'module': 'PropagationPolicy', + 'command': ('PropagationCommand', 'PropagationCommand'), + 'args': {'onlyCache': True, 'timespan': 1800} }, # ALWAYS SOMETHING POLICIES................................................... 'AlwaysActive': { - 'description' : "A Policy that always returns Active", - 'module' : 'AlwaysActivePolicy', - 'command' : None, - 'args' : None + 'description': "A Policy that always returns Active", + 'module': 'AlwaysActivePolicy', + 'command': None, + 'args': None }, 'AlwaysDegraded': { - 'description' : "A Policy that always returns Degraded", - 'module' : 'AlwaysDegradedPolicy', - 'command' : None, - 'args' : None + 'description': "A Policy that always returns Degraded", + 'module': 'AlwaysDegradedPolicy', + 'command': None, + 'args': None }, 'AlwaysProbing': { - 'description' : "A Policy that always returns Probing", - 'module' : 'AlwaysProbingPolicy', - 'command' : None, - 'args' : None + 'description': "A Policy that always returns Probing", + 'module': 'AlwaysProbingPolicy', + 'command': None, + 'args': None }, 'AlwaysBanned': { - 'description' : "A Policy that always returns Banned", - 'module' : 'AlwaysBannedPolicy', - 'command' : None, - 'args' : None + 'description': "A Policy that always returns Banned", + 'module': 'AlwaysBannedPolicy', + 'command': None, + 'args': None } } diff --git a/ResourceStatusSystem/Policy/SpaceTokenOccupancyPolicy.py b/ResourceStatusSystem/Policy/FreeDiskSpacePolicy.py similarity index 64% rename from ResourceStatusSystem/Policy/SpaceTokenOccupancyPolicy.py rename to ResourceStatusSystem/Policy/FreeDiskSpacePolicy.py index 0fceb412150..ce077e9708a 100644 --- a/ResourceStatusSystem/Policy/SpaceTokenOccupancyPolicy.py +++ b/ResourceStatusSystem/Policy/FreeDiskSpacePolicy.py @@ -1,6 +1,6 @@ -""" SpaceTokenOccupancyPolicy +""" FreeDiskSpacePolicy - SpaceTokenOccupancyPolicy.__bases__: + FreeDiskSpacePolicy.__bases__: DIRAC.ResourceStatusSystem.PolicySystem.PolicyBase.PolicyBase """ @@ -11,23 +11,23 @@ from DIRAC.ResourceStatusSystem.PolicySystem.PolicyBase import PolicyBase -class SpaceTokenOccupancyPolicy(PolicyBase): +class FreeDiskSpacePolicy(PolicyBase): """ - The SpaceTokenOccupancyPolicy class is a policy class satisfied when a SE has a + The FreeDiskSpacePolicy class is a policy class satisfied when a SE has a low occupancy. - SpaceTokenOccupancyPolicy, given the space left at the element, proposes a new status. + FreeDiskSpacePolicy, given the space left at the element, proposes a new status. """ @staticmethod def _evaluate(commandResult): """ - Evaluate policy on SE occupancy: Use SpaceTokenOccupancyCommand + Evaluate policy on SE occupancy: Use FreeDiskSpaceCommand :Parameters: **commandResult** - S_OK / S_ERROR result of the command. It is expected ( iff S_OK ) a dictionary like - { 'Total' : .., 'Free' : .., 'Guaranteed': .. } + { 'Total' : .., 'Free' : ..} :return: { @@ -52,25 +52,26 @@ def _evaluate(commandResult): commandResult = commandResult[0] - for key in ['Total', 'Free', 'Guaranteed']: + for key in ['Total', 'Free']: - if key not in commandResult.keys(): + if key not in commandResult: result['Status'] = 'Error' - result['Reason'] = 'Key %s missing' % key.lower() + result['Reason'] = 'Key %s missing' % key return S_OK(result) free = float(commandResult['Free']) - # Units are TB ! ( 0.01 == 10 GB ) + # Units (TB, GB, MB) may change, + # depending on the configuration of the command in Configurations.py if free < 0.1: result['Status'] = 'Banned' - result['Reason'] = 'Free space < 100GB' + result['Reason'] = 'Too little free space' elif free < 5: result['Status'] = 'Degraded' - result['Reason'] = 'Free space < 5TB' + result['Reason'] = 'Little free space' else: result['Status'] = 'Active' - result['Reason'] = 'Free space > 5TB' + result['Reason'] = 'Enough free space' return S_OK(result) diff --git a/ResourceStatusSystem/Policy/test/Test_RSS_Policy_FreeDiskSpacePolicy.py b/ResourceStatusSystem/Policy/test/Test_RSS_Policy_FreeDiskSpacePolicy.py new file mode 100644 index 00000000000..1b86287e65d --- /dev/null +++ b/ResourceStatusSystem/Policy/test/Test_RSS_Policy_FreeDiskSpacePolicy.py @@ -0,0 +1,104 @@ +''' Test_RSS_Policy_FreeDiskSpacePolicy +''' + +import unittest + +import DIRAC.ResourceStatusSystem.Policy.FreeDiskSpacePolicy as moduleTested + +################################################################################ + + +class FreeDiskSpacePolicy_TestCase(unittest.TestCase): + + def setUp(self): + ''' + Setup + ''' + + self.moduleTested = moduleTested + self.testClass = self.moduleTested.FreeDiskSpacePolicy + + def tearDown(self): + ''' + Tear down + ''' + + del self.moduleTested + del self.testClass + +################################################################################ + + +class FreeDiskSpacePolicy_Success(FreeDiskSpacePolicy_TestCase): + + def test_instantiate(self): + ''' tests that we can instantiate one object of the tested class + ''' + + module = self.testClass() + self.assertEqual('FreeDiskSpacePolicy', module.__class__.__name__) + + def test_evaluate(self): + ''' tests the method _evaluate + ''' + + module = self.testClass() + + res = module._evaluate({'OK': False, 'Message': 'Bo!'}) + self.assertTrue(res['OK']) + self.assertEqual('Error', res['Value']['Status']) + self.assertEqual('Bo!', res['Value']['Reason']) + + res = module._evaluate({'OK': True, 'Value': None}) + self.assertTrue(res['OK']) + self.assertEqual('Unknown', res['Value']['Status']) + self.assertEqual('No values to take a decision', res['Value']['Reason']) + + res = module._evaluate({'OK': True, 'Value': []}) + self.assertTrue(res['OK']) + self.assertEqual('Unknown', res['Value']['Status']) + self.assertEqual('No values to take a decision', res['Value']['Reason']) + + res = module._evaluate({'OK': True, 'Value': [{'A': 1}]}) + self.assertTrue(res['OK']) + self.assertEqual('Error', res['Value']['Status']) + self.assertEqual('Key Total missing', res['Value']['Reason']) + + res = module._evaluate({'OK': True, 'Value': [{'Total': 1}]}) + self.assertTrue(res['OK']) + self.assertEqual('Error', res['Value']['Status']) + self.assertEqual('Key Free missing', res['Value']['Reason']) + + res = module._evaluate({'OK': True, 'Value': [{'Total': 100, 'Fre': 0.0}]}) + self.assertTrue(res['OK']) + self.assertEqual('Error', res['Value']['Status']) + self.assertEqual('Key Free missing', res['Value']['Reason']) + + res = module._evaluate({'OK': True, 'Value': [{'Total': 100, 'Free': 0.0}]}) + self.assertTrue(res['OK']) + self.assertEqual('Banned', res['Value']['Status']) + self.assertEqual('Too little free space', res['Value']['Reason']) + + res = module._evaluate({'OK': True, 'Value': [{'Total': 100, 'Free': 4.0, + 'Guaranteed': 1}]}) + self.assertTrue(res['OK']) + self.assertEqual('Degraded', res['Value']['Status']) + self.assertEqual('Little free space', + res['Value']['Reason']) + + res = module._evaluate({'OK': True, 'Value': [{'Total': 100, 'Free': 100, + 'Guaranteed': 1}]}) + self.assertTrue(res['OK']) + self.assertEqual('Active', res['Value']['Status']) + self.assertEqual('Enough free space', + res['Value']['Reason']) + + +################################################################################ + +if __name__ == '__main__': + suite = unittest.defaultTestLoader.loadTestsFromTestCase(FreeDiskSpacePolicy_TestCase) + suite.addTest(unittest.defaultTestLoader.loadTestsFromTestCase(FreeDiskSpacePolicy_Success)) + testResult = unittest.TextTestRunner(verbosity=2).run(suite) + +# EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF diff --git a/ResourceStatusSystem/Policy/test/Test_RSS_Policy_SpaceTokenOccupancyPolicy.py b/ResourceStatusSystem/Policy/test/Test_RSS_Policy_SpaceTokenOccupancyPolicy.py deleted file mode 100644 index 2ae74a68468..00000000000 --- a/ResourceStatusSystem/Policy/test/Test_RSS_Policy_SpaceTokenOccupancyPolicy.py +++ /dev/null @@ -1,103 +0,0 @@ -''' Test_RSS_Policy_SpaceTokenOccupancyPolicy -''' - -import unittest - -import DIRAC.ResourceStatusSystem.Policy.SpaceTokenOccupancyPolicy as moduleTested - -################################################################################ - -class SpaceTokenOccupancyPolicy_TestCase( unittest.TestCase ): - - def setUp( self ): - ''' - Setup - ''' - - self.moduleTested = moduleTested - self.testClass = self.moduleTested.SpaceTokenOccupancyPolicy - - def tearDown( self ): - ''' - Tear down - ''' - - del self.moduleTested - del self.testClass - -################################################################################ - -class SpaceTokenOccupancyPolicy_Success( SpaceTokenOccupancyPolicy_TestCase ): - - def test_instantiate( self ): - ''' tests that we can instantiate one object of the tested class - ''' - - module = self.testClass() - self.assertEqual( 'SpaceTokenOccupancyPolicy', module.__class__.__name__ ) - - def test_evaluate( self ): - ''' tests the method _evaluate - ''' - - module = self.testClass() - - res = module._evaluate( { 'OK' : False, 'Message' : 'Bo!' } ) - self.assertTrue(res['OK']) - self.assertEqual( 'Error', res[ 'Value' ][ 'Status' ] ) - self.assertEqual( 'Bo!', res[ 'Value' ][ 'Reason' ] ) - - res = module._evaluate( { 'OK' : True, 'Value' : None } ) - self.assertTrue(res['OK']) - self.assertEqual( 'Unknown', res[ 'Value' ][ 'Status' ] ) - self.assertEqual( 'No values to take a decision', res[ 'Value' ][ 'Reason' ] ) - - res = module._evaluate( { 'OK' : True, 'Value' : [] } ) - self.assertTrue(res['OK']) - self.assertEqual( 'Unknown', res[ 'Value' ][ 'Status' ] ) - self.assertEqual( 'No values to take a decision', res[ 'Value' ][ 'Reason' ] ) - - res = module._evaluate( { 'OK' : True, 'Value' : [{ 'A' : 1 }] } ) - self.assertTrue(res['OK']) - self.assertEqual( 'Error', res[ 'Value' ][ 'Status' ] ) - self.assertEqual( 'Key total missing', res[ 'Value' ][ 'Reason' ] ) - - res = module._evaluate( { 'OK' : True, 'Value' : [{ 'Total' : 1 }] } ) - self.assertTrue(res['OK']) - self.assertEqual( 'Error', res[ 'Value' ][ 'Status' ] ) - self.assertEqual( 'Key free missing', res[ 'Value' ][ 'Reason' ] ) - - res = module._evaluate( { 'OK' : True, 'Value' : [{'Total' : 100, 'Free' : 0.0 }] } ) - self.assertTrue(res['OK']) - self.assertEqual( 'Error', res[ 'Value' ][ 'Status' ] ) - self.assertEqual( 'Key guaranteed missing', res[ 'Value' ][ 'Reason' ] ) - - res = module._evaluate( { 'OK' : True, 'Value' : [{'Total' : 100, 'Free' : 0.0, - 'Guaranteed' : 1 }] } ) - self.assertTrue(res['OK']) - self.assertEqual( 'Banned', res[ 'Value' ][ 'Status' ] ) - self.assertEqual( 'Free space < 100GB', res[ 'Value' ][ 'Reason' ] ) - - res = module._evaluate( { 'OK' : True, 'Value' : [{'Total' : 100, 'Free' : 4.0, - 'Guaranteed' : 1 }] } ) - self.assertTrue(res['OK']) - self.assertEqual( 'Degraded', res[ 'Value' ][ 'Status' ] ) - self.assertEqual( 'Free space < 5TB', - res[ 'Value' ][ 'Reason' ] ) - - res = module._evaluate( { 'OK' : True, 'Value' : [{'Total' : 100, 'Free' : 100, - 'Guaranteed' : 1 }] } ) - self.assertTrue(res['OK']) - self.assertEqual( 'Active', res[ 'Value' ][ 'Status' ] ) - self.assertEqual( 'Free space > 5TB', - res[ 'Value' ][ 'Reason' ] ) - - -################################################################################ - -if __name__ == '__main__': - suite = unittest.defaultTestLoader.loadTestsFromTestCase( SpaceTokenOccupancyPolicy_TestCase ) - suite.addTest( unittest.defaultTestLoader.loadTestsFromTestCase( SpaceTokenOccupancyPolicy_Success ) ) - testResult = unittest.TextTestRunner( verbosity = 2 ).run( suite ) - -#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF \ No newline at end of file diff --git a/Resources/Storage/DIPStorage.py b/Resources/Storage/DIPStorage.py index 98b9ae45765..b0c3318db81 100755 --- a/Resources/Storage/DIPStorage.py +++ b/Resources/Storage/DIPStorage.py @@ -16,48 +16,47 @@ import os import random -from DIRAC import gLogger, S_OK, S_ERROR -from DIRAC.Resources.Storage.Utilities import checkArgumentFormat -from DIRAC.Resources.Storage.StorageBase import StorageBase -from DIRAC.Core.Utilities.Pfn import pfnparse, pfnunparse -from DIRAC.Core.DISET.TransferClient import TransferClient -from DIRAC.Core.DISET.RPCClient import RPCClient -from DIRAC.Core.Utilities.File import getSize +from DIRAC import gLogger, S_OK, S_ERROR +from DIRAC.Resources.Storage.Utilities import checkArgumentFormat +from DIRAC.Resources.Storage.StorageBase import StorageBase +from DIRAC.Core.Utilities.Pfn import pfnparse, pfnunparse +from DIRAC.Core.DISET.TransferClient import TransferClient +from DIRAC.Core.DISET.RPCClient import RPCClient +from DIRAC.Core.Utilities.File import getSize -class DIPStorage( StorageBase ): +class DIPStorage(StorageBase): _INPUT_PROTOCOLS = ['file', 'dip', 'dips'] _OUTPUT_PROTOCOLS = ['dip', 'dips'] - - def __init__( self, storageName, parameters ): + def __init__(self, storageName, parameters): """ """ - StorageBase.__init__( self, storageName, parameters ) + StorageBase.__init__(self, storageName, parameters) self.pluginName = 'DIP' - self.log = gLogger.getSubLogger( "DIPStorage", True ) + self.log = gLogger.getSubLogger("DIPStorage", True) # Several ports can be specified as comma separated list, choose # randomly one of those ports - ports = self.protocolParameters['Port'].split( ',' ) - random.shuffle( ports ) + ports = self.protocolParameters['Port'].split(',') + random.shuffle(ports) self.protocolParameters['Port'] = ports[0] - pathDict = dict( self.protocolParameters ) + pathDict = dict(self.protocolParameters) pathDict['Path'] = self.basePath - result = pfnunparse( pathDict ) + result = pfnunparse(pathDict) if result['OK']: self.url = result['Value'] self.checkSum = "CheckSum" self.isok = True - def setParameters( self, parameters ): + def setParameters(self, parameters): """ Applying extra storage parameters """ - StorageBase.setParameters( self, parameters ) + StorageBase.setParameters(self, parameters) if "CheckSum" in parameters and parameters['CheckSum'].lower() in ['0', 'no', 'false', 'off']: self.checkSum = "NoCheckSum" return S_OK() @@ -67,187 +66,191 @@ def setParameters( self, parameters ): # These are the methods for file manipulation # - def exists( self, path ): + def exists(self, path): """ Check if the given path exists. The 'path' variable can be a string or a list of strings. """ - res = checkArgumentFormat( path ) + res = checkArgumentFormat(path) if not res['OK']: return res urls = res['Value'] successful = {} failed = {} - serviceClient = RPCClient( self.url ) + serviceClient = RPCClient(self.url) for url in urls: - gLogger.debug( "DIPStorage.exists: Determining existence of %s." % url ) - res = serviceClient.exists( url ) + gLogger.debug("DIPStorage.exists: Determining existence of %s." % url) + res = serviceClient.exists(url) if res['OK']: successful[url] = res['Value'] else: failed[url] = res['Message'] - resDict = {'Failed':failed, 'Successful':successful} - return S_OK( resDict ) + resDict = {'Failed': failed, 'Successful': successful} + return S_OK(resDict) - def putFile( self, path, sourceSize = 0 ): + def putFile(self, path, sourceSize=0): """Put a file to the physical storage """ - res = checkArgumentFormat( path ) + res = checkArgumentFormat(path) if not res['OK']: return res urls = res['Value'] successful = {} failed = {} for dest_url, src_file in urls.items(): - gLogger.debug( "DIPStorage.putFile: Executing transfer of %s to %s" % ( src_file, dest_url ) ) - res = self.__putFile( src_file, dest_url ) + gLogger.debug("DIPStorage.putFile: Executing transfer of %s to %s" % (src_file, dest_url)) + res = self.__putFile(src_file, dest_url) if res['OK']: successful[dest_url] = res['Value'] else: failed[dest_url] = res['Message'] - resDict = {'Failed':failed, 'Successful':successful} - return S_OK( resDict ) + resDict = {'Failed': failed, 'Successful': successful} + return S_OK(resDict) - def __putFile( self, src_file, dest_url ): - res = pfnparse( src_file ) + def __putFile(self, src_file, dest_url): + res = pfnparse(src_file) if not res['OK']: return res localCache = False srcDict = res['Value'] if srcDict['Protocol'] in ['dips', 'dip']: # Make the service URL from the file URL by stripping off the file part - serviceDict = dict( srcDict ) - serviceDict['Path'] = '/'.join( srcDict['Path'].split('/')[:3] ) + serviceDict = dict(srcDict) + serviceDict['Path'] = '/'.join(srcDict['Path'].split('/')[:3]) serviceDict['FileName'] = '' - res = pfnunparse( serviceDict ) + res = pfnunparse(serviceDict) if not res['OK']: return res srcSEURL = res['Value'] localCache = True - transferClient = TransferClient( srcSEURL ) - res = transferClient.receiveFile( srcDict['FileName'], os.path.join( srcDict['Path'], srcDict['FileName'] ) ) + transferClient = TransferClient(srcSEURL) + res = transferClient.receiveFile( + srcDict['FileName'], os.path.join( + srcDict['Path'], srcDict['FileName'])) if not res['OK']: return res src_file = srcDict['FileName'] - if not os.path.exists( src_file ): + if not os.path.exists(src_file): errStr = "DIPStorage.__putFile: The source local file does not exist." - gLogger.error( errStr, src_file ) - return S_ERROR( errStr ) - sourceSize = getSize( src_file ) + gLogger.error(errStr, src_file) + return S_ERROR(errStr) + sourceSize = getSize(src_file) if sourceSize == -1: errStr = "DIPStorage.__putFile: Failed to get file size." - gLogger.error( errStr, src_file ) - return S_ERROR( errStr ) - transferClient = TransferClient( self.url ) - res = transferClient.sendFile( src_file, dest_url, token = self.checkSum ) + gLogger.error(errStr, src_file) + return S_ERROR(errStr) + transferClient = TransferClient(self.url) + res = transferClient.sendFile(src_file, dest_url, token=self.checkSum) if localCache: - os.unlink( src_file ) + os.unlink(src_file) if res['OK']: - return S_OK( sourceSize ) + return S_OK(sourceSize) else: return res - def getFile( self, path, localPath = False ): + def getFile(self, path, localPath=False): """Get a local copy in the current directory of a physical file specified by its path """ - res = checkArgumentFormat( path ) + res = checkArgumentFormat(path) if not res['OK']: return res urls = res['Value'] successful = {} failed = {} for src_url in urls: - fileName = os.path.basename( src_url ) + fileName = os.path.basename(src_url) if localPath: - dest_file = "%s/%s" % ( localPath, fileName ) + dest_file = "%s/%s" % (localPath, fileName) else: - dest_file = "%s/%s" % ( os.getcwd(), fileName ) - gLogger.debug( "DIPStorage.getFile: Executing transfer of %s to %s" % ( src_url, dest_file ) ) - res = self.__getFile( src_url, dest_file ) + dest_file = "%s/%s" % (os.getcwd(), fileName) + gLogger.debug("DIPStorage.getFile: Executing transfer of %s to %s" % (src_url, dest_file)) + res = self.__getFile(src_url, dest_file) if res['OK']: successful[src_url] = res['Value'] else: failed[src_url] = res['Message'] - resDict = {'Failed':failed, 'Successful':successful} - return S_OK( resDict ) + resDict = {'Failed': failed, 'Successful': successful} + return S_OK(resDict) - def __getFile( self, src_url, dest_file ): - transferClient = TransferClient( self.url ) - res = transferClient.receiveFile( dest_file, src_url, token = self.checkSum ) + def __getFile(self, src_url, dest_file): + transferClient = TransferClient(self.url) + res = transferClient.receiveFile(dest_file, src_url, token=self.checkSum) if not res['OK']: return res - if not os.path.exists( dest_file ): + if not os.path.exists(dest_file): errStr = "DIPStorage.__getFile: The destination local file does not exist." - gLogger.error( errStr, dest_file ) - return S_ERROR( errStr ) - destSize = getSize( dest_file ) + gLogger.error(errStr, dest_file) + return S_ERROR(errStr) + destSize = getSize(dest_file) if destSize == -1: errStr = "DIPStorage.__getFile: Failed to get the local file size." - gLogger.error( errStr, dest_file ) - return S_ERROR( errStr ) - return S_OK( destSize ) + gLogger.error(errStr, dest_file) + return S_ERROR(errStr) + return S_OK(destSize) - def removeFile( self, path ): + def removeFile(self, path): """Remove physically the file specified by its path """ - res = checkArgumentFormat( path ) + res = checkArgumentFormat(path) if not res['OK']: return res urls = res['Value'] - if not len( urls ) > 0: - return S_ERROR( "DIPStorage.removeFile: No surls supplied." ) + if not len(urls) > 0: + return S_ERROR("DIPStorage.removeFile: No surls supplied.") successful = {} failed = {} - serviceClient = RPCClient( self.url ) + serviceClient = RPCClient(self.url) for url in urls: - gLogger.debug( "DIPStorage.removeFile: Attempting to remove %s." % url ) - res = serviceClient.remove( url, '' ) + gLogger.debug("DIPStorage.removeFile: Attempting to remove %s." % url) + res = serviceClient.remove(url, '') if res['OK']: successful[url] = True else: failed[url] = res['Message'] - resDict = {'Failed':failed, 'Successful':successful} - return S_OK( resDict ) + resDict = {'Failed': failed, 'Successful': successful} + return S_OK(resDict) - def isFile( self, path ): + def isFile(self, path): """ Determine whether the path is a directory """ - res = checkArgumentFormat( path ) + res = checkArgumentFormat(path) if not res['OK']: return res urls = res['Value'] successful = {} failed = {} - gLogger.debug( "DIPStorage.isFile: Attempting to determine whether %s paths are files." % len( urls ) ) - serviceClient = RPCClient( self.url ) + gLogger.debug( + "DIPStorage.isFile: Attempting to determine whether %s paths are files." % + len(urls)) + serviceClient = RPCClient(self.url) for url in urls: - res = serviceClient.getMetadata( url ) + res = serviceClient.getMetadata(url) if res['OK']: if res['Value']['Exists']: if res['Value']['Type'] == 'File': - gLogger.debug( "DIPStorage.isFile: Successfully obtained metadata for %s." % url ) + gLogger.debug("DIPStorage.isFile: Successfully obtained metadata for %s." % url) successful[url] = True else: successful[url] = False else: failed[url] = 'File does not exist' else: - gLogger.error( "DIPStorage.isFile: Failed to get metadata for url", - "%s: %s" % ( url, res['Message'] ) ) + gLogger.error("DIPStorage.isFile: Failed to get metadata for url", + "%s: %s" % (url, res['Message'])) failed[url] = res['Message'] - resDict = {'Failed':failed, 'Successful':successful} - return S_OK( resDict ) + resDict = {'Failed': failed, 'Successful': successful} + return S_OK(resDict) - def getFileSize( self, path ): + def getFileSize(self, path): """ Get size of supplied files """ - res = checkArgumentFormat( path ) + res = checkArgumentFormat(path) if not res['OK']: return res urls = res['Value'] successful = {} failed = {} - gLogger.debug( "DIPStorage.getFileSize: Attempting to obtain size for %s files." % len( urls ) ) - res = self.getFileMetadata( urls ) + gLogger.debug("DIPStorage.getFileSize: Attempting to obtain size for %s files." % len(urls)) + res = self.getFileMetadata(urls) if not res['OK']: return res for url, urlDict in res['Value']['Successful'].items(): @@ -257,59 +260,63 @@ def getFileSize( self, path ): failed[url] = 'File does not exist' for url, error in res['Value']['Failed'].items(): failed[url] = error - resDict = {'Failed':failed, 'Successful':successful} - return S_OK( resDict ) + resDict = {'Failed': failed, 'Successful': successful} + return S_OK(resDict) - def getFileMetadata( self, path ): + def getFileMetadata(self, path): """ Get metadata associated to the file """ - res = checkArgumentFormat( path ) + res = checkArgumentFormat(path) if not res['OK']: return res urls = res['Value'] successful = {} failed = {} - gLogger.debug( "DIPStorage.getFileMetadata: Attempting to obtain metadata for %s files." % len( urls ) ) - serviceClient = RPCClient( self.url ) + gLogger.debug( + "DIPStorage.getFileMetadata: Attempting to obtain metadata for %s files." % + len(urls)) + serviceClient = RPCClient(self.url) for url in urls: pfn = url - if url.find( self.url ) == 0: - pfn = url[ ( len( self.url ) ):] - res = serviceClient.getMetadata( pfn ) + if url.find(self.url) == 0: + pfn = url[(len(self.url)):] + res = serviceClient.getMetadata(pfn) if res['OK']: if res['Value']['Exists']: if res['Value']['Type'] == 'File': - gLogger.debug( "DIPStorage.getFileMetadata: Successfully obtained metadata for %s." % url ) + gLogger.debug( + "DIPStorage.getFileMetadata: Successfully obtained metadata for %s." % + url) successful[url] = res['Value'] else: failed[url] = 'Supplied path is not a file' else: failed[url] = 'File does not exist' else: - gLogger.error( "DIPStorage.getFileMetadata: Failed to get metadata for url", - "%s: %s" % ( url, res['Message'] ) ) + gLogger.error("DIPStorage.getFileMetadata: Failed to get metadata for url", + "%s: %s" % (url, res['Message'])) failed[url] = res['Message'] - resDict = {'Failed':failed, 'Successful':successful} - return S_OK( resDict ) + resDict = {'Failed': failed, 'Successful': successful} + return S_OK(resDict) ############################################################# # # These are the methods for directory manipulation # - def listDirectory( self, path ): + def listDirectory(self, path): """ List the contents of the directory """ - res = checkArgumentFormat( path ) + res = checkArgumentFormat(path) if not res['OK']: return res urls = res['Value'] successful = {} failed = {} - gLogger.debug( "DIPStorage.listDirectory: Attempting to list %s directories." % len( urls ) ) - serviceClient = RPCClient( self.url ) + gLogger.debug("DIPStorage.listDirectory: Attempting to list %s directories." % len(urls)) + serviceClient = RPCClient(self.url) for url in urls: - res = serviceClient.listDirectory( url, 'l' ) + res = serviceClient.listDirectory(url, 'l') if not res['OK']: failed[url] = res['Message'] else: @@ -323,180 +330,217 @@ def listDirectory( self, path ): successful[url] = {} successful[url]['SubDirs'] = subDirs successful[url]['Files'] = files - resDict = {'Failed':failed, 'Successful':successful} - return S_OK( resDict ) + resDict = {'Failed': failed, 'Successful': successful} + return S_OK(resDict) - def isDirectory( self, path ): + def isDirectory(self, path): """ Determine whether the path is a directory """ - res = checkArgumentFormat( path ) + res = checkArgumentFormat(path) if not res['OK']: return res urls = res['Value'] successful = {} failed = {} - gLogger.debug( "DIPStorage.isDirectory: Attempting to determine whether %s paths are directories." % len( urls ) ) - serviceClient = RPCClient( self.url ) + gLogger.debug( + "DIPStorage.isDirectory: Attempting to determine whether %s paths are directories." % + len(urls)) + serviceClient = RPCClient(self.url) for url in urls: - res = serviceClient.getMetadata( url ) + res = serviceClient.getMetadata(url) if res['OK']: if res['Value']['Exists']: if res['Value']['Type'] == 'Directory': - gLogger.debug( "DIPStorage.isDirectory: Successfully obtained metadata for %s." % url ) + gLogger.debug("DIPStorage.isDirectory: Successfully obtained metadata for %s." % url) successful[url] = True else: successful[url] = False else: failed[url] = 'Path does not exist' else: - gLogger.error( "DIPStorage.isDirectory: Failed to get metadata for url", - "%s: %s" % ( url, res['Message'] ) ) + gLogger.error("DIPStorage.isDirectory: Failed to get metadata for url", + "%s: %s" % (url, res['Message'])) failed[url] = res['Message'] - resDict = {'Failed':failed, 'Successful':successful} - return S_OK( resDict ) + resDict = {'Failed': failed, 'Successful': successful} + return S_OK(resDict) - def getDirectorySize( self, path ): + def getDirectorySize(self, path): """ Get the size of the contents of the directory """ - res = checkArgumentFormat( path ) + res = checkArgumentFormat(path) if not res['OK']: return res urls = res['Value'] successful = {} failed = {} - gLogger.debug( "DIPStorage.isDirectory: Attempting to determine whether %s paths are directories." % len( urls ) ) - serviceClient = RPCClient( self.url ) + gLogger.debug( + "DIPStorage.isDirectory: Attempting to determine whether %s paths are directories." % + len(urls)) + serviceClient = RPCClient(self.url) for url in urls: - res = serviceClient.getDirectorySize( url ) + res = serviceClient.getDirectorySize(url) if not res['OK']: failed[url] = res['Message'] else: - successful[url] = {'Files':0, 'Size':res['Value'], 'SubDirs':0} - resDict = {'Failed':failed, 'Successful':successful} - return S_OK( resDict ) + successful[url] = {'Files': 0, 'Size': res['Value'], 'SubDirs': 0} + resDict = {'Failed': failed, 'Successful': successful} + return S_OK(resDict) - def getDirectoryMetadata( self, path ): + def getDirectoryMetadata(self, path): """ Get metadata associated to the directory """ - res = checkArgumentFormat( path ) + res = checkArgumentFormat(path) if not res['OK']: return res urls = res['Value'] successful = {} failed = {} - gLogger.debug( "DIPStorage.getFileMetadata: Attempting to obtain metadata for %s directories." % len( urls ) ) - serviceClient = RPCClient( self.url ) + gLogger.debug( + "DIPStorage.getFileMetadata: Attempting to obtain metadata for %s directories." % + len(urls)) + serviceClient = RPCClient(self.url) for url in urls: - res = serviceClient.getMetadata( url ) + res = serviceClient.getMetadata(url) if res['OK']: if res['Value']['Exists']: if res['Value']['Type'] == 'Directory': res['Value']['Directory'] = True - gLogger.debug( "DIPStorage.getFileMetadata: Successfully obtained metadata for %s." % url ) + gLogger.debug( + "DIPStorage.getFileMetadata: Successfully obtained metadata for %s." % + url) successful[url] = res['Value'] else: failed[url] = 'Supplied path is not a directory' else: failed[url] = 'Directory does not exist' else: - gLogger.error( "DIPStorage.getFileMetadata: Failed to get metadata for url", - "%s: %s" % ( url, res['Message'] ) ) + gLogger.error("DIPStorage.getFileMetadata: Failed to get metadata for url", + "%s: %s" % (url, res['Message'])) failed[url] = res['Message'] - resDict = {'Failed':failed, 'Successful':successful} - return S_OK( resDict ) + resDict = {'Failed': failed, 'Successful': successful} + return S_OK(resDict) - def createDirectory( self, path ): + def createDirectory(self, path): """ Create the remote directory """ - res = checkArgumentFormat( path ) + res = checkArgumentFormat(path) if not res['OK']: return res urls = res['Value'] successful = {} failed = {} - gLogger.debug( "DIPStorage.createDirectory: Attempting to create %s directories." % len( urls ) ) - serviceClient = RPCClient( self.url ) + gLogger.debug("DIPStorage.createDirectory: Attempting to create %s directories." % len(urls)) + serviceClient = RPCClient(self.url) for url in urls: - res = serviceClient.createDirectory( url ) + res = serviceClient.createDirectory(url) if res['OK']: - gLogger.debug( "DIPStorage.createDirectory: Successfully created directory on storage: %s" % url ) + gLogger.debug( + "DIPStorage.createDirectory: Successfully created directory on storage: %s" % + url) successful[url] = True else: - gLogger.error( "DIPStorage.createDirectory: Failed to create directory on storage.", "%s: %s" % ( url, res['Message'] ) ) + gLogger.error( + "DIPStorage.createDirectory: Failed to create directory on storage.", "%s: %s" % + (url, res['Message'])) failed[url] = res['Message'] - resDict = {'Failed':failed, 'Successful':successful} - return S_OK( resDict ) + resDict = {'Failed': failed, 'Successful': successful} + return S_OK(resDict) - def putDirectory( self, path ): + def putDirectory(self, path): """ Put a local directory to the physical storage together with all its files and subdirectories. """ - res = checkArgumentFormat( path ) + res = checkArgumentFormat(path) if not res['OK']: return res urls = res['Value'] successful = {} failed = {} - gLogger.debug( "DIPStorage.putDirectory: Attemping to put %s directories to remote storage." % len( urls ) ) - transferClient = TransferClient( self.url ) + gLogger.debug( + "DIPStorage.putDirectory: Attemping to put %s directories to remote storage." % + len(urls)) + transferClient = TransferClient(self.url) for destDir, sourceDir in urls.items(): - tmpList = os.listdir( sourceDir ) - sourceFiles = [ "%s/%s" % ( sourceDir, x ) for x in tmpList ] - res = transferClient.sendBulk( sourceFiles, destDir ) + tmpList = os.listdir(sourceDir) + sourceFiles = ["%s/%s" % (sourceDir, x) for x in tmpList] + res = transferClient.sendBulk(sourceFiles, destDir) if res['OK']: - successful[destDir] = {'Files':0, 'Size':0} + successful[destDir] = {'Files': 0, 'Size': 0} else: failed[destDir] = res['Message'] - resDict = {'Failed':failed, 'Successful':successful} - return S_OK( resDict ) + resDict = {'Failed': failed, 'Successful': successful} + return S_OK(resDict) - def removeDirectory( self, path, recursive = False ): + def removeDirectory(self, path, recursive=False): """ Remove a directory from the storage together with all its files and subdirectories. """ - res = checkArgumentFormat( path ) + res = checkArgumentFormat(path) if not res['OK']: return res urls = res['Value'] successful = {} failed = {} - gLogger.debug( "DIPStorage.removeDirectory: Attempting to remove %s directories." % len( urls ) ) - serviceClient = RPCClient( self.url ) + gLogger.debug("DIPStorage.removeDirectory: Attempting to remove %s directories." % len(urls)) + serviceClient = RPCClient(self.url) for url in urls: - res = serviceClient.removeDirectory( url, '' ) + res = serviceClient.removeDirectory(url, '') if res['OK']: - gLogger.debug( "DIPStorage.removeDirectory: Successfully removed directory on storage: %s" % url ) - successful[url] = {'FilesRemoved':0, 'SizeRemoved':0} + gLogger.debug( + "DIPStorage.removeDirectory: Successfully removed directory on storage: %s" % + url) + successful[url] = {'FilesRemoved': 0, 'SizeRemoved': 0} else: - gLogger.error( "DIPStorage.removeDirectory: Failed to remove directory from storage.", "%s: %s" % ( url, res['Message'] ) ) + gLogger.error( + "DIPStorage.removeDirectory: Failed to remove directory from storage.", "%s: %s" % + (url, res['Message'])) failed[url] = res['Message'] - resDict = {'Failed':failed, 'Successful':successful} - return S_OK( resDict ) + resDict = {'Failed': failed, 'Successful': successful} + return S_OK(resDict) - def getDirectory( self, path, localPath = False ): + def getDirectory(self, path, localPath=False): """ Get a local copy in the current directory of a physical file specified by its path """ - res = checkArgumentFormat( path ) + res = checkArgumentFormat(path) if not res['OK']: return res urls = res['Value'] failed = {} successful = {} - gLogger.debug( "DIPStorage.getDirectory: Attempting to get local copies of %s directories." % len( urls ) ) - transferClient = TransferClient( self.url ) + gLogger.debug( + "DIPStorage.getDirectory: Attempting to get local copies of %s directories." % + len(urls)) + transferClient = TransferClient(self.url) for src_dir in urls: if localPath: dest_dir = localPath else: dest_dir = os.getcwd() - if not os.path.exists( dest_dir ): - os.mkdir( dest_dir ) - res = transferClient.receiveBulk( dest_dir, src_dir ) + if not os.path.exists(dest_dir): + os.mkdir(dest_dir) + res = transferClient.receiveBulk(dest_dir, src_dir) if res['OK']: - gLogger.debug( "DIPStorage.getDirectory: Successfully got local copy of %s" % src_dir ) - successful[src_dir] = {'Files':0, 'Size':0} + gLogger.debug("DIPStorage.getDirectory: Successfully got local copy of %s" % src_dir) + successful[src_dir] = {'Files': 0, 'Size': 0} else: - gLogger.error( "DIPStorage.getDirectory: Failed to get entire directory.", src_dir ) + gLogger.error("DIPStorage.getDirectory: Failed to get entire directory.", src_dir) failed[src_dir] = res['Message'] - resDict = {'Failed':failed, 'Successful':successful} - return S_OK( resDict ) + resDict = {'Failed': failed, 'Successful': successful} + return S_OK(resDict) + + def getOccupancy(self, *parms, **kws): + """ Gets the DIPStorage occupancy info + + :return: S_OK/S_ERROR (free and total space, in MB) + """ + + rpc = RPCClient(self.url, timeout=120) + + free = rpc.getFreeDiskSpace() + if not free['OK']: + return free + + total = rpc.getTotalDiskSpace() + if not total['OK']: + return total + return S_OK({'Free': free['Value'], 'Total': total['Value']}) diff --git a/Resources/Storage/GFAL2_SRM2Storage.py b/Resources/Storage/GFAL2_SRM2Storage.py index f4f223ba8c4..02e70e5987e 100644 --- a/Resources/Storage/GFAL2_SRM2Storage.py +++ b/Resources/Storage/GFAL2_SRM2Storage.py @@ -9,28 +9,31 @@ # pylint: disable=invalid-name +import errno +import json + +import gfal2 # from DIRAC -from DIRAC.Resources.Storage.GFAL2_StorageBase import GFAL2_StorageBase from DIRAC import gLogger, gConfig, S_OK, S_ERROR +from DIRAC.Resources.Storage.GFAL2_StorageBase import GFAL2_StorageBase from DIRAC.Resources.Storage.Utilities import checkArgumentFormat __RCSID__ = "$Id$" -class GFAL2_SRM2Storage( GFAL2_StorageBase ): + +class GFAL2_SRM2Storage(GFAL2_StorageBase): """ SRM2 SE class that inherits from GFAL2StorageBase """ _INPUT_PROTOCOLS = ['file', 'root', 'srm', 'gsiftp'] _OUTPUT_PROTOCOLS = ['file', 'root', 'dcap', 'gsidcap', 'rfio', 'srm', 'gsiftp'] - - - def __init__( self, storageName, parameters ): + def __init__(self, storageName, parameters): """ """ - super( GFAL2_SRM2Storage, self ).__init__( storageName, parameters ) - self.log = gLogger.getSubLogger( "GFAL2_SRM2Storage", True ) - self.log.debug( "GFAL2_SRM2Storage.__init__: Initializing object" ) + super(GFAL2_SRM2Storage, self).__init__(storageName, parameters) + self.log = gLogger.getSubLogger("GFAL2_SRM2Storage", True) + self.log.debug("GFAL2_SRM2Storage.__init__: Initializing object") self.pluginName = 'GFAL2_SRM2' # This attribute is used to know the file status (OFFLINE,NEARLINE,ONLINE) @@ -42,35 +45,31 @@ def __init__( self, storageName, parameters ): # the proper values and then calling the base class method. # ## - self.gfal2requestLifetime = gConfig.getValue( '/Resources/StorageElements/RequestLifeTime', 100 ) + self.gfal2requestLifetime = gConfig.getValue('/Resources/StorageElements/RequestLifeTime', 100) self.__setSRMOptionsToDefault() - # This lists contains the list of protocols to ask to SRM to get a URL # It can be either defined in the plugin of the SE, or as a global option if 'ProtocolsList' in parameters: self.protocolsList = parameters['ProtocolsList'].split(',') else: - self.log.debug( "GFAL2_SRM2Storage: No protocols provided, using the default protocols." ) + self.log.debug("GFAL2_SRM2Storage: No protocols provided, using the default protocols.") self.protocolsList = self.defaultLocalProtocols - self.log.debug( 'GFAL2_SRM2Storage: protocolsList = %s' % self.protocolsList ) - - + self.log.debug('GFAL2_SRM2Storage: protocolsList = %s' % self.protocolsList) - def __setSRMOptionsToDefault( self ): + def __setSRMOptionsToDefault(self): ''' Resetting the SRM options back to default ''' - self.ctx.set_opt_integer( "SRM PLUGIN", "OPERATION_TIMEOUT", self.gfal2Timeout ) - self.ctx.set_opt_string( "SRM PLUGIN", "SPACETOKENDESC", self.spaceToken ) - self.ctx.set_opt_integer( "SRM PLUGIN", "REQUEST_LIFETIME", self.gfal2requestLifetime ) + self.ctx.set_opt_integer("SRM PLUGIN", "OPERATION_TIMEOUT", self.gfal2Timeout) + self.ctx.set_opt_string("SRM PLUGIN", "SPACETOKENDESC", self.spaceToken) + self.ctx.set_opt_integer("SRM PLUGIN", "REQUEST_LIFETIME", self.gfal2requestLifetime) # Setting the TURL protocol to gsiftp because with other protocols we have authorisation problems # self.ctx.set_opt_string_list( "SRM PLUGIN", "TURL_PROTOCOLS", self.defaultLocalProtocols ) - self.ctx.set_opt_string_list( "SRM PLUGIN", "TURL_PROTOCOLS", ['gsiftp'] ) + self.ctx.set_opt_string_list("SRM PLUGIN", "TURL_PROTOCOLS", ['gsiftp']) - - def _updateMetadataDict( self, metadataDict, attributeDict ): + def _updateMetadataDict(self, metadataDict, attributeDict): """ Updating the metadata dictionary with srm specific attributes :param self: self reference @@ -79,15 +78,14 @@ def _updateMetadataDict( self, metadataDict, attributeDict ): """ # 'user.status' is the extended attribute we are interested in - user_status = attributeDict.get( 'user.status', '' ) - metadataDict['Cached'] = int( 'ONLINE' in user_status ) - metadataDict['Migrated'] = int( 'NEARLINE' in user_status ) - metadataDict['Lost'] = int( user_status == 'LOST' ) - metadataDict['Unavailable'] = int( user_status == 'UNAVAILABLE' ) + user_status = attributeDict.get('user.status', '') + metadataDict['Cached'] = int('ONLINE' in user_status) + metadataDict['Migrated'] = int('NEARLINE' in user_status) + metadataDict['Lost'] = int(user_status == 'LOST') + metadataDict['Unavailable'] = int(user_status == 'UNAVAILABLE') metadataDict['Accessible'] = not metadataDict['Lost'] and metadataDict['Cached'] and not metadataDict['Unavailable'] - - def getTransportURL( self, path, protocols = False ): + def getTransportURL(self, path, protocols=False): """ obtain the tURLs for the supplied path and protocols :param self: self reference @@ -97,60 +95,59 @@ def getTransportURL( self, path, protocols = False ): Successful dict {path : transport url} S_ERROR in case of argument problems """ - res = checkArgumentFormat( path ) + res = checkArgumentFormat(path) if not res['OK']: return res urls = res['Value'] - self.log.debug( 'GFAL2_SRM2Storage.getTransportURL: Attempting to retrieve tURL for %s paths' % len( urls ) ) + self.log.debug( + 'GFAL2_SRM2Storage.getTransportURL: Attempting to retrieve tURL for %s paths' % + len(urls)) failed = {} successful = {} if not protocols: listProtocols = self.protocolsList if not listProtocols: - return S_ERROR( "GFAL2_SRM2Storage.getTransportURL: No local protocols defined and no defaults found." ) - elif isinstance( protocols, basestring ): + return S_ERROR( + "GFAL2_SRM2Storage.getTransportURL: No local protocols defined and no defaults found.") + elif isinstance(protocols, basestring): listProtocols = [protocols] - elif isinstance( protocols, list ): + elif isinstance(protocols, list): listProtocols = protocols else: - return S_ERROR( "getTransportURL: Must supply desired protocols to this plug-in." ) + return S_ERROR("getTransportURL: Must supply desired protocols to this plug-in.") # Compatibility because of castor returning a castor: url if you ask # for a root URL, and a root: url if you ask for a xroot url... if 'root' in listProtocols and 'xroot' not in listProtocols: - listProtocols.insert( listProtocols.index( 'root' ), 'xroot' ) + listProtocols.insert(listProtocols.index('root'), 'xroot') elif 'xroot' in listProtocols and 'root' not in listProtocols: - listProtocols.insert( listProtocols.index( 'xroot' ) + 1, 'root' ) - + listProtocols.insert(listProtocols.index('xroot') + 1, 'root') if self.protocolParameters['Protocol'] in listProtocols: successful = {} failed = {} for url in urls: - if self.isURL( url )['Value']: + if self.isURL(url)['Value']: successful[url] = url else: failed[url] = 'getTransportURL: Failed to obtain turls.' - return S_OK( {'Successful' : successful, 'Failed' : failed} ) + return S_OK({'Successful': successful, 'Failed': failed}) for url in urls: - res = self.__getSingleTransportURL( url, listProtocols ) - self.log.debug( 'res = %s' % res ) + res = self.__getSingleTransportURL(url, listProtocols) + self.log.debug('res = %s' % res) if not res['OK']: failed[url] = res['Message'] else: successful[url] = res['Value'] - return S_OK( { 'Failed' : failed, 'Successful' : successful } ) + return S_OK({'Failed': failed, 'Successful': successful}) - - - - def __getSingleTransportURL( self, path, protocols = False ): + def __getSingleTransportURL(self, path, protocols=False): """ Get the tURL from path with getxattr from gfal2 :param self: self reference @@ -158,16 +155,61 @@ def __getSingleTransportURL( self, path, protocols = False ): :returns: S_OK( Transport_URL ) in case of success S_ERROR( errStr ) in case of a failure """ - self.log.debug( 'GFAL2_SRM2Storage.__getSingleTransportURL: trying to retrieve tURL for %s' % path ) + self.log.debug( + 'GFAL2_SRM2Storage.__getSingleTransportURL: trying to retrieve tURL for %s' % + path) if protocols: - self.ctx.set_opt_string_list( "SRM PLUGIN", "TURL_PROTOCOLS", protocols ) + self.ctx.set_opt_string_list("SRM PLUGIN", "TURL_PROTOCOLS", protocols) - res = self._getExtendedAttributes( path, attributes = ['user.replicas'] ) + res = self._getExtendedAttributes(path, attributes=['user.replicas']) self.__setSRMOptionsToDefault() if res['OK']: - return S_OK( res['Value']['user.replicas'] ) - else: - errStr = 'GFAL2_SRM2Storage.__getSingleTransportURL: Extended attribute tURL is not set.' - self.log.debug( errStr, res['Message'] ) - return res + return S_OK(res['Value']['user.replicas']) + + errStr = 'GFAL2_SRM2Storage.__getSingleTransportURL: Extended attribute tURL is not set.' + self.log.debug(errStr, res['Message']) + return res + + def getOccupancy(self, *parms, **kws): + """ Gets the GFAL2_SRM2Storage occupancy info. + + TODO: needs gfal2.15 because of bugs: + https://its.cern.ch/jira/browse/DMC-979 + https://its.cern.ch/jira/browse/DMC-977 + + It queries the srm interface for a given space token. + Out of the results, we keep totalsize, guaranteedsize, and unusedsize all in MB. + """ + + # Gfal2 extended parameter name to query the space token occupancy + spaceTokenAttr = 'spacetoken.description?%s' % self.protocolParameters['SpaceToken'] + # gfal2 can take any srm url as a base. + spaceTokenEndpoint = self.getURLBase()['Value'] + print "%s %s" % (spaceTokenEndpoint, spaceTokenAttr) + try: + occupancyStr = self.ctx.getxattr(spaceTokenEndpoint, spaceTokenAttr) + try: + occupancyDict = json.loads(occupancyStr) + except ValueError: + # https://its.cern.ch/jira/browse/DMC-977 + # a closing bracket is missing, so we retry after adding it + occupancyStr = occupancyStr[:-1] + '}]' + occupancyDict = json.loads(occupancyStr)[0] + + # https://its.cern.ch/jira/browse/DMC-979 + # We set totalsize to guaranteed size + # (it is anyway true for all the SEs I could test) + occupancyDict['totalsize'] = occupancyDict.get('guaranteedsize', 0) + + except (gfal2.GError, ValueError) as e: + errStr = 'Something went wrong while checking for spacetoken occupancy.' + self.log.verbose(errStr, e.message) + return S_ERROR(getattr(e, 'code', errno.EINVAL), "%s %s" % (errStr, repr(e))) + + sTokenDict = {} + + sTokenDict['Total'] = float(occupancyDict.get('totalsize', '0')) / 1e6 + sTokenDict['Free'] = float(occupancyDict.get('unusedsize', '0')) / 1e6 + + return S_OK(sTokenDict) diff --git a/Resources/Storage/GFAL2_StorageBase.py b/Resources/Storage/GFAL2_StorageBase.py index 4db7be974e0..281338d8c62 100644 --- a/Resources/Storage/GFAL2_StorageBase.py +++ b/Resources/Storage/GFAL2_StorageBase.py @@ -11,7 +11,6 @@ # # imports import os -import sys import datetime import errno from stat import S_ISREG, S_ISDIR, S_IXUSR, S_IRUSR, S_IWUSR, \ diff --git a/Resources/Storage/StorageBase.py b/Resources/Storage/StorageBase.py index c22bceb394c..b3008d82d6a 100755 --- a/Resources/Storage/StorageBase.py +++ b/Resources/Storage/StorageBase.py @@ -28,6 +28,10 @@ getName() getParameters() getCurrentURL() + +These are the methods for getting information about the Storage: + getOccupancy() + """ __RCSID__ = "$Id$" @@ -35,46 +39,43 @@ from DIRAC.Core.Utilities.Pfn import pfnparse, pfnunparse from DIRAC.Resources.Storage.Utilities import checkArgumentFormat -import os -class StorageBase( object ): +class StorageBase(object): """ .. class:: StorageBase """ - PROTOCOL_PARAMETERS = [ "Protocol", "Host", "Path", "Port", "SpaceToken", "WSUrl" ] + PROTOCOL_PARAMETERS = ["Protocol", "Host", "Path", "Port", "SpaceToken", "WSUrl"] # Options to be prepended in the URL # keys are the name of the parameters in the CS # values are the name of the options as they appear in the URL DYNAMIC_OPTIONS = {} - def __init__( self, name, parameterDict ): + def __init__(self, name, parameterDict): self.name = name self.pluginName = '' self.protocolParameters = {} - self.__updateParameters( parameterDict ) + self.__updateParameters(parameterDict) # Keep the list of all parameters passed for constructions # Taken from the CS # In a further major release, this could be nerged together # with protocolParameters. There is no reason for it to - # be so strict about the possible content. + # be so strict about the possible content. self._allProtocolParameters = parameterDict - - - if hasattr( self, '_INPUT_PROTOCOLS' ): - self.protocolParameters['InputProtocols'] = getattr( self, '_INPUT_PROTOCOLS' ) + if hasattr(self, '_INPUT_PROTOCOLS'): + self.protocolParameters['InputProtocols'] = getattr(self, '_INPUT_PROTOCOLS') else: - self.protocolParameters['InputProtocols'] = [ self.protocolParameters['Protocol'], 'file'] + self.protocolParameters['InputProtocols'] = [self.protocolParameters['Protocol'], 'file'] - if hasattr( self, '_OUTPUT_PROTOCOLS' ): - self.protocolParameters['OutputProtocols'] = getattr( self, '_OUTPUT_PROTOCOLS' ) + if hasattr(self, '_OUTPUT_PROTOCOLS'): + self.protocolParameters['OutputProtocols'] = getattr(self, '_OUTPUT_PROTOCOLS') else: - self.protocolParameters['OutputProtocols'] = [ self.protocolParameters['Protocol']] + self.protocolParameters['OutputProtocols'] = [self.protocolParameters['Protocol']] self.basePath = parameterDict['Path'] self.cwd = self.basePath @@ -84,223 +85,222 @@ def __init__( self, name, parameterDict ): # use True for backward compatibility self.srmSpecificParse = True - def setStorageElement( self, se ): + def setStorageElement(self, se): self.se = se - def setParameters( self, parameterDict ): + def setParameters(self, parameterDict): """ Set standard parameters, method can be overriden in subclasses to process specific parameters """ - self.__updateParameters( parameterDict ) + self.__updateParameters(parameterDict) - def __updateParameters( self, parameterDict ): + def __updateParameters(self, parameterDict): """ setParameters implementation method """ for item in self.PROTOCOL_PARAMETERS: - self.protocolParameters[item] = parameterDict.get( item, '' ) + self.protocolParameters[item] = parameterDict.get(item, '') - def getParameters( self ): + def getParameters(self): """ Get the parameters with which the storage was instantiated """ - parameterDict = dict( self.protocolParameters ) + parameterDict = dict(self.protocolParameters) parameterDict["StorageName"] = self.name parameterDict["PluginName"] = self.pluginName - parameterDict['URLBase'] = self.getURLBase().get( 'Value', '' ) + parameterDict['URLBase'] = self.getURLBase().get('Value', '') return parameterDict - def exists( self, *parms, **kws ): + def exists(self, *parms, **kws): """Check if the given path exists """ - return S_ERROR( "Storage.exists: implement me!" ) + return S_ERROR("Storage.exists: implement me!") ############################################################# # # These are the methods for file manipulation # - def isFile( self, *parms, **kws ): + def isFile(self, *parms, **kws): """Check if the given path exists and it is a file """ - return S_ERROR( "Storage.isFile: implement me!" ) + return S_ERROR("Storage.isFile: implement me!") - def getFile( self, *parms, **kws ): + def getFile(self, *parms, **kws): """Get a local copy of the file specified by its path """ - return S_ERROR( "Storage.getFile: implement me!" ) + return S_ERROR("Storage.getFile: implement me!") - def putFile( self, *parms, **kws ): + def putFile(self, *parms, **kws): """Put a copy of the local file to the current directory on the physical storage """ - return S_ERROR( "Storage.putFile: implement me!" ) + return S_ERROR("Storage.putFile: implement me!") - def removeFile( self, *parms, **kws ): + def removeFile(self, *parms, **kws): """Remove physically the file specified by its path """ - return S_ERROR( "Storage.removeFile: implement me!" ) + return S_ERROR("Storage.removeFile: implement me!") - def getFileMetadata( self, *parms, **kws ): + def getFileMetadata(self, *parms, **kws): """ Get metadata associated to the file """ - return S_ERROR( "Storage.getFileMetadata: implement me!" ) + return S_ERROR("Storage.getFileMetadata: implement me!") - def getFileSize( self, *parms, **kws ): + def getFileSize(self, *parms, **kws): """Get the physical size of the given file """ - return S_ERROR( "Storage.getFileSize: implement me!" ) + return S_ERROR("Storage.getFileSize: implement me!") - def prestageFile( self, *parms, **kws ): + def prestageFile(self, *parms, **kws): """ Issue prestage request for file """ - return S_ERROR( "Storage.prestageFile: implement me!" ) + return S_ERROR("Storage.prestageFile: implement me!") - def prestageFileStatus( self, *parms, **kws ): + def prestageFileStatus(self, *parms, **kws): """ Obtain the status of the prestage request """ - return S_ERROR( "Storage.prestageFileStatus: implement me!" ) + return S_ERROR("Storage.prestageFileStatus: implement me!") - def pinFile( self, *parms, **kws ): + def pinFile(self, *parms, **kws): """ Pin the file on the destination storage element """ - return S_ERROR( "Storage.pinFile: implement me!" ) + return S_ERROR("Storage.pinFile: implement me!") - def releaseFile( self, *parms, **kws ): + def releaseFile(self, *parms, **kws): """ Release the file on the destination storage element """ - return S_ERROR( "Storage.releaseFile: implement me!" ) + return S_ERROR("Storage.releaseFile: implement me!") ############################################################# # # These are the methods for directory manipulation # - def isDirectory( self, *parms, **kws ): + def isDirectory(self, *parms, **kws): """Check if the given path exists and it is a directory """ - return S_ERROR( "Storage.isDirectory: implement me!" ) + return S_ERROR("Storage.isDirectory: implement me!") - def getDirectory( self, *parms, **kws ): + def getDirectory(self, *parms, **kws): """Get locally a directory from the physical storage together with all its files and subdirectories. """ - return S_ERROR( "Storage.getDirectory: implement me!" ) + return S_ERROR("Storage.getDirectory: implement me!") - def putDirectory( self, *parms, **kws ): + def putDirectory(self, *parms, **kws): """Put a local directory to the physical storage together with all its files and subdirectories. """ - return S_ERROR( "Storage.putDirectory: implement me!" ) + return S_ERROR("Storage.putDirectory: implement me!") - def createDirectory( self, *parms, **kws ): + def createDirectory(self, *parms, **kws): """ Make a new directory on the physical storage """ - return S_ERROR( "Storage.createDirectory: implement me!" ) + return S_ERROR("Storage.createDirectory: implement me!") - def removeDirectory( self, *parms, **kws ): + def removeDirectory(self, *parms, **kws): """Remove a directory on the physical storage together with all its files and subdirectories. """ - return S_ERROR( "Storage.removeDirectory: implement me!" ) + return S_ERROR("Storage.removeDirectory: implement me!") - def listDirectory( self, *parms, **kws ): + def listDirectory(self, *parms, **kws): """ List the supplied path """ - return S_ERROR( "Storage.listDirectory: implement me!" ) + return S_ERROR("Storage.listDirectory: implement me!") - def getDirectoryMetadata( self, *parms, **kws ): + def getDirectoryMetadata(self, *parms, **kws): """ Get the metadata for the directory """ - return S_ERROR( "Storage.getDirectoryMetadata: implement me!" ) + return S_ERROR("Storage.getDirectoryMetadata: implement me!") - def getDirectorySize( self, *parms, **kws ): + def getDirectorySize(self, *parms, **kws): """ Get the size of the directory on the storage """ - return S_ERROR( "Storage.getDirectorySize: implement me!" ) - + return S_ERROR("Storage.getDirectorySize: implement me!") ############################################################# # # These are the methods for manipulating the client # - def isOK( self ): + def isOK(self): return self.isok - def resetCurrentDirectory( self ): + def resetCurrentDirectory(self): """ Reset the working directory to the base dir """ self.cwd = self.basePath - def changeDirectory( self, directory ): + def changeDirectory(self, directory): """ Change the directory to the supplied directory """ - if directory.startswith( '/' ): - self.cwd = "%s/%s" % ( self.basePath, directory ) + if directory.startswith('/'): + self.cwd = "%s/%s" % (self.basePath, directory) else: - self.cwd = '%s/%s' % ( self.cwd, directory ) + self.cwd = '%s/%s' % (self.cwd, directory) - def getCurrentDirectory( self ): + def getCurrentDirectory(self): """ Get the current directory """ return self.cwd - def getCurrentURL( self, fileName ): + def getCurrentURL(self, fileName): """ Obtain the current file URL from the current working directory and the filename :param self: self reference :param str fileName: path on storage """ - urlDict = dict( self.protocolParameters ) - if not fileName.startswith( '/' ): + urlDict = dict(self.protocolParameters) + if not fileName.startswith('/'): # Relative path is given urlDict['Path'] = self.cwd - result = pfnunparse( urlDict, srmSpecific = self.srmSpecificParse ) + result = pfnunparse(urlDict, srmSpecific=self.srmSpecificParse) if not result['OK']: return result cwdUrl = result['Value'] - fullUrl = '%s%s' % ( cwdUrl, fileName ) - return S_OK( fullUrl ) + fullUrl = '%s%s' % (cwdUrl, fileName) + return S_OK(fullUrl) - def getName( self ): + def getName(self): """ The name with which the storage was instantiated """ return self.name - def getURLBase( self, withWSUrl = False ): + def getURLBase(self, withWSUrl=False): """ This will get the URL base. This is then appended with the LFN in DIRAC convention. :param self: self reference :param bool withWSUrl: flag to include Web Service part of the url :returns: URL """ - urlDict = dict( self.protocolParameters ) + urlDict = dict(self.protocolParameters) if not withWSUrl: urlDict['WSUrl'] = '' - return pfnunparse( urlDict, srmSpecific = self.srmSpecificParse ) + return pfnunparse(urlDict, srmSpecific=self.srmSpecificParse) - def isURL( self, path ): + def isURL(self, path): """ Guess if the path looks like a URL :param self: self reference :param string path: input file LFN or URL :returns boolean: True if URL, False otherwise """ - if self.basePath and path.startswith( self.basePath ): - return S_OK( True ) + if self.basePath and path.startswith(self.basePath): + return S_OK(True) - result = pfnparse( path, srmSpecific = self.srmSpecificParse ) + result = pfnparse(path, srmSpecific=self.srmSpecificParse) if not result['OK']: return result - if len( result['Value']['Protocol'] ) != 0: - return S_OK( True ) + if len(result['Value']['Protocol']) != 0: + return S_OK(True) - if result['Value']['Path'].startswith( self.basePath ): - return S_OK( True ) + if result['Value']['Path'].startswith(self.basePath): + return S_OK(True) - return S_OK( False ) + return S_OK(False) - def getTransportURL( self, pathDict, protocols ): + def getTransportURL(self, pathDict, protocols): """ Get a transport URL for a given URL. For a simple storage plugin it is just returning input URL if the plugin protocol is one of the requested protocols @@ -310,7 +310,7 @@ def getTransportURL( self, pathDict, protocols ): :param protocols: a list of acceptable transport protocols in priority order :type protocols: `python:list` """ - res = checkArgumentFormat( pathDict ) + res = checkArgumentFormat(pathDict) if not res['OK']: return res urls = res['Value'] @@ -318,15 +318,15 @@ def getTransportURL( self, pathDict, protocols ): failed = {} if protocols and not self.protocolParameters['Protocol'] in protocols: - return S_ERROR( 'No native protocol requested' ) + return S_ERROR('No native protocol requested') for url in urls: successful[url] = url - resDict = {'Failed':failed, 'Successful':successful} - return S_OK( resDict ) + resDict = {'Failed': failed, 'Successful': successful} + return S_OK(resDict) - def constructURLFromLFN( self, lfn, withWSUrl = False ): + def constructURLFromLFN(self, lfn, withWSUrl=False): """ Construct URL from the given LFN according to the VO convention for the primary protocol of the storage plagin @@ -338,27 +338,28 @@ def constructURLFromLFN( self, lfn, withWSUrl = False ): # Check the LFN convention: # 1. LFN must start with the VO name as the top level directory # 2. VO name must not appear as any subdirectory or file name - lfnSplitList = lfn.split( '/' ) + lfnSplitList = lfn.split('/') voLFN = lfnSplitList[1] - # TODO comparison to Sandbox below is for backward compatibility, should be removed in the next release + # TODO comparison to Sandbox below is for backward compatibility, should + # be removed in the next release if voLFN != self.se.vo and voLFN != "SandBox" and voLFN != "Sandbox": - return S_ERROR( 'LFN does not follow the DIRAC naming convention %s' % lfn ) + return S_ERROR('LFN does not follow the DIRAC naming convention %s' % lfn) - urlDict = dict( self.protocolParameters ) - urlDict['Options'] = '&'.join( "%s=%s" % ( optionName, urlDict[paramName] ) - for paramName, optionName in self.DYNAMIC_OPTIONS.iteritems() - if urlDict.get( paramName ) ) + urlDict = dict(self.protocolParameters) + urlDict['Options'] = '&'.join("%s=%s" % (optionName, urlDict[paramName]) + for paramName, optionName in self.DYNAMIC_OPTIONS.iteritems() + if urlDict.get(paramName)) if not withWSUrl: urlDict['WSUrl'] = '' - urlDict['FileName'] = lfn.lstrip( '/' ) + urlDict['FileName'] = lfn.lstrip('/') - return pfnunparse( urlDict, srmSpecific = self.srmSpecificParse ) + return pfnunparse(urlDict, srmSpecific=self.srmSpecificParse) - def updateURL( self, url, withWSUrl = False ): + def updateURL(self, url, withWSUrl=False): """ Update the URL according to the current SE parameters """ - result = pfnparse( url, srmSpecific = self.srmSpecificParse ) + result = pfnparse(url, srmSpecific=self.srmSpecificParse) if not result['OK']: return result urlDict = result['Value'] @@ -370,22 +371,22 @@ def updateURL( self, url, withWSUrl = False ): if withWSUrl: urlDict['WSUrl'] = self.protocolParameters['WSUrl'] - return pfnunparse( urlDict, srmSpecific = self.srmSpecificParse ) + return pfnunparse(urlDict, srmSpecific=self.srmSpecificParse) - def isNativeURL( self, url ): + def isNativeURL(self, url): """ Check if URL :url: is valid for :self.protocol: :param self: self reference :param str url: URL """ - res = pfnparse( url, srmSpecific = self.srmSpecificParse ) + res = pfnparse(url, srmSpecific=self.srmSpecificParse) if not res['OK']: return res urlDict = res['Value'] - return S_OK( urlDict['Protocol'] == self.protocolParameters['Protocol'] ) + return S_OK(urlDict['Protocol'] == self.protocolParameters['Protocol']) @staticmethod - def _addCommonMetadata( metadataDict ): + def _addCommonMetadata(metadataDict): """ To make the output of getFileMetadata uniform throughout the protocols this returns a minimum set of metadata with default value, that are then complemented with the protocol specific metadata @@ -394,32 +395,43 @@ def _addCommonMetadata( metadataDict ): :returns: dictionnary with all the metadata (specific and basic) """ - commonMetadata = { 'Checksum' : '', - 'Directory' : False, - 'File' : False, - 'Mode' : 0o000, - 'Size' : 0, - 'Accessible' : True, + commonMetadata = {'Checksum': '', + 'Directory': False, + 'File': False, + 'Mode': 0o000, + 'Size': 0, + 'Accessible': True, } - commonMetadata.update( metadataDict ) + commonMetadata.update(metadataDict) return commonMetadata - - def _isInputURL( self, url ): + def _isInputURL(self, url): """ Check if the given url can be taken as input :param self: self reference :param str url: URL """ - res = pfnparse( url ) + res = pfnparse(url) if not res['OK']: return res urlDict = res['Value'] # Special case of 'file' protocol which can be just a URL if not urlDict['Protocol'] and 'file' in self.protocolParameters['InputProtocols']: - return S_OK( True ) + return S_OK(True) - return S_OK( urlDict['Protocol'] == self.protocolParameters['Protocol'] ) + return S_OK(urlDict['Protocol'] == self.protocolParameters['Protocol']) + + ############################################################# + # + # These are the methods for getting information about the Storage element: + # + + def getOccupancy(self, *parms, **kws): + """ Get the StorageElement occupancy info in MB. + :returns: S_OK/S_ERROR dictionary + """ + # FIXME: put an implementation that just gets a file and check its content + return S_ERROR("Storage.occupancy: implement me!") diff --git a/Resources/Storage/StorageElement.py b/Resources/Storage/StorageElement.py index 6d66b37f362..e5913bfe554 100755 --- a/Resources/Storage/StorageElement.py +++ b/Resources/Storage/StorageElement.py @@ -13,6 +13,7 @@ # # from DIRAC from DIRAC import gLogger, gConfig, siteName from DIRAC.Core.Utilities import DErrno +from DIRAC.Core.Utilities.File import convertSizeUnits from DIRAC.Core.Utilities.ReturnValues import S_OK, S_ERROR, returnSingleResult from DIRAC.Resources.Storage.StorageFactory import StorageFactory from DIRAC.Core.Utilities.Pfn import pfnparse @@ -26,17 +27,18 @@ from DIRAC.AccountingSystem.Client.Types.DataOperation import DataOperation from DIRAC.AccountingSystem.Client.DataStoreClient import gDataStoreClient from DIRAC.DataManagementSystem.Utilities.DMSHelpers import DMSHelpers +from functools import reduce __RCSID__ = "$Id$" -class StorageElementCache( object ): +class StorageElementCache(object): - def __init__( self ): + def __init__(self): self.seCache = DictCache() - def __call__( self, name, plugins = None, vo = None, hideExceptions = False ): - self.seCache.purgeExpired( expiredInSeconds = 60 ) + def __call__(self, name, plugins=None, vo=None, hideExceptions=False): + self.seCache.purgeExpired(expiredInSeconds=60) tId = threading.current_thread().ident if not vo: @@ -45,17 +47,18 @@ def __call__( self, name, plugins = None, vo = None, hideExceptions = False ): return vo = result['Value'] - argTuple = ( tId, name, plugins, vo ) - seObj = self.seCache.get( argTuple ) + argTuple = (tId, name, plugins, vo) + seObj = self.seCache.get(argTuple) if not seObj: - seObj = StorageElementItem( name, plugins, vo, hideExceptions = hideExceptions ) + seObj = StorageElementItem(name, plugins, vo, hideExceptions=hideExceptions) # Add the StorageElement to the cache for 1/2 hour - self.seCache.add( argTuple, 1800, seObj ) + self.seCache.add(argTuple, 1800, seObj) return seObj -class StorageElementItem( object ): + +class StorageElementItem(object): """ .. class:: StorageElement @@ -103,38 +106,38 @@ class StorageElementItem( object ): # Some methods have a different name in the StorageElement and the plugins... # We could avoid this static list in the __getattr__ by checking the storage plugin and so on # but fine... let's not be too smart, otherwise it becomes unreadable :-) - __equivalentMethodNames = { "exists" : "exists", - "isFile" : "isFile", - "getFile" : "getFile", - "putFile" : "putFile", - "replicateFile" : "putFile", - "getFileMetadata" : "getFileMetadata", - "getFileSize" : "getFileSize", - "removeFile" : "removeFile", - "prestageFile" : "prestageFile", - "prestageFileStatus" : "prestageFileStatus", - "pinFile" : "pinFile", - "releaseFile" : "releaseFile", - "isDirectory" : "isDirectory", - "getDirectoryMetadata" : "getDirectoryMetadata", - "getDirectorySize" : "getDirectorySize", - "listDirectory" : "listDirectory", - "removeDirectory" : "removeDirectory", - "createDirectory" : "createDirectory", - "putDirectory" : "putDirectory", - "getDirectory" : "getDirectory", - } + __equivalentMethodNames = {"exists": "exists", + "isFile": "isFile", + "getFile": "getFile", + "putFile": "putFile", + "replicateFile": "putFile", + "getFileMetadata": "getFileMetadata", + "getFileSize": "getFileSize", + "removeFile": "removeFile", + "prestageFile": "prestageFile", + "prestageFileStatus": "prestageFileStatus", + "pinFile": "pinFile", + "releaseFile": "releaseFile", + "isDirectory": "isDirectory", + "getDirectoryMetadata": "getDirectoryMetadata", + "getDirectorySize": "getDirectorySize", + "listDirectory": "listDirectory", + "removeDirectory": "removeDirectory", + "createDirectory": "createDirectory", + "putDirectory": "putDirectory", + "getDirectory": "getDirectory", + } # We can set default argument in the __executeFunction which impacts all plugins - __defaultsArguments = {"putFile" : {"sourceSize" : 0 }, - "getFile": { "localPath": False }, - "prestageFile" : { "lifetime" : 86400 }, - "pinFile" : { "lifetime" : 60 * 60 * 24 }, - "removeDirectory" : { "recursive" : False }, - "getDirectory" : { "localPath" : False }, - } - - def __init__( self, name, plugins = None, vo = None, hideExceptions = False ): + __defaultsArguments = {"putFile": {"sourceSize": 0}, + "getFile": {"localPath": False}, + "prestageFile": {"lifetime": 86400}, + "pinFile": {"lifetime": 60 * 60 * 24}, + "removeDirectory": {"recursive": False}, + "getDirectory": {"localPath": False}, + } + + def __init__(self, name, plugins=None, vo=None, hideExceptions=False): """ c'tor :param str name: SE name @@ -151,24 +154,35 @@ def __init__( self, name, plugins = None, vo = None, hideExceptions = False ): if not result['OK']: return self.vo = result['Value'] - self.opHelper = Operations( vo = self.vo ) + self.opHelper = Operations(vo=self.vo) # These things will soon have to go as well. 'AccessProtocol.1' is all but flexible. - proxiedProtocols = gConfig.getValue( '/LocalSite/StorageElements/ProxyProtocols', "" ).split( ',' ) - self.useProxy = ( gConfig.getValue( "/Resources/StorageElements/%s/AccessProtocol.1/Protocol" % name, "UnknownProtocol" ) - in proxiedProtocols ) + proxiedProtocols = gConfig.getValue('/LocalSite/StorageElements/ProxyProtocols', "").split(',') + self.useProxy = ( + gConfig.getValue( + "/Resources/StorageElements/%s/AccessProtocol.1/Protocol" % + name, "UnknownProtocol") in proxiedProtocols) if not self.useProxy: - self.useProxy = gConfig.getValue( '/LocalSite/StorageElements/%s/UseProxy' % name, False ) + self.useProxy = gConfig.getValue('/LocalSite/StorageElements/%s/UseProxy' % name, False) if not self.useProxy: - self.useProxy = self.opHelper.getValue( '/Services/StorageElements/%s/UseProxy' % name, False ) + self.useProxy = self.opHelper.getValue('/Services/StorageElements/%s/UseProxy' % name, False) self.valid = True - if plugins == None: - res = StorageFactory( useProxy = self.useProxy, vo = self.vo ).getStorages( name, pluginList = [], hideExceptions = hideExceptions ) + if plugins is None: + res = StorageFactory( + useProxy=self.useProxy, + vo=self.vo).getStorages( + name, + pluginList=[], + hideExceptions=hideExceptions) else: - res = StorageFactory( useProxy = self.useProxy, vo = self.vo ).getStorages( name, pluginList = plugins, hideExceptions = hideExceptions ) - + res = StorageFactory( + useProxy=self.useProxy, + vo=self.vo).getStorages( + name, + pluginList=plugins, + hideExceptions=hideExceptions) if not res['OK']: self.valid = False @@ -186,113 +200,114 @@ def __init__( self, name, plugins = None, vo = None, hideExceptions = False ): for storage in self.storages: - storage.setStorageElement( self ) - + storage.setStorageElement(self) - self.log = gLogger.getSubLogger( "SE[%s]" % self.name ) + self.log = gLogger.getSubLogger("SE[%s]" % self.name) if self.valid: - self.useCatalogURL = gConfig.getValue( '/Resources/StorageElements/%s/UseCatalogURL' % self.name, False ) - self.log.debug( "useCatalogURL: %s" % self.useCatalogURL ) + self.useCatalogURL = gConfig.getValue( + '/Resources/StorageElements/%s/UseCatalogURL' % + self.name, False) + self.log.debug("useCatalogURL: %s" % self.useCatalogURL) - self.__dmsHelper = DMSHelpers( vo = vo ) + self.__dmsHelper = DMSHelpers(vo=vo) # Allow SE to overwrite general operation config - accessProto = self.options.get( 'AccessProtocols' ) + accessProto = self.options.get('AccessProtocols') self.localAccessProtocolList = accessProto if accessProto else self.__dmsHelper.getAccessProtocols() - self.log.debug( "localAccessProtocolList %s" % self.localAccessProtocolList ) + self.log.debug("localAccessProtocolList %s" % self.localAccessProtocolList) - writeProto = self.options.get( 'WriteProtocols' ) + writeProto = self.options.get('WriteProtocols') self.localWriteProtocolList = writeProto if writeProto else self.__dmsHelper.getWriteProtocols() - self.log.debug( "localWriteProtocolList %s" % self.localWriteProtocolList ) - - - + self.log.debug("localWriteProtocolList %s" % self.localWriteProtocolList) # 'getTransportURL', - self.readMethods = [ 'getFile', - 'prestageFile', - 'prestageFileStatus', - 'getDirectory'] - - self.writeMethods = [ 'retransferOnlineFile', - 'putFile', - 'replicateFile', - 'pinFile', - 'releaseFile', - 'createDirectory', - 'putDirectory' ] - - self.removeMethods = [ 'removeFile', 'removeDirectory' ] - - self.checkMethods = [ 'exists', - 'getDirectoryMetadata', - 'getDirectorySize', - 'getFileSize', - 'getFileMetadata', - 'listDirectory', - 'isDirectory', - 'isFile', - ] - - self.okMethods = [ 'getLocalProtocols', - 'getProtocols', - 'getRemoteProtocols', - 'storageElementName', - 'getStorageParameters', - 'getTransportURL', - 'isLocalSE' ] + self.readMethods = ['getFile', + 'prestageFile', + 'prestageFileStatus', + 'getDirectory'] + + self.writeMethods = ['retransferOnlineFile', + 'putFile', + 'replicateFile', + 'pinFile', + 'releaseFile', + 'createDirectory', + 'putDirectory'] + + self.removeMethods = ['removeFile', + 'removeDirectory'] + + self.checkMethods = ['exists', + 'getDirectoryMetadata', + 'getDirectorySize', + 'getFileSize', + 'getFileMetadata', + 'listDirectory', + 'isDirectory', + 'isFile', + 'getOccupancy' + ] + + self.okMethods = ['getLocalProtocols', + 'getProtocols', + 'getRemoteProtocols', + 'storageElementName', + 'getStorageParameters', + 'getTransportURL', + 'isLocalSE'] self.__fileCatalog = None - def dump( self ): + def dump(self): """ Dump to the logger a summary of the StorageElement items. """ - log = self.log.getSubLogger( 'dump', True ) - log.verbose( "Preparing dump for StorageElement %s." % self.name ) + log = self.log.getSubLogger('dump', True) + log.verbose("Preparing dump for StorageElement %s." % self.name) if not self.valid: - log.debug( "Failed to create StorageElement plugins.", self.errorReason ) + log.debug("Failed to create StorageElement plugins.", self.errorReason) return i = 1 outStr = "\n\n============ Options ============\n" - for key in sorted( self.options ): - outStr = "%s%s: %s\n" % ( outStr, key.ljust( 15 ), self.options[key] ) + for key in sorted(self.options): + outStr = "%s%s: %s\n" % (outStr, key.ljust(15), self.options[key]) for storage in self.storages: - outStr = "%s============Protocol %s ============\n" % ( outStr, i ) + outStr = "%s============Protocol %s ============\n" % (outStr, i) storageParameters = storage.getParameters() - for key in sorted( storageParameters ): - outStr = "%s%s: %s\n" % ( outStr, key.ljust( 15 ), storageParameters[key] ) + for key in sorted(storageParameters): + outStr = "%s%s: %s\n" % (outStr, key.ljust(15), storageParameters[key]) i = i + 1 - log.verbose( outStr ) + log.verbose(outStr) ################################################################################################# # # These are the basic get functions for storage configuration # - def getStorageElementName( self ): + def getStorageElementName(self): """ SE name getter for backward compatibility """ - return S_OK( self.storageElementName() ) + return S_OK(self.storageElementName()) - def storageElementName( self ): + def storageElementName(self): """ SE name getter """ - self.log.getSubLogger( 'storageElementName' ).verbose( "The Storage Element name is %s." % self.name ) + self.log.getSubLogger('storageElementName').verbose( + "The Storage Element name is %s." % self.name) return self.name - def getChecksumType( self ): + def getChecksumType(self): """ Checksum type getter for backward compatibility """ - return S_OK( self.checksumType() ) + return S_OK(self.checksumType()) - def checksumType( self ): + def checksumType(self): """ get specific /Resources/StorageElements//ChecksumType option if defined, otherwise global /Resources/StorageElements/ChecksumType """ - self.log.getSubLogger( 'checksumType' ).verbose( "get checksum type for %s." % self.name ) - return self.options["ChecksumType"].upper() \ - if "ChecksumType" in self.options else gConfig.getValue( "/Resources/StorageElements/ChecksumType", "ADLER32" ).upper() + self.log.getSubLogger('checksumType').verbose("get checksum type for %s." % self.name) + return self.options["ChecksumType"].upper() if "ChecksumType" in self.options else gConfig.getValue( + "/Resources/StorageElements/ChecksumType", "ADLER32").upper() - def getStatus( self ): + def getStatus(self): """ Return Status of the SE only if the SE is valid It returns an S_OK/S_ERROR structure @@ -300,9 +315,42 @@ def getStatus( self ): valid = self.isValid() if not valid['OK']: return valid - return S_OK( self.status() ) + return S_OK(self.status()) + + def getOccupancy(self, unit='MB', **kwargs): + """ Retrieves the space information about the storage. + It returns the Total, Guaranteed and Free space . + + It loops over the different Storage Plugins to query it. - def status( self ): + :returns: S_OK with dict (keys: Total, Guaranteed, Free) + """ + log = self.log.getSubLogger('getOccupancy', True) + + filteredPlugins = self.__filterPlugins('getOccupancy') + if not filteredPlugins: + return S_ERROR(errno.EPROTONOSUPPORT, "No storage plugins to query the occupancy") + # Try all of the storages one by one + for storage in filteredPlugins: + # The result of the plugin is always in MB + res = storage.getOccupancy(**kwargs) + if res['OK']: + occupancyDict = res['Value'] + if unit != 'MB': + for space in ['Total', 'Free']: + convertedSpace = convertSizeUnits(occupancyDict[space], 'MB', unit) + # If we have a conversion error, we go to the next plugin + if convertedSpace == -sys.maxsize: + log.verbose( + "Error converting %s space from MB to %s: %s" % + (space, unit, occupancyDict[space])) + break + occupancyDict[space] = convertedSpace + return res + + return S_ERROR("Could not retrieve the occupancy from any plugin") + + def status(self): """ Return Status of the SE, a dictionary with: @@ -311,7 +359,8 @@ def status( self ): * Remove: True (is allowed), False (it is not allowed) * Check: True (is allowed), False (it is not allowed). - .. note:: Check is always allowed IF Read is allowed (regardless of what set in the Check option of the configuration) + .. note:: Check is always allowed IF Read is allowed + (regardless of what set in the Check option of the configuration) * DiskSE: True if TXDY with Y > 0 (defaults to True) * TapeSE: True if TXDY with X > 0 (defaults to False) @@ -320,7 +369,7 @@ def status( self ): It returns directly the dictionary """ - self.log.getSubLogger( 'getStatus' ).verbose( "determining status of %s." % self.name ) + self.log.getSubLogger('getStatus').verbose("determining status of %s." % self.name) retDict = {} if not self.valid: @@ -336,126 +385,137 @@ def status( self ): # If nothing is defined in the CS Access is allowed # If something is defined, then it must be set to Active - retDict['Read'] = not ( 'ReadAccess' in self.options and self.options['ReadAccess'] not in ( 'Active', 'Degraded' ) ) - retDict['Write'] = not ( 'WriteAccess' in self.options and self.options['WriteAccess'] not in ( 'Active', 'Degraded' ) ) - retDict['Remove'] = not ( 'RemoveAccess' in self.options and self.options['RemoveAccess'] not in ( 'Active', 'Degraded' ) ) + retDict['Read'] = not ( + 'ReadAccess' in self.options and self.options['ReadAccess'] not in ( + 'Active', 'Degraded')) + retDict['Write'] = not ( + 'WriteAccess' in self.options and self.options['WriteAccess'] not in ( + 'Active', 'Degraded')) + retDict['Remove'] = not ( + 'RemoveAccess' in self.options and self.options['RemoveAccess'] not in ( + 'Active', 'Degraded')) if retDict['Read']: retDict['Check'] = True else: - retDict['Check'] = not ( 'CheckAccess' in self.options and self.options['CheckAccess'] not in ( 'Active', 'Degraded' ) ) + retDict['Check'] = not ( + 'CheckAccess' in self.options and self.options['CheckAccess'] not in ( + 'Active', 'Degraded')) diskSE = True tapeSE = False if 'SEType' in self.options: # Type should follow the convention TXDY seType = self.options['SEType'] - diskSE = re.search( 'D[1-9]', seType ) != None - tapeSE = re.search( 'T[1-9]', seType ) != None + diskSE = re.search('D[1-9]', seType) is not None + tapeSE = re.search('T[1-9]', seType) is not None retDict['DiskSE'] = diskSE retDict['TapeSE'] = tapeSE try: - retDict['TotalCapacityTB'] = float( self.options['TotalCapacityTB'] ) + retDict['TotalCapacityTB'] = float(self.options['TotalCapacityTB']) except Exception: retDict['TotalCapacityTB'] = -1 try: - retDict['DiskCacheTB'] = float( self.options['DiskCacheTB'] ) + retDict['DiskCacheTB'] = float(self.options['DiskCacheTB']) except Exception: retDict['DiskCacheTB'] = -1 return retDict - def isValid( self, operation = None ): + def isValid(self, operation=None): """ check CS/RSS statuses for :operation: :param str operation: operation name """ - log = self.log.getSubLogger( 'isValid', True ) - log.verbose( "Determining if the StorageElement %s is valid for VO %s" % ( self.name, self.vo ) ) + log = self.log.getSubLogger('isValid', True) + log.verbose("Determining if the StorageElement %s is valid for VO %s" % (self.name, self.vo)) if not self.valid: - log.debug( "Failed to create StorageElement plugins.", self.errorReason ) - return S_ERROR( "SE.isValid: Failed to create StorageElement plugins: %s" % self.errorReason ) + log.debug("Failed to create StorageElement plugins.", self.errorReason) + return S_ERROR("SE.isValid: Failed to create StorageElement plugins: %s" % self.errorReason) # Check if the Storage Element is eligible for the user's VO - if 'VO' in self.options and not self.vo in self.options['VO']: - log.debug( "StorageElement is not allowed for VO", self.vo ) - return S_ERROR( errno.EACCES, "StorageElement.isValid: StorageElement is not allowed for VO" ) - log.verbose( "Determining if the StorageElement %s is valid for operation '%s'" % ( self.name, operation ) ) - if ( not operation ) or ( operation in self.okMethods ): + if 'VO' in self.options and self.vo not in self.options['VO']: + log.debug("StorageElement is not allowed for VO", self.vo) + return S_ERROR(errno.EACCES, "StorageElement.isValid: StorageElement is not allowed for VO") + log.verbose( + "Determining if the StorageElement %s is valid for operation '%s'" % + (self.name, operation)) + if (not operation) or (operation in self.okMethods): return S_OK() # Determine whether the StorageElement is valid for checking, reading, writing status = self.status() - checking = status[ 'Check' ] - reading = status[ 'Read' ] - writing = status[ 'Write' ] - removing = status[ 'Remove' ] + checking = status['Check'] + reading = status['Read'] + writing = status['Write'] + removing = status['Remove'] # Determine whether the requested operation can be fulfilled - if ( not operation ) and ( not reading ) and ( not writing ) and ( not checking ): - log.debug( "Read, write and check access not permitted." ) - return S_ERROR( errno.EACCES, "SE.isValid: Read, write and check access not permitted." ) - + if (not operation) and (not reading) and (not writing) and (not checking): + log.debug("Read, write and check access not permitted.") + return S_ERROR(errno.EACCES, "SE.isValid: Read, write and check access not permitted.") # The supplied operation can be 'Read','Write' or any of the possible StorageElement methods. - if ( operation in self.readMethods ) or ( operation.lower() in ( 'read', 'readaccess' ) ): + if (operation in self.readMethods) or (operation.lower() in ('read', 'readaccess')): operation = 'ReadAccess' - elif operation in self.writeMethods or ( operation.lower() in ( 'write', 'writeaccess' ) ): + elif operation in self.writeMethods or (operation.lower() in ('write', 'writeaccess')): operation = 'WriteAccess' - elif operation in self.removeMethods or ( operation.lower() in ( 'remove', 'removeaccess' ) ): + elif operation in self.removeMethods or (operation.lower() in ('remove', 'removeaccess')): operation = 'RemoveAccess' - elif operation in self.checkMethods or ( operation.lower() in ( 'check', 'checkaccess' ) ): + elif operation in self.checkMethods or (operation.lower() in ('check', 'checkaccess')): operation = 'CheckAccess' else: - log.debug( "The supplied operation is not known.", operation ) - return S_ERROR( DErrno.ENOMETH , "SE.isValid: The supplied operation is not known." ) - log.debug( "check the operation: %s " % operation ) + log.debug("The supplied operation is not known.", operation) + return S_ERROR(DErrno.ENOMETH, "SE.isValid: The supplied operation is not known.") + log.debug("check the operation: %s " % operation) # Check if the operation is valid if operation == 'CheckAccess': if not reading: if not checking: - log.debug( "Check access not currently permitted." ) - return S_ERROR( errno.EACCES, "SE.isValid: Check access not currently permitted." ) + log.debug("Check access not currently permitted.") + return S_ERROR(errno.EACCES, "SE.isValid: Check access not currently permitted.") if operation == 'ReadAccess': if not reading: - log.debug( "Read access not currently permitted." ) - return S_ERROR( errno.EACCES, "SE.isValid: Read access not currently permitted." ) + log.debug("Read access not currently permitted.") + return S_ERROR(errno.EACCES, "SE.isValid: Read access not currently permitted.") if operation == 'WriteAccess': if not writing: - log.debug( "Write access not currently permitted." ) - return S_ERROR( errno.EACCES, "SE.isValid: Write access not currently permitted." ) + log.debug("Write access not currently permitted.") + return S_ERROR(errno.EACCES, "SE.isValid: Write access not currently permitted.") if operation == 'RemoveAccess': if not removing: - log.debug( "Remove access not currently permitted." ) - return S_ERROR( errno.EACCES, "SE.isValid: Remove access not currently permitted." ) + log.debug("Remove access not currently permitted.") + return S_ERROR(errno.EACCES, "SE.isValid: Remove access not currently permitted.") return S_OK() - def getPlugins( self ): + def getPlugins(self): """ Get the list of all the plugins defined for this Storage Element """ - self.log.getSubLogger( 'getPlugins' ).verbose( "Obtaining all plugins of %s." % self.name ) + self.log.getSubLogger('getPlugins').verbose("Obtaining all plugins of %s." % self.name) if not self.valid: - return S_ERROR( self.errorReason ) + return S_ERROR(self.errorReason) allPlugins = self.localPlugins + self.remotePlugins - return S_OK( allPlugins ) + return S_OK(allPlugins) - def getRemotePlugins( self ): + def getRemotePlugins(self): """ Get the list of all the remote access protocols defined for this Storage Element """ - self.log.getSubLogger( 'getRemotePlugins' ).verbose( "Obtaining remote protocols for %s." % self.name ) + self.log.getSubLogger('getRemotePlugins').verbose( + "Obtaining remote protocols for %s." % self.name) if not self.valid: - return S_ERROR( self.errorReason ) - return S_OK( self.remotePlugins ) + return S_ERROR(self.errorReason) + return S_OK(self.remotePlugins) - def getLocalPlugins( self ): + def getLocalPlugins(self): """ Get the list of all the local access protocols defined for this Storage Element """ - self.log.getSubLogger( 'getLocalPlugins' ).verbose( "Obtaining local protocols for %s." % self.name ) + self.log.getSubLogger('getLocalPlugins').verbose( + "Obtaining local protocols for %s." % self.name) if not self.valid: - return S_ERROR( self.errorReason ) - return S_OK( self.localPlugins ) + return S_ERROR(self.errorReason) + return S_OK(self.localPlugins) - def getStorageParameters( self, plugin = None, protocol = None ): + def getStorageParameters(self, plugin=None, protocol=None): """ Get plugin specific options :param plugin : plugin we are interested in :param protocol: protocol we are interested in @@ -465,49 +525,47 @@ def getStorageParameters( self, plugin = None, protocol = None ): # both set if plugin and protocol: - return S_ERROR( errno.EINVAL, "plugin and protocol cannot be set together." ) + return S_ERROR(errno.EINVAL, "plugin and protocol cannot be set together.") # both None - elif not ( plugin or protocol ): - return S_ERROR( errno.EINVAL, "plugin and protocol cannot be None together." ) + elif not (plugin or protocol): + return S_ERROR(errno.EINVAL, "plugin and protocol cannot be None together.") - log = self.log.getSubLogger( 'getStorageParameters' ) + log = self.log.getSubLogger('getStorageParameters') reqStr = "plugin %s" % plugin if plugin else "protocol %s" % protocol - log.verbose( "Obtaining storage parameters for %s for %s." % ( self.name, - reqStr ) ) - + log.verbose("Obtaining storage parameters for %s for %s." % (self.name, + reqStr)) for storage in self.storages: storageParameters = storage.getParameters() if plugin and storageParameters['PluginName'] == plugin: - return S_OK( storageParameters ) + return S_OK(storageParameters) elif protocol and storageParameters['Protocol'] == protocol: - return S_OK( storageParameters ) + return S_OK(storageParameters) errStr = "Requested plugin or protocol not available." - log.debug( errStr, "%s for %s" % ( reqStr, self.name ) ) - return S_ERROR( errStr ) + log.debug(errStr, "%s for %s" % (reqStr, self.name)) + return S_ERROR(errStr) - def __getAllProtocols( self, protoType ): + def __getAllProtocols(self, protoType): """ Returns the list of all protocols for Input or Output :param proto = InputProtocols or OutputProtocols """ - return set( reduce( lambda x, y:x + y, [plugin.protocolParameters[protoType] for plugin in self.storages ] ) ) - + return set(reduce(lambda x, y: x + + y, [plugin.protocolParameters[protoType] for plugin in self.storages])) - def _getAllInputProtocols( self ): + def _getAllInputProtocols(self): """ Returns all the protocols supported by the SE for Input """ - return self.__getAllProtocols( 'InputProtocols' ) + return self.__getAllProtocols('InputProtocols') - def _getAllOutputProtocols( self ): + def _getAllOutputProtocols(self): """ Returns all the protocols supported by the SE for Output """ - return self.__getAllProtocols( 'OutputProtocols' ) + return self.__getAllProtocols('OutputProtocols') - - def generateTransferURLsBetweenSEs(self, lfns, sourceSE, protocols = None): + def generateTransferURLsBetweenSEs(self, lfns, sourceSE, protocols=None): """ This negociate the URLs to be used for third party copy. This is mostly useful for FTS. If protocols is given, it restricts the list of plugins to use @@ -518,18 +576,18 @@ def generateTransferURLsBetweenSEs(self, lfns, sourceSE, protocols = None): :return:dictionnary Successful/Failed with pair (src, dest) urls """ - log = self.log.getSubLogger( 'generateTransferURLsBetweenSEs' ) + log = self.log.getSubLogger('generateTransferURLsBetweenSEs') - result = checkArgumentFormat( lfns ) + result = checkArgumentFormat(lfns) if result['OK']: lfns = result['Value'] else: errStr = "Supplied urls must be string, list of strings or a dictionary." - log.debug( errStr ) - return S_ERROR( errno.EINVAL, errStr ) + log.debug(errStr) + return S_ERROR(errno.EINVAL, errStr) # First, find common protocols to use - res = self.negociateProtocolWithOtherSE(sourceSE, protocols = protocols) + res = self.negociateProtocolWithOtherSE(sourceSE, protocols=protocols) if not res['OK']: return res @@ -541,7 +599,7 @@ def generateTransferURLsBetweenSEs(self, lfns, sourceSE, protocols = None): srcPlugin = None destPlugin = None - log.debug("Trying to find plugins for protocol %s"%proto) + log.debug("Trying to find plugins for protocol %s" % proto) # Finding the source storage plugin for storagePlugin in sourceSE.storages: @@ -549,15 +607,15 @@ def generateTransferURLsBetweenSEs(self, lfns, sourceSE, protocols = None): storageParameters = storagePlugin.getParameters() nativeProtocol = storageParameters['Protocol'] # If the native protocol of the plugin is allowed for read - if nativeProtocol in sourceSE.localAccessProtocolList: + if nativeProtocol in sourceSE.localAccessProtocolList: # If the plugin can generate the protocol we are interested in if proto in storageParameters['OutputProtocols']: log.debug("Selecting it") srcPlugin = storagePlugin break # If we did not find a source plugin, continue - if srcPlugin is None : - log.debug("Could not find a source plugin for protocol %s"%proto) + if srcPlugin is None: + log.debug("Could not find a source plugin for protocol %s" % proto) continue # Finding the destination storage plugin @@ -567,7 +625,7 @@ def generateTransferURLsBetweenSEs(self, lfns, sourceSE, protocols = None): storageParameters = storagePlugin.getParameters() nativeProtocol = storageParameters['Protocol'] # If the native protocol of the plugin is allowed for write - if nativeProtocol in self.localWriteProtocolList: + if nativeProtocol in self.localWriteProtocolList: # If the plugin can accept the protocol we are interested in if proto in storageParameters['InputProtocols']: log.debug("Selecting it") @@ -577,42 +635,40 @@ def generateTransferURLsBetweenSEs(self, lfns, sourceSE, protocols = None): # If we found both a source and destination plugin, we are happy, # otherwise we continue with the next protocol if destPlugin is None: - log.debug("Could not find a destination plugin for protocol %s"%proto) + log.debug("Could not find a destination plugin for protocol %s" % proto) srcPlugin = None continue - failed = {} successful = {} # Generate the URLs for lfn in lfns: # Source URL first - res = srcPlugin.constructURLFromLFN( lfn, withWSUrl = True ) + res = srcPlugin.constructURLFromLFN(lfn, withWSUrl=True) if not res['OK']: - errMsg = "Error generating source url: %s"%res['Message'] - gLogger.debug("Error generating source url",errMsg) + errMsg = "Error generating source url: %s" % res['Message'] + gLogger.debug("Error generating source url", errMsg) failed[lfn] = errMsg continue srcURL = res['Value'] # Destination URL - res = destPlugin.constructURLFromLFN( lfn, withWSUrl = True ) + res = destPlugin.constructURLFromLFN(lfn, withWSUrl=True) if not res['OK']: - errMsg = "Error generating destination url: %s"%res['Message'] - gLogger.debug("Error generating destination url",errMsg) + errMsg = "Error generating destination url: %s" % res['Message'] + gLogger.debug("Error generating destination url", errMsg) failed[lfn] = errMsg continue destURL = res['Value'] successful[lfn] = (srcURL, destURL) - return S_OK({'Successful':successful, 'Failed':failed}) + return S_OK({'Successful': successful, 'Failed': failed}) return S_ERROR(errno.ENOPROTOOPT, "Could not find a protocol ") - - def negociateProtocolWithOtherSE( self, sourceSE, protocols = None ): + def negociateProtocolWithOtherSE(self, sourceSE, protocols=None): """ Negotiate what protocol could be used for a third party transfer between the sourceSE and ourselves. If protocols is given, the chosen protocol has to be among those @@ -626,22 +682,23 @@ def negociateProtocolWithOtherSE( self, sourceSE, protocols = None ): # No common protocols if this is a proxy storage if self.useProxy: - return S_OK( [] ) - - log = self.log.getSubLogger( 'negociateProtocolWithOtherSE', child = True ) + return S_OK([]) - log.debug( "Negociating protocols between %s and %s (protocols %s)" % ( sourceSE.name, self.name, protocols ) ) + log = self.log.getSubLogger('negociateProtocolWithOtherSE', child=True) + log.debug( + "Negociating protocols between %s and %s (protocols %s)" % + (sourceSE.name, self.name, protocols)) # Take all the protocols the destination can accept as input destProtocols = self._getAllInputProtocols() - log.debug( "Destination input protocols %s" % destProtocols ) + log.debug("Destination input protocols %s" % destProtocols) # Take all the protocols the source can provide sourceProtocols = sourceSE._getAllOutputProtocols() - log.debug( "Source output protocols %s" % sourceProtocols ) + log.debug("Source output protocols %s" % sourceProtocols) commonProtocols = destProtocols & sourceProtocols @@ -649,35 +706,39 @@ def negociateProtocolWithOtherSE( self, sourceSE, protocols = None ): # take the intersection, and sort the commonProtocols # based on the protocolList order if protocols: - protocolList = list( protocols ) - commonProtocols = sorted( commonProtocols & set( protocolList ), key = lambda x : self.__getIndexInList( x, protocolList ) ) + protocolList = list(protocols) + commonProtocols = sorted( + commonProtocols & set(protocolList), + key=lambda x: self.__getIndexInList( + x, + protocolList)) - log.debug( "Common protocols %s" % commonProtocols ) + log.debug("Common protocols %s" % commonProtocols) - return S_OK( list( commonProtocols ) ) + return S_OK(list(commonProtocols)) ################################################################################################# # # These are the basic get functions for lfn manipulation # - def __getURLPath( self, url ): + def __getURLPath(self, url): """ Get the part of the URL path below the basic storage path. This path must coincide with the LFN of the file in order to be compliant with the DIRAC conventions. """ - log = self.log.getSubLogger( '__getURLPath' ) - log.verbose( "Getting path from url in %s." % self.name ) + log = self.log.getSubLogger('__getURLPath') + log.verbose("Getting path from url in %s." % self.name) if not self.valid: - return S_ERROR( self.errorReason ) - res = pfnparse( url ) + return S_ERROR(self.errorReason) + res = pfnparse(url) if not res['OK']: return res - fullURLPath = '%s/%s' % ( res['Value']['Path'], res['Value']['FileName'] ) + fullURLPath = '%s/%s' % (res['Value']['Path'], res['Value']['FileName']) # Check all available storages and check whether the url is for that protocol urlPath = '' for storage in self.storages: - res = storage.isNativeURL( url ) + res = storage.isNativeURL(url) if res['OK']: if res['Value']: parameters = storage.getParameters() @@ -686,91 +747,92 @@ def __getURLPath( self, url ): # If the sa path doesn't exist then the url path is the entire string urlPath = fullURLPath else: - if re.search( saPath, fullURLPath ): + if re.search(saPath, fullURLPath): # Remove the sa path from the fullURLPath - urlPath = fullURLPath.replace( saPath, '' ) + urlPath = fullURLPath.replace(saPath, '') if urlPath: - return S_OK( urlPath ) + return S_OK(urlPath) # This should never happen. DANGER!! errStr = "Failed to get the url path for any of the protocols!!" - log.debug( errStr ) - return S_ERROR( errStr ) + log.debug(errStr) + return S_ERROR(errStr) - def getLFNFromURL( self, urls ): + def getLFNFromURL(self, urls): """ Get the LFN from the PFNS . :param lfn : input lfn or lfns (list/dict) """ - result = checkArgumentFormat( urls ) + result = checkArgumentFormat(urls) if result['OK']: urlDict = result['Value'] else: errStr = "Supplied urls must be string, list of strings or a dictionary." - self.log.getSubLogger( 'getLFNFromURL' ).debug( errStr ) - return S_ERROR( errno.EINVAL, errStr ) + self.log.getSubLogger('getLFNFromURL').debug(errStr) + return S_ERROR(errno.EINVAL, errStr) - retDict = { "Successful" : {}, "Failed" : {} } + retDict = {"Successful": {}, "Failed": {}} for url in urlDict: - res = self.__getURLPath( url ) + res = self.__getURLPath(url) if res["OK"]: retDict["Successful"][url] = res["Value"] else: retDict["Failed"][url] = res["Message"] - return S_OK( retDict ) + return S_OK(retDict) ########################################################################################### # # This is the generic wrapper for file operations # - def getURL( self, lfn, protocol = False, replicaDict = None ): + def getURL(self, lfn, protocol=False, replicaDict=None): """ execute 'getTransportURL' operation. :param str lfn: string, list or dictionary of lfns :param protocol: if no protocol is specified, we will request self.turlProtocols :param replicaDict: optional results from the File Catalog replica query """ - self.log.getSubLogger( 'getURL' ).verbose( "Getting accessUrl %s for lfn in %s." % ( "(%s)" % protocol if protocol else "", self.name ) ) + self.log.getSubLogger('getURL').verbose("Getting accessUrl %s for lfn in %s." % + ("(%s)" % protocol if protocol else "", self.name)) if not protocol: # This turlProtocols seems totally useless. # Get ride of it when gfal2 is totally ready # and replace it with the localAccessProtocol list protocols = self.turlProtocols - elif isinstance( protocol, list ): + elif isinstance(protocol, list): protocols = protocol - elif isinstance( protocol, basestring ): + elif isinstance(protocol, basestring): protocols = [protocol] self.methodName = "getTransportURL" - result = self.__executeMethod( lfn, protocols = protocols ) + result = self.__executeMethod(lfn, protocols=protocols) return result - def __isLocalSE( self ): + def __isLocalSE(self): """ Test if the Storage Element is local in the current context """ - self.log.getSubLogger( 'LocalSE' ).verbose( "Determining whether %s is a local SE." % self.name ) + self.log.getSubLogger('LocalSE').verbose("Determining whether %s is a local SE." % self.name) import DIRAC - localSEs = getSEsForSite( DIRAC.siteName() )['Value'] + localSEs = getSEsForSite(DIRAC.siteName())['Value'] if self.name in localSEs: - return S_OK( True ) + return S_OK(True) else: - return S_OK( False ) + return S_OK(False) - def __getFileCatalog( self ): + def __getFileCatalog(self): if not self.__fileCatalog: - self.__fileCatalog = FileCatalog( vo = self.vo ) + self.__fileCatalog = FileCatalog(vo=self.vo) return self.__fileCatalog - def __generateURLDict( self, lfns, storage, replicaDict = None ): + def __generateURLDict(self, lfns, storage, replicaDict=None): """ Generates a dictionary (url : lfn ), where the url are constructed from the lfn using the constructURLFromLFN method of the storage plugins. :param: lfns : dictionary {lfn:whatever} :returns dictionary {constructed url : lfn} """ - log = self.log.getSubLogger( "__generateURLDict" ) - log.verbose( "generating url dict for %s lfn in %s." % ( len( lfns ), self.name ) ) + log = self.log.getSubLogger("__generateURLDict") + log.verbose("generating url dict for %s lfn in %s." % (len(lfns), self.name)) if not replicaDict: replicaDict = {} @@ -780,7 +842,7 @@ def __generateURLDict( self, lfns, storage, replicaDict = None ): for lfn in lfns: if self.useCatalogURL: # Is this self.name alias proof? - url = replicaDict.get( lfn, {} ).get( self.name, '' ) + url = replicaDict.get(lfn, {}).get(self.name, '') if url: urlDict[url] = lfn continue @@ -789,32 +851,32 @@ def __generateURLDict( self, lfns, storage, replicaDict = None ): result = fc.getReplicas() if not result['OK']: failed[lfn] = result['Message'] - url = result['Value']['Successful'].get( lfn, {} ).get( self.name, '' ) + url = result['Value']['Successful'].get(lfn, {}).get(self.name, '') if not url: failed[lfn] = 'Failed to get catalog replica' else: # Update the URL according to the current SE description - result = returnSingleResult( storage.updateURL( url ) ) + result = returnSingleResult(storage.updateURL(url)) if not result['OK']: failed[lfn] = result['Message'] else: urlDict[result['Value']] = lfn else: - result = storage.constructURLFromLFN( lfn, withWSUrl = True ) + result = storage.constructURLFromLFN(lfn, withWSUrl=True) if not result['OK']: errStr = result['Message'] - log.debug( errStr, 'for %s' % ( lfn ) ) - failed[lfn] = "%s %s" % ( failed[lfn], errStr ) if lfn in failed else errStr + log.debug(errStr, 'for %s' % (lfn)) + failed[lfn] = "%s %s" % (failed[lfn], errStr) if lfn in failed else errStr else: urlDict[result['Value']] = lfn - res = S_OK( {'Successful': urlDict, 'Failed' : failed} ) + res = S_OK({'Successful': urlDict, 'Failed': failed}) # res['Failed'] = failed return res @staticmethod - def __getIndexInList( x, l ): + def __getIndexInList(x, l): """ Return the index of the element x in the list l or sys.maxint if it does not exist @@ -824,11 +886,11 @@ def __getIndexInList( x, l ): :return: the index or sys.maxint """ try: - return l.index( x ) + return l.index(x) except ValueError: - return sys.maxint + return sys.maxsize - def __filterPlugins( self, methodName, protocols = None, inputProtocol = None ): + def __filterPlugins(self, methodName, protocols=None, inputProtocol=None): """ Determine the list of plugins that can be used for a particular action @@ -842,11 +904,13 @@ def __filterPlugins( self, methodName, protocols = None, inputProtocol = None ): list: list of storage plugins """ - log = self.log.getSubLogger( '__filterPlugins', child = True ) + log = self.log.getSubLogger('__filterPlugins', child=True) - log.debug( "Filtering plugins for %s (protocol = %s ; inputProtocol = %s)" % ( methodName, protocols, inputProtocol ) ) + log.debug( + "Filtering plugins for %s (protocol = %s ; inputProtocol = %s)" % + (methodName, protocols, inputProtocol)) - if isinstance( protocols, basestring ): + if isinstance(protocols, basestring): protocols = [protocols] pluginsToUse = [] @@ -856,7 +920,7 @@ def __filterPlugins( self, methodName, protocols = None, inputProtocol = None ): if methodName in self.readMethods + self.checkMethods: allowedProtocols = self.localAccessProtocolList - elif methodName in self.removeMethods + self.writeMethods : + elif methodName in self.removeMethods + self.writeMethods: allowedProtocols = self.localWriteProtocolList else: # OK methods @@ -864,66 +928,74 @@ def __filterPlugins( self, methodName, protocols = None, inputProtocol = None ): # can generate such protocol # otherwise we return them all if protocols: - setProtocol = set( protocols ) + setProtocol = set(protocols) for plugin in self.storages: - if set( plugin.protocolParameters.get( "OutputProtocols", [] ) ) & setProtocol: - log.debug( "Plugin %s can generate compatible protocol" % plugin.pluginName ) - pluginsToUse.append( plugin ) + if set(plugin.protocolParameters.get("OutputProtocols", [])) & setProtocol: + log.debug("Plugin %s can generate compatible protocol" % plugin.pluginName) + pluginsToUse.append(plugin) else: pluginsToUse = self.storages # The closest list for "OK" methods is the AccessProtocol preference, so we sort based on that - pluginsToUse.sort( key = lambda x: self.__getIndexInList( x.protocolParameters['Protocol'] , self.localAccessProtocolList ) ) - log.debug( "Plugins to be used for %s: %s" % ( methodName, [p.pluginName for p in pluginsToUse] ) ) + pluginsToUse.sort( + key=lambda x: self.__getIndexInList( + x.protocolParameters['Protocol'], + self.localAccessProtocolList)) + log.debug("Plugins to be used for %s: %s" % + (methodName, [p.pluginName for p in pluginsToUse])) return pluginsToUse - log.debug( "Allowed protocol: %s" % allowedProtocols ) + log.debug("Allowed protocol: %s" % allowedProtocols) # if a list of protocol is specified, take it into account if protocols: - potentialProtocols = list( set( allowedProtocols ) & set( protocols ) ) + potentialProtocols = list(set(allowedProtocols) & set(protocols)) else: potentialProtocols = allowedProtocols - log.debug( 'Potential protocols %s' % potentialProtocols ) + log.debug('Potential protocols %s' % potentialProtocols) localSE = self.__isLocalSE()['Value'] for plugin in self.storages: # Determine whether to use this storage object pluginParameters = plugin.getParameters() - pluginName = pluginParameters.get( 'PluginName' ) + pluginName = pluginParameters.get('PluginName') if not pluginParameters: - log.debug( "Failed to get storage parameters.", "%s %s" % ( self.name, pluginName ) ) + log.debug("Failed to get storage parameters.", "%s %s" % (self.name, pluginName)) continue - if not ( pluginName in self.remotePlugins ) and not localSE and not pluginName == "Proxy": + if not (pluginName in self.remotePlugins) and not localSE and not pluginName == "Proxy": # If the SE is not local then we can't use local protocols - log.debug( "Local protocol not appropriate for remote use: %s." % pluginName ) + log.debug("Local protocol not appropriate for remote use: %s." % pluginName) continue if pluginParameters['Protocol'] not in potentialProtocols: - log.debug( "Plugin %s not allowed for %s." % ( pluginName, methodName ) ) + log.debug("Plugin %s not allowed for %s." % (pluginName, methodName)) continue # If we are attempting a putFile and we know the inputProtocol if methodName == 'putFile' and inputProtocol: if inputProtocol not in pluginParameters['InputProtocols']: - log.debug( "Plugin %s not appropriate for %s protocol as input." % ( pluginName, inputProtocol ) ) + log.debug( + "Plugin %s not appropriate for %s protocol as input." % + (pluginName, inputProtocol)) continue - pluginsToUse.append( plugin ) + pluginsToUse.append(plugin) # sort the plugins according to the lists in the CS - pluginsToUse.sort( key = lambda x: self.__getIndexInList( x.protocolParameters['Protocol'] , allowedProtocols ) ) + pluginsToUse.sort( + key=lambda x: self.__getIndexInList( + x.protocolParameters['Protocol'], + allowedProtocols)) - log.debug( "Plugins to be used for %s: %s" % ( methodName, [p.pluginName for p in pluginsToUse] ) ) + log.debug("Plugins to be used for %s: %s" % (methodName, [p.pluginName for p in pluginsToUse])) return pluginsToUse - - def __executeMethod( self, lfn, *args, **kwargs ): + def __executeMethod(self, lfn, *args, **kwargs): """ Forward the call to each storage in turn until one works. The method to be executed is stored in self.methodName :param lfn : string, list or dictionary @@ -937,68 +1009,67 @@ def __executeMethod( self, lfn, *args, **kwargs ): the protocol used as source protocol, since there is in principle only one. """ - removedArgs = {} - log = self.log.getSubLogger( '__executeMethod' ) - log.verbose( "preparing the execution of %s" % ( self.methodName ) ) + log = self.log.getSubLogger('__executeMethod') + log.verbose("preparing the execution of %s" % (self.methodName)) # args should normaly be empty to avoid problem... - if len( args ): - log.verbose( "args should be empty!%s" % args ) + if len(args): + log.verbose("args should be empty!%s" % args) # because there is normally only one kw argument, I can move it from args to kwargs - methDefaultArgs = StorageElementItem.__defaultsArguments.get( self.methodName, {} ).keys() - if len( methDefaultArgs ): - kwargs[methDefaultArgs[0] ] = args[0] + methDefaultArgs = StorageElementItem.__defaultsArguments.get(self.methodName, {}).keys() + if len(methDefaultArgs): + kwargs[methDefaultArgs[0]] = args[0] args = args[1:] - log.verbose( "put it in kwargs, but dirty and might be dangerous!args %s kwargs %s" % ( args, kwargs ) ) - + log.verbose( + "put it in kwargs, but dirty and might be dangerous!args %s kwargs %s" % + (args, kwargs)) # We check the deprecated arguments for depArg in StorageElementItem.__deprecatedArguments: if depArg in kwargs: - log.verbose( "%s is not an allowed argument anymore. Please change your code!" % depArg ) + log.verbose("%s is not an allowed argument anymore. Please change your code!" % depArg) removedArgs[depArg] = kwargs[depArg] del kwargs[depArg] - - # Set default argument if any - methDefaultArgs = StorageElementItem.__defaultsArguments.get( self.methodName, {} ) + methDefaultArgs = StorageElementItem.__defaultsArguments.get(self.methodName, {}) for argName in methDefaultArgs: if argName not in kwargs: - log.debug( "default argument %s for %s not present.\ - Setting value %s." % ( argName, self.methodName, methDefaultArgs[argName] ) ) + log.debug("default argument %s for %s not present.\ + Setting value %s." % (argName, self.methodName, methDefaultArgs[argName])) kwargs[argName] = methDefaultArgs[argName] - res = checkArgumentFormat( lfn ) + res = checkArgumentFormat(lfn) if not res['OK']: errStr = "Supplied lfns must be string, list of strings or a dictionary." - log.debug( errStr ) + log.debug(errStr) return res lfnDict = res['Value'] - log.verbose( "Attempting to perform '%s' operation with %s lfns." % ( self.methodName, len( lfnDict ) ) ) + log.verbose( + "Attempting to perform '%s' operation with %s lfns." % + (self.methodName, len(lfnDict))) - res = self.isValid( operation = self.methodName ) + res = self.isValid(operation=self.methodName) if not res['OK']: return res else: if not self.valid: - return S_ERROR( self.errorReason ) + return S_ERROR(self.errorReason) # In case executing putFile, we can assume that all the source urls # are from the same protocol. This optional parameter, if defined # can be used to ignore some storage plugins and thus save time # and avoid fake failures showing in the accounting - inputProtocol = kwargs.pop( 'inputProtocol', None ) - + inputProtocol = kwargs.pop('inputProtocol', None) successful = {} failed = {} - filteredPlugins = self.__filterPlugins( self.methodName, kwargs.get( 'protocols' ), inputProtocol ) + filteredPlugins = self.__filterPlugins(self.methodName, kwargs.get('protocols'), inputProtocol) if not filteredPlugins: - return S_ERROR( errno.EPROTONOSUPPORT, "No storage plugins matching the requirements\ - (operation %s protocols %s inputProtocol %s)"\ - % ( self.methodName, kwargs.get( 'protocols' ), inputProtocol ) ) + return S_ERROR(errno.EPROTONOSUPPORT, "No storage plugins matching the requirements\ + (operation %s protocols %s inputProtocol %s)" % + (self.methodName, kwargs.get('protocols'), inputProtocol)) # Try all of the storages one by one for storage in filteredPlugins: # Determine whether to use this storage object @@ -1006,45 +1077,48 @@ def __executeMethod( self, lfn, *args, **kwargs ): pluginName = storageParameters['PluginName'] if not lfnDict: - log.debug( "No lfns to be attempted for %s protocol." % pluginName ) + log.debug("No lfns to be attempted for %s protocol." % pluginName) continue - log.verbose( "Generating %s protocol URLs for %s." % ( len( lfnDict ), pluginName ) ) - replicaDict = kwargs.pop( 'replicaDict', {} ) + log.verbose("Generating %s protocol URLs for %s." % (len(lfnDict), pluginName)) + replicaDict = kwargs.pop('replicaDict', {}) if storage.pluginName != "Proxy": - res = self.__generateURLDict( lfnDict, storage, replicaDict = replicaDict ) + res = self.__generateURLDict(lfnDict, storage, replicaDict=replicaDict) urlDict = res['Value']['Successful'] # url : lfn - failed.update( res['Value']['Failed'] ) + failed.update(res['Value']['Failed']) else: - urlDict = dict( [ ( lfn, lfn ) for lfn in lfnDict ] ) - if not len( urlDict ): - log.verbose( "__executeMethod No urls generated for protocol %s." % pluginName ) + urlDict = dict([(lfn, lfn) for lfn in lfnDict]) + if not len(urlDict): + log.verbose("__executeMethod No urls generated for protocol %s." % pluginName) else: - log.verbose( "Attempting to perform '%s' for %s physical files" % ( self.methodName, len( urlDict ) ) ) + log.verbose( + "Attempting to perform '%s' for %s physical files" % + (self.methodName, len(urlDict))) fcn = None - if hasattr( storage, self.methodName ) and callable( getattr( storage, self.methodName ) ): - fcn = getattr( storage, self.methodName ) + if hasattr(storage, self.methodName) and callable(getattr(storage, self.methodName)): + fcn = getattr(storage, self.methodName) if not fcn: - return S_ERROR( DErrno.ENOMETH, "SE.__executeMethod: unable to invoke %s, it isn't a member function of storage" ) + return S_ERROR( + DErrno.ENOMETH, + "SE.__executeMethod: unable to invoke %s, it isn't a member function of storage") urlsToUse = {} # url : the value of the lfn dictionary for the lfn of this url for url in urlDict: urlsToUse[url] = lfnDict[urlDict[url]] startDate = datetime.datetime.utcnow() startTime = time.time() - res = fcn( urlsToUse, *args, **kwargs ) + res = fcn(urlsToUse, *args, **kwargs) elapsedTime = time.time() - startTime - - self.addAccountingOperation( urlsToUse, startDate, elapsedTime, storageParameters, res ) + self.addAccountingOperation(urlsToUse, startDate, elapsedTime, storageParameters, res) if not res['OK']: errStr = "Completely failed to perform %s." % self.methodName - log.debug( errStr, 'with plugin %s: %s' % ( pluginName, res['Message'] ) ) + log.debug(errStr, 'with plugin %s: %s' % (pluginName, res['Message'])) for lfn in urlDict.values(): if lfn not in failed: failed[lfn] = '' - failed[lfn] = "%s %s" % ( failed[lfn], res['Message'] ) if failed[lfn] else res['Message'] + failed[lfn] = "%s %s" % (failed[lfn], res['Message']) if failed[lfn] else res['Message'] else: for url, lfn in urlDict.items(): @@ -1052,40 +1126,33 @@ def __executeMethod( self, lfn, *args, **kwargs ): if lfn not in failed: failed[lfn] = '' if url in res['Value']['Failed']: - self.log.debug( res['Value']['Failed'][url] ) - failed[lfn] = "%s %s" % ( failed[lfn], res['Value']['Failed'][url] ) if failed[lfn] else res['Value']['Failed'][url] + self.log.debug(res['Value']['Failed'][url]) + failed[lfn] = "%s %s" % (failed[lfn], res['Value']['Failed'][url] + ) if failed[lfn] else res['Value']['Failed'][url] else: errStr = 'No error returned from plug-in' - failed[lfn] = "%s %s" % ( failed[lfn], errStr ) if failed[lfn] else errStr + failed[lfn] = "%s %s" % (failed[lfn], errStr) if failed[lfn] else errStr else: successful[lfn] = res['Value']['Successful'][url] if lfn in failed: - failed.pop( lfn ) - lfnDict.pop( lfn ) - - + failed.pop(lfn) + lfnDict.pop(lfn) gDataStoreClient.commit() + return S_OK({'Failed': failed, 'Successful': successful}) - - return S_OK( { 'Failed': failed, 'Successful': successful } ) - - - def __getattr__( self, name ): + def __getattr__(self, name): """ Forwards the equivalent Storage calls to __executeMethod""" # We take either the equivalent name, or the name itself - self.methodName = StorageElementItem.__equivalentMethodNames.get( name, None ) + self.methodName = StorageElementItem.__equivalentMethodNames.get(name, None) if self.methodName: return self.__executeMethod - raise AttributeError( "StorageElement does not have a method '%s'" % name ) - + raise AttributeError("StorageElement does not have a method '%s'" % name) - - - def addAccountingOperation( self, lfns, startDate, elapsedTime, storageParameters, callRes ): + def addAccountingOperation(self, lfns, startDate, elapsedTime, storageParameters, callRes): """ Generates a DataOperation accounting if needs to be, and adds it to the DataStore client cache @@ -1103,12 +1170,12 @@ def addAccountingOperation( self, lfns, startDate, elapsedTime, storageParameter """ - if self.methodName not in ( self.readMethods + self.writeMethods + self.removeMethods ): + if self.methodName not in (self.readMethods + self.writeMethods + self.removeMethods): return baseAccountingDict = {} baseAccountingDict['OperationType'] = 'se.%s' % self.methodName - baseAccountingDict['User'] = getProxyInfo().get( 'Value', {} ).get( 'username', 'unknown' ) + baseAccountingDict['User'] = getProxyInfo().get('Value', {}).get('username', 'unknown') baseAccountingDict['RegistrationTime'] = 0.0 baseAccountingDict['RegistrationOK'] = 0 baseAccountingDict['RegistrationTotal'] = 0 @@ -1116,7 +1183,7 @@ def addAccountingOperation( self, lfns, startDate, elapsedTime, storageParameter # if it is a get method, then source and destination of the transfer should be inverted if self.methodName == 'getFile': baseAccountingDict['Destination'] = siteName() - baseAccountingDict[ 'Source'] = self.name + baseAccountingDict['Source'] = self.name else: baseAccountingDict['Destination'] = self.name baseAccountingDict['Source'] = siteName() @@ -1128,58 +1195,56 @@ def addAccountingOperation( self, lfns, startDate, elapsedTime, storageParameter baseAccountingDict['FinalStatus'] = 'Successful' oDataOperation = DataOperation() - oDataOperation.setValuesFromDict( baseAccountingDict ) - oDataOperation.setStartTime( startDate ) - oDataOperation.setEndTime( startDate + datetime.timedelta( seconds = elapsedTime ) ) - oDataOperation.setValueByKey( 'TransferTime', elapsedTime ) - oDataOperation.setValueByKey( 'Protocol', storageParameters.get( 'Protocol', 'unknown' ) ) + oDataOperation.setValuesFromDict(baseAccountingDict) + oDataOperation.setStartTime(startDate) + oDataOperation.setEndTime(startDate + datetime.timedelta(seconds=elapsedTime)) + oDataOperation.setValueByKey('TransferTime', elapsedTime) + oDataOperation.setValueByKey('Protocol', storageParameters.get('Protocol', 'unknown')) if not callRes['OK']: # Everything failed - oDataOperation.setValueByKey( 'TransferTotal', len( lfns ) ) - oDataOperation.setValueByKey( 'FinalStatus', 'Failed' ) + oDataOperation.setValueByKey('TransferTotal', len(lfns)) + oDataOperation.setValueByKey('FinalStatus', 'Failed') else: - succ = callRes.get( 'Value', {} ).get( 'Successful', {} ) - failed = callRes.get( 'Value', {} ).get( 'Failed', {} ) + succ = callRes.get('Value', {}).get('Successful', {}) + failed = callRes.get('Value', {}).get('Failed', {}) totalSize = 0 # We don't take len(lfns) in order to make two # separate entries in case of few failures - totalSucc = len( succ ) + totalSucc = len(succ) - if self.methodName in ( 'putFile', 'getFile' ): + if self.methodName in ('putFile', 'getFile'): # putFile and getFile return for each entry # in the successful dir the size of the corresponding file - totalSize = sum( succ.values() ) + totalSize = sum(succ.values()) - elif self.methodName in ( 'putDirectory', 'getDirectory' ): + elif self.methodName in ('putDirectory', 'getDirectory'): # putDirectory and getDirectory return for each dir name # a dictionnary with the keys 'Files' and 'Size' - totalSize = sum( val.get( 'Size', 0 ) for val in succ.values() if isinstance( val, dict ) ) - totalSucc = sum( val.get( 'Files', 0 ) for val in succ.values() if isinstance( val, dict ) ) - oDataOperation.setValueByKey( 'TransferOK', len( succ ) ) + totalSize = sum(val.get('Size', 0) for val in succ.values() if isinstance(val, dict)) + totalSucc = sum(val.get('Files', 0) for val in succ.values() if isinstance(val, dict)) + oDataOperation.setValueByKey('TransferOK', len(succ)) - oDataOperation.setValueByKey( 'TransferSize', totalSize ) - oDataOperation.setValueByKey( 'TransferTotal', totalSucc ) - oDataOperation.setValueByKey( 'TransferOK', totalSucc ) + oDataOperation.setValueByKey('TransferSize', totalSize) + oDataOperation.setValueByKey('TransferTotal', totalSucc) + oDataOperation.setValueByKey('TransferOK', totalSucc) if callRes['Value']['Failed']: - oDataOperationFailed = copy.deepcopy( oDataOperation ) - oDataOperationFailed.setValueByKey( 'TransferTotal', len( failed ) ) - oDataOperationFailed.setValueByKey( 'TransferOK', 0 ) - oDataOperationFailed.setValueByKey( 'TransferSize', 0 ) - oDataOperationFailed.setValueByKey( 'FinalStatus', 'Failed' ) + oDataOperationFailed = copy.deepcopy(oDataOperation) + oDataOperationFailed.setValueByKey('TransferTotal', len(failed)) + oDataOperationFailed.setValueByKey('TransferOK', 0) + oDataOperationFailed.setValueByKey('TransferSize', 0) + oDataOperationFailed.setValueByKey('FinalStatus', 'Failed') - accRes = gDataStoreClient.addRegister( oDataOperationFailed ) + accRes = gDataStoreClient.addRegister(oDataOperationFailed) if not accRes['OK']: - self.log.error( "Could not send failed accounting report", accRes['Message'] ) + self.log.error("Could not send failed accounting report", accRes['Message']) - - accRes = gDataStoreClient.addRegister( oDataOperation ) + accRes = gDataStoreClient.addRegister(oDataOperation) if not accRes['OK']: - self.log.error( "Could not send accounting report", accRes['Message'] ) - + self.log.error("Could not send accounting report", accRes['Message']) StorageElement = StorageElementCache() diff --git a/WorkloadManagementSystem/Service/SandboxStoreHandler.py b/WorkloadManagementSystem/Service/SandboxStoreHandler.py index 0bbb78c577e..763c8f8a3b4 100755 --- a/WorkloadManagementSystem/Service/SandboxStoreHandler.py +++ b/WorkloadManagementSystem/Service/SandboxStoreHandler.py @@ -12,7 +12,7 @@ from DIRAC.Core.DISET.RequestHandler import RequestHandler from DIRAC.Core.Security import Properties from DIRAC.WorkloadManagementSystem.DB.SandboxMetadataDB import SandboxMetadataDB -from DIRAC.DataManagementSystem.Client.DataManager import DataManager +from DIRAC.DataManagementSystem.Client.DataManager import DataManager from DIRAC.DataManagementSystem.Service.StorageElementHandler import getDiskSpace from DIRAC.RequestManagementSystem.Client.ReqClient import ReqClient from DIRAC.RequestManagementSystem.Client.Request import Request @@ -29,16 +29,17 @@ def initializeSandboxStoreHandler( serviceInfo ): sandboxDB = SandboxMetadataDB() return S_OK() -class SandboxStoreHandler( RequestHandler ): + +class SandboxStoreHandler(RequestHandler): __purgeCount = -1 __purgeLock = threading.Lock() __purgeWorking = False - def initialize( self ): - self.__backend = self.getCSOption( "Backend", "local" ) - self.__localSEName = self.getCSOption( "LocalSE", "SandboxSE" ) - self.__maxUploadBytes = self.getCSOption( "MaxSandboxSizeMiB", 10 ) * 1048576 + def initialize(self): + self.__backend = self.getCSOption("Backend", "local") + self.__localSEName = self.getCSOption("LocalSE", "SandboxSE") + self.__maxUploadBytes = self.getCSOption("MaxSandboxSizeMiB", 10) * 1048576 if self.__backend.lower() == "local" or self.__backend == self.__localSEName: self.__useLocalStorage = True self.__seNameToUse = self.__localSEName @@ -48,450 +49,452 @@ def initialize( self ): self.__seNameToUse = self.__backend # Execute the purge once every 1000 calls SandboxStoreHandler.__purgeCount += 1 - if SandboxStoreHandler.__purgeCount > self.getCSOption( "QueriesBeforePurge", 1000 ): + if SandboxStoreHandler.__purgeCount > self.getCSOption("QueriesBeforePurge", 1000): SandboxStoreHandler.__purgeCount = 0 if SandboxStoreHandler.__purgeCount == 0: - threading.Thread( target = self.purgeUnusedSandboxes ).start() + threading.Thread(target=self.purgeUnusedSandboxes).start() - def __getSandboxPath( self, md5 ): + def __getSandboxPath(self, md5): """ Generate the sandbox path """ # prefix = self.getCSOption( "SandboxPrefix", "SandBox" ) prefix = "SandBox" credDict = self.getRemoteCredentials() - if Properties.JOB_SHARING in credDict[ 'properties' ]: - idField = credDict[ 'group' ] + if Properties.JOB_SHARING in credDict['properties']: + idField = credDict['group'] else: - idField = "%s.%s" % ( credDict[ 'username' ], credDict[ 'group' ] ) - pathItems = [ "/", prefix, idField[0], idField ] - pathItems.extend( [ md5[0:3], md5[3:6], md5 ] ) - return os.path.join( *pathItems ) - + idField = "%s.%s" % (credDict['username'], credDict['group']) + pathItems = ["/", prefix, idField[0], idField] + pathItems.extend([md5[0:3], md5[3:6], md5]) + return os.path.join(*pathItems) - def transfer_fromClient( self, fileId, token, fileSize, fileHelper ): + def transfer_fromClient(self, fileId, token, fileSize, fileHelper): """ Receive a file as a sandbox """ if self.__maxUploadBytes and fileSize > self.__maxUploadBytes: fileHelper.markAsTransferred() - return S_ERROR( "Sandbox is too big. Please upload it to a grid storage element" ) + return S_ERROR("Sandbox is too big. Please upload it to a grid storage element") - if isinstance( fileId, ( list, tuple ) ): - if len( fileId ) > 1: + if isinstance(fileId, (list, tuple)): + if len(fileId) > 1: assignTo = fileId[1] fileId = fileId[0] else: - return S_ERROR( "File identified tuple has to have length greater than 1" ) + return S_ERROR("File identified tuple has to have length greater than 1") else: assignTo = {} - extPos = fileId.find( ".tar" ) + extPos = fileId.find(".tar") if extPos > -1: - extension = fileId[ extPos + 1: ] - aHash = fileId[ :extPos ] + extension = fileId[extPos + 1:] + aHash = fileId[:extPos] else: extension = "" aHash = fileId - gLogger.info( "Upload requested for %s [%s]" % ( aHash, extension ) ) + gLogger.info("Upload requested for %s [%s]" % (aHash, extension)) credDict = self.getRemoteCredentials() - sbPath = self.__getSandboxPath( "%s.%s" % ( aHash, extension ) ) + sbPath = self.__getSandboxPath("%s.%s" % (aHash, extension)) # Generate the location - result = self.__generateLocation( sbPath ) - if not result[ 'OK' ]: + result = self.__generateLocation(sbPath) + if not result['OK']: return result - seName, sePFN = result[ 'Value' ] + seName, sePFN = result['Value'] - result = sandboxDB.getSandboxId( seName, sePFN, credDict[ 'username' ], credDict[ 'group' ] ) - if result[ 'OK' ]: - gLogger.info( "Sandbox already exists. Skipping upload" ) + result = sandboxDB.getSandboxId(seName, sePFN, credDict['username'], credDict['group']) + if result['OK']: + gLogger.info("Sandbox already exists. Skipping upload") fileHelper.markAsTransferred() - sbURL = "SB:%s|%s" % ( seName, sePFN ) - assignTo = dict( [ ( key, [ ( sbURL, assignTo[ key ] ) ] ) for key in assignTo ] ) - result = self.export_assignSandboxesToEntities( assignTo ) - if not result[ 'OK' ]: + sbURL = "SB:%s|%s" % (seName, sePFN) + assignTo = dict([(key, [(sbURL, assignTo[key])]) for key in assignTo]) + result = self.export_assignSandboxesToEntities(assignTo) + if not result['OK']: return result - return S_OK( sbURL ) + return S_OK(sbURL) if self.__useLocalStorage: - hdPath = self.__sbToHDPath( sbPath ) + hdPath = self.__sbToHDPath(sbPath) else: hdPath = False # Write to local file - result = self.__networkToFile( fileHelper, hdPath ) - if not result[ 'OK' ]: - gLogger.error( "Error while receiving sandbox file", "%s" % result['Message'] ) + result = self.__networkToFile(fileHelper, hdPath) + if not result['OK']: + gLogger.error("Error while receiving sandbox file", "%s" % result['Message']) return result - hdPath = result[ 'Value' ] - gLogger.info( "Wrote sandbox to file %s" % hdPath ) + hdPath = result['Value'] + gLogger.info("Wrote sandbox to file %s" % hdPath) # Check hash! if fileHelper.getHash() != aHash: - self.__secureUnlinkFile( hdPath ) - gLogger.error( "Hashes don't match! Client defined hash is different with received data hash!" ) - return S_ERROR( "Hashes don't match!" ) + self.__secureUnlinkFile(hdPath) + gLogger.error("Hashes don't match! Client defined hash is different with received data hash!") + return S_ERROR("Hashes don't match!") # If using remote storage, copy there! if not self.__useLocalStorage: - gLogger.info( "Uploading sandbox to external storage" ) - result = self.__copyToExternalSE( hdPath, sbPath ) - self.__secureUnlinkFile( hdPath ) - if not result[ 'OK' ]: + gLogger.info("Uploading sandbox to external storage") + result = self.__copyToExternalSE(hdPath, sbPath) + self.__secureUnlinkFile(hdPath) + if not result['OK']: return result - sbPath = result[ 'Value' ][1] + sbPath = result['Value'][1] # Register! - gLogger.info( "Registering sandbox in the DB with", "SB:%s|%s" % ( self.__seNameToUse, sbPath ) ) - result = sandboxDB.registerAndGetSandbox( credDict[ 'username' ], credDict[ 'DN' ], credDict[ 'group' ], - self.__seNameToUse, sbPath, fileHelper.getTransferedBytes() ) - if not result[ 'OK' ]: - self.__secureUnlinkFile( hdPath ) + gLogger.info("Registering sandbox in the DB with", "SB:%s|%s" % (self.__seNameToUse, sbPath)) + result = sandboxDB.registerAndGetSandbox(credDict['username'], credDict['DN'], credDict['group'], + self.__seNameToUse, sbPath, fileHelper.getTransferedBytes()) + if not result['OK']: + self.__secureUnlinkFile(hdPath) return result - sbURL = "SB:%s|%s" % ( self.__seNameToUse, sbPath ) - assignTo = dict( [ ( key, [ ( sbURL, assignTo[ key ] ) ] ) for key in assignTo ] ) - result = self.export_assignSandboxesToEntities( assignTo ) - if not result[ 'OK' ]: + sbURL = "SB:%s|%s" % (self.__seNameToUse, sbPath) + assignTo = dict([(key, [(sbURL, assignTo[key])]) for key in assignTo]) + result = self.export_assignSandboxesToEntities(assignTo) + if not result['OK']: return result - return S_OK( sbURL ) + return S_OK(sbURL) - def transfer_bulkFromClient( self, fileId, token, fileSize, fileHelper ): + def transfer_bulkFromClient(self, fileId, token, fileSize, fileHelper): """ Receive files packed into a tar archive by the fileHelper logic. token is used for access rights confirmation. """ - result = self.__networkToFile( fileHelper ) - if not result[ 'OK' ]: + result = self.__networkToFile(fileHelper) + if not result['OK']: return result - tmpFilePath = result[ 'OK' ] - gLogger.info( "Got Sandbox to local storage", tmpFilePath ) + tmpFilePath = result['OK'] + gLogger.info("Got Sandbox to local storage", tmpFilePath) - extension = fileId[ fileId.find( ".tar" ) + 1: ] - sbPath = "%s.%s" % ( self.__getSandboxPath( fileHelper.getHash() ), extension ) - gLogger.info( "Sandbox path will be", sbPath ) + extension = fileId[fileId.find(".tar") + 1:] + sbPath = "%s.%s" % (self.__getSandboxPath(fileHelper.getHash()), extension) + gLogger.info("Sandbox path will be", sbPath) # Generate the location - result = self.__generateLocation( sbPath ) - if not result[ 'OK' ]: + result = self.__generateLocation(sbPath) + if not result['OK']: return result - seName, sePFN = result[ 'Value' ] + seName, sePFN = result['Value'] # Register in DB credDict = self.getRemoteCredentials() - result = sandboxDB.getSandboxId( seName, sePFN, credDict[ 'username' ], credDict[ 'group' ] ) - if result[ 'OK' ]: - return S_OK( "SB:%s|%s" % ( seName, sePFN ) ) - - result = sandboxDB.registerAndGetSandbox( credDict[ 'username' ], credDict[ 'DN' ], credDict[ 'group' ], - seName, sePFN, fileHelper.getTransferedBytes() ) - if not result[ 'OK' ]: - self.__secureUnlinkFile( tmpFilePath ) + result = sandboxDB.getSandboxId(seName, sePFN, credDict['username'], credDict['group']) + if result['OK']: + return S_OK("SB:%s|%s" % (seName, sePFN)) + + result = sandboxDB.registerAndGetSandbox(credDict['username'], credDict['DN'], credDict['group'], + seName, sePFN, fileHelper.getTransferedBytes()) + if not result['OK']: + self.__secureUnlinkFile(tmpFilePath) return result - sbid, _newSandbox = result[ 'Value' ] - gLogger.info( "Registered in DB", "with SBId %s" % sbid ) + sbid, _newSandbox = result['Value'] + gLogger.info("Registered in DB", "with SBId %s" % sbid) - result = self.__moveToFinalLocation( tmpFilePath, sbPath ) - self.__secureUnlinkFile( tmpFilePath ) - if not result[ 'OK' ]: - gLogger.error( "Could not move sandbox to final destination", result[ 'Message' ] ) + result = self.__moveToFinalLocation(tmpFilePath, sbPath) + self.__secureUnlinkFile(tmpFilePath) + if not result['OK']: + gLogger.error("Could not move sandbox to final destination", result['Message']) return result - gLogger.info( "Moved to final destination" ) + gLogger.info("Moved to final destination") # Unlink temporal file if it's there - self.__secureUnlinkFile( tmpFilePath ) - return S_OK( "SB:%s|%s" % ( seName, sePFN ) ) + self.__secureUnlinkFile(tmpFilePath) + return S_OK("SB:%s|%s" % (seName, sePFN)) - def __generateLocation( self, sbPath ): + def __generateLocation(self, sbPath): """ Generate the location string """ if self.__useLocalStorage: - return S_OK( ( self.__localSEName, sbPath ) ) + return S_OK((self.__localSEName, sbPath)) # It's external storage - storageElement = StorageElement( self.__externalSEName ) + storageElement = StorageElement(self.__externalSEName) res = storageElement.isValid() if not res['OK']: errStr = "Failed to instantiate destination StorageElement" - gLogger.error( errStr, self.__externalSEName ) - return S_ERROR( errStr ) - result = storageElement.getURL( sbPath ) + gLogger.error(errStr, self.__externalSEName) + return S_ERROR(errStr) + result = storageElement.getURL(sbPath) if not result['OK'] or sbPath not in result['Value']['Successful']: errStr = "Failed to generate PFN" - gLogger.error( errStr, self.__externalSEName ) - return S_ERROR( errStr ) + gLogger.error(errStr, self.__externalSEName) + return S_ERROR(errStr) destPfn = result['Value']['Successful'][sbPath] - return S_OK( ( self.__externalSEName, destPfn ) ) + return S_OK((self.__externalSEName, destPfn)) - def __sbToHDPath( self, sbPath ): + def __sbToHDPath(self, sbPath): while sbPath and sbPath[0] == "/": sbPath = sbPath[1:] - basePath = self.getCSOption( "BasePath", "/opt/dirac/storage/sandboxes" ) - return os.path.join( basePath, sbPath ) + basePath = self.getCSOption("BasePath", "/opt/dirac/storage/sandboxes") + return os.path.join(basePath, sbPath) - def __networkToFile( self, fileHelper, destFileName = False ): + def __networkToFile(self, fileHelper, destFileName=False): """ Dump incoming network data to temporal file """ tfd = None if not destFileName: try: - tfd, destFileName = tempfile.mkstemp( prefix = "DSB." ) + tfd, destFileName = tempfile.mkstemp(prefix="DSB.") tfd.close() except Exception as e: - gLogger.error( "%s" % repr( e ).replace( ',)', ')' ) ) - return S_ERROR( "Cannot create temporary file" ) + gLogger.error("%s" % repr(e).replace(',)', ')')) + return S_ERROR("Cannot create temporary file") - destFileName = os.path.realpath( destFileName ) - mkDir( os.path.dirname( destFileName ) ) + destFileName = os.path.realpath(destFileName) + mkDir(os.path.dirname(destFileName)) try: if tfd is not None: fd = tfd else: - fd = open( destFileName, "wb" ) - result = fileHelper.networkToDataSink( fd, maxFileSize = self.__maxUploadBytes ) + fd = open(destFileName, "wb") + result = fileHelper.networkToDataSink(fd, maxFileSize=self.__maxUploadBytes) fd.close() except Exception as e: - gLogger.error( "Cannot open to write destination file", "%s: %s" % ( destFileName, repr( e ).replace( ',)', ')' ) ) ) - return S_ERROR( "Cannot open to write destination file" ) - if not result[ 'OK' ]: + gLogger.error("Cannot open to write destination file", "%s: %s" % (destFileName, repr(e).replace(',)', ')'))) + return S_ERROR("Cannot open to write destination file") + if not result['OK']: return result - return S_OK( destFileName ) + return S_OK(destFileName) - def __secureUnlinkFile( self, filePath ): + def __secureUnlinkFile(self, filePath): try: - os.unlink( filePath ) + os.unlink(filePath) except Exception as e: - gLogger.warn( "Could not unlink file %s: %s" % ( filePath, repr( e ).replace( ',)', ')' ) ) ) + gLogger.warn("Could not unlink file %s: %s" % (filePath, repr(e).replace(',)', ')'))) return False return True - def __moveToFinalLocation( self, localFilePath, sbPath ): + def __moveToFinalLocation(self, localFilePath, sbPath): if self.__useLocalStorage: - hdFilePath = self.__sbToHDPath( sbPath ) - result = S_OK( ( self.__localSEName, sbPath ) ) - if os.path.isfile( hdFilePath ): - gLogger.info( "There was already a sandbox with that name, skipping copy", sbPath ) + hdFilePath = self.__sbToHDPath(sbPath) + result = S_OK((self.__localSEName, sbPath)) + if os.path.isfile(hdFilePath): + gLogger.info("There was already a sandbox with that name, skipping copy", sbPath) else: - hdDirPath = os.path.dirname( hdFilePath ) + hdDirPath = os.path.dirname(hdFilePath) mkDir(hdDirPath) try: - os.rename( localFilePath, hdFilePath ) + os.rename(localFilePath, hdFilePath) except OSError as e: errMsg = "Cannot move temporal file to final path" - gLogger.error( errMsg, repr( e ).replace( ',)', ')' ) ) - result = S_ERROR( errMsg ) + gLogger.error(errMsg, repr(e).replace(',)', ')')) + result = S_ERROR(errMsg) else: - result = self.__copyToExternalSE( localFilePath, sbPath ) + result = self.__copyToExternalSE(localFilePath, sbPath) return result - def __copyToExternalSE( self, localFilePath, sbPath ): + def __copyToExternalSE(self, localFilePath, sbPath): """ Copy uploaded file to external SE """ try: dm = DataManager() - result = dm.put( sbPath, localFilePath, self.__externalSEName ) - if not result[ 'OK' ]: + result = dm.put(sbPath, localFilePath, self.__externalSEName) + if not result['OK']: return result - if 'Successful' not in result[ 'Value' ]: - gLogger.verbose( "Oops, no successful transfers there", str( result ) ) - return S_ERROR( "RM returned OK to the action but no successful transfers were there" ) - okTrans = result[ 'Value' ][ 'Successful' ] + if 'Successful' not in result['Value']: + gLogger.verbose("Oops, no successful transfers there", str(result)) + return S_ERROR("RM returned OK to the action but no successful transfers were there") + okTrans = result['Value']['Successful'] if sbPath not in okTrans: - gLogger.verbose( "Ooops, SB transfer wasn't in the successful ones", str( result ) ) - return S_ERROR( "RM returned OK to the action but SB transfer wasn't in the successful ones" ) - return S_OK( ( self.__externalSEName, okTrans[ sbPath ] ) ) + gLogger.verbose("Ooops, SB transfer wasn't in the successful ones", str(result)) + return S_ERROR("RM returned OK to the action but SB transfer wasn't in the successful ones") + return S_OK((self.__externalSEName, okTrans[sbPath])) except Exception as e: - gLogger.error( "Error while moving sandbox to SE", "%s" % repr( e ).replace( ',)', ')' ) ) - return S_ERROR( "Error while moving sandbox to SE" ) + gLogger.error("Error while moving sandbox to SE", "%s" % repr(e).replace(',)', ')')) + return S_ERROR("Error while moving sandbox to SE") ################## # Assigning sbs to jobs - types_assignSandboxesToEntities = [ dict ] - def export_assignSandboxesToEntities( self, enDict, ownerName = "", ownerGroup = "", entitySetup = False ): + types_assignSandboxesToEntities = [dict] + + def export_assignSandboxesToEntities(self, enDict, ownerName="", ownerGroup="", entitySetup=False): """ Assign sandboxes to jobs. Expects a dict of { entityId : [ ( SB, SBType ), ... ] } """ if not entitySetup: - entitySetup = self.serviceInfoDict[ 'clientSetup' ] + entitySetup = self.serviceInfoDict['clientSetup'] credDict = self.getRemoteCredentials() - return sandboxDB.assignSandboxesToEntities( enDict, credDict[ 'username' ], credDict[ 'group' ], entitySetup, - ownerName, ownerGroup ) + return sandboxDB.assignSandboxesToEntities(enDict, credDict['username'], credDict['group'], entitySetup, + ownerName, ownerGroup) ################## # Unassign sbs to jobs - types_unassignEntities = [ ( list, tuple ) ] - def export_unassignEntities( self, entitiesList, entitiesSetup = False ): + types_unassignEntities = [(list, tuple)] + + def export_unassignEntities(self, entitiesList, entitiesSetup=False): """ Unassign a list of jobs """ if not entitiesSetup: - entitiesSetup = self.serviceInfoDict[ 'clientSetup' ] + entitiesSetup = self.serviceInfoDict['clientSetup'] credDict = self.getRemoteCredentials() - return sandboxDB.unassignEntities( { entitiesSetup : entitiesList }, credDict[ 'username' ], credDict[ 'group' ] ) + return sandboxDB.unassignEntities({entitiesSetup: entitiesList}, credDict['username'], credDict['group']) ################## # Getting assigned sandboxes - types_getSandboxesAssignedToEntity = [ basestring ] - def export_getSandboxesAssignedToEntity( self, entityId, entitySetup = False ): + types_getSandboxesAssignedToEntity = [basestring] + + def export_getSandboxesAssignedToEntity(self, entityId, entitySetup=False): """ Get the sandboxes associated to a job and the association type """ if not entitySetup: - entitySetup = self.serviceInfoDict[ 'clientSetup' ] + entitySetup = self.serviceInfoDict['clientSetup'] credDict = self.getRemoteCredentials() - result = sandboxDB.getSandboxesAssignedToEntity( entityId, entitySetup, - credDict[ 'username' ], credDict[ 'group' ] ) - if not result[ 'OK' ]: + result = sandboxDB.getSandboxesAssignedToEntity(entityId, entitySetup, + credDict['username'], credDict['group']) + if not result['OK']: return result sbDict = {} - for SEName, SEPFN, SBType in result[ 'Value' ]: + for SEName, SEPFN, SBType in result['Value']: if SBType not in sbDict: - sbDict[ SBType ] = [] - sbDict[ SBType ].append( "SB:%s|%s" % ( SEName, SEPFN ) ) - return S_OK( sbDict ) - + sbDict[SBType] = [] + sbDict[SBType].append("SB:%s|%s" % (SEName, SEPFN)) + return S_OK(sbDict) ################## # Disk space left management types_getFreeDiskSpace = [basestring] - def export_getFreeDiskSpace( self, path, size = 'TB' ): + + def export_getFreeDiskSpace(self, path): """ Get the free disk space of the storage element If no size is specified, terabytes will be used by default. """ - return getDiskSpace(path, size) + return getDiskSpace(path) types_getTotalDiskSpace = [basestring] - def export_getTotalDiskSpace( self, path, size = 'TB' ): + + def export_getTotalDiskSpace(self, path): """ Get the total disk space of the storage element If no size is specified, terabytes will be used by default. """ - return getDiskSpace(path, size, total = True) - + return getDiskSpace(path, total=True) ################## # Download sandboxes - def transfer_toClient( self, fileID, token, fileHelper ): + def transfer_toClient(self, fileID, token, fileHelper): """ Method to send files to clients. fileID is the local file name in the SE. token is used for access rights confirmation. """ credDict = self.getRemoteCredentials() serviceURL = self.serviceInfoDict['URL'] - filePath = fileID.replace( serviceURL, '' ) - result = sandboxDB.getSandboxId( self.__localSEName, filePath, credDict[ 'username' ], credDict[ 'group' ] ) - if not result[ 'OK' ]: + filePath = fileID.replace(serviceURL, '') + result = sandboxDB.getSandboxId(self.__localSEName, filePath, credDict['username'], credDict['group']) + if not result['OK']: return result - sbId = result[ 'Value' ] - sandboxDB.accessedSandboxById( sbId ) + sbId = result['Value'] + sandboxDB.accessedSandboxById(sbId) # If it's a local file - hdPath = self.__sbToHDPath( filePath ) - if not os.path.isfile( hdPath ): - return S_ERROR( "Sandbox does not exist" ) - result = fileHelper.getFileDescriptor( hdPath, 'rb' ) - if not result[ 'OK' ]: - return S_ERROR( 'Failed to get file descriptor: %s' % result[ 'Message' ] ) - fd = result[ 'Value' ] - result = fileHelper.FDToNetwork( fd ) + hdPath = self.__sbToHDPath(filePath) + if not os.path.isfile(hdPath): + return S_ERROR("Sandbox does not exist") + result = fileHelper.getFileDescriptor(hdPath, 'rb') + if not result['OK']: + return S_ERROR('Failed to get file descriptor: %s' % result['Message']) + fd = result['Value'] + result = fileHelper.FDToNetwork(fd) fileHelper.oFile.close() return result ################## # Purge sandboxes - def purgeUnusedSandboxes( self ): + def purgeUnusedSandboxes(self): # If a purge is already working skip SandboxStoreHandler.__purgeLock.acquire() try: if SandboxStoreHandler.__purgeWorking: if time.time() - SandboxStoreHandler.__purgeWorking < 86400: - gLogger.info( "Sandbox purge still working" ) + gLogger.info("Sandbox purge still working") return S_OK() SandboxStoreHandler.__purgeWorking = time.time() finally: SandboxStoreHandler.__purgeLock.release() - gLogger.info( "Purging sandboxes" ) + gLogger.info("Purging sandboxes") result = sandboxDB.getUnusedSandboxes() - if not result[ 'OK' ]: - gLogger.error( "Error while retrieving sandboxes to purge", result[ 'Message' ] ) + if not result['OK']: + gLogger.error("Error while retrieving sandboxes to purge", result['Message']) SandboxStoreHandler.__purgeWorking = False return result - sbList = result[ 'Value' ] - gLogger.info( "Got %s sandboxes to purge" % len( sbList ) ) + sbList = result['Value'] + gLogger.info("Got %s sandboxes to purge" % len(sbList)) for sbId, SEName, SEPFN in sbList: - self.__purgeSandbox( sbId, SEName, SEPFN ) + self.__purgeSandbox(sbId, SEName, SEPFN) SandboxStoreHandler.__purgeWorking = False return S_OK() - def __purgeSandbox( self, sbId, SEName, SEPFN ): - result = self.__deleteSandboxFromBackend( SEName, SEPFN ) - if not result[ 'OK' ]: - gLogger.error( "Cannot delete sandbox from backend", result[ 'Message' ] ) + def __purgeSandbox(self, sbId, SEName, SEPFN): + result = self.__deleteSandboxFromBackend(SEName, SEPFN) + if not result['OK']: + gLogger.error("Cannot delete sandbox from backend", result['Message']) return - result = sandboxDB.deleteSandboxes( [ sbId ] ) - if not result[ 'OK' ]: - gLogger.error( "Cannot delete sandbox from DB", result[ 'Message' ] ) + result = sandboxDB.deleteSandboxes([sbId]) + if not result['OK']: + gLogger.error("Cannot delete sandbox from DB", result['Message']) - def __deleteSandboxFromBackend( self, SEName, SEPFN ): - gLogger.info( "Purging sandbox" "SB:%s|%s" % ( SEName, SEPFN ) ) + def __deleteSandboxFromBackend(self, SEName, SEPFN): + gLogger.info("Purging sandbox" "SB:%s|%s" % (SEName, SEPFN)) if SEName != self.__localSEName: - return self.__deleteSandboxFromExternalBackend( SEName, SEPFN ) + return self.__deleteSandboxFromExternalBackend(SEName, SEPFN) else: - hdPath = self.__sbToHDPath( SEPFN ) + hdPath = self.__sbToHDPath(SEPFN) try: - if not os.path.isfile( hdPath ): + if not os.path.isfile(hdPath): return S_OK() except Exception as e: - gLogger.error( "Cannot perform isfile", "%s : %s" % ( hdPath, repr( e ).replace( ',)', ')' ) ) ) - return S_ERROR( "Error checking %s" % hdPath ) + gLogger.error("Cannot perform isfile", "%s : %s" % (hdPath, repr(e).replace(',)', ')'))) + return S_ERROR("Error checking %s" % hdPath) try: - os.unlink( hdPath ) + os.unlink(hdPath) except Exception as e: - gLogger.error( "Cannot delete local sandbox", "%s : %s" % ( hdPath, repr( e ).replace( ',)', ')' ) ) ) + gLogger.error("Cannot delete local sandbox", "%s : %s" % (hdPath, repr(e).replace(',)', ')'))) while hdPath: - hdPath = os.path.dirname( hdPath ) - gLogger.info( "Checking if dir %s is empty" % hdPath ) + hdPath = os.path.dirname(hdPath) + gLogger.info("Checking if dir %s is empty" % hdPath) try: - if not os.path.isdir( hdPath ): + if not os.path.isdir(hdPath): break - if len( os.listdir( hdPath ) ) > 0: + if os.listdir(hdPath): break - gLogger.info( "Trying to clean dir %s" % hdPath ) + gLogger.info("Trying to clean dir %s" % hdPath) # Empty dir! - os.rmdir( hdPath ) + os.rmdir(hdPath) except Exception as e: - gLogger.error( "Cannot clean directory", "%s : %s" % ( hdPath, repr( e ).replace( ',)', ')' ) ) ) + gLogger.error("Cannot clean directory", "%s : %s" % (hdPath, repr(e).replace(',)', ')'))) break return S_OK() - def __deleteSandboxFromExternalBackend( self, SEName, SEPFN ): - if self.getCSOption( "DelayedExternalDeletion", True ): - gLogger.info( "Setting deletion request" ) + def __deleteSandboxFromExternalBackend(self, SEName, SEPFN): + if self.getCSOption("DelayedExternalDeletion", True): + gLogger.info("Setting deletion request") try: request = Request() - request.RequestName = "RemoteSBDeletion:%s|%s:%s" % ( SEName, SEPFN, time.time() ) + request.RequestName = "RemoteSBDeletion:%s|%s:%s" % (SEName, SEPFN, time.time()) physicalRemoval = Operation() physicalRemoval.Type = "PhysicalRemoval" physicalRemoval.TargetSE = SEName fileToRemove = File() fileToRemove.PFN = SEPFN - physicalRemoval.addFile( fileToRemove ) - request.addOperation( physicalRemoval ) - return ReqClient().putRequest( request ) + physicalRemoval.addFile(fileToRemove) + request.addOperation(physicalRemoval) + return ReqClient().putRequest(request) except Exception as e: - gLogger.exception( "Exception while setting deletion request" ) - return S_ERROR( "Cannot set deletion request: %s" % str( e ) ) + gLogger.exception("Exception while setting deletion request") + return S_ERROR("Cannot set deletion request: %s" % str(e)) else: - gLogger.info( "Deleting external Sandbox" ) + gLogger.info("Deleting external Sandbox") try: - return StorageElement( SEName ).removeFile( SEPFN ) + return StorageElement(SEName).removeFile(SEPFN) except Exception as e: - gLogger.exception( "RM raised an exception while trying to delete a remote sandbox" ) - return S_ERROR( "RM raised an exception while trying to delete a remote sandbox" ) + gLogger.exception("RM raised an exception while trying to delete a remote sandbox") + return S_ERROR("RM raised an exception while trying to delete a remote sandbox")