diff --git a/.github/workflows/basic-python3.yml b/.github/workflows/basic-python3.yml new file mode 100644 index 00000000000..172322c786c --- /dev/null +++ b/.github/workflows/basic-python3.yml @@ -0,0 +1,42 @@ +name: Python 3 tests + +on: [push, pull_request] + +jobs: + check: + runs-on: ubuntu-latest + if: github.event_name != 'push' || github.repository == 'DIRACGrid/DIRAC' + timeout-minutes: 30 + + strategy: + fail-fast: False + matrix: + command: + # TODO These three tests fail on Python 3: + # * `test_BaseType_Unicode` and `test_nestedStructure` fail due to + # DISET's string and unicode types being poorly defined + # * `testLockedClass` deadlocks frequently due to the use of + # self.__processThread._Thread__stop(). Officially "threads + # cannot be destroyed, stopped, suspended, resumed, or + # interrupted" so the code should be rewritten to follow the + # supported alternative: "If you want your threads to stop + # gracefully, make them non-daemonic and use a suitable + # signalling mechanism such as an Event." + - pytest --no-cov -k 'not test_BaseType_Unicode and not test_nestedStructure and not testLockedClass' + + steps: + - uses: actions/checkout@v2 + - name: Fail-fast for outdated pipelines + run: .github/workflows/fail-fast.sh + - name: Prepare environment + run: | + conda env create --name dirac-testing --file environment-py3.yml + - name: Run tests + run: | + source "${CONDA}/bin/activate" + conda activate dirac-testing + set -euxo pipefail + export PYTHONPATH=${PWD%/*} + ${{ matrix.command }} + env: + REFERENCE_BRANCH: ${{ github['base_ref'] || github['head_ref'] }} diff --git a/AccountingSystem/Agent/NetworkAgent.py b/AccountingSystem/Agent/NetworkAgent.py index f6fb95a62cc..6c633199b07 100644 --- a/AccountingSystem/Agent/NetworkAgent.py +++ b/AccountingSystem/Agent/NetworkAgent.py @@ -102,7 +102,7 @@ def updateNameDictionary(self): return S_ERROR('Unable to fetch perfSONAR endpoints from CS.') tmpDict = {} - for path, value in result['Value'].iteritems(): + for path, value in result['Value'].items(): if value == 'True': elements = path.split('/') diracName = elements[4] @@ -211,7 +211,7 @@ def processMessage(self, headers, body): OWDMax = 0 total = 0 count = 0 - for value, items in metricData.iteritems(): + for value, items in metricData.items(): floatValue = float(value) total += floatValue * items count += items diff --git a/AccountingSystem/Service/DataStoreHandler.py b/AccountingSystem/Service/DataStoreHandler.py index 35820d3640a..f08cbfaeba3 100644 --- a/AccountingSystem/Service/DataStoreHandler.py +++ b/AccountingSystem/Service/DataStoreHandler.py @@ -220,7 +220,7 @@ def export_removeRegisters(self, entriesList): for entry in entriesList: if len(entry) != 4: return S_ERROR("Invalid records") - for i in xrange(len(entry)): + for i in range(len(entry)): if not isinstance(entry[i], expectedTypes[i]): return S_ERROR("%s field in the records should be %s" % (i, expectedTypes[i])) ok = 0 diff --git a/AccountingSystem/private/MainReporter.py b/AccountingSystem/private/MainReporter.py index 846592b72a3..fc2d46669aa 100644 --- a/AccountingSystem/private/MainReporter.py +++ b/AccountingSystem/private/MainReporter.py @@ -46,8 +46,8 @@ def __calculateReportHash(self, reportRequest): epoch = requestToHash[key] requestToHash[key] = epoch - epoch % granularity md5Hash = hashlib.md5() - md5Hash.update(repr(requestToHash)) - md5Hash.update(self.setup) + md5Hash.update(repr(requestToHash).encode()) + md5Hash.update(self.setup.encode()) return md5Hash.hexdigest() def generate(self, reportRequest, credDict): diff --git a/AccountingSystem/private/Plotters/PilotPlotter.py b/AccountingSystem/private/Plotters/PilotPlotter.py index f6d92ac6f79..6531ad03817 100644 --- a/AccountingSystem/private/Plotters/PilotPlotter.py +++ b/AccountingSystem/private/Plotters/PilotPlotter.py @@ -168,7 +168,7 @@ def _plotJobsPerPilot(self, reportRequest, plotInfo, filename): 'endtime': reportRequest['endTime'], 'span': plotInfo['granularity'], 'ylabel': "jobs/pilot", - 'normalization': max(x for y in plotInfo['data'].itervalues() for x in y.itervalues())} + 'normalization': max(x for y in plotInfo['data'].values() for x in y.values())} return self._generateQualityPlot(filename, plotInfo['data'], metadata) def _reportTotalNumberOfPilots(self, reportRequest): diff --git a/AccountingSystem/scripts/dirac-accounting-decode-fileid.py b/AccountingSystem/scripts/dirac-accounting-decode-fileid.py index e8aee28809b..0dd71abdacf 100755 --- a/AccountingSystem/scripts/dirac-accounting-decode-fileid.py +++ b/AccountingSystem/scripts/dirac-accounting-decode-fileid.py @@ -13,7 +13,7 @@ import pprint import sys -import urlparse +from six.moves.urllib import parse as urlparse import cgi from DIRAC import gLogger from DIRAC.Core.Base import Script diff --git a/ConfigurationSystem/Agent/GOCDB2CSAgent.py b/ConfigurationSystem/Agent/GOCDB2CSAgent.py index d0cc4e5809b..0ba87db16b6 100644 --- a/ConfigurationSystem/Agent/GOCDB2CSAgent.py +++ b/ConfigurationSystem/Agent/GOCDB2CSAgent.py @@ -58,7 +58,7 @@ def execute(self): """ # __functionMap is at the end of the class definition - for option, functionCall in GOCDB2CSAgent.__functionMap.iteritems(): + for option, functionCall in GOCDB2CSAgent.__functionMap.items(): optionValue = self.am_getOption(option, True) if optionValue: result = functionCall(self) @@ -157,12 +157,12 @@ def __preparePerfSONARConfiguration(self, endpointList): split = endpoint['DIRACSITENAME'].split('.') path = cfgPath(rootPath, split[0], endpoint['DIRACSITENAME'], extPath, endpoint['HOSTNAME']) - for name, defaultValue in options.iteritems(): + for name, defaultValue in options.items(): newConfiguration[cfgPath(path, name)] = defaultValue # get current configuration currentConfiguration = {} - for option in options.iterkeys(): + for option in options.keys(): result = gConfig.getConfigurationTree(rootPath, extPath + '/', '/' + option) if not result['OK']: log.error("getConfigurationTree() failed with message: %s" % result['Message']) @@ -242,7 +242,7 @@ def __updateConfiguration(self, setElements=None, delElements=None): log.debug('Begin function ...') # assure existence and proper value of a section or an option - for path, value in setElements.iteritems(): + for path, value in setElements.items(): if value is None: section = path diff --git a/ConfigurationSystem/Client/Helpers/CSGlobals.py b/ConfigurationSystem/Client/Helpers/CSGlobals.py index 52ca0409cec..bff05030703 100644 --- a/ConfigurationSystem/Client/Helpers/CSGlobals.py +++ b/ConfigurationSystem/Client/Helpers/CSGlobals.py @@ -11,12 +11,12 @@ __RCSID__ = "$Id$" import imp +import six from DIRAC.Core.Utilities.DIRACSingleton import DIRACSingleton +@six.add_metaclass(DIRACSingleton) class Extensions(object): - __metaclass__ = DIRACSingleton - def __init__(self): self.__modules = {} self.__orderedExtNames = [] diff --git a/ConfigurationSystem/Client/Helpers/Operations.py b/ConfigurationSystem/Client/Helpers/Operations.py index 4bdc0e03e16..9e8d7da8900 100644 --- a/ConfigurationSystem/Client/Helpers/Operations.py +++ b/ConfigurationSystem/Client/Helpers/Operations.py @@ -74,7 +74,7 @@ from __future__ import print_function import six -import thread +from six.moves import _thread as thread import os from diraccfg import CFG from DIRAC import S_OK, S_ERROR, gConfig diff --git a/ConfigurationSystem/Client/Helpers/Resources.py b/ConfigurationSystem/Client/Helpers/Resources.py index 07204ede5b5..9ac2559959c 100644 --- a/ConfigurationSystem/Client/Helpers/Resources.py +++ b/ConfigurationSystem/Client/Helpers/Resources.py @@ -6,7 +6,7 @@ __RCSID__ = "$Id$" -import urlparse +from six.moves.urllib import parse as urlparse from distutils.version import LooseVersion # pylint: disable=no-name-in-module,import-error import six diff --git a/ConfigurationSystem/Client/PathFinder.py b/ConfigurationSystem/Client/PathFinder.py index 1b5386378ff..4a748aad28e 100755 --- a/ConfigurationSystem/Client/PathFinder.py +++ b/ConfigurationSystem/Client/PathFinder.py @@ -6,7 +6,7 @@ __RCSID__ = "$Id$" -import urlparse +from six.moves.urllib import parse as urlparse from DIRAC.Core.Utilities import List from DIRAC.ConfigurationSystem.Client.ConfigurationData import gConfigurationData diff --git a/ConfigurationSystem/Client/Utilities.py b/ConfigurationSystem/Client/Utilities.py index a8f788cea1a..14ccda480f0 100644 --- a/ConfigurationSystem/Client/Utilities.py +++ b/ConfigurationSystem/Client/Utilities.py @@ -17,7 +17,7 @@ import re import socket -from urlparse import urlparse +from six.moves.urllib.parse import urlparse import six @@ -274,9 +274,7 @@ def addToChangeSet(entry, changeSet): if submissionMode == "Unknown" and newSubmissionMode: addToChangeSet((ceSection, 'SubmissionMode', submissionMode, newSubmissionMode), changeSet) - queues = ceInfo['Queues'].keys() - for queue in queues: - queueInfo = ceInfo['Queues'][queue] + for queue, queueInfo in ceInfo['Queues'].items(): queueStatus = queueInfo['GlueCEStateStatus'] queueSection = cfgPath(ceSection, 'Queues', queue) queueDict = {} @@ -455,8 +453,7 @@ def getGridSRMs(vo, bdiiInfo=None, srmBlackList=None, unUsed=False): srmSeDict = {} for site in siteSRMDict: - srms = siteSRMDict[site].keys() - for srm in srms: + for srm in siteSRMDict[site]: if seBdiiDict.get(site, {}).get('SEs', {}).get(srm, {}): srmSeDict.setdefault(site, {}) srmSeDict[site].setdefault(srm, {}) diff --git a/ConfigurationSystem/private/ConfigurationClient.py b/ConfigurationSystem/private/ConfigurationClient.py index c0654251c62..38501230911 100755 --- a/ConfigurationSystem/private/ConfigurationClient.py +++ b/ConfigurationSystem/private/ConfigurationClient.py @@ -281,7 +281,7 @@ def getConfigurationTree(self, root='', *filters): if not options['OK']: return S_ERROR("getOptionsDict() failed with message: %s" % options['Message']) - for key, value in options['Value'].iteritems(): + for key, value in options['Value'].items(): path = cfgPath(root, key) addOption = True for substr in filters: diff --git a/ConfigurationSystem/private/ConfigurationData.py b/ConfigurationSystem/private/ConfigurationData.py index 21515de514b..4b9aeaa2ca6 100755 --- a/ConfigurationSystem/private/ConfigurationData.py +++ b/ConfigurationSystem/private/ConfigurationData.py @@ -7,7 +7,7 @@ import os.path import zlib import zipfile -import thread +from six.moves import _thread as thread import time import DIRAC diff --git a/ConfigurationSystem/private/Refresher.py b/ConfigurationSystem/private/Refresher.py index 4af11e81469..afd31d2023a 100755 --- a/ConfigurationSystem/private/Refresher.py +++ b/ConfigurationSystem/private/Refresher.py @@ -8,7 +8,7 @@ __RCSID__ = "$Id$" import threading -import thread +from six.moves import _thread as thread import time import random from DIRAC.ConfigurationSystem.Client.ConfigurationData import gConfigurationData diff --git a/ConfigurationSystem/scripts/dirac-admin-add-resources.py b/ConfigurationSystem/scripts/dirac-admin-add-resources.py index 862e9a52b64..15311713785 100755 --- a/ConfigurationSystem/scripts/dirac-admin-add-resources.py +++ b/ConfigurationSystem/scripts/dirac-admin-add-resources.py @@ -16,7 +16,7 @@ import re import os import shlex -from urlparse import urlparse +from six.moves.urllib.parse import urlparse from DIRAC.Core.Base import Script from DIRAC import gLogger, exit as DIRACExit, S_OK @@ -97,7 +97,7 @@ def checkUnusedCEs(): result = getDIRACSiteName(site) if result['OK']: diracSite = ','.join(result['Value']) - ces = siteDict[site].keys() # pylint: disable=no-member + ces = list(siteDict[site]) if ces: gLogger.notice(" %s, DIRAC site %s" % (site, diracSite)) for ce in ces: @@ -120,8 +120,7 @@ def checkUnusedCEs(): for site in siteDict: # Get the country code: country = '' - ces = siteDict[site].keys() # pylint: disable=no-member - for ce in ces: + for ce in siteDict[site]: country = ce.strip().split('.')[-1].lower() if len(country) == 2: break diff --git a/ConfigurationSystem/scripts/dirac-admin-bdii-info.py b/ConfigurationSystem/scripts/dirac-admin-bdii-info.py index e16a5b59c90..3509e5cd4ef 100755 --- a/ConfigurationSystem/scripts/dirac-admin-bdii-info.py +++ b/ConfigurationSystem/scripts/dirac-admin-bdii-info.py @@ -129,7 +129,7 @@ def showInfo(result, info): if info == 'site-se' or info == 'all': print("SE: %s \n{" % element.get('GlueSEUniqueID', 'Unknown')) - for item in element.iteritems(): + for item in element.items(): print(" %s: %s" % item) print("}") diff --git a/Core/Base/API.py b/Core/Base/API.py index 7d86ebe5d28..f73f92650c6 100644 --- a/Core/Base/API.py +++ b/Core/Base/API.py @@ -88,6 +88,8 @@ def __getstate__(self): # because of the thread locks if isinstance(state['log'], Logging): state['log'] = state['log'].getSubName() + # In Python 3 PrettyPrinter can't be pickled + del state['pPrint'] return state def __setstate__(self, state): @@ -101,6 +103,7 @@ def __setstate__(self, state): # due to the thread locks if isinstance(state['log'], six.string_types): self.log = gLogger.getSubLogger(state['log']) + self.pPrint = pprint.PrettyPrinter() ############################################################################# diff --git a/Core/Base/ExecutorMindHandler.py b/Core/Base/ExecutorMindHandler.py index 9105b8cb9df..317e8ab8079 100644 --- a/Core/Base/ExecutorMindHandler.py +++ b/Core/Base/ExecutorMindHandler.py @@ -4,9 +4,9 @@ from __future__ import division from __future__ import print_function -import types import pprint +import six from DIRAC import gLogger from DIRAC.Core.Utilities.ReturnValues import S_OK, S_ERROR, isReturnStructure from DIRAC.Core.Utilities.ThreadScheduler import gThreadScheduler @@ -16,21 +16,21 @@ class ExecutorMindHandler(RequestHandler): - MSG_DEFINITIONS = {'ProcessTask': {'taskId': (types.IntType, types.LongType), - 'taskStub': types.StringTypes, - 'eType': types.StringTypes}, - 'TaskDone': {'taskId': (types.IntType, types.LongType), - 'taskStub': types.StringTypes}, - 'TaskFreeze': {'taskId': (types.IntType, types.LongType), - 'taskStub': types.StringTypes, - 'freezeTime': (types.IntType, types.LongType)}, - 'TaskError': {'taskId': (types.IntType, types.LongType), - 'errorMsg': types.StringTypes, - 'taskStub': types.StringTypes, - 'eType': types.StringTypes}, - 'ExecutorError': {'taskId': (types.IntType, types.LongType), - 'errorMsg': types.StringTypes, - 'eType': types.StringTypes}} + MSG_DEFINITIONS = {'ProcessTask': {'taskId': six.integer_types, + 'taskStub': six.string_types, + 'eType': six.string_types}, + 'TaskDone': {'taskId': six.integer_types, + 'taskStub': six.string_types}, + 'TaskFreeze': {'taskId': six.integer_types, + 'taskStub': six.string_types, + 'freezeTime': six.integer_types}, + 'TaskError': {'taskId': six.integer_types, + 'errorMsg': six.string_types, + 'taskStub': six.string_types, + 'eType': six.string_types}, + 'ExecutorError': {'taskId': six.integer_types, + 'errorMsg': six.string_types, + 'eType': six.string_types}} class MindCallbacks(ExecutorDispatcherCallbacks): diff --git a/Core/Base/SQLAlchemyDB.py b/Core/Base/SQLAlchemyDB.py index b63d14f55ec..bd2fe0df05a 100644 --- a/Core/Base/SQLAlchemyDB.py +++ b/Core/Base/SQLAlchemyDB.py @@ -9,6 +9,7 @@ __RCSID__ = "$Id$" +import six import datetime from sqlalchemy import create_engine, desc, exc from sqlalchemy.engine.reflection import Inspector @@ -189,13 +190,13 @@ def select(self, table, params): select = Query(columns, session=session) # query conditions - for columnName, columnValue in params.iteritems(): + for columnName, columnValue in params.items(): if not columnValue: continue column_a = getattr(table_c, columnName.lower()) if isinstance(columnValue, (list, tuple)): select = select.filter(column_a.in_(list(columnValue))) - elif isinstance(columnValue, (basestring, datetime.datetime, bool)): + elif isinstance(columnValue, six.string_types + (datetime.datetime, bool)): select = select.filter(column_a == columnValue) else: self.log.error("type(columnValue) == %s" % type(columnValue)) @@ -206,7 +207,7 @@ def select(self, table, params): column_a = getattr(table_c, newer[0].lower()) select = select.filter(column_a > newer[1]) if order: - order = [order] if isinstance(order, basestring) else list(order) + order = [order] if isinstance(order, six.string_types) else list(order) column_a = getattr(table_c, order[0].lower()) if len(order) == 2 and order[1].lower() == 'desc': select = select.order_by(desc(column_a)) @@ -268,13 +269,13 @@ def delete(self, table, params): try: deleteQuery = Query(table_c, session=session) - for columnName, columnValue in params.iteritems(): + for columnName, columnValue in params.items(): if not columnValue: continue column_a = getattr(table_c, columnName.lower()) if isinstance(columnValue, (list, tuple)): deleteQuery = deleteQuery.filter(column_a.in_(list(columnValue))) - elif isinstance(columnValue, (basestring, datetime.datetime, bool)): + elif isinstance(columnValue, six.string_types + (datetime.datetime, bool)): deleteQuery = deleteQuery.filter(column_a == columnValue) else: self.log.error("type(columnValue) == %s" % type(columnValue)) @@ -285,7 +286,7 @@ def delete(self, table, params): column_a = getattr(table_c, newer[0].lower()) deleteQuery = deleteQuery.filter(column_a > newer[1]) if order: - order = [order] if isinstance(order, basestring) else list(order) + order = [order] if isinstance(order, six.string_types) else list(order) column_a = getattr(table_c, order[0].lower()) if len(order) == 2 and order[1].lower() == 'desc': deleteQuery = deleteQuery.order_by(desc(column_a)) diff --git a/Core/DISET/MessageClient.py b/Core/DISET/MessageClient.py index 6bf67eae2f7..afb8460e9d8 100755 --- a/Core/DISET/MessageClient.py +++ b/Core/DISET/MessageClient.py @@ -33,7 +33,7 @@ def _initialize(self): def __generateUniqueClientName(self): hashStr = ":".join((Time.toString(), str(random.random()), Network.getFQDN(), gLogger.getName())) - hexHash = md5(hashStr).hexdigest() + hexHash = md5(hashStr.encode()).hexdigest() return hexHash def setUniqueName(self, uniqueName): diff --git a/Core/DISET/RequestHandler.py b/Core/DISET/RequestHandler.py index 0d423b92ac1..82b83daa549 100755 --- a/Core/DISET/RequestHandler.py +++ b/Core/DISET/RequestHandler.py @@ -9,6 +9,7 @@ import os import time import psutil +import six import DIRAC @@ -344,7 +345,7 @@ def __checkExpectedArgumentTypes(self, method, args): # #### - __connectionCallbackTypes = {'new': [basestring, dict], + __connectionCallbackTypes = {'new': list(six.string_types) + [dict], 'connected': [], 'drop': []} @@ -499,7 +500,7 @@ def export_ping(self): return S_OK(dInfo) - types_echo = [basestring] + types_echo = [six.string_types] @staticmethod def export_echo(data): diff --git a/Core/DISET/ThreadConfig.py b/Core/DISET/ThreadConfig.py index bc86d58936a..77d3d19e89d 100644 --- a/Core/DISET/ThreadConfig.py +++ b/Core/DISET/ThreadConfig.py @@ -4,9 +4,11 @@ import threading import functools +import six from DIRAC.Core.Utilities.DIRACSingleton import DIRACSingleton +@six.add_metaclass(DIRACSingleton) class ThreadConfig(threading.local): """ This class allows to contain extra information when a call is done on behalf of somebody else. Typically, when a host performs the request on behalf of a user. @@ -18,9 +20,6 @@ class ThreadConfig(threading.local): Also, this class has to be populated manually, no Client class will do it for you. """ - - __metaclass__ = DIRACSingleton - def __init__(self): self.reset() diff --git a/Core/DISET/private/BaseClient.py b/Core/DISET/private/BaseClient.py index d5ac0a2eebc..3346b4638b4 100755 --- a/Core/DISET/private/BaseClient.py +++ b/Core/DISET/private/BaseClient.py @@ -9,7 +9,7 @@ import six import time -import thread +from six.moves import _thread as thread import DIRAC from DIRAC.Core.DISET.private.Protocols import gProtocolDict from DIRAC.FrameworkSystem.Client.Logger import gLogger @@ -645,8 +645,11 @@ def _getBaseStub(self): del newKwargs['useCertificates'] return [self._destinationSrv, newKwargs] - def __nonzero__(self): + def __bool__(self): return True + # For Python 2 compatibility + __nonzero__ = __bool__ + def __str__(self): return "" % (self.serviceURL, self.__extraCredentials) diff --git a/Core/DISET/private/FileHelper.py b/Core/DISET/private/FileHelper.py index 55230abaed2..5f29f8647e2 100755 --- a/Core/DISET/private/FileHelper.py +++ b/Core/DISET/private/FileHelper.py @@ -7,7 +7,7 @@ import os import hashlib import threading -import cStringIO +from six import StringIO import tarfile import tempfile @@ -132,7 +132,7 @@ def networkToString(self, maxFileSize=0): """ Receive the input from a DISET client and return it as a string """ - stringIO = cStringIO.StringIO() + stringIO = StringIO() result = self.networkToDataSink(stringIO, maxFileSize=maxFileSize) if not result['OK']: return result @@ -184,7 +184,7 @@ def stringToNetwork(self, stringVal): """ Send a given string to the DISET client over the network """ - stringIO = cStringIO.StringIO(stringVal) + stringIO = StringIO(stringVal) iPacketSize = self.packetSize ioffset = 0 @@ -234,7 +234,7 @@ def FDToNetwork(self, iFD): return S_OK() def BufferToNetwork(self, stringToSend): - sIO = cStringIO.StringIO(stringToSend) + sIO = StringIO(stringToSend) try: return self.DataSourceToNetwork(sIO) finally: diff --git a/Core/DISET/private/GatewayService.py b/Core/DISET/private/GatewayService.py index 809bff37501..bca7c39119a 100644 --- a/Core/DISET/private/GatewayService.py +++ b/Core/DISET/private/GatewayService.py @@ -20,7 +20,7 @@ import sys -import cStringIO +from six import BytesIO import os # TODO: Remove ThreadPool later @@ -312,7 +312,7 @@ def errMsg(self, msg, dynMsg=""): gLogger.error("[%s] %s" % (self.__currentMethod, msg), dynMsg) def getDataFromClient(self, clientFileHelper): - sIO = cStringIO.StringIO() + sIO = BytesIO() self.infoMsg("About to get data from client") result = clientFileHelper.networkToDataSink(sIO, self.__transferBytesLimit) if not result['OK']: @@ -363,7 +363,7 @@ def getDataFromService(self, srvMethod, params): _, srvTransport = result['Value'] srvFileHelper = FileHelper(srvTransport) srvFileHelper.setDirection("receive") - sIO = cStringIO.StringIO() + sIO = BytesIO() result = srvFileHelper.networkToDataSink(sIO, self.__transferBytesLimit) if not result['OK']: self.errMsg("Could not receive data from server", result['Message']) diff --git a/Core/DISET/private/Transports/BaseTransport.py b/Core/DISET/private/Transports/BaseTransport.py index a431578e27f..ee57a78e180 100755 --- a/Core/DISET/private/Transports/BaseTransport.py +++ b/Core/DISET/private/Transports/BaseTransport.py @@ -25,7 +25,8 @@ import time import select -import cStringIO +import six +from six import BytesIO from hashlib import md5 from DIRAC.Core.Utilities.ReturnValues import S_ERROR, S_OK @@ -43,19 +44,19 @@ class BaseTransport(object): # you may want to read the man page before tuning it... iListenQueueSize = 128 iReadTimeout = 600 - keepAliveMagic = "dka" + keepAliveMagic = b"dka" def __init__(self, stServerAddress, bServerMode=False, **kwargs): self.bServerMode = bServerMode self.extraArgsDict = kwargs - self.byteStream = "" + self.byteStream = b"" self.packetSize = 1048576 # 1MiB self.stServerAddress = stServerAddress self.peerCredentials = {} self.remoteAddress = False self.appData = "" self.startedKeepAlives = set() - self.keepAliveId = md5(str(stServerAddress) + str(bServerMode)).hexdigest() + self.keepAliveId = md5((str(stServerAddress) + str(bServerMode)).encode()).hexdigest() self.receivedMessages = [] self.sentKeepAlives = 0 self.waitingForKeepAlivePong = False @@ -165,13 +166,12 @@ def _read(self, bufSize=4096, skipReadyCheck=False): def _write(self, buf): return S_OK(self.oSocket.send(buf)) - def sendData(self, uData, prefix=False): + def sendData(self, uData, prefix=b""): self.__updateLastActionTimestamp() sCodedData = MixedEncode.encode(uData) - if prefix: - dataToSend = "%s%s:%s" % (prefix, len(sCodedData), sCodedData) - else: - dataToSend = "%s:%s" % (len(sCodedData), sCodedData) + if isinstance(sCodedData, six.text_type): + sCodedData = sCodedData.encode() + dataToSend = b"".join([prefix, str(len(sCodedData)).encode(), b":", sCodedData]) for index in range(0, len(dataToSend), self.packetSize): bytesToSend = min(self.packetSize, len(dataToSend) - index) packSentBytes = 0 @@ -198,7 +198,7 @@ def receiveData(self, maxBufferSize=0, blockAfterKeepAlive=True, idleReceive=Fal maxBufferSize = max(maxBufferSize, 0) try: # Look either for message length of keep alive magic string - iSeparatorPosition = self.byteStream.find(":", 0, 10) + iSeparatorPosition = self.byteStream.find(b":", 0, 10) keepAliveMagicLen = len(BaseTransport.keepAliveMagic) isKeepAlive = self.byteStream.find(BaseTransport.keepAliveMagic, 0, keepAliveMagicLen) == 0 # While not found the message length or the ka, keep receiving @@ -213,7 +213,7 @@ def receiveData(self, maxBufferSize=0, blockAfterKeepAlive=True, idleReceive=Fal # New data! self.byteStream += retVal['Value'] # Look again for either message length of ka magic string - iSeparatorPosition = self.byteStream.find(":", 0, 10) + iSeparatorPosition = self.byteStream.find(b":", 0, 10) isKeepAlive = self.byteStream.find(BaseTransport.keepAliveMagic, 0, keepAliveMagicLen) == 0 # Over the limit? if maxBufferSize and len(self.byteStream) > maxBufferSize and iSeparatorPosition == -1: @@ -235,7 +235,7 @@ def receiveData(self, maxBufferSize=0, blockAfterKeepAlive=True, idleReceive=Fal self.byteStream = pkgData[pkgSize:] else: # If we still need to read stuff - pkgMem = cStringIO.StringIO() + pkgMem = BytesIO() pkgMem.write(pkgData) # Receive while there's still data to be received while readSize < pkgSize: @@ -252,7 +252,7 @@ def receiveData(self, maxBufferSize=0, blockAfterKeepAlive=True, idleReceive=Fal # Data is here! take it out from the bytestream, dencode and return if readSize == pkgSize: data = pkgMem.getvalue() - self.byteStream = "" + self.byteStream = b"" else: # readSize > pkgSize: pkgMem.seek(0, 0) data = pkgMem.read(pkgSize) diff --git a/Core/DISET/private/Transports/M2SSLTransport.py b/Core/DISET/private/Transports/M2SSLTransport.py index 4e8bf22fe7a..8164ca12831 100755 --- a/Core/DISET/private/Transports/M2SSLTransport.py +++ b/Core/DISET/private/Transports/M2SSLTransport.py @@ -163,7 +163,7 @@ def initAsServer(self): # a server session ID in the context host = self.stServerAddress[0] port = self.stServerAddress[1] - self.__ctx.set_session_id_ctx("DIRAC-%s-%s" % (host, port)) + self.__ctx.set_session_id_ctx(("DIRAC-%s-%s" % (host, port)).encode()) self.oSocket = self.__getConnection() # Make sure reuse address is set correctly if self.bAllowReuseAddress: diff --git a/Core/DISET/private/Transports/SSL/M2Utils.py b/Core/DISET/private/Transports/SSL/M2Utils.py index 8274a58d879..9830b73ff13 100644 --- a/Core/DISET/private/Transports/SSL/M2Utils.py +++ b/Core/DISET/private/Transports/SSL/M2Utils.py @@ -112,6 +112,16 @@ def getM2SSLContext(ctx=None, **kwargs): raise RuntimeError("CA path (%s) is not a valid directory" % caPath) ctx.load_verify_locations(capath=caPath) + # If the version of M2Crypto is recent enough, there is an API + # to accept proxy certificate, and we do not need to rely on + # OPENSSL_ALLOW_PROXY_CERT environment variable + # which was removed as of openssl 1.1 + # We need this to be merged in M2Crypto: https://gitlab.com/m2crypto/m2crypto/merge_requests/236 + # We set the proper verify flag to the X509Store of the context + # as described here https://www.openssl.org/docs/man1.1.1/man7/proxy-certificates.html + if hasattr(SSL, 'verify_allow_proxy_certs'): + ctx.get_cert_store().set_flags(SSL.verify_allow_proxy_certs) # pylint: disable=no-member + # Other parameters sslMethod = kwargs.get('sslMethod', None) if sslMethod: diff --git a/Core/DISET/private/Transports/test/Test_SSLTransport.py b/Core/DISET/private/Transports/test/Test_SSLTransport.py index 85734140c0f..17fedbeb35d 100644 --- a/Core/DISET/private/Transports/test/Test_SSLTransport.py +++ b/Core/DISET/private/Transports/test/Test_SSLTransport.py @@ -109,8 +109,9 @@ def __acceptIncomingConnection(self): try: _inList, _outList, _exList = select.select(sockets, [], [], 2) - - clientTransport = self.transport.acceptConnection()['Value'] + result = self.transport.acceptConnection() + assert result["OK"], result + clientTransport = result['Value'] self.handleConnection(clientTransport) @@ -189,14 +190,14 @@ def ping_server(clientTransport): """ clientTransport.setSocketTimeout(5) - clientTransport.sendData(MAGIC_QUESTION) + result = clientTransport.sendData(MAGIC_QUESTION) + assert result["OK"] serverReturn = clientTransport.receiveData() return serverReturn def test_simpleMessage(create_serverAndClient): """ Send a message, wait for an answer """ - serv, client = create_serverAndClient serverAnswer = ping_server(client) assert serv.receivedMessage == MAGIC_QUESTION diff --git a/Core/LCG/GGUSTicketsClient.py b/Core/LCG/GGUSTicketsClient.py index 13ecca59e79..099174417cc 100644 --- a/Core/LCG/GGUSTicketsClient.py +++ b/Core/LCG/GGUSTicketsClient.py @@ -4,7 +4,7 @@ from __future__ import division from __future__ import print_function -import urllib2 +from six.moves.urllib_error import URLError from suds import WebFault from suds.client import Client @@ -76,7 +76,7 @@ def getTicketsList(self, siteName=None, startDate=None, endDate=None): ticketList = self.gclient.service.TicketGetList(query) except WebFault as e: return S_ERROR(e) - except urllib2.URLError as e: + except URLError as e: return S_ERROR(e) return self.globalStatistics(ticketList) diff --git a/Core/LCG/GOCDBClient.py b/Core/LCG/GOCDBClient.py index 30e854e4aa9..15a1a497a0a 100644 --- a/Core/LCG/GOCDBClient.py +++ b/Core/LCG/GOCDBClient.py @@ -37,7 +37,7 @@ def _parseSingleElement(element, attributes=None): if child.nodeName == "SCOPES": for subchild in child.childNodes: if subchild.childNodes: - handler.setdefault('SCOPES', []).append(subchild.childNodes[0].nodeValue.encode('utf-8')) + handler.setdefault('SCOPES', []).append(subchild.childNodes[0].nodeValue) continue if child.nodeName == "EXTENSIONS": @@ -46,13 +46,12 @@ def _parseSingleElement(element, attributes=None): dct = {} for subsubchild in subchild.childNodes: if subsubchild.childNodes: - dct[subsubchild.nodeName.encode('utf-8')] = subsubchild.childNodes[0].nodeValue.encode('utf-8') + dct[subsubchild.nodeName] = subsubchild.childNodes[0].nodeValue handler.setdefault('EXTENSIONS', []).append(dct) continue try: - nodeValue = child.childNodes[0].nodeValue - attrValue = nodeValue.encode('utf-8') + attrValue = child.childNodes[0].nodeValue except IndexError: continue handler[attrName] = attrValue @@ -334,7 +333,6 @@ def _getServiceEndpointCurlDownload(self, granularity, entity): def _downTimeXMLParsing(self, dt, siteOrRes, entities=None, startDateMax=None): """ Performs xml parsing from the dt string (returns a dictionary) """ - dt = dt.encode('utf-8') doc = minidom.parseString(dt) downtimeElements = doc.getElementsByTagName("DOWNTIME") diff --git a/Core/LCG/test/Test_LCG.py b/Core/LCG/test/Test_LCG.py index 18f89aeef6f..528c9cd872e 100644 --- a/Core/LCG/test/Test_LCG.py +++ b/Core/LCG/test/Test_LCG.py @@ -234,7 +234,7 @@ def test__downTimeXMLParsing_affected(): res = GOCCli._downTimeXMLParsing(xml_endpoint_and_affected_ongoing, 'Resource') - assert list(res)[0] == '109962G0 lhcbsrm-kit.gridka.deSRM' + assert set(res) == {'109962G0 lhcbsrm-kit.gridka.deSRM'} assert res['109962G0 lhcbsrm-kit.gridka.deSRM']['HOSTNAME'] == 'lhcbsrm-kit.gridka.de' assert res['109962G0 lhcbsrm-kit.gridka.deSRM']['URL'] == 'lhcbsrm-disk-kit.gridka.de' res = GOCCli._downTimeXMLParsing(xml_endpoint_and_affected_ongoing, 'Site') @@ -244,11 +244,11 @@ def test__downTimeXMLParsing_affected(): def test__downTimeXMLParsing(): res = GOCCli._downTimeXMLParsing(XML_site_ongoing, 'Site') - assert list(res)[0] == '28490G0 GRISU-ENEA-GRID' + assert set(res) == {'28490G0 GRISU-ENEA-GRID'} assert res['28490G0 GRISU-ENEA-GRID']['SITENAME'] == 'GRISU-ENEA-GRID' res = GOCCli._downTimeXMLParsing(XML_node_ongoing, 'Resource') - assert list(res)[0] == '28490G0 egse-cresco.portici.enea.it' + assert set(res) == {'28490G0 egse-cresco.portici.enea.it'} assert res['28490G0 egse-cresco.portici.enea.it']['HOSTNAME'] == 'egse-cresco.portici.enea.it' assert res['28490G0 egse-cresco.portici.enea.it']['HOSTED_BY'] == 'GRISU-ENEA-GRID' @@ -259,12 +259,12 @@ def test__downTimeXMLParsing(): res = GOCCli._downTimeXMLParsing(XML_nodesite_ongoing, 'Site') assert len(res) == 1 - assert list(res)[0] == '28490G0 GRISU-ENEA-GRID' + assert set(res) == {'28490G0 GRISU-ENEA-GRID'} assert res['28490G0 GRISU-ENEA-GRID']['SITENAME'] == 'GRISU-ENEA-GRID' res = GOCCli._downTimeXMLParsing(XML_nodesite_ongoing, 'Resource') assert len(res) == 1 - assert list(res)[0] == '28490G0 egse-cresco.portici.enea.it' + assert set(res) == {'28490G0 egse-cresco.portici.enea.it'} assert res['28490G0 egse-cresco.portici.enea.it']['HOSTNAME'] == 'egse-cresco.portici.enea.it' res = GOCCli._downTimeXMLParsing(XML_site_startingIn8h, 'Site', None, now) @@ -273,7 +273,7 @@ def test__downTimeXMLParsing(): assert res == {} res = GOCCli._downTimeXMLParsing(XML_site_ongoing_and_site_starting_in_24_hours, 'Site', None, now) - assert list(res)[0] == '28490G1 GRISU-ENEA-GRID' + assert set(res) == {'28490G1 GRISU-ENEA-GRID'} assert res['28490G1 GRISU-ENEA-GRID']['SITENAME'] == 'GRISU-ENEA-GRID' res = GOCCli._downTimeXMLParsing(XML_site_ongoing_and_site_starting_in_24_hours, 'Resource', None, now) @@ -282,7 +282,7 @@ def test__downTimeXMLParsing(): assert res == {} res = GOCCli._downTimeXMLParsing(XML_site_startingIn24h_and_site_startingIn50h, 'Site', None, tomorrow) - assert list(res)[0] == '28490G1 GRISU-ENEA-GRID' + assert set(res) == {'28490G1 GRISU-ENEA-GRID'} assert res['28490G1 GRISU-ENEA-GRID']['SITENAME'] == 'GRISU-ENEA-GRID' res = GOCCli._downTimeXMLParsing( @@ -298,23 +298,23 @@ def test__downTimeXMLParsing(): assert res['28490G1 GRISU-ENEA-GRID']['SITENAME'] == 'GRISU-ENEA-GRID' assert res['28490G0 CERN-PROD']['SITENAME'] == 'CERN-PROD' res = GOCCli._downTimeXMLParsing(XML_site_ongoing_and_other_site_starting_in_24_hours, 'Site', 'CERN-PROD') - assert list(res)[0] == '28490G0 CERN-PROD' + assert set(res) == {'28490G0 CERN-PROD'} assert res['28490G0 CERN-PROD']['SITENAME'] == 'CERN-PROD' res = GOCCli._downTimeXMLParsing(XML_site_ongoing_and_other_site_starting_in_24_hours, 'Site', 'CNAF-T1') assert res == {} res = GOCCli._downTimeXMLParsing(XML_site_ongoing_and_other_site_starting_in_24_hours, 'Site', ['GRISU-ENEA-GRID', 'CERN-PROD'], now) - assert list(res)[0] == '28490G1 GRISU-ENEA-GRID' + assert set(res) == {'28490G1 GRISU-ENEA-GRID'} assert res['28490G1 GRISU-ENEA-GRID']['SITENAME'] == 'GRISU-ENEA-GRID' res = GOCCli._downTimeXMLParsing(XML_site_ongoing_and_other_site_starting_in_24_hours, 'Site', ['GRISU-ENEA-GRID', 'CERN-PROD'], inAWeek) - assert list(res)[0] == '28490G0 CERN-PROD' + assert set(res) == {'28490G0 CERN-PROD', '28490G1 GRISU-ENEA-GRID'} assert res['28490G0 CERN-PROD']['SITENAME'] == 'CERN-PROD' res = GOCCli._downTimeXMLParsing(XML_node_ongoing_and_other_node_starting_in_24_hours, 'Resource', ['egse-cresco.portici.enea.it']) - assert list(res)[0] == '28490G1 egse-cresco.portici.enea.it' + assert set(res) == {'28490G1 egse-cresco.portici.enea.it'} assert res['28490G1 egse-cresco.portici.enea.it']['HOSTNAME'] == 'egse-cresco.portici.enea.it' res = GOCCli._downTimeXMLParsing(XML_node_ongoing_and_other_node_starting_in_24_hours, 'Resource', ['egse-cresco.portici.enea.it', 'ce112.cern.ch']) @@ -326,7 +326,7 @@ def test__downTimeXMLParsing(): XML_node_ongoing_and_other_node_starting_in_24_hours, 'Resource', 'ce112.cern.ch') - assert list(res)[0] == '28490G0 ce112.cern.ch' + assert set(res) == {'28490G0 ce112.cern.ch'} assert res['28490G0 ce112.cern.ch']['HOSTNAME'] == 'ce112.cern.ch' res = GOCCli._downTimeXMLParsing( XML_node_ongoing_and_other_node_starting_in_24_hours, @@ -341,5 +341,5 @@ def test__downTimeXMLParsing(): assert res['28490G1 egse-cresco.portici.enea.it']['HOSTNAME'] == 'egse-cresco.portici.enea.it' res = GOCCli._downTimeXMLParsing(XML_node_ongoing_and_other_node_starting_in_24_hours, 'Resource', ['egse-cresco.portici.enea.it', 'ce112.cern.ch'], inAWeek) - assert list(res)[0] == '28490G0 ce112.cern.ch' + assert set(res) == {'28490G1 egse-cresco.portici.enea.it', '28490G0 ce112.cern.ch'} assert res['28490G0 ce112.cern.ch']['HOSTNAME'] == 'ce112.cern.ch' diff --git a/Core/Security/m2crypto/X509Certificate.py b/Core/Security/m2crypto/X509Certificate.py index 8f8b8addb34..f0e99a64599 100644 --- a/Core/Security/m2crypto/X509Certificate.py +++ b/Core/Security/m2crypto/X509Certificate.py @@ -440,7 +440,7 @@ def getExtensions(self): :returns: S_OK( list of tuple (extensionName, extensionValue)) """ extList = [] - for i in xrange(self.__certObj.get_ext_count()): + for i in range(self.__certObj.get_ext_count()): sn = self.__certObj.get_ext_at(i).get_name() try: value = self.__certObj.get_ext_at(i).get_value() @@ -469,7 +469,7 @@ def asPem(self): :returns: pem string """ - return self.__certObj.as_pem() + return self.__certObj.as_pem().decode() @executeOnlyIfCertLoaded def getExtension(self, name): diff --git a/Core/Security/m2crypto/X509Chain.py b/Core/Security/m2crypto/X509Chain.py index fa08118dc0e..38d8ed4d077 100644 --- a/Core/Security/m2crypto/X509Chain.py +++ b/Core/Security/m2crypto/X509Chain.py @@ -245,7 +245,7 @@ def loadChainFromString(self, data): @staticmethod def __certListFromPemString(certString): """ - Create certificates list from string. String sould contain certificates, just like plain text proxy file. + Create certificates list from string. String should contain certificates, just like plain text proxy file. """ # To get list of X509 certificates (not X509 Certificate Chain) from string it has to be parsed like that # (constructors are not able to deal with big string) @@ -291,7 +291,7 @@ def loadKeyFromString(self, pemData, password=False): """ self._keyObj = None try: - self._keyObj = M2Crypto.EVP.load_key_string(pemData, lambda x: password) + self._keyObj = M2Crypto.EVP.load_key_string(pemData.encode(), lambda x: password) except BaseException as e: return S_ERROR(DErrno.ECERTREAD, "%s (Probably bad pass phrase?)" % repr(e).replace(',)', ')')) @@ -328,7 +328,6 @@ def loadProxyFromString(self, pemData): :returns: S_OK / S_ERROR """ - retVal = self.loadChainFromString(pemData) if not retVal['OK']: return retVal @@ -463,7 +462,7 @@ def generateProxyToString(self, lifetime, diracGroup=False, strength=1024, limit # Generate the proxy string proxyString = "%s%s" % (proxyCert.asPem(), proxyKey.as_pem( - cipher=None, callback=M2Crypto.util.no_passphrase_callback)) + cipher=None, callback=M2Crypto.util.no_passphrase_callback).decode()) for i in range(len(self._certList)): crt = self._certList[i] proxyString += crt.asPem() @@ -847,7 +846,7 @@ def dumpAllToString(self): """ data = self._certList[0].asPem() if self._keyObj: - data += self._keyObj.as_pem(cipher=None, callback=M2Crypto.util.no_passphrase_callback) + data += self._keyObj.as_pem(cipher=None, callback=M2Crypto.util.no_passphrase_callback).decode() for cert in self._certList[1:]: data += cert.asPem() return S_OK(data) @@ -900,7 +899,7 @@ def dumpPKeyToString(self): :returns: S_OK(PEM encoded key) """ - return S_OK(self._keyObj.as_pem(cipher=None, callback=M2Crypto.util.no_passphrase_callback)) + return S_OK(self._keyObj.as_pem(cipher=None, callback=M2Crypto.util.no_passphrase_callback).decode()) def __str__(self): """ String representation diff --git a/Core/Security/m2crypto/X509Request.py b/Core/Security/m2crypto/X509Request.py index 5e0b5db6770..12050c62fdb 100644 --- a/Core/Security/m2crypto/X509Request.py +++ b/Core/Security/m2crypto/X509Request.py @@ -66,7 +66,7 @@ def dumpRequest(self): if not self.__valid: return S_ERROR(DErrno.ENOCERT) try: - reqStr = self.__reqObj.as_pem() + reqStr = self.__reqObj.as_pem().decode() except Exception as e: return S_ERROR(DErrno.EX509, "Can't serialize request: %s" % e) return S_OK(reqStr) @@ -95,7 +95,7 @@ def dumpPKey(self): if not self.__valid: return S_ERROR(DErrno.ENOCERT) try: - pkeyStr = self.__pkeyObj.as_pem(cipher=None, callback=M2Crypto.util.no_passphrase_callback) + pkeyStr = self.__pkeyObj.as_pem(cipher=None, callback=M2Crypto.util.no_passphrase_callback).decode() except Exception as e: return S_ERROR(DErrno.EX509, "Can't serialize pkey: %s" % e) return S_OK(pkeyStr) @@ -126,11 +126,11 @@ def loadAllFromString(self, pemData): :returns: S_OK() """ try: - self.__reqObj = M2Crypto.X509.load_request_string(pemData) + self.__reqObj = M2Crypto.X509.load_request_string(pemData.encode()) except Exception as e: return S_ERROR(DErrno.ENOCERT, str(e)) try: - self.__pkeyObj = M2Crypto.EVP.load_key_string(pemData) + self.__pkeyObj = M2Crypto.EVP.load_key_string(pemData.encode()) except Exception as e: return S_ERROR(DErrno.ENOPKEY, str(e)) self.__valid = True @@ -177,8 +177,8 @@ def checkChain(self, chain): chainPubKey = lastCert.getPublicKey() if not chainPubKey['OK']: return chainPubKey - chainPubKey = chainPubKey['Value'].as_pem(cipher=None, callback=M2Crypto.util.no_passphrase_callback) - reqPubKey = self.__reqObj.get_pubkey().as_pem(cipher=None, callback=M2Crypto.util.no_passphrase_callback) + chainPubKey = chainPubKey['Value'].as_pem(cipher=None, callback=M2Crypto.util.no_passphrase_callback).decode() + reqPubKey = self.__reqObj.get_pubkey().as_pem(cipher=None, callback=M2Crypto.util.no_passphrase_callback).decode() if not chainPubKey == reqPubKey: return S_ERROR(DErrno.EX509, "Public keys do not match") diff --git a/Core/Utilities/Adler.py b/Core/Utilities/Adler.py index 3dadfa4c426..56b141f5be2 100755 --- a/Core/Utilities/Adler.py +++ b/Core/Utilities/Adler.py @@ -94,7 +94,7 @@ def readChunk(fd, size=1048576): yield data try: - with open(fileName) as inputFile: + with open(fileName, "rb") as inputFile: myAdler = 1 for data in readChunk(inputFile): myAdler = adler32(data, myAdler) @@ -110,7 +110,7 @@ def stringAdler(string): :param str string: data """ try: - intAdler = adler32(string) + intAdler = adler32(string.encode()) return intAdlerToHex(intAdler) except Exception as error: print(repr(error).replace(',)', ')')) diff --git a/Core/Utilities/ClassAd/ClassAdLight.py b/Core/Utilities/ClassAd/ClassAdLight.py index 34635a29549..746ca81532b 100755 --- a/Core/Utilities/ClassAd/ClassAdLight.py +++ b/Core/Utilities/ClassAd/ClassAdLight.py @@ -254,7 +254,7 @@ def asJDL(self): """ result = '' - for name, value in self.contents.items(): + for name, value in sorted(self.contents.items()): if value[0:1] == "{": result = result + 4 * ' ' + name + " = \n" result = result + 8 * ' ' + '{\n' diff --git a/Core/Utilities/DAG.py b/Core/Utilities/DAG.py index 5fcdcfda194..5cfd8d2704c 100644 --- a/Core/Utilities/DAG.py +++ b/Core/Utilities/DAG.py @@ -50,7 +50,7 @@ def addEdge(self, fromNode, toNode): gLogger.error("Missing node to where the edge lands") return - for node, toNodes in self.graph.iteritems(): + for node, toNodes in self.graph.items(): # This is clearly not enough to assure that it's really acyclic... if toNode == node and fromNode in toNodes: gLogger.error("Can't insert this edge") @@ -61,7 +61,7 @@ def getIndexNodes(self): """ Return a list of index nodes """ notIndexNodes = set() - for depNodes in self.graph.itervalues(): + for depNodes in self.graph.values(): [notIndexNodes.add(depNode) for depNode in depNodes] indexNodes = list(set(self.graph.keys()) - notIndexNodes) return [unHashNode(inu) for inu in indexNodes] diff --git a/Core/Utilities/DEncode.py b/Core/Utilities/DEncode.py index 7b6018a1ca6..55224500664 100755 --- a/Core/Utilities/DEncode.py +++ b/Core/Utilities/DEncode.py @@ -19,7 +19,6 @@ from past.builtins import long import six -import types import datetime import os @@ -31,6 +30,34 @@ from pprint import pprint +def _ord(char): + """ Convert a single character string to it's byte value + + In Python 2 a single byte is represented as a string whereas in Python 3 + it is an integer. This function converts it as appropriate. + """ + if six.PY2: + return char + else: + return ord(char) + + +# This is a hack for Python 3 to make it possible to import DEncode +# There is not point in porting DEncode to Python 3 as it will be removed as +# part of the HTTPS transition. +class types(object): + IntType = int + LongType = long if six.PY2 else int + FloatType = float + BooleanType = bool + StringType = str + UnicodeType = type(u"") + NoneType = type(None) + ListType = list + TupleType = tuple + DictType = dict + + # Setting this environment variable to any value will enable the dump of the debugging # call stack DIRAC_DEBUG_DENCODE_CALLSTACK = bool(os.environ.get('DIRAC_DEBUG_DENCODE_CALLSTACK', False)) @@ -209,126 +236,134 @@ def stripArgs(frame): def encodeInt(iValue, eList): """Encoding ints """ - eList.extend(("i", str(iValue), "e")) + eList.extend((b"i", str(iValue).encode(), b"e")) def decodeInt(data, i): """Decoding ints """ i += 1 - end = data.index('e', i) + end = data.index(b'e', i) value = int(data[i:end]) return (value, end + 1) g_dEncodeFunctions[types.IntType] = encodeInt -g_dDecodeFunctions["i"] = decodeInt +g_dDecodeFunctions[_ord("i")] = decodeInt def encodeLong(iValue, eList): """ Encoding longs """ # corrected by KGG eList.extend( ( "l", str( iValue ), "e" ) ) - eList.extend(("I", str(iValue), "e")) + eList.extend((b"I", str(iValue).encode(), b"e")) def decodeLong(data, i): """ Decoding longs """ i += 1 - end = data.index('e', i) + end = data.index(_ord('e'), i) value = long(data[i:end]) return (value, end + 1) g_dEncodeFunctions[types.LongType] = encodeLong -g_dDecodeFunctions["I"] = decodeLong +g_dDecodeFunctions[_ord("I")] = decodeLong def encodeFloat(iValue, eList): """ Encoding floats """ - eList.extend(("f", str(iValue), "e")) + eList.extend((b"f", str(iValue).encode(), b"e")) def decodeFloat(data, i): """ Decoding floats """ i += 1 - end = data.index('e', i) - if end + 1 < len(data) and data[end + 1] in ('+', '-'): + end = data.index(b'e', i) + if end + 1 < len(data) and data[end + 1] in (_ord('+'), _ord('-')): eI = end - end = data.index('e', end + 1) - value = float(data[i:eI]) * 10 ** int(data[eI + 1:end]) + end = data.index(b'e', end + 1) + value = float(data[i:eI].decode()) * 10 ** int(data[eI + 1:end].decode()) else: - value = float(data[i:end]) + value = float(data[i:end].decode()) return (value, end + 1) g_dEncodeFunctions[types.FloatType] = encodeFloat -g_dDecodeFunctions["f"] = decodeFloat +g_dDecodeFunctions[_ord("f")] = decodeFloat def encodeBool(bValue, eList): """ Encoding booleans """ if bValue: - eList.append("b1") + eList.append(b"b1") else: - eList.append("b0") + eList.append(b"b0") def decodeBool(data, i): """ Decoding booleans """ - if data[i + 1] == "0": + if data[i + 1] == _ord("0"): return (False, i + 2) else: return (True, i + 2) g_dEncodeFunctions[types.BooleanType] = encodeBool -g_dDecodeFunctions["b"] = decodeBool +g_dDecodeFunctions[_ord("b")] = decodeBool def encodeString(sValue, eList): """ Encoding strings """ - eList.extend(('s', str(len(sValue)), ':', sValue)) + if not six.PY2: + sValue = sValue.encode() + eList.extend((b's', str(len(sValue)).encode(), b':', sValue)) def decodeString(data, i): """ Decoding strings """ i += 1 - colon = data.index(":", i) - value = int(data[i: colon]) + colon = data.index(b":", i) + value = int(data[i: colon].decode()) colon += 1 end = colon + value - return (data[colon: end], end) + if six.PY2: + return (data[colon: end], end) + else: + return (data[colon: end].decode(), end) g_dEncodeFunctions[types.StringType] = encodeString -g_dDecodeFunctions["s"] = decodeString +g_dDecodeFunctions[_ord("s")] = decodeString def encodeUnicode(sValue, eList): """ Encoding unicode strings """ valueStr = sValue.encode('utf-8') - eList.extend(('u', str(len(valueStr)), ':', valueStr)) + eList.extend((b'u', str(len(valueStr)).encode(), b':', valueStr)) def decodeUnicode(data, i): """ Decoding unicode strings """ i += 1 - colon = data.index(":", i) + colon = data.index(b":", i) value = int(data[i: colon]) colon += 1 end = colon + value - return (unicode(data[colon: end], 'utf-8'), end) + return (six.text_type(data[colon: end].decode('utf-8')), end) -g_dEncodeFunctions[types.UnicodeType] = encodeUnicode -g_dDecodeFunctions["u"] = decodeUnicode +if six.PY2: + g_dEncodeFunctions[types.UnicodeType] = encodeUnicode + g_dDecodeFunctions[_ord("u")] = decodeUnicode +else: + g_dDecodeFunctions[_ord("u")] = decodeString def encodeDateTime(oValue, eList): @@ -338,17 +373,17 @@ def encodeDateTime(oValue, eList): tDateTime = (oValue.year, oValue.month, oValue.day, oValue.hour, oValue.minute, oValue.second, oValue.microsecond, oValue.tzinfo) - eList.append("za") + eList.append(b"za") # corrected by KGG encode( tDateTime, eList ) g_dEncodeFunctions[type(tDateTime)](tDateTime, eList) elif isinstance(oValue, _dateType): tData = (oValue.year, oValue.month, oValue. day) - eList.append("zd") + eList.append(b"zd") # corrected by KGG encode( tData, eList ) g_dEncodeFunctions[type(tData)](tData, eList) elif isinstance(oValue, _timeType): tTime = (oValue.hour, oValue.minute, oValue.second, oValue.microsecond, oValue.tzinfo) - eList.append("zt") + eList.append(b"zt") # corrected by KGG encode( tTime, eList ) g_dEncodeFunctions[type(tTime)](tTime, eList) else: @@ -362,11 +397,11 @@ def decodeDateTime(data, i): dataType = data[i] # corrected by KGG tupleObject, i = decode( data, i + 1 ) tupleObject, i = g_dDecodeFunctions[data[i + 1]](data, i + 1) - if dataType == 'a': + if dataType == _ord('a'): dtObject = datetime.datetime(*tupleObject) - elif dataType == 'd': + elif dataType == _ord('d'): dtObject = datetime.date(*tupleObject) - elif dataType == 't': + elif dataType == _ord('t'): dtObject = datetime.time(*tupleObject) else: raise Exception("Unexpected type %s while decoding a datetime object" % dataType) @@ -376,13 +411,13 @@ def decodeDateTime(data, i): g_dEncodeFunctions[_dateTimeType] = encodeDateTime g_dEncodeFunctions[_dateType] = encodeDateTime g_dEncodeFunctions[_timeType] = encodeDateTime -g_dDecodeFunctions['z'] = decodeDateTime +g_dDecodeFunctions[_ord("z")] = decodeDateTime def encodeNone(_oValue, eList): """ Encoding None """ - eList.append("n") + eList.append(b"n") def decodeNone(_data, i): @@ -392,16 +427,16 @@ def decodeNone(_data, i): g_dEncodeFunctions[types.NoneType] = encodeNone -g_dDecodeFunctions['n'] = decodeNone +g_dDecodeFunctions[_ord("n")] = decodeNone def encodeList(lValue, eList): """ Encoding list """ - eList.append("l") + eList.append(b"l") for uObject in lValue: g_dEncodeFunctions[type(uObject)](uObject, eList) - eList.append("e") + eList.append(b"e") def decodeList(data, i): @@ -409,14 +444,14 @@ def decodeList(data, i): oL = [] i += 1 - while data[i] != "e": + while data[i] != _ord("e"): ob, i = g_dDecodeFunctions[data[i]](data, i) oL.append(ob) return(oL, i + 1) g_dEncodeFunctions[types.ListType] = encodeList -g_dDecodeFunctions["l"] = decodeList +g_dDecodeFunctions[_ord("l")] = decodeList def encodeTuple(lValue, eList): @@ -425,10 +460,10 @@ def encodeTuple(lValue, eList): if DIRAC_DEBUG_DENCODE_CALLSTACK: printDebugCallstack('Encoding tuples') - eList.append("t") + eList.append(b"t") for uObject in lValue: g_dEncodeFunctions[type(uObject)](uObject, eList) - eList.append("e") + eList.append(b"e") def decodeTuple(data, i): @@ -442,7 +477,7 @@ def decodeTuple(data, i): g_dEncodeFunctions[types.TupleType] = encodeTuple -g_dDecodeFunctions["t"] = decodeTuple +g_dDecodeFunctions[_ord("t")] = decodeTuple def encodeDict(dValue, eList): @@ -453,11 +488,11 @@ def encodeDict(dValue, eList): if any([isinstance(x, six.integer_types + (float,)) for x in dValue]): printDebugCallstack("Encoding dict with numeric keys") - eList.append("d") + eList.append(b"d") for key in sorted(dValue): g_dEncodeFunctions[type(key)](key, eList) g_dEncodeFunctions[type(dValue[key])](dValue[key], eList) - eList.append("e") + eList.append(b"e") def decodeDict(data, i): @@ -465,11 +500,11 @@ def decodeDict(data, i): oD = {} i += 1 - while data[i] != "e": + while data[i] != _ord("e"): if DIRAC_DEBUG_DENCODE_CALLSTACK: # If we have numbers as keys - if data[i] in ('i', 'I', 'f'): + if data[i] in (_ord('i'), _ord('I'), _ord('f')): printDebugCallstack("Decoding dict with numeric keys") k, i = g_dDecodeFunctions[data[i]](data, i) @@ -478,31 +513,24 @@ def decodeDict(data, i): g_dEncodeFunctions[types.DictType] = encodeDict -g_dDecodeFunctions["d"] = decodeDict +g_dDecodeFunctions[_ord("d")] = decodeDict # Encode function def encode(uObject): """ Generic encoding function """ - - try: - eList = [] - # print "ENCODE FUNCTION : %s" % g_dEncodeFunctions[ type( uObject ) ] - g_dEncodeFunctions[type(uObject)](uObject, eList) - return "".join(eList) - except Exception: - raise + eList = [] + # print("ENCODE FUNCTION : %s" % g_dEncodeFunctions[ type( uObject ) ]) + g_dEncodeFunctions[type(uObject)](uObject, eList) + return b"".join(eList) def decode(data): """ Generic decoding function """ if not data: return data - try: - # print "DECODE FUNCTION : %s" % g_dDecodeFunctions[ sStream [ iIndex ] ] - return g_dDecodeFunctions[data[0]](data, 0) - except Exception: - raise + # print("DECODE FUNCTION : %s" % g_dDecodeFunctions[ sStream [ iIndex ] ]) + return g_dDecodeFunctions[data[0]](data, 0) if __name__ == "__main__": diff --git a/Core/Utilities/DIRACSingleton.py b/Core/Utilities/DIRACSingleton.py index e6b76a2b687..64f17bf5e58 100644 --- a/Core/Utilities/DIRACSingleton.py +++ b/Core/Utilities/DIRACSingleton.py @@ -36,9 +36,10 @@ class DIRACSingleton(type): If you want make your class a singleton, just set its __metaclass__ to DIRACSingleton, i.e.:: + import six from DIRAC.Core.Utilities.DIRACSingleton import DIRACSingleton - class CheesShop( object ): - __metaclass__ = DIRACSingleton + @six.add_metaclass(DIRACSingleton) + class CheesShop(object): ... """ def __init__(cls, name, bases, dic): diff --git a/Core/Utilities/Devloader.py b/Core/Utilities/Devloader.py index e7cbdfaa96b..9328e7d7314 100644 --- a/Core/Utilities/Devloader.py +++ b/Core/Utilities/Devloader.py @@ -9,14 +9,13 @@ import types import threading import time - +import six from DIRAC import gLogger from DIRAC.Core.Utilities.DIRACSingleton import DIRACSingleton +@six.add_metaclass(DIRACSingleton) class Devloader(object): - __metaclass__ = DIRACSingleton - def __init__(self): self.__log = gLogger.getSubLogger("Devloader") self.__reloaded = False diff --git a/Core/Utilities/DictCache.py b/Core/Utilities/DictCache.py index b4bde787ea3..5eddb9770ee 100644 --- a/Core/Utilities/DictCache.py +++ b/Core/Utilities/DictCache.py @@ -235,8 +235,7 @@ def purgeAll(self, useLock=True): if useLock: self.lock.acquire() try: - keys = self.__cache.keys() - for cKey in keys: + for cKey in list(self.__cache): if self.__deleteFunction: self.__deleteFunction(self.__cache[cKey]['value']) del self.__cache[cKey] diff --git a/Core/Utilities/Dictionaries.py b/Core/Utilities/Dictionaries.py index ea1cdc22ced..0c7ca709cc4 100644 --- a/Core/Utilities/Dictionaries.py +++ b/Core/Utilities/Dictionaries.py @@ -15,5 +15,5 @@ def breakDictionaryIntoChunks(aDict, chunkSize): :return: a generator object, that generates chunks of the original dictionary in aDict """ iterDict = iter(aDict) - for _ in xrange(0, len(aDict), chunkSize): + for _ in range(0, len(aDict), chunkSize): yield {k: aDict[k] for k in islice(iterDict, chunkSize)} diff --git a/Core/Utilities/File.py b/Core/Utilities/File.py index 47f8f4ce76b..d5b0b0ea921 100755 --- a/Core/Utilities/File.py +++ b/Core/Utilities/File.py @@ -78,13 +78,13 @@ def makeGuid(fileName=None): myMd5 = hashlib.md5() if fileName: try: - with open(fileName, 'r') as fd: + with open(fileName, 'rb') as fd: data = fd.read(10 * 1024 * 1024) myMd5.update(data) except BaseException: return None else: - myMd5.update(str(random.getrandbits(128))) + myMd5.update(str(random.getrandbits(128)).encode()) md5HexString = myMd5.hexdigest().upper() return generateGuid(md5HexString, "MD5") @@ -112,7 +112,7 @@ def generateGuid(checksum, checksumtype): # Failed to use the check sum, generate a new guid myMd5 = hashlib.md5() - myMd5.update(str(random.getrandbits(128))) + myMd5.update(str(random.getrandbits(128)).encode()) md5HexString = myMd5.hexdigest() guid = "%s-%s-%s-%s-%s" % (md5HexString[0:8], md5HexString[8:12], diff --git a/Core/Utilities/Graph.py b/Core/Utilities/Graph.py index eda89e00f05..57871322106 100644 --- a/Core/Utilities/Graph.py +++ b/Core/Utilities/Graph.py @@ -18,6 +18,7 @@ from __future__ import division from __future__ import print_function __RCSID__ = "$Id$" +import six # # # @file Graph.py # @author Krzysztof.Ciba@NOSPAMgmail.com @@ -73,12 +74,11 @@ def _getProperty(self, name): return type.__new__(cls, name, bases, classdict) +@six.add_metaclass(DynamicProps) class Node(object): """ graph node """ - __metaclass__ = DynamicProps - def __init__(self, name, rwAttrs=None, roAttrs=None): """ c'tor @@ -139,12 +139,11 @@ def connect(self, other, rwAttrs=None, roAttrs=None): return edge +@six.add_metaclass(DynamicProps) class Edge(object): """ Directed link between two nodes """ - __metaclass__ = DynamicProps - def __init__(self, fromNode, toNode, rwAttrs=None, roAttrs=None): """ c'tor @@ -186,12 +185,11 @@ def __repr__(self): ######################################################################## +@six.add_metaclass(DynamicProps) class Graph(object): """ A generic directed graph with attributes attached to its nodes and edges """ - # # metaclass - __metaclass__ = DynamicProps # # flag to indicate PREORDER traversal __PREORDER = False # # flag to indicate POSTORDER traversal diff --git a/Core/Utilities/Graphs/GraphData.py b/Core/Utilities/Graphs/GraphData.py index 9ca94b35c00..8de64709378 100644 --- a/Core/Utilities/Graphs/GraphData.py +++ b/Core/Utilities/Graphs/GraphData.py @@ -382,8 +382,7 @@ class PlotData(object): def __init__(self, data, single=True, key_type=None): self.key_type = "unknown" - keys = data.keys() - if not keys: + if not data: print("PlotData Error: empty data") return @@ -460,7 +459,7 @@ def expandKeys(self, all_keys): self.parsed_data[k] = 0. self.sorted_keys = [] - self.keys = self.parsed_data.keys() + self.keys = list(self.parsed_data) self.initialize() def sortKeys(self, sort_type='alpha'): @@ -543,7 +542,7 @@ def parseData(self, key_type=None): if key_type: self.key_type = key_type else: - self.key_type = get_key_type(self.data.keys()) + self.key_type = get_key_type(list(self.data)) new_parsed_data = {} new_passed_errors = {} for key, data in self.data.items(): @@ -555,7 +554,7 @@ def parseData(self, key_type=None): self.parsed_data = new_parsed_data self.parsed_errors = new_passed_errors - self.keys = self.parsed_data.keys() + self.keys = list(self.parsed_data) def makeCumulativePlot(self): diff --git a/Core/Utilities/Graphs/GraphUtilities.py b/Core/Utilities/Graphs/GraphUtilities.py index 24904ff8da9..93af80d6273 100644 --- a/Core/Utilities/Graphs/GraphUtilities.py +++ b/Core/Utilities/Graphs/GraphUtilities.py @@ -395,8 +395,8 @@ def statistics(results, span=None, is_timestamp=False): results = dict(results) if span is not None: parsed_data = {} - min_key = min(results.keys()) - max_key = max(results.keys()) + min_key = min(results) + max_key = max(results) for i in range(min_key, max_key + span, span): if i in results: parsed_data[i] = results[i] @@ -412,7 +412,7 @@ def statistics(results, span=None, is_timestamp=False): data_max = max(values) data_avg = numpy.average(values) if is_timestamp: - current_time = max(parsed_data.keys()) + current_time = max(parsed_data) data_current = parsed_data[current_time] return data_min, data_max, data_avg, data_current else: diff --git a/Core/Utilities/Grid.py b/Core/Utilities/Grid.py index bf4814f6e56..66b4bc4226e 100644 --- a/Core/Utilities/Grid.py +++ b/Core/Utilities/Grid.py @@ -147,7 +147,7 @@ def ldapsearchBDII(filt=None, attr=None, host=None, base=None, selectionString=" def ldapSite(site, attr=None, host=None): """ Site information from bdii. -:param site: Site as it defined in GOCDB or part of it with globing, for example: \UKI-* +:param site: Site as it defined in GOCDB or part of it with globing, for example: \\UKI-* :return: standard DIRAC answer with Value equals to list of sites. Each site is dictionary which contains attributes of site. diff --git a/Core/Utilities/JEncode.py b/Core/Utilities/JEncode.py index c40fb18364b..5d089745078 100644 --- a/Core/Utilities/JEncode.py +++ b/Core/Utilities/JEncode.py @@ -150,7 +150,7 @@ def dict_to_object(dataDict): obj = cl() # Set each attribute - for attrName, attrValue in dataDict.iteritems(): + for attrName, attrValue in dataDict.items(): # If the value is None, do not set it # This is needed to play along well with SQLalchemy if attrValue is None: @@ -201,4 +201,4 @@ def strToIntDict(inDict): """ - return {int(key): value for key, value in inDict.iteritems()} + return {int(key): value for key, value in inDict.items()} diff --git a/Core/Utilities/List.py b/Core/Utilities/List.py index d6b07b92ddd..80b77056725 100755 --- a/Core/Utilities/List.py +++ b/Core/Utilities/List.py @@ -111,7 +111,7 @@ def getChunk(aList, chunkSize): process( chunk ) """ - for i in xrange(0, len(aList), chunkSize): + for i in range(0, len(aList), chunkSize): yield aList[i:i + chunkSize] @@ -127,7 +127,8 @@ def breakListIntoChunks(aList, chunkSize): """ if chunkSize < 1: raise RuntimeError("chunkSize cannot be less than 1") - if isinstance(aList, (set, dict, tuple)): + if isinstance(aList, (set, dict, tuple, {}.keys().__class__, + {}.items().__class__, {}.values().__class__)): aList = list(aList) return [chunk for chunk in getChunk(aList, chunkSize)] @@ -141,7 +142,10 @@ def getIndexInList(anItem, aList): :return: the index or sys.maxint """ - try: + # try: + if anItem in aList: return aList.index(anItem) - except ValueError: + else: return sys.maxsize + # except ValueError: + # return sys.maxsize diff --git a/Core/Utilities/LockRing.py b/Core/Utilities/LockRing.py index 9168d0dcc09..aad3ca14785 100644 --- a/Core/Utilities/LockRing.py +++ b/Core/Utilities/LockRing.py @@ -5,26 +5,27 @@ import random import time import threading -import thread +import six +from six.moves import _thread as thread from hashlib import md5 from DIRAC.Core.Utilities.ReturnValues import S_ERROR, S_OK from DIRAC.Core.Utilities.DIRACSingleton import DIRACSingleton +@six.add_metaclass(DIRACSingleton) class LockRing(object): - __metaclass__ = DIRACSingleton - def __init__(self): random.seed() self.__locks = {} self.__events = {} def __genName(self, container): - name = md5(str(time.time() + random.random())).hexdigest() + # TODO: Shouldn't this be a UUID? + name = md5(str(time.time() + random.random()).encode()).hexdigest() retries = 10 while name in container and retries: - name = md5(str(time.time() + random.random())).hexdigest() + name = md5(str(time.time() + random.random()).encode()).hexdigest() retries -= 1 return name diff --git a/Core/Utilities/MySQL.py b/Core/Utilities/MySQL.py index 4e601370d13..ed00ba8948b 100755 --- a/Core/Utilities/MySQL.py +++ b/Core/Utilities/MySQL.py @@ -161,11 +161,15 @@ from DIRAC.Core.Utilities.Time import fromString from DIRAC.Core.Utilities import DErrno -# This is for proper initialization of embedded server, it should only be called once -try: - MySQLdb.server_init(['--defaults-file=/opt/dirac/etc/my.cnf', '--datadir=/opt/mysql/db'], ['mysqld']) -except MySQLdb.ProgrammingError: - pass +# The mysql_server_init function called here is deprecated and not used by mysqlclient +# https://dev.mysql.com/doc/c-api/8.0/en/mysql-server-init.html +# TODO: Check if any changes are needed +if six.PY2: + # This is for proper initialization of embedded server, it should only be called once + try: + MySQLdb.server_init(['--defaults-file=/opt/dirac/etc/my.cnf', '--datadir=/opt/mysql/db'], ['mysqld']) + except MySQLdb.ProgrammingError: + pass gInstancesCount = 0 @@ -709,12 +713,12 @@ def _createViews(self, viewsDict, force=False): if force: # gLogger.debug(viewsDict) - for viewName, viewDict in viewsDict.iteritems(): + for viewName, viewDict in viewsDict.items(): viewQuery = ["CREATE OR REPLACE VIEW `%s`.`%s` AS" % (self.__dbName, viewName)] columns = ",".join(["%s AS %s" % (colDef, colName) - for colName, colDef in viewDict.get("Fields", {}).iteritems()]) + for colName, colDef in viewDict.get("Fields", {}).items()]) tables = viewDict.get("SelectFrom", "") if columns and tables: viewQuery.append("SELECT %s FROM %s" % (columns, tables)) @@ -782,7 +786,7 @@ def _createTables(self, tableDict, force=False): return S_ERROR(DErrno.EMYSQL, 'Argument is not a dictionary: %s( %s )' % (type(tableDict), tableDict)) - tableList = tableDict.keys() + tableList = list(tableDict) if len(tableList) == 0: return S_OK(0) for table in tableList: @@ -812,7 +816,7 @@ def _createTables(self, tableDict, force=False): thisTable = tableDict[table] if 'ForeignKeys' in thisTable: thisKeys = thisTable['ForeignKeys'] - for key, auxTable in thisKeys.iteritems(): + for key, auxTable in thisKeys.items(): forTable = auxTable.split('.')[0] forKey = key if forTable != auxTable: @@ -867,7 +871,7 @@ def _createTables(self, tableDict, force=False): cmdList.append('UNIQUE INDEX `%s` ( `%s` )' % (index, indexedFields)) if 'ForeignKeys' in thisTable: thisKeys = thisTable['ForeignKeys'] - for key, auxTable in thisKeys.iteritems(): + for key, auxTable in thisKeys.items(): forTable = auxTable.split('.')[0] forKey = key @@ -1081,7 +1085,7 @@ def buildCondition(self, condDict=None, older=None, newer=None, conjunction = "WHERE" if condDict is not None: - for aName, attrValue in condDict.iteritems(): + for aName, attrValue in condDict.items(): if isinstance(aName, six.string_types): attrName = _quotedList([aName]) elif isinstance(aName, tuple): @@ -1147,7 +1151,7 @@ def buildCondition(self, condDict=None, older=None, newer=None, escapeInValue) if isinstance(greater, dict): - for attrName, attrValue in greater.iteritems(): + for attrName, attrValue in greater.items(): attrName = _quotedList([attrName]) if not attrName: error = 'Invalid greater argument' @@ -1167,7 +1171,7 @@ def buildCondition(self, condDict=None, older=None, newer=None, conjunction = "AND" if isinstance(smaller, dict): - for attrName, attrValue in smaller.iteritems(): + for attrName, attrValue in smaller.items(): attrName = _quotedList([attrName]) if not attrName: error = 'Invalid smaller argument' @@ -1348,8 +1352,8 @@ def updateFields(self, tableName, updateFields=None, updateValues=None, # self.log.debug('updateFields:', error) return S_ERROR(DErrno.EMYSQL, error) try: - updateFields += updateDict.keys() - updateValues += [updateDict[k] for k in updateDict.keys()] + updateFields += list(updateDict) + updateValues += [updateDict[k] for k in updateFields] except TypeError: error = 'updateFields and updateValues must be a list' # self.log.debug('updateFields:', error) @@ -1404,8 +1408,8 @@ def insertFields(self, tableName, inFields=None, inValues=None, conn=None, inDic # self.log.debug('insertFields:', error) return S_ERROR(DErrno.EMYSQL, error) try: - inFields += inDict.keys() - inValues += [inDict[k] for k in inDict.keys()] + inFields += list(inDict) + inValues += [inDict[k] for k in inFields] except TypeError: error = 'inFields and inValues must be a list' # self.log.debug('insertFields:', error) diff --git a/Core/Utilities/Network.py b/Core/Utilities/Network.py index 3f1671feeab..d9a284fb1c0 100755 --- a/Core/Utilities/Network.py +++ b/Core/Utilities/Network.py @@ -8,7 +8,7 @@ __RCSID__ = "$Id$" import socket -import urlparse +from six.moves.urllib import parse as urlparse import os import struct import array diff --git a/Core/Utilities/ObjectLoader.py b/Core/Utilities/ObjectLoader.py index 50163f70aff..d0f27e0a62a 100644 --- a/Core/Utilities/ObjectLoader.py +++ b/Core/Utilities/ObjectLoader.py @@ -11,6 +11,7 @@ import imp import pkgutil import collections +import six from DIRAC import gLogger, S_OK, S_ERROR from DIRAC.Core.Utilities import DErrno @@ -18,6 +19,7 @@ from DIRAC.ConfigurationSystem.Client.Helpers import CSGlobals +@six.add_metaclass(DIRACSingleton.DIRACSingleton) class ObjectLoader(object): """ Class for loading objects. Example: @@ -25,8 +27,6 @@ class ObjectLoader(object): ol = ObjectLoader() ol.loadObject('TransformationSystem.Client.TransformationClient') """ - __metaclass__ = DIRACSingleton.DIRACSingleton - def __init__(self, baseModules=False): """ init """ @@ -100,7 +100,7 @@ def __recurseImport(self, modName, parentModule=None, hideExceptions=False, full if impData[0]: impData[0].close() except ImportError as excp: - if str(excp).find("No module named %s" % modName[0]) == 0: + if "No module named" in str(excp) and modName[0] in str(excp): return S_OK(None) errMsg = "Can't load %s in %s" % (".".join(modName), parentModule.__path__[0]) if not hideExceptions: diff --git a/Core/Utilities/Pfn.py b/Core/Utilities/Pfn.py index 713c51e9cae..8a42297eace 100755 --- a/Core/Utilities/Pfn.py +++ b/Core/Utilities/Pfn.py @@ -20,7 +20,7 @@ import os # # from DIRAC from DIRAC import S_OK, S_ERROR, gLogger -import urlparse +from six.moves.urllib import parse as urlparse def pfnunparse(pfnDict, srmSpecific=True): diff --git a/Core/Utilities/Plotting/Plots.py b/Core/Utilities/Plotting/Plots.py index 7dce87ba74b..7d31517dc48 100644 --- a/Core/Utilities/Plotting/Plots.py +++ b/Core/Utilities/Plotting/Plots.py @@ -5,10 +5,7 @@ from __future__ import division from __future__ import print_function -try: - from StringIO import StringIO -except ImportError: - from io import BytesIO as StringIO +from six import BytesIO import errno from DIRAC.Core.Utilities.Graphs import barGraph, lineGraph, pieGraph, qualityGraph, textGraph, histogram @@ -53,7 +50,7 @@ def generateErrorMessagePlot(msgText): :param str msgText: the text which will appear on the plot. :return: the plot. """ - fn = StringIO() + fn = BytesIO() textGraph(msgText, fn, {}) data = fn.getvalue() fn.close() diff --git a/Core/Utilities/PrettyPrint.py b/Core/Utilities/PrettyPrint.py index 957d129947a..22a3d394d9f 100644 --- a/Core/Utilities/PrettyPrint.py +++ b/Core/Utilities/PrettyPrint.py @@ -11,7 +11,7 @@ __RCSID__ = '$Id$' import six -import StringIO +from six import StringIO def int_with_commas(inputValue): @@ -159,7 +159,7 @@ def __writeField(buffer, value, length, columnSeparator, lastColumn=False): totalLength += (numberWidth + separatorWidth) # Accumulate the table output in the stringBuffer now - stringBuffer = StringIO.StringIO() + stringBuffer = StringIO() topLength = (numberWidth + separatorWidth) if numbering else 0 stringBuffer.write(' ' * (topLength)) diff --git a/Core/Utilities/ProcessPool.py b/Core/Utilities/ProcessPool.py index dede0e63414..cbf03720802 100644 --- a/Core/Utilities/ProcessPool.py +++ b/Core/Utilities/ProcessPool.py @@ -102,15 +102,16 @@ __RCSID__ = "$Id$" +import errno +import inspect import multiprocessing -import sys -import time -import threading import os import signal -import Queue -import errno -from types import FunctionType, TypeType, ClassType +import sys +import threading +import time + +from six.moves import queue as Queue try: from DIRAC.FrameworkSystem.Client.Logger import gLogger @@ -521,10 +522,10 @@ def process(self): self.__done = True try: # # it's a function? - if isinstance(self.__taskFunction, FunctionType): + if inspect.isfunction(self.__taskFunction): self.__taskResult = self.__taskFunction(*self.__taskArgs, **self.__taskKwArgs) # # or a class? - elif isinstance(self.__taskFunction, (type, ClassType)): + elif inspect.isclass(self.__taskFunction): # # create new instance taskObj = self.__taskFunction(*self.__taskArgs, **self.__taskKwArgs) # ## check if it is callable, raise TypeError if not @@ -705,7 +706,7 @@ def getNumWorkingProcesses(self): counter = 0 self.__prListLock.acquire() try: - counter = len([pid for pid, worker in self.__workersDict.iteritems() if worker.isWorking()]) + counter = len([pid for pid, worker in self.__workersDict.items() if worker.isWorking()]) finally: self.__prListLock.release() return counter @@ -719,7 +720,7 @@ def getNumIdleProcesses(self): counter = 0 self.__prListLock.acquire() try: - counter = len([pid for pid, worker in self.__workersDict.iteritems() if not worker.isWorking()]) + counter = len([pid for pid, worker in self.__workersDict.items() if not worker.isWorking()]) finally: self.__prListLock.release() return counter @@ -753,7 +754,7 @@ def __cleanDeadProcesses(self): # # check wounded processes self.__prListLock.acquire() try: - for pid, worker in self.__workersDict.items(): + for pid, worker in list(self.__workersDict.items()): if not worker.is_alive(): del self.__workersDict[pid] finally: @@ -990,7 +991,7 @@ def __filicide(self): :param self: self reference """ while self.__workersDict: - pid = self.__workersDict.keys().pop(0) + pid = list(self.__workersDict).pop(0) worker = self.__workersDict[pid] if worker.is_alive(): os.kill(pid, signal.SIGKILL) diff --git a/Core/Utilities/ReturnValues.py b/Core/Utilities/ReturnValues.py index 82efe934c9d..feee8585def 100755 --- a/Core/Utilities/ReturnValues.py +++ b/Core/Utilities/ReturnValues.py @@ -126,7 +126,7 @@ def returnSingleResult(dictRes): return dictRes # if there is a Failed, we return the first one in an S_ERROR if "Failed" in dictRes['Value'] and len(dictRes['Value']['Failed']): - errorMessage = dictRes['Value']['Failed'].values()[0] + errorMessage = list(dictRes['Value']['Failed'].values())[0] if isinstance(errorMessage, dict): if isReturnStructure(errorMessage): return errorMessage @@ -135,6 +135,6 @@ def returnSingleResult(dictRes): return S_ERROR(errorMessage) # if there is a Successful, we return the first one in an S_OK elif "Successful" in dictRes['Value'] and len(dictRes['Value']['Successful']): - return S_OK(dictRes['Value']['Successful'].values()[0]) + return S_OK(list(dictRes['Value']['Successful'].values())[0]) else: return S_ERROR("returnSingleResult: Failed and Successful dictionaries are empty") diff --git a/Core/Utilities/SitesDIRACGOCDBmapping.py b/Core/Utilities/SitesDIRACGOCDBmapping.py index 443811b2cac..38e7e9718e3 100644 --- a/Core/Utilities/SitesDIRACGOCDBmapping.py +++ b/Core/Utilities/SitesDIRACGOCDBmapping.py @@ -142,7 +142,7 @@ def getDIRACGOCDictionary(): # /Resource/Sites///Name # [0]/[1] /[2] /[3] /[4] /[5] - for path, gocdbSiteName in siteNamesTree.iteritems(): + for path, gocdbSiteName in siteNamesTree.items(): elements = path.split('/') if len(elements) != PATHELEMENTS: continue diff --git a/Core/Utilities/Subprocess.py b/Core/Utilities/Subprocess.py index a11daff3b3a..d84d527b2f0 100644 --- a/Core/Utilities/Subprocess.py +++ b/Core/Utilities/Subprocess.py @@ -21,7 +21,7 @@ ( returncode, stdout, stderr ) the tuple will also be available upon timeout error or buffer overflow error. - - pythonCall( iTimeOut, function, \*stArgs, \*\*stKeyArgs ) + - pythonCall( iTimeOut, function, \\*stArgs, \\*\\*stKeyArgs ) calls function with given arguments within a timeout Wrapper should be used to wrap third party python functions @@ -191,7 +191,7 @@ def __readFromFD(self, fd, baseLength=0): dataString = '' redBuf = " " while len(redBuf) > 0: - redBuf = os.read(fd, 8192) + redBuf = os.read(fd, 8192).decode() dataString += redBuf if len(dataString) + baseLength > self.bufferLimit: self.log.error('Maximum output buffer length reached', @@ -344,7 +344,7 @@ def pythonCall(self, function, *stArgs, **stKeyArgs): dataStub = retDict['Value'] if not dataStub: return S_ERROR("Error decoding data coming from call") - retObj, stubLen = DEncode.decode(dataStub) + retObj, stubLen = DEncode.decode(dataStub.encode()) if stubLen == len(dataStub): return retObj return S_ERROR("Error decoding data coming from call") @@ -377,9 +377,9 @@ def __readFromFile(self, fd, baseLength): nB = os.read(fn, self.bufferLimit) else: nB = fd.read(1) - if nB == "": + if not nB: break - dataString += nB + dataString += nB.decode() # break out of potential infinite loop, indicated by dataString growing beyond reason if len(dataString) + baseLength > self.bufferLimit: self.log.error("DataString is getting too long (%s): %s " % (len(dataString), dataString[-10000:])) diff --git a/Core/Utilities/ThreadPool.py b/Core/Utilities/ThreadPool.py index af16db5ce85..8fad28b7965 100755 --- a/Core/Utilities/ThreadPool.py +++ b/Core/Utilities/ThreadPool.py @@ -75,7 +75,7 @@ import time import sys -import Queue +from six.moves import queue as Queue import threading try: from DIRAC.FrameworkSystem.Client.Logger import gLogger diff --git a/Core/Utilities/ThreadScheduler.py b/Core/Utilities/ThreadScheduler.py index 033f240db3f..709b453150c 100644 --- a/Core/Utilities/ThreadScheduler.py +++ b/Core/Utilities/ThreadScheduler.py @@ -45,7 +45,7 @@ def addPeriodicTask(self, period, taskFunc, taskArgs=(), executions=0, elapsedTi 'func': taskFunc, 'args': taskArgs, } - md.update(str(task)) + md.update(str(task).encode()) taskId = md.hexdigest() if taskId in self.__taskDict: return S_ERROR("Task %s is already added" % taskId) diff --git a/Core/Utilities/Time.py b/Core/Utilities/Time.py index 634b20252e0..9000999d7be 100755 --- a/Core/Utilities/Time.py +++ b/Core/Utilities/Time.py @@ -30,7 +30,7 @@ from __future__ import division import time as nativetime import datetime -from types import StringTypes +import six import sys __RCSID__ = "$Id$" @@ -83,8 +83,8 @@ def timed(*args, **kw): except IndexError: if kw: try: - if isinstance(kw.items()[0][1], (list, dict)): - argsLen = "arguments len: %d" % len(kw.items()[0][1]) + if isinstance(list(list(kw.items())[0])[1], (list, dict)): + argsLen = "arguments len: %d" % len(list(list(kw.items())[0])[1]) except IndexError: argsLen = '' @@ -194,7 +194,7 @@ def fromString(myDate=None): See notice on toString method On Error, return None """ - if StringTypes.__contains__(type(myDate)): + if isinstance(myDate, six.string_types): if myDate.find(' ') > 0: dateTimeTuple = myDate.split(' ') dateTuple = dateTimeTuple[0].split('-') diff --git a/Core/Utilities/Traced.py b/Core/Utilities/Traced.py index 0437e301883..1b5e72c737d 100644 --- a/Core/Utilities/Traced.py +++ b/Core/Utilities/Traced.py @@ -22,6 +22,7 @@ __RCSID__ = "$Id: $" +import six ## # @file Traced.py # @author Krzysztof.Ciba@NOSPAMgmail.com @@ -100,11 +101,11 @@ def trace_setitem(self, ind, item): return type.__new__(cls, cls_name, bases, classdict) +@six.add_metaclass(Traced) class TracedDict(dict): """ traced dict """ - __metaclass__ = Traced +@six.add_metaclass(Traced) class TracedList(list): """ traced list """ - __metaclass__ = Traced diff --git a/Core/Utilities/TypedList.py b/Core/Utilities/TypedList.py index 8e35b29707a..ed293f35225 100644 --- a/Core/Utilities/TypedList.py +++ b/Core/Utilities/TypedList.py @@ -289,4 +289,4 @@ def __init__(self, iterable=None): :param self: self reference :param mixed iterable: initial values """ - TypedList.__init__(self, iterable, allowedTypes=(str, unicode)) + TypedList.__init__(self, iterable, allowedTypes=six.string_types) diff --git a/Core/Utilities/test/Test_Adler.py b/Core/Utilities/test/Test_Adler.py index 09a3e73ecc6..5fd4acd9450 100644 --- a/Core/Utilities/test/Test_Adler.py +++ b/Core/Utilities/test/Test_Adler.py @@ -47,8 +47,8 @@ class AdlerTestCase(unittest.TestCase): """ def setUp(self): - self.emptyAdler = hex(zlib.adler32("") & 0xffffffff)[2:] - self.lettersAdler = hex(zlib.adler32(string.letters) & 0xffffffff)[2:] + self.emptyAdler = hex(zlib.adler32(b"") & 0xffffffff)[2:] + self.lettersAdler = hex(zlib.adler32(string.ascii_letters.encode()) & 0xffffffff)[2:] def testStringAdler(self): """ stringAdler tests """ @@ -62,7 +62,7 @@ def testStringAdler(self): # empty string self.assertEqual(int(Adler.stringAdler("")), int(self.emptyAdler)) # all letters - self.assertEqual(Adler.stringAdler(string.letters), self.lettersAdler) + self.assertEqual(Adler.stringAdler(string.ascii_letters), self.lettersAdler) def testConversion(self): """ intAdlerToHex and hexAdlerToInt tests """ @@ -93,7 +93,7 @@ def testFileAdler(self): # normal operation fd, path = tempfile.mkstemp("_adler32", "norewgian_blue") self.assertEqual(int(Adler.fileAdler(path)), int(self.emptyAdler)) - os.write(fd, string.letters) + os.write(fd, string.ascii_letters.encode()) self.assertEqual(Adler.fileAdler(path), self.lettersAdler) def testCompareAdler(self): @@ -101,7 +101,7 @@ def testCompareAdler(self): # same adlers self.assertEqual(Adler.compareAdler(Adler.stringAdler(""), Adler.stringAdler("")), True) # diff adlers - self.assertEqual(Adler.compareAdler(Adler.stringAdler(""), Adler.stringAdler(string.letters)), False) + self.assertEqual(Adler.compareAdler(Adler.stringAdler(""), Adler.stringAdler(string.ascii_letters)), False) # test suite execution diff --git a/Core/Utilities/test/Test_DAG.py b/Core/Utilities/test/Test_DAG.py index 3dab16a0768..1f0ddf02615 100644 --- a/Core/Utilities/test/Test_DAG.py +++ b/Core/Utilities/test/Test_DAG.py @@ -14,6 +14,18 @@ __RCSID__ = "$Id $" +def listToSet(obj, recursive=False): + """ Convert a list of objects into a set which can be used for comparison """ + result = set() + for x in obj: + if isinstance(x, (list, tuple)): + x = frozenset(x) if recursive else tuple(x) + elif isinstance(x, dict): + x = tuple(map(tuple, sorted(x.items()))) + result.add(x) + return result + + ######################################################################## class DAGTestCase( unittest.TestCase ): """ Test case for DIRAC.Core.Utilities.DAG module @@ -121,7 +133,7 @@ def test_full(self): dag.addEdge('A', 'B') self.assertEqual(dag.graph, {'A': {'B'}, 'B': set()}) l = dag.getList() - self.assertEqual(l, ['A', 'B']) + self.assertEqual(set(l), {'A', 'B'}) dag.addEdge('A', 'B') self.assertEqual(dag.graph, {'A': {'B'}, 'B': set()}) dag.addEdge('A', 'C') @@ -135,10 +147,10 @@ def test_full(self): self.assertEqual(dag.graph, {'A': {'B', 'C'}, 'B': set(), 'C': set()}) dag.addNode('D') i_n = dag.getIndexNodes() - self.assertEqual(i_n, ['A', 'D']) + self.assertEqual(set(i_n), {'A', 'D'}) dag.addNode('E') i_n = dag.getIndexNodes() - self.assertEqual(sorted(i_n), sorted(['A', 'D', 'E'])) + self.assertEqual(listToSet(i_n), listToSet(['A', 'D', 'E'])) dag.addEdge('A', 'D') dag.addEdge('D', 'E') self.assertEqual(dag.graph, {'A': {'B', 'C', 'D'}, 'B': set(), 'C': set(), 'D': {'E'}, 'E': set()} ) @@ -222,7 +234,7 @@ class forTest(object): ) i_n = dag.getIndexNodes() - self.assertEqual(i_n, ['A', l]) + self.assertEqual(listToSet(i_n), listToSet(['A', l])) d1 = {'a':'b'} dag.addNode(d1) @@ -248,7 +260,10 @@ class forTest(object): ) i_n = dag.getIndexNodes() - self.assertEqual(sorted(i_n), sorted(['A', l, d1, l1])) + self.assertEqual( + listToSet(i_n, recursive=True), + listToSet(['A', l, d1, l1], recursive=True) + ) s1 = set() dag.addNode(s1) diff --git a/Core/Utilities/test/Test_Dictionaries.py b/Core/Utilities/test/Test_Dictionaries.py index ea333f84a8b..139d58bf6fa 100644 --- a/Core/Utilities/test/Test_Dictionaries.py +++ b/Core/Utilities/test/Test_Dictionaries.py @@ -10,13 +10,18 @@ from DIRAC.Core.Utilities.Dictionaries import breakDictionaryIntoChunks -@pytest.mark.parametrize("aDict, chunkSize, expected", [ +@pytest.mark.parametrize("aDict, chunkSize, expectedSizes", [ ({}, 1, []), - ({'a': 'aa', 'b': 'bb', 'c': 'cc', 'd': 'dd'}, 5, [{'a': 'aa', 'b': 'bb', 'c': 'cc', 'd': 'dd'}]), - ({'a': 'aa', 'b': 'bb', 'c': 'cc', 'd': 'dd'}, 2, [{'a': 'aa', 'c': 'cc'}, {'b': 'bb', 'd': 'dd'}]), - ({'a': 'aa', 'b': 'bb', 'c': 'cc', 'd': 'dd'}, 1, [{'a': 'aa'}, {'c': 'cc'}, {'b': 'bb'}, {'d': 'dd'}]), - ({'a': 'aa', 'b': 'bb', 'c': 'cc', 'd': 'dd'}, 3, [{'a': 'aa', 'b': 'bb', 'c': 'cc'}, {'d': 'dd'}]), + ({'a': 'aa', 'b': 'bb', 'c': 'cc', 'd': 'dd'}, 5, [4]), + ({'a': 'aa', 'b': 'bb', 'c': 'cc', 'd': 'dd'}, 2, [2, 2]), + ({'a': 'aa', 'b': 'bb', 'c': 'cc', 'd': 'dd'}, 1, [1, 1, 1, 1]), + ({'a': 'aa', 'b': 'bb', 'c': 'cc', 'd': 'dd'}, 3, [3, 1]), ]) -def test_breakDictionaryIntoChunks_normal(aDict, chunkSize, expected): +def test_breakDictionaryIntoChunks_normal(aDict, chunkSize, expectedSizes): """ breakDictIntoChunks tests""" - assert list(breakDictionaryIntoChunks(aDict, chunkSize)) == expected + result = list(breakDictionaryIntoChunks(aDict, chunkSize)) + assert list(map(len, result)) == expectedSizes + mergedResult = {} + for partialDict in result: + mergedResult.update(partialDict) + assert aDict == mergedResult diff --git a/Core/Utilities/test/Test_Encode.py b/Core/Utilities/test/Test_Encode.py index 951a73ef17d..dbf4538651d 100644 --- a/Core/Utilities/test/Test_Encode.py +++ b/Core/Utilities/test/Test_Encode.py @@ -23,8 +23,8 @@ from hypothesis import given, settings, HealthCheck from hypothesis.strategies import builds, integers, lists, recursive, floats, text,\ booleans, none, dictionaries, tuples, datetimes - -from pytest import mark, approx, raises, fixture +import six +from pytest import mark, approx, raises, fixture, skip parametrize = mark.parametrize @@ -99,7 +99,7 @@ def test_everyBaseTypeIsTested(): """ current_module = sys.modules[__name__] - for encodeFunc in g_dEncodeFunctions.itervalues(): + for encodeFunc in g_dEncodeFunctions.values(): testFuncName = ('test_BaseType_%s' % encodeFunc.__name__).replace('encode', '') getattr(current_module, testFuncName) @@ -314,3 +314,23 @@ def test_nestedSerializable(data): subObj = Serializable(instAttr=data) objData = Serializable(instAttr=subObj) agnosticTestFunction(jsonTuple, objData) + + +def test_types(): + """ Ensure that the DEncode types object matches the Python 2 types module + """ + if not six.PY2: + skip("This test only makes sense on Python 2") + import types as pythonTypes + from DIRAC.Core.Utilities.DEncode import types as DIRACTypes + + assert DIRACTypes.IntType is pythonTypes.IntType + assert DIRACTypes.LongType is pythonTypes.LongType + assert DIRACTypes.FloatType is pythonTypes.FloatType + assert DIRACTypes.BooleanType is pythonTypes.BooleanType + assert DIRACTypes.StringType is pythonTypes.StringType + assert DIRACTypes.UnicodeType is pythonTypes.UnicodeType + assert DIRACTypes.NoneType is pythonTypes.NoneType + assert DIRACTypes.ListType is pythonTypes.ListType + assert DIRACTypes.TupleType is pythonTypes.TupleType + assert DIRACTypes.DictType is pythonTypes.DictType diff --git a/Core/Utilities/test/Test_ExecutorDispatcher.py b/Core/Utilities/test/Test_ExecutorDispatcher.py index cd206b34e0d..f7dae3e9f0e 100644 --- a/Core/Utilities/test/Test_ExecutorDispatcher.py +++ b/Core/Utilities/test/Test_ExecutorDispatcher.py @@ -38,8 +38,8 @@ def test_ExecutorState(): def test_execQueues(): """ test of ExecutorQueues """ - for y in xrange(2): - for i in xrange(3): + for y in range(2): + for i in range(3): assert eQ.pushTask("type%s" % y, "t%s%s" % (y, i)) == i + 1 assert "DONE IN" assert eQ.pushTask("type0", "t01") == 3 @@ -51,6 +51,6 @@ def test_execQueues(): assert eQ.getState() assert eQ.deleteTask("t02") assert eQ.getState() - for i in xrange(3): + for i in range(3): assert eQ.popTask("type1")[0] == "t1%s" % i assert eQ._internals() diff --git a/Core/Utilities/test/Test_Graph.py b/Core/Utilities/test/Test_Graph.py index 59d18c9f85c..5c08bacc0ae 100644 --- a/Core/Utilities/test/Test_Graph.py +++ b/Core/Utilities/test/Test_Graph.py @@ -22,6 +22,7 @@ # @brief Definition of GraphTests class. # # imports import unittest +import six # # SUT from DIRAC.Core.Utilities.Graph import Node, Edge, Graph, DynamicProps # , topologicalSort, topoSort @@ -31,13 +32,14 @@ class DynamicPropTests( unittest.TestCase ): """ def testDynamicProps( self ): """ test dynamic props """ + @six.add_metaclass(DynamicProps) class TestClass( object ): """ .. class:: TestClass dummy class """ - __metaclass__ = DynamicProps + # # dummy instance testObj = TestClass() # # makeProperty in @@ -207,11 +209,11 @@ def testGraph( self ): # # nodes and edges for node in self.nodes: - self.assertEqual( node in gr, True ) + self.assertEqual(node in gr, True) for edge in self.edges: - self.assertEqual( edge in gr, True ) - self.assertEqual( sorted( self.nodes ), sorted( gr.nodes() ) ) - self.assertEqual( sorted( self.edges ), sorted( gr.edges() ) ) + self.assertEqual(edge in gr, True) + self.assertEqual(sorted(self.nodes, key=lambda x: x.name), sorted(gr.nodes(), key=lambda x: x.name)) + self.assertEqual(sorted(self.edges, key=lambda x: x.name), sorted(gr.edges(), key=lambda x: x.name)) # # getNode for node in self.nodes: diff --git a/Core/Utilities/test/Test_List.py b/Core/Utilities/test/Test_List.py index 1ea783c931b..9cfea3251f6 100644 --- a/Core/Utilities/test/Test_List.py +++ b/Core/Utilities/test/Test_List.py @@ -138,11 +138,11 @@ def testBreakListIntoChunks(self): self.assertEqual(isinstance(val, RuntimeError), True) self.assertEqual(str(val), "chunkSize cannot be less than 1") - # normal behaviour - aList = range(10) + # normal behavior + aList = list(range(10)) self.assertEqual(List.breakListIntoChunks(aList, 5), [[0, 1, 2, 3, 4], [5, 6, 7, 8, 9]]) # and once again this time with a rest - aList = range(10) + aList = list(range(10)) self.assertEqual(List.breakListIntoChunks(aList, 4), [[0, 1, 2, 3], [4, 5, 6, 7], [8, 9]]) # almost empty list, too many chunks aList = [1] diff --git a/Core/Utilities/test/Test_Pfn.py b/Core/Utilities/test/Test_Pfn.py index 8f884b1fe02..3f6b79fe1a6 100644 --- a/Core/Utilities/test/Test_Pfn.py +++ b/Core/Utilities/test/Test_Pfn.py @@ -69,7 +69,7 @@ def test_01_srm_parse( self ): :param self: self reference """ - for pfn, result in self.srm_pfns.iteritems(): + for pfn, result in self.srm_pfns.items(): parseResult = pfnparse( pfn ) self.assertEqual( parseResult['OK'], result['OK'] ) if result['OK']: @@ -82,7 +82,7 @@ def test_02_default_parse( self ): :param self: self reference """ - for pfn, result in self.default_pfns.iteritems(): + for pfn, result in self.default_pfns.items(): parseResult = pfnparse( pfn, srmSpecific = False ) self.assertEqual( parseResult['OK'], result['OK'] ) if result['OK']: diff --git a/Core/Utilities/test/Test_ProcessPool.py b/Core/Utilities/test/Test_ProcessPool.py index b681253e016..dc70bc6db86 100644 --- a/Core/Utilities/test/Test_ProcessPool.py +++ b/Core/Utilities/test/Test_ProcessPool.py @@ -24,11 +24,14 @@ ## imports import os +import sys import unittest import random import time import threading +import pytest + ## from DIRAC # from DIRAC.Core.Base import Script # Script.parseCommandLine() @@ -36,6 +39,15 @@ ## SUT from DIRAC.Core.Utilities.ProcessPool import ProcessPool + +@pytest.fixture(autouse=True) +def capture_wrap(): + """ Avoid https://github.com/pytest-dev/pytest/issues/5502 """ + sys.stderr.close = lambda *args: None + sys.stdout.close = lambda *args: None + yield + + def ResultCallback( task, taskResult ): """ dummy result callback """ print("callback for %s result is %s" % (task.getTaskID(), taskResult)) diff --git a/Core/Utilities/test/Test_Profiler.py b/Core/Utilities/test/Test_Profiler.py index c61178bb23c..99ef9bacd2c 100644 --- a/Core/Utilities/test/Test_Profiler.py +++ b/Core/Utilities/test/Test_Profiler.py @@ -8,7 +8,8 @@ import time from subprocess import Popen -# sut +from flaky import flaky + from DIRAC.Core.Utilities.Profiler import Profiler @@ -71,6 +72,7 @@ def test_base(): assert resWC['Value'] >= res['Value'] +@flaky(max_runs=10, min_passes=2) def test_cpuUsage(): mainProcess = Popen(['python', 'tests/Utilities/ProcessesCreator_withChildren.py']) time.sleep(2) diff --git a/Core/Utilities/test/Test_Traced.py b/Core/Utilities/test/Test_Traced.py index 5a009be7627..d5cbd479a6d 100644 --- a/Core/Utilities/test/Test_Traced.py +++ b/Core/Utilities/test/Test_Traced.py @@ -29,6 +29,7 @@ ## imports import unittest +import six ## SUT from DIRAC.Core.Utilities.Traced import Traced, TracedDict, TracedList @@ -38,17 +39,19 @@ class TracedTests(unittest.TestCase): .. class:: TracedTests """ - def setUp( self ): + def setUp(self): """c'tor :param self: self reference """ - self.tracedDict = TracedDict( { 1 : 1 } ) - self.tracedList = TracedList( [ 1 ] ) - class TracedClass( object ): - __metaclass__ = Traced + self.tracedDict = TracedDict({1: 1}) + self.tracedList = TracedList([1]) + + @six.add_metaclass(Traced) + class TracedClass(object): classArg = None - def __init__( self ): + + def __init__(self): instanceArg = None self.tracedClass = TracedClass() diff --git a/Core/Workflow/Module.py b/Core/Workflow/Module.py index 5509a968953..e142d517921 100755 --- a/Core/Workflow/Module.py +++ b/Core/Workflow/Module.py @@ -162,7 +162,7 @@ def __init__(self, parent, pool=None): dict.__init__(self) self.parent = parent # this is a cache value, we propagate it into next level if isinstance(pool, DefinitionsPool): - for k in pool.keys(): + for k in pool: v = pool[k] if isinstance(v, ModuleDefinition): obj = ModuleDefinition(None, v, self.parent) @@ -198,9 +198,9 @@ def compare(self, s): if len(s) != len(self): return False # checkin size # we need to compare the keys of dictionaries - if self.keys() != s.keys(): + if set(self) != set(s): return False - for k in self.keys(): + for k in self: if (k not in s) or (not self[k].compare(s[k])): return False return True @@ -208,7 +208,7 @@ def compare(self, s): def __str__(self): ret = str(type(self)) + ': number of Definitions:' + str(len(self)) + '\n' index = 0 - for k in self.keys(): + for k in self: ret = ret + 'definition(' + str(index) + ')=' + str(self[k]) + '\n' index = index + 1 return ret @@ -216,7 +216,7 @@ def __str__(self): def setParent(self, parent): self.parent = parent # we need to propagate it just in case it was different one - for k in self.keys(): + for k in self: self[k].setParent(parent) def getParent(self): @@ -224,24 +224,24 @@ def getParent(self): def updateParents(self, parent): self.parent = parent - for k in self.keys(): + for k in self: self[k].updateParents(parent) def toXML(self): ret = '' - for k in self.keys(): + for k in self: ret = ret + self[k].toXML() return ret def createCode(self): str = '' - for k in self.keys(): + for k in self: # str=str+indent(2)+'# flush code for instance\n' str = str + self[k].createCode() return str def loadCode(self): - for k in self.keys(): + for k in self: # load code of the modules self[k].loadCode() diff --git a/Core/Workflow/Parameter.py b/Core/Workflow/Parameter.py index 3c8155de3b2..94498635427 100755 --- a/Core/Workflow/Parameter.py +++ b/Core/Workflow/Parameter.py @@ -7,6 +7,7 @@ class which is the base class for the main Workflow classes. from __future__ import absolute_import from __future__ import division from past.builtins import long +import six from DIRAC.Core.Workflow.Utility import * __RCSID__ = "$Id$" @@ -172,13 +173,13 @@ def isOutput(self): return self.typeout def setInput(self, i): - if isinstance(i, str) or isinstance(i, unicode): + if isinstance(i, six.string_types): self.typein = self.__setBooleanFromString(i) else: self.typein = bool(i) def setOutput(self, i): - if isinstance(i, str) or isinstance(i, unicode): + if isinstance(i, six.string_types): self.typeout = self.__setBooleanFromString(i) else: self.typeout = bool(i) @@ -638,7 +639,7 @@ def toXMLFile(self, filename): def toXML(self): ret = "" - for v in self.keys(): + for v in sorted(self): if v == 'parent': continue # doing nothing elif v == 'body' or v == 'description': diff --git a/Core/Workflow/Utility.py b/Core/Workflow/Utility.py index 653a0db27b4..d630e07ce9b 100644 --- a/Core/Workflow/Utility.py +++ b/Core/Workflow/Utility.py @@ -39,11 +39,11 @@ def resolveVariables(varDict): """ Resolve variables defined in terms of others within the same dictionary """ max_tries = 10 - variables = varDict.keys() + variables = list(varDict) ntry = 0 while ntry < max_tries: substFlag = False - for var, value in varDict.iteritems(): + for var, value in list(varDict.items()): if isinstance(value, six.string_types): substitute_vars = getSubstitute(value) for substitute_var in substitute_vars: diff --git a/Core/scripts/dirac-externals-requirements.py b/Core/scripts/dirac-externals-requirements.py index 61013149cea..c32a6f53f26 100755 --- a/Core/scripts/dirac-externals-requirements.py +++ b/Core/scripts/dirac-externals-requirements.py @@ -15,7 +15,12 @@ import os import sys -import commands +# TODO: This should be moderised to use subprocess(32) +try: + import commands +except ImportError: + # Python 3's subprocess module contains a compatibility layer + import subprocess as commands from diraccfg import CFG from DIRAC.Core.Base import Script diff --git a/Core/scripts/dirac-platform.py b/Core/scripts/dirac-platform.py index 9064a297b75..fa0b9f1deef 100755 --- a/Core/scripts/dirac-platform.py +++ b/Core/scripts/dirac-platform.py @@ -59,7 +59,7 @@ def libc_ver(executable=sys.executable, lib='', version='', chunksize=2048): lib = 'libc' elif glibc: glibcversion_parts = glibcversion.split('.') - for i in xrange(len(glibcversion_parts)): + for i in range(len(glibcversion_parts)): try: glibcversion_parts[i] = int(glibcversion_parts[i]) except ValueError: @@ -106,7 +106,7 @@ def getPlatformString(): newest_lib = [0, 0, 0] for lib in libs: lib_parts = libc_ver(lib)[1].split('.') - for i in xrange(len(lib_parts)): + for i in range(len(lib_parts)): try: lib_parts[i] = int(lib_parts[i]) except ValueError: diff --git a/DataManagementSystem/Agent/RequestOperations/ArchiveFiles.py b/DataManagementSystem/Agent/RequestOperations/ArchiveFiles.py index 96376ed7fc9..2d2cb27757a 100644 --- a/DataManagementSystem/Agent/RequestOperations/ArchiveFiles.py +++ b/DataManagementSystem/Agent/RequestOperations/ArchiveFiles.py @@ -72,7 +72,7 @@ def _run(self): self.parameterDict = DEncode.decode(self.operation.Arguments)[0] # tuple: dict, number of characters self.cacheFolder = os.path.join(self.cacheFolder, self.request.RequestName) self._checkArchiveLFN() - for parameter, value in self.parameterDict.iteritems(): + for parameter, value in self.parameterDict.items(): self.log.info('Parameters: %s = %s' % (parameter, value)) self.log.info('Cache folder: %r' % self.cacheFolder) self.waitingFiles = self.getWaitingFilesList() @@ -103,14 +103,14 @@ def _checkReplicas(self): notAt = [] failed = [] sourceSE = self.parameterDict['SourceSE'] - for lfn, replInfo in resReplica['Value']['Successful'].iteritems(): + for lfn, replInfo in resReplica['Value']['Successful'].items(): if sourceSE in replInfo: atSource.append(lfn) else: self.log.warn('LFN %r not found at source, only at: %s' % (lfn, ','.join(replInfo.keys()))) notAt.append(lfn) - for lfn, errorMessage in resReplica['Value']['Failed'].iteritems(): + for lfn, errorMessage in resReplica['Value']['Failed'].items(): self.log.warn('Failed to get replica info', '%s: %s' % (lfn, errorMessage)) if 'No such file or directory' in errorMessage: continue diff --git a/DataManagementSystem/Agent/RequestOperations/MoveReplica.py b/DataManagementSystem/Agent/RequestOperations/MoveReplica.py index 460cb685c5f..317f37ee103 100644 --- a/DataManagementSystem/Agent/RequestOperations/MoveReplica.py +++ b/DataManagementSystem/Agent/RequestOperations/MoveReplica.py @@ -163,7 +163,7 @@ def __checkReplicas(self): targetSESet = set(self.operation.targetSEList) # Check replicas - res = self.ci._getCatalogReplicas(waitingFiles.keys()) + res = self.ci._getCatalogReplicas(list(waitingFiles)) if not res["OK"]: self.log.error('Failed to get catalog replicas', res["Message"]) @@ -171,7 +171,7 @@ def __checkReplicas(self): allReplicas = res['Value'][0] - replicas = self.ci.compareChecksum(waitingFiles.keys()) + replicas = self.ci.compareChecksum(list(waitingFiles)) if not replicas["OK"]: self.log.error('Failed to check replicas', replicas["Message"]) @@ -452,7 +452,7 @@ def bulkRemoval(self, toRemoveDict, targetSE): :param str targetSE: target SE name :return: toRemoveDict with updated errors """ - removeReplicas = self.dm.removeReplica(targetSE, toRemoveDict.keys()) + removeReplicas = self.dm.removeReplica(targetSE, list(toRemoveDict)) if not removeReplicas["OK"]: for opFile in toRemoveDict.values(): diff --git a/DataManagementSystem/Agent/RequestOperations/RemoveFile.py b/DataManagementSystem/Agent/RequestOperations/RemoveFile.py index 49fdb199e19..c9e0f7b0b33 100644 --- a/DataManagementSystem/Agent/RequestOperations/RemoveFile.py +++ b/DataManagementSystem/Agent/RequestOperations/RemoveFile.py @@ -162,7 +162,7 @@ def __call__(self): gMonitor.addMark("RemoveFileOK", len(toRemoveDict) - len(bulkRemoval["Value"])) # # 2nd step - single file removal - for lfn, opFile in toRemoveDict.iteritems(): + for lfn, opFile in toRemoveDict.items(): self.log.info("removing single file %s" % lfn) singleRemoval = self.singleRemoval(opFile) if not singleRemoval["OK"]: @@ -183,7 +183,7 @@ def __call__(self): gMonitor.addMark("RemoveFileOK", 1) # # set - failedFiles = [(lfn, opFile) for (lfn, opFile) in toRemoveDict.iteritems() + failedFiles = [(lfn, opFile) for (lfn, opFile) in toRemoveDict.items() if opFile.Status in ("Failed", "Waiting")] if failedFiles: self.operation.Error = "failed to remove %d files" % len(failedFiles) @@ -201,7 +201,7 @@ def bulkRemoval(self, toRemoveDict): :param dict toRemoveDict: { lfn: opFile, ... } :return: S_ERROR or S_OK( { lfn: opFile, ... } ) -- dict with files still waiting to be removed """ - bulkRemoval = self.dm.removeFile(toRemoveDict.keys(), force=True) + bulkRemoval = self.dm.removeFile(list(toRemoveDict), force=True) if not bulkRemoval["OK"]: error = bulkRemoval["Message"] self.log.error("Bulk file removal failed", error) @@ -211,20 +211,20 @@ def bulkRemoval(self, toRemoveDict): return bulkRemoval bulkRemoval = bulkRemoval["Value"] # # filter results - for lfn, opFile in toRemoveDict.iteritems(): + for lfn, opFile in toRemoveDict.items(): if lfn in bulkRemoval["Successful"]: opFile.Status = "Done" elif lfn in bulkRemoval["Failed"]: error = bulkRemoval["Failed"][lfn] if isinstance(error, dict): - error = ";".join(["%s-%s" % (k, v) for k, v in error.iteritems()]) + error = ";".join(["%s-%s" % (k, v) for k, v in error.items()]) opFile.Error = error if self.reNotExisting.search(opFile.Error): opFile.Status = "Done" # # return files still waiting - toRemoveDict = dict((lfn, opFile) for lfn, opFile in toRemoveDict.iteritems() if opFile.Status == "Waiting") + toRemoveDict = dict((lfn, opFile) for lfn, opFile in toRemoveDict.items() if opFile.Status == "Waiting") return S_OK(toRemoveDict) def singleRemoval(self, opFile): @@ -258,7 +258,7 @@ def singleRemoval(self, opFile): if opFile.LFN in removeFile["Failed"]: error = removeFile["Failed"][opFile.LFN] if isinstance(error, dict): - error = ";".join(["%s-%s" % (k, v) for k, v in error.iteritems()]) + error = ";".join(["%s-%s" % (k, v) for k, v in error.items()]) if self.reNotExisting.search(error): # This should never happen due to the "force" flag opFile.Status = "Done" diff --git a/DataManagementSystem/Agent/RequestOperations/RemoveReplica.py b/DataManagementSystem/Agent/RequestOperations/RemoveReplica.py index c81b6c9a0a5..d54a0750162 100644 --- a/DataManagementSystem/Agent/RequestOperations/RemoveReplica.py +++ b/DataManagementSystem/Agent/RequestOperations/RemoveReplica.py @@ -129,15 +129,15 @@ def __call__(self): # # report removal status for successful files if self.rmsMonitoring: self.rmsMonitoringReporter.addRecord( - self.createRMSRecord("Successful", len(([opFile for opFile in toRemoveDict.itervalues() + self.createRMSRecord("Successful", len(([opFile for opFile in toRemoveDict.values() if not opFile.Error]))) ) else: - gMonitor.addMark("RemoveReplicaOK", len([opFile for opFile in toRemoveDict.itervalues() if not opFile.Error])) + gMonitor.addMark("RemoveReplicaOK", len([opFile for opFile in toRemoveDict.values() if not opFile.Error])) # # 2nd step - process the rest again - toRetry = dict((lfn, opFile) for lfn, opFile in toRemoveDict.iteritems() if opFile.Error) - for lfn, opFile in toRetry.iteritems(): + toRetry = dict((lfn, opFile) for lfn, opFile in toRemoveDict.items() if opFile.Error) + for lfn, opFile in toRetry.items(): self._removeWithOwnerProxy(opFile, targetSE) if opFile.Error: if self.rmsMonitoring: @@ -159,7 +159,7 @@ def __call__(self): failed = 0 for opFile in self.operation: if opFile.Status == "Waiting": - errors = list(set(error for error in removalStatus[opFile.LFN].itervalues() if error)) + errors = list(set(error for error in removalStatus[opFile.LFN].values() if error)) if errors: opFile.Error = "\n".join(errors) # This seems to be the only unrecoverable error @@ -184,16 +184,16 @@ def _bulkRemoval(self, toRemoveDict, targetSE): :param str targetSE: target SE name """ # Clear the error - for opFile in toRemoveDict.itervalues(): + for opFile in toRemoveDict.values(): opFile.Error = '' - removeReplicas = self.dm.removeReplica(targetSE, toRemoveDict.keys()) + removeReplicas = self.dm.removeReplica(targetSE, list(toRemoveDict)) if not removeReplicas["OK"]: - for opFile in toRemoveDict.itervalues(): + for opFile in toRemoveDict.values(): opFile.Error = removeReplicas["Message"] return removeReplicas removeReplicas = removeReplicas["Value"] # # filter out failed - for lfn, opFile in toRemoveDict.iteritems(): + for lfn, opFile in toRemoveDict.items(): if lfn in removeReplicas["Failed"]: errorReason = str(removeReplicas['Failed'][lfn]) # If the reason is that the file does not exist, diff --git a/DataManagementSystem/Agent/RequestOperations/ReplicateAndRegister.py b/DataManagementSystem/Agent/RequestOperations/ReplicateAndRegister.py index 559e1c2ef70..1355bb60041 100644 --- a/DataManagementSystem/Agent/RequestOperations/ReplicateAndRegister.py +++ b/DataManagementSystem/Agent/RequestOperations/ReplicateAndRegister.py @@ -215,13 +215,13 @@ def __checkReplicas(self): if opFile.Status in ("Waiting", "Scheduled")]) targetSESet = set(self.operation.targetSEList) - replicas = self.fc.getReplicas(waitingFiles.keys()) + replicas = self.fc.getReplicas(list(waitingFiles)) if not replicas["OK"]: self.log.error('Failed to get replicas', replicas["Message"]) return replicas reMissing = re.compile(r".*such file.*") - for failedLFN, errStr in replicas["Value"]["Failed"].iteritems(): + for failedLFN, errStr in replicas["Value"]["Failed"].items(): waitingFiles[failedLFN].Error = errStr if reMissing.search(errStr.lower()): self.log.error("File does not exists", failedLFN) @@ -233,7 +233,7 @@ def __checkReplicas(self): gMonitor.addMark("ReplicateFail", len(targetSESet)) waitingFiles[failedLFN].Status = "Failed" - for successfulLFN, reps in replicas["Value"]["Successful"].iteritems(): + for successfulLFN, reps in replicas["Value"]["Successful"].items(): if targetSESet.issubset(set(reps)): self.log.info("file replicated to all targets", successfulLFN) waitingFiles[successfulLFN].Status = "Done" @@ -252,7 +252,7 @@ def _addMetadataToFiles(self, toSchedule): self.log.verbose("No files to schedule") return S_OK([]) - res = self.fc.getFileMetadata(toSchedule.keys()) + res = self.fc.getFileMetadata(list(toSchedule)) if not res['OK']: return res else: @@ -263,7 +263,7 @@ def _addMetadataToFiles(self, toSchedule): filesToSchedule = {} - for lfn, lfnMetadata in metadata.iteritems(): + for lfn, lfnMetadata in metadata.items(): opFileToSchedule = toSchedule[lfn][0] opFileToSchedule.GUID = lfnMetadata['GUID'] # In principle this is defined already in filterReplicas() @@ -703,7 +703,7 @@ def dmTransfer(self, fromFTS=False): if delayExecution: self.log.info("Delay execution of the request by %d minutes" % delayExecution) self.request.delayNextExecution(delayExecution) - for error, count in errors.iteritems(): + for error, count in errors.items(): self.log.error(error, 'for %d files' % count) if self.rmsMonitoring: diff --git a/DataManagementSystem/Client/ConsistencyInspector.py b/DataManagementSystem/Client/ConsistencyInspector.py index cb686fe3464..f0c280ad8bd 100644 --- a/DataManagementSystem/Client/ConsistencyInspector.py +++ b/DataManagementSystem/Client/ConsistencyInspector.py @@ -117,7 +117,7 @@ def getReplicasPresence(self, lfns): for chunk in breakListIntoChunks(lfns, chunkSize): if printProgress: self.__write('.') - for _ in xrange(1, 10): + for _ in range(1, 10): res = self.fileCatalog.getReplicas(chunk) if res['OK']: present.update(res['Value']['Successful']) @@ -322,7 +322,7 @@ def _selectByFileType(self, lfnDict, fileTypes=None, fileTypesExcluded=None): fileTypes = [] # and loop on the original dictionaries for ancestor in lfnDict: - for desc in lfnDict[ancestor].keys(): + for desc in list(lfnDict[ancestor]): ft = lfnDict[ancestor][desc]['FileType'] if ft in fileTypesExcluded or (fileTypes and ft not in fileTypes): ancDict[ancestor].pop(desc) @@ -583,7 +583,7 @@ def catalogFileToSE(self, lfns): if not res['OK']: return res catalogMetadata, _missingCatalogFiles, _zeroSizeFiles = res['Value'] - res = self._getCatalogReplicas(catalogMetadata.keys()) + res = self._getCatalogReplicas(list(catalogMetadata)) if not res['OK']: return res replicas, _zeroReplicaFiles = res['Value'] @@ -605,8 +605,8 @@ def checkPhysicalFiles(self, replicas, catalogMetadata, ses=None): gLogger.info("Performing the FC->SE check") gLogger.info("-" * 40) seLfns = {} - for lfn, replicaDict in replicas.iteritems(): - for se, _url in replicaDict.iteritems(): + for lfn, replicaDict in replicas.items(): + for se, _url in replicaDict.items(): if (ses) and (se not in ses): continue seLfns.setdefault(se, []).append(lfn) @@ -622,7 +622,7 @@ def checkPhysicalFiles(self, replicas, catalogMetadata, ses=None): if not res['OK']: gLogger.error('Failed to get physical file metadata.', res['Message']) return res - for lfn, metadata in res['Value'].iteritems(): + for lfn, metadata in res['Value'].items(): if lfn in catalogMetadata: # and ( metadata['Size'] != 0 ): if metadata['Size'] != catalogMetadata[lfn]['Size']: @@ -647,7 +647,7 @@ def __checkPhysicalFileMetadata(self, lfns, se): pfnMetadata = res['Value']['Successful'] # If the replicas are completely missing missingReplicas = [] - for lfn, reason in res['Value']['Failed'].iteritems(): + for lfn, reason in res['Value']['Failed'].items(): if re.search('File does not exist', reason): missingReplicas.append((lfn, 'deprecatedUrl', se, 'PFNMissing')) if missingReplicas: @@ -656,7 +656,7 @@ def __checkPhysicalFileMetadata(self, lfns, se): unavailableReplicas = [] zeroSizeReplicas = [] # If the files are not accessible - for lfn, metadata in pfnMetadata.iteritems(): + for lfn, metadata in pfnMetadata.items(): if metadata.get('Lost'): lostReplicas.append((lfn, 'deprecatedUrl', se, 'PFNLost')) if metadata.get('Unavailable') or not metadata['Accessible']: @@ -753,7 +753,7 @@ def _getCatalogReplicas(self, lfns): gLogger.error('Failed to get catalog replicas', res['Message']) return res allReplicas = res['Value']['Successful'] - for lfn, error in res['Value']['Failed'].iteritems(): + for lfn, error in res['Value']['Failed'].items(): if re.search('File has zero replicas', error): zeroReplicaFiles.append(lfn) gLogger.info('Obtaining the replicas for files complete') @@ -773,7 +773,7 @@ def _getCatalogMetadata(self, lfns): gLogger.error('Failed to get catalog metadata', res['Message']) return res allMetadata = res['Value']['Successful'] - for lfn, error in res['Value']['Failed'].iteritems(): + for lfn, error in res['Value']['Failed'].items(): if re.search('No such file or directory', error): missingCatalogFiles.append(lfn) gLogger.info('Obtaining the catalog metadata complete') diff --git a/DataManagementSystem/Client/FTS3Job.py b/DataManagementSystem/Client/FTS3Job.py index b433c8725b4..e523e4fc8c0 100644 --- a/DataManagementSystem/Client/FTS3Job.py +++ b/DataManagementSystem/Client/FTS3Job.py @@ -261,7 +261,7 @@ def _constructTransferJob(self, pinTime, allLFNs, target_spacetoken, protocols=N if not res['OK']: return res - for lfn, reason in res['Value']['Failed'].iteritems(): + for lfn, reason in res['Value']['Failed'].items(): failedLFNs.add(lfn) log.error("Could not get source SURL", "%s %s" % (lfn, reason)) @@ -410,7 +410,7 @@ def _constructStagingJob(self, pinTime, allLFNs, target_spacetoken): if not res['OK']: return res - for lfn, reason in res['Value']['Failed'].iteritems(): + for lfn, reason in res['Value']['Failed'].items(): failedLFNs.add(lfn) log.error("Could not get target SURL", "%s %s" % (lfn, reason)) diff --git a/DataManagementSystem/Client/FTS3Operation.py b/DataManagementSystem/Client/FTS3Operation.py index 6265bb08d16..f12dba5588f 100644 --- a/DataManagementSystem/Client/FTS3Operation.py +++ b/DataManagementSystem/Client/FTS3Operation.py @@ -388,7 +388,7 @@ def prepareNewJobs(self, maxFilesPerJob=100, maxAttemptsPerFile=10): return res filesGroupedByTarget = res['Value'] - for targetSE, ftsFiles in filesGroupedByTarget.iteritems(): + for targetSE, ftsFiles in filesGroupedByTarget.items(): res = self._checkSEAccess(targetSE, 'WriteAccess', vo=self.vo) @@ -412,7 +412,7 @@ def prepareNewJobs(self, maxFilesPerJob=100, maxAttemptsPerFile=10): uniqueTransfersBySource, failedFiles = res['Value'] # Treat the errors of the failed files - for ftsFile, errMsg in failedFiles.iteritems(): + for ftsFile, errMsg in failedFiles.items(): log.error("Error when selecting random sources", "%s, %s" % (ftsFile.lfn, errMsg)) # If the error is that the file does not exist in the catalog # fail it ! @@ -421,7 +421,7 @@ def prepareNewJobs(self, maxFilesPerJob=100, maxAttemptsPerFile=10): ftsFile.status = 'Defunct' # We don't need to check the source, since it is already filtered by the DataManager - for sourceSE, ftsFiles in uniqueTransfersBySource.iteritems(): + for sourceSE, ftsFiles in uniqueTransfersBySource.items(): for ftsFilesChunk in breakListIntoChunks(ftsFiles, maxFilesPerJob): @@ -483,7 +483,7 @@ def _callback(self): log.info("will create %s 'RegisterReplica' operations" % len(ftsFilesByTarget)) - for target, ftsFileList in ftsFilesByTarget.iteritems(): + for target, ftsFileList in ftsFilesByTarget.items(): log.info( "creating 'RegisterReplica' operation for targetSE %s with %s files..." % (target, len(ftsFileList))) @@ -533,7 +533,7 @@ def prepareNewJobs(self, maxFilesPerJob=100, maxAttemptsPerFile=10): # {targetSE : [FTS3Files] } filesGroupedByTarget = FTS3Utilities.groupFilesByTarget(filesToSubmit) - for targetSE, ftsFiles in filesGroupedByTarget.iteritems(): + for targetSE, ftsFiles in filesGroupedByTarget.items(): res = self._checkSEAccess(targetSE, 'ReadAccess', vo=self.vo) if not res['OK']: diff --git a/DataManagementSystem/Client/FileCatalogClientCLI.py b/DataManagementSystem/Client/FileCatalogClientCLI.py index d3f1b33653c..a0dfd9254b0 100644 --- a/DataManagementSystem/Client/FileCatalogClientCLI.py +++ b/DataManagementSystem/Client/FileCatalogClientCLI.py @@ -6,7 +6,12 @@ from __future__ import division __RCSID__ = "$Id$" -import commands +# TODO: This should be moderised to use subprocess(32) +try: + import commands +except ImportError: + # Python 3's subprocess module contains a compatability layer + import subprocess as commands import os.path import time import sys @@ -2157,7 +2162,7 @@ def dataset_show(self, argss): print(dName) else: fields = ['Key', 'Value'] - datasets = datasetDict.keys() + datasets = list(datasetDict.keys()) dsAnnotations = {} resultAnno = returnSingleResult(self.fc.getDatasetAnnotation(datasets)) if resultAnno['OK']: diff --git a/DataManagementSystem/Client/test/Test_Client_DataManagementSystem.py b/DataManagementSystem/Client/test/Test_Client_DataManagementSystem.py index 04cd7d192a9..84d8e9041bf 100644 --- a/DataManagementSystem/Client/test/Test_Client_DataManagementSystem.py +++ b/DataManagementSystem/Client/test/Test_Client_DataManagementSystem.py @@ -41,7 +41,7 @@ class ConsistencyInspectorSuccess(UtilitiesTestCase): def test_getReplicasPresence(self): res = self.ci.getReplicasPresence(['/this/is/dir1/file1.txt', '/this/is/dir1/file2.foo.bar']) - self.assertEqual(res, (['/this/is/dir1/file1.txt', '/this/is/dir1/file2.foo.bar'], [])) + self.assertEqual(tuple(map(set, res)), ({'/this/is/dir1/file1.txt', '/this/is/dir1/file2.foo.bar'}, set())) def test__selectByFileType(self): lfnDict = {'aa.raw': {'bb.raw': {'FileType': 'RAW', 'RunNumber': 97019}, diff --git a/DataManagementSystem/DB/FTS3DB.py b/DataManagementSystem/DB/FTS3DB.py index 8947bef1c82..f16c3d07ffe 100644 --- a/DataManagementSystem/DB/FTS3DB.py +++ b/DataManagementSystem/DB/FTS3DB.py @@ -347,7 +347,7 @@ def updateFileStatus(self, fileStatusDict, ftsGUID=None): # However, commiting every time is the recommendation of MySQL # (https://dev.mysql.com/doc/refman/5.7/en/innodb-deadlocks-handling.html) - for fileID, valueDict in fileStatusDict.iteritems(): + for fileID, valueDict in fileStatusDict.items(): session = self.dbSession() try: @@ -409,7 +409,7 @@ def updateJobStatus(self, jobStatusDict): session = self.dbSession() try: - for jobID, valueDict in jobStatusDict.iteritems(): + for jobID, valueDict in jobStatusDict.items(): updateDict = {FTS3Job.status: valueDict['status']} diff --git a/DataManagementSystem/DB/FileCatalogComponents/DatasetManager/DatasetManager.py b/DataManagementSystem/DB/FileCatalogComponents/DatasetManager/DatasetManager.py index c7f6677f530..8a2b0973ed8 100644 --- a/DataManagementSystem/DB/FileCatalogComponents/DatasetManager/DatasetManager.py +++ b/DataManagementSystem/DB/FileCatalogComponents/DatasetManager/DatasetManager.py @@ -96,7 +96,7 @@ def addDataset(self, datasets, credDict): failed = dict() successful = dict() - for datasetName, metaQuery in datasets.iteritems(): + for datasetName, metaQuery in datasets.items(): result = self.__addDataset(datasetName, metaQuery, credDict, uid, gid) if result['OK']: successful[datasetName] = True @@ -193,7 +193,7 @@ def __findFullPathDatasets(self, datasets, connection): dirDict = self._getDatasetDirectories(datasets) failed = {} successful = {} - result = self.db.dtree.findDirs(dirDict.keys()) + result = self.db.dtree.findDirs(list(dirDict)) if not result['OK']: return result directoryIDs = result['Value'] @@ -264,12 +264,12 @@ def addDatasetAnnotation(self, datasets, credDict): """ connection = self._getConnection() successful = {} - result = self._findDatasets(datasets.keys(), connection) + result = self._findDatasets(list(datasets), connection) if not result['OK']: return result failed = result['Value']['Failed'] datasetDict = result['Value']['Successful'] - for dataset, annotation in datasets.iteritems(): + for dataset, annotation in datasets.items(): if dataset in datasetDict: req = "REPLACE FC_DatasetAnnotations (Annotation,DatasetID) VALUE ('%s',%d)" % ( annotation, datasetDict[dataset]['DatasetID']) @@ -296,7 +296,7 @@ def getDatasetAnnotation(self, datasets, credDict={}): for dataset in result['Value']['Failed']: failed[dataset] = "Dataset not found" - idString = ','.join([str(x) for x in dsDict.keys()]) + idString = ','.join([str(x) for x in dsDict]) req = "SELECT DatasetID, Annotation FROM FC_DatasetAnnotations WHERE DatasetID in (%s)" % idString else: req = "SELECT DatasetID, Annotation FROM FC_DatasetAnnotations" @@ -329,10 +329,10 @@ def __getMetaQueryParameters(self, metaQuery, credDict): return S_ERROR('Failed to apply the metaQuery') idLfnDict = result['Value'] - lfnIDList = idLfnDict.keys() + lfnIDList = list(idLfnDict) lfnList = sorted(idLfnDict.values()) myMd5 = hashlib.md5() - myMd5.update(str(lfnList)) + myMd5.update(str(lfnList).encode()) datasetHash = myMd5.hexdigest().upper() numberOfFiles = len(lfnList) result = self.db.fileManager.getFileSize(lfnList) @@ -570,7 +570,7 @@ def getDatasetsInDirectory(self, dirID, verbose=False, connection=False): datasets[dsName]['Metadata'] = dsDict if verbose and datasets: - result = self.getDatasetAnnotation(datasets.keys()) + result = self.getDatasetAnnotation(list(datasets)) if result['OK']: for dataset in result['Value']['Successful']: datasets[dataset]['Annotation'] = result['Value']['Successful'][dataset] @@ -712,9 +712,9 @@ def __getFrozenDatasetFiles(self, datasetID, credDict): return result lfnDict = result['Value']['Successful'] - lfnList = [lfnDict[i] for i in lfnDict.keys()] - result = S_OK(lfnList) - result['FileIDList'] = lfnDict.keys() + fileIDList = list(lfnDict) + result = S_OK([lfnDict[i] for i in fileIDList]) + result['FileIDList'] = fileIDList return result def getDatasetFiles(self, datasets, credDict): diff --git a/DataManagementSystem/DB/FileCatalogComponents/DirectoryManager/DirectoryClosure.py b/DataManagementSystem/DB/FileCatalogComponents/DirectoryManager/DirectoryClosure.py index eaef5430185..cc6f9874468 100644 --- a/DataManagementSystem/DB/FileCatalogComponents/DirectoryManager/DirectoryClosure.py +++ b/DataManagementSystem/DB/FileCatalogComponents/DirectoryManager/DirectoryClosure.py @@ -547,10 +547,9 @@ def _setDirectoryMode(self, path, mode, recursive=False): return self._setDirectoryParameter(path, 'Mode', mode, recursive=recursive) def __getLogicalSize(self, lfns, ps_name, connection): - paths = lfns.keys() successful = {} failed = {} - for path in paths: + for path in lfns: result = self.findDir(path) if not result['OK'] or not result['Value']: failed[path] = "Directory not found" @@ -591,10 +590,9 @@ def __getPhysicalSize(self, lfns, ps_name, connection): """ Get the total size of the requested directories """ - paths = lfns.keys() successful = {} failed = {} - for path in paths: + for path in lfns: result = self.findDir(path) if not result['OK']: failed[path] = "Directory not found" @@ -652,7 +650,7 @@ def _changeDirectoryParameter(self, paths, arguments = paths successful = {} failed = {} - for path, attribute in arguments.iteritems(): + for path, attribute in arguments.items(): result = directoryFunction(path, attribute, recursive=recursive) if not result['OK']: failed[path] = result['Message'] diff --git a/DataManagementSystem/DB/FileCatalogComponents/DirectoryManager/DirectoryFlatTree.py b/DataManagementSystem/DB/FileCatalogComponents/DirectoryManager/DirectoryFlatTree.py index 921a4723973..0dbc335dfc4 100644 --- a/DataManagementSystem/DB/FileCatalogComponents/DirectoryManager/DirectoryFlatTree.py +++ b/DataManagementSystem/DB/FileCatalogComponents/DirectoryManager/DirectoryFlatTree.py @@ -76,7 +76,7 @@ def getPathPermissions(self, paths, credDict): if not res['OK']: return res successful = {} - for dirName, dirDict in res['Value']['Successful'].iteritems(): + for dirName, dirDict in res['Value']['Successful'].items(): mode = dirDict['Mode'] p_uid = dirDict['UID'] p_gid = dirDict['GID'] diff --git a/DataManagementSystem/DB/FileCatalogComponents/DirectoryManager/DirectoryLevelTree.py b/DataManagementSystem/DB/FileCatalogComponents/DirectoryManager/DirectoryLevelTree.py index 86cafbaa89f..eeed9c1a37d 100644 --- a/DataManagementSystem/DB/FileCatalogComponents/DirectoryManager/DirectoryLevelTree.py +++ b/DataManagementSystem/DB/FileCatalogComponents/DirectoryManager/DirectoryLevelTree.py @@ -332,7 +332,7 @@ def getPathIDsByID(self, dirID): lpaths = result['Value'] lpathSelects = [] - for lev in xrange(level): + for lev in range(level): sel = ' AND '.join(["Level=%d" % lev] + ['LPATH%d=%d' % (ll + 1, lpaths[ll]) for ll in range(lev)]) lpathSelects.append(sel) selection = '(' + ') OR ('.join(lpathSelects) + ')' @@ -485,7 +485,7 @@ def recoverOrphanDirectories(self, credDict): parentDict[parentPath]['DirList'].append(dirID) parentDict[parentPath]['OldParentID'] = parentID - for parentPath, dirDict in parentDict.iteritems(): + for parentPath, dirDict in parentDict.items(): dirIDList = dirDict['DirList'] oldParentID = dirDict['OldParentID'] result = self.findDir(parentPath) diff --git a/DataManagementSystem/DB/FileCatalogComponents/DirectoryManager/DirectoryTreeBase.py b/DataManagementSystem/DB/FileCatalogComponents/DirectoryManager/DirectoryTreeBase.py index e236311a33c..f3bdbc45091 100755 --- a/DataManagementSystem/DB/FileCatalogComponents/DirectoryManager/DirectoryTreeBase.py +++ b/DataManagementSystem/DB/FileCatalogComponents/DirectoryManager/DirectoryTreeBase.py @@ -196,10 +196,9 @@ def existsDir(self, path): def isDirectory(self, paths): """ Checking for existence of directories """ - dirs = paths.keys() successful = {} failed = {} - for dir in dirs: + for dir in paths: result = self.existsDir(dir) if not result['OK']: failed[dir] = result['Message'] @@ -265,9 +264,8 @@ def removeDirectory(self, dirs, force=False): for d in dirs: if d not in dirDict: successful[d] = "Directory does not exist" - dirList = dirDict.keys() - for dir in dirList: + for dir in dirDict: result = self.isEmpty(dir) if not result['OK']: return result @@ -453,7 +451,7 @@ def _changeDirectoryParameter(self, paths, arguments = paths successful = {} failed = {} - for path, attribute in arguments.iteritems(): + for path, attribute in arguments.items(): result = directoryFunction(path, attribute) if not result['OK']: failed[path] = result['Message'] @@ -692,10 +690,9 @@ def _getDirectoryContents(self, path, details=False): def listDirectory(self, lfns, verbose=False): """ Get the directory listing """ - paths = lfns.keys() successful = {} failed = {} - for path in paths: + for path in lfns: result = self._getDirectoryContents(path, details=verbose) if not result['OK']: failed[path] = result['Message'] @@ -707,10 +704,9 @@ def listDirectory(self, lfns, verbose=False): def getDirectoryReplicas(self, lfns, allStatus=False): """ Get replicas for files in the given directories """ - paths = lfns.keys() successful = {} failed = {} - for path in paths: + for path in lfns: result = self.findDir(path) if not result['OK']: failed[path] = result['Message'] @@ -774,10 +770,9 @@ def getDirectorySize(self, lfns, longOutput=False, rawFileTables=False): def _getDirectoryLogicalSizeFromUsage(self, lfns, connection): """ Get the total "logical" size of the requested directories """ - paths = lfns.keys() successful = {} failed = {} - for path in paths: + for path in lfns: result = self.findDir(path) if not result['OK']: failed[path] = "Directory not found" @@ -810,11 +805,10 @@ def _getDirectoryLogicalSizeFromUsage(self, lfns, connection): def _getDirectoryLogicalSize(self, lfns, connection): """ Get the total "logical" size of the requested directories """ - paths = lfns.keys() successful = {} failed = {} treeTable = self.getTreeTable() - for path in paths: + for path in lfns: if path == "/": req = "SELECT SUM(Size),COUNT(*) FROM FC_Files" @@ -859,10 +853,9 @@ def _getDirectoryLogicalSize(self, lfns, connection): def _getDirectoryPhysicalSizeFromUsage(self, lfns, connection): """ Get the total size of the requested directories """ - paths = lfns.keys() successful = {} failed = {} - for path in paths: + for path in lfns: result = self.findDir(path) if not result['OK']: failed[path] = "Directory not found" @@ -904,10 +897,9 @@ def _getDirectoryPhysicalSizeFromUsage(self, lfns, connection): def _getDirectoryPhysicalSizeFromUsage_old(self, lfns, connection): """ Get the total size of the requested directories """ - paths = lfns.keys() successful = {} failed = {} - for path in paths: + for path in lfns: if path == '/': req = "SELECT S.SEName, D.SESize, D.SEFiles FROM FC_DirectoryUsage as D, FC_StorageElements as S" @@ -956,10 +948,9 @@ def _getDirectoryPhysicalSizeFromUsage_old(self, lfns, connection): def _getDirectoryPhysicalSize(self, lfns, connection): """ Get the total size of the requested directories """ - paths = lfns.keys() successful = {} failed = {} - for path in paths: + for path in lfns: if path == '/': req = "SELECT SUM(F.Size),COUNT(F.Size),S.SEName from FC_Files as F, FC_Replicas as R, FC_StorageElements as S " req += "WHERE R.SEID=S.SEID AND F.FileID=R.FileID " diff --git a/DataManagementSystem/DB/FileCatalogComponents/DirectoryMetadata/DirectoryMetadata.py b/DataManagementSystem/DB/FileCatalogComponents/DirectoryMetadata/DirectoryMetadata.py index d39e1e7b285..944ecd577b8 100644 --- a/DataManagementSystem/DB/FileCatalogComponents/DirectoryMetadata/DirectoryMetadata.py +++ b/DataManagementSystem/DB/FileCatalogComponents/DirectoryMetadata/DirectoryMetadata.py @@ -155,7 +155,7 @@ def addMetadataSet(self, metaSetName, metaSetDict, credDict): req = "INSERT INTO FC_MetaSets (MetaSetID,MetaKey,MetaValue) VALUES %s" vList = [] - for key, value in metaSetDict.iteritems(): + for key, value in metaSetDict.items(): vList.append("(%d,'%s','%s')" % (metaSetID, key, str(value))) vString = ','.join(vList) result = self.db._update(req % vString) @@ -231,7 +231,7 @@ def setMetadata(self, dPath, metaDict, credDict): if not dirmeta['OK']: return dirmeta - for metaName, metaValue in metaDict.iteritems(): + for metaName, metaValue in metaDict.items(): if metaName not in metaFields: result = self.setMetaParameter(dPath, metaName, metaValue, credDict) if not result['OK']: @@ -485,7 +485,7 @@ def __createMetaSelection(self, value, table=''): if isinstance(value, dict): selectList = [] - for operation, operand in value.iteritems(): + for operation, operand in value.items(): if operation in ['>', '<', '>=', '<=']: if isinstance(operand, list): return S_ERROR('Illegal query: list of values for comparison operation') @@ -610,7 +610,7 @@ def __expandMetaDictionary(self, metaDict, credDict): metaTypeDict = result['Value'] resultDict = {} extraDict = {} - for key, value in metaDict.iteritems(): + for key, value in metaDict.items(): if key not in metaTypeDict: # return S_ERROR( 'Unknown metadata field %s' % key ) extraDict[key] = value @@ -623,7 +623,7 @@ def __expandMetaDictionary(self, metaDict, credDict): if not result['OK']: return result mDict = result['Value'] - for mk, mv in mDict.iteritems(): + for mk, mv in mDict.items(): if mk in resultDict: return S_ERROR('Contradictory query for key %s' % mk) else: @@ -710,7 +710,7 @@ def findDirIDsByMetadata(self, queryDict, path, credDict): pathSelection = result['Value'] dirList = [] first = True - for meta, value in finalMetaDict.iteritems(): + for meta, value in finalMetaDict.items(): if value == "Missing": result = self.__findSubdirMissingMeta(meta, pathSelection) else: @@ -960,7 +960,7 @@ def getCompatibleMetadata(self, queryDict, path, credDict): anyMeta = True if metaDict: anyMeta = False - for meta, value in metaDict.iteritems(): + for meta, value in metaDict.items(): result = self.__findCompatibleDirectories(meta, value, fromList) if not result['OK']: return result diff --git a/DataManagementSystem/DB/FileCatalogComponents/DirectoryMetadata/MultiVODirectoryMetadata.py b/DataManagementSystem/DB/FileCatalogComponents/DirectoryMetadata/MultiVODirectoryMetadata.py index f732d73c1ce..017456d40bb 100644 --- a/DataManagementSystem/DB/FileCatalogComponents/DirectoryMetadata/MultiVODirectoryMetadata.py +++ b/DataManagementSystem/DB/FileCatalogComponents/DirectoryMetadata/MultiVODirectoryMetadata.py @@ -39,7 +39,7 @@ def _getMetaNameDict(metaDict, credDict): """ fMetaDict = {} - for meta, value in metaDict.iteritems(): + for meta, value in metaDict.items(): fMetaDict[_getMetaName(meta, credDict)] = value return fMetaDict @@ -65,7 +65,7 @@ def _stripSuffix(metaDict, credDict): """ suffix = _getMetaNameSuffix(credDict) - smetaDict = {key.rsplit(suffix, 1)[0]: value for key, value in metaDict.iteritems() + smetaDict = {key.rsplit(suffix, 1)[0]: value for key, value in metaDict.items() if key.endswith(suffix)} return smetaDict diff --git a/DataManagementSystem/DB/FileCatalogComponents/FileManager/FileManager.py b/DataManagementSystem/DB/FileCatalogComponents/FileManager/FileManager.py index d8e3860093f..628427bd2e5 100755 --- a/DataManagementSystem/DB/FileCatalogComponents/FileManager/FileManager.py +++ b/DataManagementSystem/DB/FileCatalogComponents/FileManager/FileManager.py @@ -31,7 +31,7 @@ def _findFiles(self, lfns, metadata=['FileID'], allStatus=False, connection=Fals connection = self._getConnection(connection) dirDict = self._getFileDirectories(lfns) failed = {} - result = self.db.dtree.findDirs(dirDict.keys()) + result = self.db.dtree.findDirs(list(dirDict)) if not result['OK']: return result directoryIDs = result['Value'] @@ -59,7 +59,7 @@ def _findFiles(self, lfns, metadata=['FileID'], allStatus=False, connection=Fals fname = fname.replace('//', '/') failed[fname] = error else: - for fileName, fileDict in res['Value'].iteritems(): + for fileName, fileDict in res['Value'].items(): fname = '%s/%s' % (dirPath, fileName) fname = fname.replace('//', '/') successful[fname] = fileDict @@ -77,7 +77,7 @@ def _findFileIDs(self, lfns, connection=False): dirDict = self._getFileDirectories(lfns) failed = {} successful = {} - result = self.db.dtree.findDirs(dirDict.keys()) + result = self.db.dtree.findDirs(list(dirDict)) if not result['OK']: return result directoryIDs = result['Value'] @@ -91,9 +91,8 @@ def _findFileIDs(self, lfns, connection=False): failed[fname] = 'No such file or directory' else: directoryPaths[directoryIDs[dirPath]] = dirPath - directoryIDList = directoryIDs.keys() - for dirIDs in breakListIntoChunks(directoryIDList, 1000): + for dirIDs in breakListIntoChunks(list(directoryIDs), 1000): wheres = [] for dirPath in dirIDs: fileNames = dirDict[dirPath] @@ -280,20 +279,20 @@ def _insertFiles(self, lfns, uid, gid, connection=False): if not res['OK']: return res # Get the fileIDs for the inserted files - res = self._findFiles(lfns.keys(), ['FileID'], connection=connection) + res = self._findFiles(list(lfns), ['FileID'], connection=connection) if not res['OK']: for lfn in lfns.keys(): failed[lfn] = 'Failed post insert check' lfns.pop(lfn) else: failed.update(res['Value']['Failed']) - for lfn in res['Value']['Failed'].keys(): + for lfn in res['Value']['Failed']: lfns.pop(lfn) - for lfn, fileDict in res['Value']['Successful'].iteritems(): + for lfn, fileDict in res['Value']['Successful'].items(): lfns[lfn]['FileID'] = fileDict['FileID'] insertTuples = [] toDelete = [] - for lfn in lfns.keys(): + for lfn in lfns: fileInfo = lfns[lfn] fileID = fileInfo['FileID'] dirID = fileInfo['DirID'] @@ -311,7 +310,7 @@ def _insertFiles(self, lfns, uid, gid, connection=False): res = self.db._update(req) if not res['OK']: self._deleteFiles(toDelete, connection=connection) - for lfn in lfns.keys(): + for lfn in list(lfns): failed[lfn] = res['Message'] lfns.pop(lfn) else: @@ -359,7 +358,7 @@ def getLFNForGUID(self, guids, connection=False): dirIDFileIDs.setdefault(dirID, []).append(fileID) fileIDName[fileID] = fileName - res = self.db.dtree.getDirectoryPaths(dirIDFileIDs.keys()) + res = self.db.dtree.getDirectoryPaths(list(dirIDFileIDs)) if not res['OK']: return res @@ -393,8 +392,7 @@ def __deleteFileReplicas(self, fileIDs, connection=False): res = self.__getFileIDReplicas(fileIDs, connection=connection) if not res['OK']: return res - repIDs = res['Value'].keys() - return self.__deleteReplicas(repIDs, connection=connection) + return self.__deleteReplicas(list(res['Value']), connection=connection) def __deleteFiles(self, fileIDs, connection=False): connection = self._getConnection(connection) @@ -452,8 +450,8 @@ def _insertReplicas(self, lfns, master=False, connection=False): res = self._getRepIDsForReplica(insertTuples, connection=connection) if not res['OK']: return res - for fileID, repDict in res['Value'].iteritems(): - for seID, repID in repDict.iteritems(): + for fileID, repDict in res['Value'].items(): + for seID, repID in repDict.items(): successful[fileIDLFNs[fileID]] = True insertTuples.remove((fileID, seID)) @@ -470,11 +468,11 @@ def _insertReplicas(self, lfns, master=False, connection=False): return res replicaDict = res['Value'] directorySESizeDict = {} - for fileID, repDict in replicaDict.iteritems(): + for fileID, repDict in replicaDict.items(): lfn = fileIDLFNs[fileID] dirID = lfns[lfn]['DirID'] directorySESizeDict.setdefault(dirID, {}) - for seID, repID in repDict.iteritems(): + for seID, repID in repDict.items(): lfns[lfn]['RepID'] = repID directorySESizeDict[dirID].setdefault(seID, {'Files': 0, 'Size': 0}) directorySESizeDict[dirID][seID]['Size'] += lfns[lfn]['Size'] @@ -532,10 +530,10 @@ def _deleteReplicas(self, lfns, connection=False): connection = self._getConnection(connection) failed = {} successful = {} - res = self._findFiles(lfns.keys(), ['DirID', 'FileID', 'Size'], connection=connection) + res = self._findFiles(list(lfns), ['DirID', 'FileID', 'Size'], connection=connection) # If the file does not exist we consider the deletion successful - for lfn, error in res['Value']['Failed'].iteritems(): + for lfn, error in res['Value']['Failed'].items(): if error == 'No such file or directory': successful[lfn] = True else: @@ -544,7 +542,7 @@ def _deleteReplicas(self, lfns, connection=False): lfnFileIDDict = res['Value']['Successful'] toRemove = [] directorySESizeDict = {} - for lfn, fileDict in lfnFileIDDict.iteritems(): + for lfn, fileDict in lfnFileIDDict.items(): fileID = fileDict['FileID'] se = lfns[lfn]['SE'] if isinstance(se, str): @@ -565,8 +563,8 @@ def _deleteReplicas(self, lfns, connection=False): failed[lfn] = res['Message'] else: repIDs = [] - for fileID, seDict in res['Value'].iteritems(): - for seID, repID in seDict.iteritems(): + for fileID, seDict in res['Value'].items(): + for seID, repID in seDict.items(): repIDs.append(repID) res = self.__deleteReplicas(repIDs, connection=connection) if not res['OK']: diff --git a/DataManagementSystem/DB/FileCatalogComponents/FileManager/FileManagerBase.py b/DataManagementSystem/DB/FileCatalogComponents/FileManager/FileManagerBase.py index 86dbd5f84d8..20382cb4fc9 100755 --- a/DataManagementSystem/DB/FileCatalogComponents/FileManager/FileManagerBase.py +++ b/DataManagementSystem/DB/FileCatalogComponents/FileManager/FileManagerBase.py @@ -226,7 +226,7 @@ def addFile(self, lfns, credDict, connection=False): connection = self._getConnection(connection) successful = {} failed = {} - for lfn, info in lfns.items(): + for lfn, info in list(lfns.items()): res = self._checkInfo(info, ['PFN', 'SE', 'Size', 'Checksum']) if not res['OK']: failed[lfn] = res['Message'] @@ -262,7 +262,7 @@ def _addFiles(self, lfns, credDict, connection=False): extraLfns[lfn]['SE'] = lfns[lfn]['SE'][1:] # Check whether the supplied files have been registered already - res = self._getExistingMetadata(masterLfns.keys(), connection=connection) + res = self._getExistingMetadata(list(masterLfns), connection=connection) if not res['OK']: return res existingMetadata, failed = res['Value'] @@ -270,7 +270,7 @@ def _addFiles(self, lfns, credDict, connection=False): success, fail = self._checkExistingMetadata(existingMetadata, masterLfns) successful.update(success) failed.update(fail) - for lfn in (success.keys() + fail.keys()): + for lfn in list(success) + list(fail): masterLfns.pop(lfn) # If GUIDs are supposed to be unique check their pre-existance @@ -283,8 +283,8 @@ def _addFiles(self, lfns, credDict, connection=False): # If we have files left to register if masterLfns: # Create the directories for the supplied files and store their IDs - directories = self._getFileDirectories(masterLfns.keys()) - for directory, fileNames in directories.iteritems(): + directories = self._getFileDirectories(list(masterLfns)) + for directory, fileNames in directories.items(): res = self.db.dtree.makeDirectories(directory, credDict) if not res['OK']: for fileName in fileNames: @@ -316,7 +316,7 @@ def _addFiles(self, lfns, credDict, connection=False): failed[lfn] = res['Message'] masterLfns.pop(lfn) else: - for lfn, error in res['Value']['Failed'].iteritems(): + for lfn, error in res['Value']['Failed'].items(): failed[lfn] = error masterLfns.pop(lfn) masterLfns = res['Value']['Successful'] @@ -331,7 +331,7 @@ def _addFiles(self, lfns, credDict, connection=False): toPurge.append(masterLfns[lfn]['FileID']) else: failed.update(res['Value']['Failed']) - for lfn, error in res['Value']['Failed'].iteritems(): + for lfn, error in res['Value']['Failed'].items(): toPurge.append(masterLfns[lfn]['FileID']) if toPurge: self._deleteFiles(toPurge, connection=connection) @@ -349,7 +349,7 @@ def _addFiles(self, lfns, credDict, connection=False): newlyRegistered = res['Value']['Successful'] successful.update(newlyRegistered) failed.update(res['Value']['Failed']) - for lfn, error in res['Value']['Failed'].iteritems(): + for lfn, error in res['Value']['Failed'].items(): toPurge.append(masterLfns[lfn]['FileID']) if toPurge: self._deleteFiles(toPurge, connection=connection) @@ -360,7 +360,7 @@ def _addFiles(self, lfns, credDict, connection=False): extraLfns.pop(lfn) if extraLfns: - res = self._findFiles(extraLfns.keys(), ['FileID', 'DirID'], connection=connection) + res = self._findFiles(list(extraLfns), ['FileID', 'DirID'], connection=connection) if not res['OK']: for lfn in lfns.keys(): failed[lfn] = 'Failed while registering extra replicas' @@ -368,17 +368,17 @@ def _addFiles(self, lfns, credDict, connection=False): extraLfns.pop(lfn) else: failed.update(res['Value']['Failed']) - for lfn in res['Value']['Failed'].keys(): + for lfn in res['Value']['Failed']: successful.pop(lfn) extraLfns.pop(lfn) - for lfn, fileDict in res['Value']['Successful'].iteritems(): + for lfn, fileDict in res['Value']['Successful'].items(): extraLfns[lfn]['FileID'] = fileDict['FileID'] extraLfns[lfn]['DirID'] = fileDict['DirID'] if extraLfns: res = self._insertReplicas(extraLfns, master=False, connection=connection) if not res['OK']: - for lfn in extraLfns.keys(): # pylint: disable=consider-iterating-dictionary + for lfn in extraLfns: # pylint: disable=consider-iterating-dictionary failed[lfn] = "Failed while registering extra replicas" successful.pop(lfn) else: @@ -417,7 +417,7 @@ def _populateFileAncestors(self, lfns, connection=False): connection = self._getConnection(connection) successful = {} failed = {} - for lfn, lfnDict in lfns.iteritems(): + for lfn, lfnDict in lfns.items(): originalFileID = lfnDict['FileID'] originalDepth = lfnDict.get('AncestorDepth', 1) ancestors = lfnDict.get('Ancestors', []) @@ -438,13 +438,13 @@ def _populateFileAncestors(self, lfns, connection=False): for ancestor in ancestorIDs.keys(): fileIDLFNs[ancestorIDs[ancestor]['FileID']] = ancestor toInsert[ancestorIDs[ancestor]['FileID']] = originalDepth - res = self._getFileAncestors(fileIDLFNs.keys()) + res = self._getFileAncestors(list(fileIDLFNs)) if not res['OK']: failed[lfn] = "Failed to obtain all ancestors" continue fileIDAncestorDict = res['Value'] for fileIDDict in fileIDAncestorDict.values(): - for ancestorID, relativeDepth in fileIDDict.iteritems(): + for ancestorID, relativeDepth in fileIDDict.items(): toInsert[ancestorID] = relativeDepth + originalDepth res = self._insertFileAncestors(originalFileID, toInsert, connection=connection) if not res['OK']: @@ -459,7 +459,7 @@ def _populateFileAncestors(self, lfns, connection=False): def _insertFileAncestors(self, fileID, ancestorDict, connection=False): connection = self._getConnection(connection) ancestorTuples = [] - for ancestorID, depth in ancestorDict.iteritems(): + for ancestorID, depth in ancestorDict.items(): ancestorTuples.append("(%d,%d,%d)" % (fileID, ancestorID, depth)) if not ancestorTuples: return S_OK() @@ -504,7 +504,7 @@ def addFileAncestors(self, lfns, connection=False): connection = self._getConnection(connection) failed = {} successful = {} - result = self._findFiles(lfns.keys(), connection=connection) + result = self._findFiles(list(lfns), connection=connection) if not result['OK']: return result if result['Value']['Failed']: @@ -528,7 +528,7 @@ def _getFileRelatives(self, lfns, depths, relation, connection=False): connection = self._getConnection(connection) failed = {} successful = {} - result = self._findFiles(lfns.keys(), connection=connection) + result = self._findFiles(list(lfns), connection=connection) if not result['OK']: return result if result['Value']['Failed']: @@ -542,7 +542,7 @@ def _getFileRelatives(self, lfns, depths, relation, connection=False): for lfn in result['Value']['Successful']: inputIDDict[result['Value']['Successful'][lfn]['FileID']] = lfn - inputIDs = inputIDDict.keys() + inputIDs = list(inputIDDict) if relation == 'ancestor': result = self._getFileAncestors(inputIDs, depths, connection) else: @@ -556,8 +556,7 @@ def _getFileRelatives(self, lfns, depths, relation, connection=False): relDict = result['Value'] for id_ in inputIDs: if id_ in relDict: - aList = relDict[id_].keys() - result = self._getFileLFNs(aList) + result = self._getFileLFNs(list(relDict[id_])) if not result['OK']: failed[inputIDDict[id]] = "Failed to find %s" % relation else: @@ -596,16 +595,16 @@ def _checkExistingMetadata(self, existingLfns, lfns): failed = {} successful = {} fileIDLFNs = {} - for lfn, fileDict in existingLfns.iteritems(): + for lfn, fileDict in existingLfns.items(): fileIDLFNs[fileDict['FileID']] = lfn # For those that exist get the replicas to determine whether they are already registered - res = self._getFileReplicas(fileIDLFNs.keys()) + res = self._getFileReplicas(list(fileIDLFNs)) if not res['OK']: - for lfn in fileIDLFNs.itervalues(): + for lfn in fileIDLFNs.values(): failed[lfn] = 'Failed checking pre-existing replicas' else: replicaDict = res['Value'] - for fileID, lfn in fileIDLFNs.iteritems(): + for fileID, lfn in fileIDLFNs.items(): fileMetadata = existingLfns[lfn] existingGuid = fileMetadata['GUID'] existingSize = fileMetadata['Size'] @@ -633,12 +632,12 @@ def _checkUniqueGUID(self, lfns, connection=False): connection = self._getConnection(connection) guidLFNs = {} failed = {} - for lfn, fileDict in lfns.iteritems(): + for lfn, fileDict in lfns.items(): guidLFNs[fileDict['GUID']] = lfn - res = self._getFileIDFromGUID(guidLFNs.keys(), connection=connection) + res = self._getFileIDFromGUID(list(guidLFNs), connection=connection) if not res['OK']: return dict.fromkeys(lfns, res['Message']) - for guid, fileID in res['Value'].iteritems(): + for guid, fileID in res['Value'].items(): # resolve this to LFN failed[guidLFNs[guid]] = "GUID already registered for another file %s" % fileID return failed @@ -651,14 +650,14 @@ def removeFile(self, lfns, connection=False): res = self._findFiles(lfns, ['DirID', 'FileID', 'Size'], connection=connection) if not res['OK']: return res - for lfn, error in res['Value']['Failed'].iteritems(): + for lfn, error in res['Value']['Failed'].items(): if error == 'No such file or directory': successful[lfn] = True else: failed[lfn] = error fileIDLfns = {} lfns = res['Value']['Successful'] - for lfn, lfnDict in lfns.iteritems(): + for lfn, lfnDict in lfns.items(): fileIDLfns[lfnDict['FileID']] = lfn res = self._computeStorageUsageOnRemoveFile(lfns, connection=connection) @@ -667,7 +666,7 @@ def removeFile(self, lfns, connection=False): directorySESizeDict = res['Value'] # Now do removal - res = self._deleteFiles(fileIDLfns.keys(), connection=connection) + res = self._deleteFiles(list(fileIDLfns), connection=connection) if not res['OK']: for lfn in fileIDLfns.values(): failed[lfn] = res['Message'] @@ -681,13 +680,13 @@ def removeFile(self, lfns, connection=False): def _computeStorageUsageOnRemoveFile(self, lfns, connection=False): # Resolve the replicas to calculate reduction in storage usage fileIDLfns = {} - for lfn, lfnDict in lfns.iteritems(): + for lfn, lfnDict in lfns.items(): fileIDLfns[lfnDict['FileID']] = lfn - res = self._getFileReplicas(fileIDLfns.keys(), connection=connection) + res = self._getFileReplicas(list(fileIDLfns), connection=connection) if not res['OK']: return res directorySESizeDict = {} - for fileID, seDict in res['Value'].iteritems(): + for fileID, seDict in res['Value'].items(): dirID = lfns[fileIDLfns[fileID]]['DirID'] size = lfns[fileIDLfns[fileID]]['Size'] directorySESizeDict.setdefault(dirID, {}) @@ -743,14 +742,14 @@ def addReplica(self, lfns, connection=False): connection = self._getConnection(connection) successful = {} failed = {} - for lfn, info in lfns.items(): + for lfn, info in list(lfns.items()): res = self._checkInfo(info, ['PFN', 'SE']) if not res['OK']: failed[lfn] = res['Message'] lfns.pop(lfn) res = self._addReplicas(lfns, connection=connection) if not res['OK']: - for lfn in lfns.keys(): + for lfn in lfns: failed[lfn] = res['Message'] else: failed.update(res['Value']['Failed']) @@ -761,18 +760,18 @@ def _addReplicas(self, lfns, connection=False): connection = self._getConnection(connection) successful = {} - res = self._findFiles(lfns.keys(), ['DirID', 'FileID', 'Size'], connection=connection) + res = self._findFiles(list(lfns), ['DirID', 'FileID', 'Size'], connection=connection) if not res['OK']: return res failed = res['Value']['Failed'] - for lfn in failed.keys(): + for lfn in failed: lfns.pop(lfn) lfnFileIDDict = res['Value']['Successful'] - for lfn, fileDict in lfnFileIDDict.iteritems(): + for lfn, fileDict in lfnFileIDDict.items(): lfns[lfn].update(fileDict) res = self._insertReplicas(lfns, connection=connection) if not res['OK']: - for lfn in lfns.keys(): + for lfn in lfns: failed[lfn] = res['Message'] else: successful = res['Value']['Successful'] @@ -784,7 +783,7 @@ def removeReplica(self, lfns, connection=False): connection = self._getConnection(connection) successful = {} failed = {} - for lfn, info in lfns.items(): + for lfn, info in list(lfns.items()): res = self._checkInfo(info, ['SE']) if not res['OK']: failed[lfn] = res['Message'] @@ -803,7 +802,7 @@ def setReplicaStatus(self, lfns, connection=False): connection = self._getConnection(connection) successful = {} failed = {} - for lfn, info in lfns.iteritems(): + for lfn, info in lfns.items(): res = self._checkInfo(info, ['SE', 'Status']) if not res['OK']: failed[lfn] = res['Message'] @@ -827,7 +826,7 @@ def setReplicaHost(self, lfns, connection=False): connection = self._getConnection(connection) successful = {} failed = {} - for lfn, info in lfns.iteritems(): + for lfn, info in lfns.items(): res = self._checkInfo(info, ['SE', 'NewSE']) if not res['OK']: failed[lfn] = res['Message'] @@ -891,10 +890,10 @@ def exists(self, lfns, connection=False): if not res['OK']: return res guidLfns = res['Value']['Successful'] - for guid, realLfn in guidLfns.iteritems(): + for guid, realLfn in guidLfns.items(): successful[guidToGivenLfn[guid]] = realLfn - for lfn, error in origFailed.iteritems(): + for lfn, error in origFailed.items(): # It could be in successful because the guid exists with another lfn if lfn in successful: continue @@ -948,7 +947,7 @@ def getPathPermissions(self, paths, credDict, connection=False): if not res['OK']: return res successful = {} - for dirName, dirDict in res['Value']['Successful'].iteritems(): + for dirName, dirDict in res['Value']['Successful'].items(): mode = dirDict['Mode'] p_uid = dirDict['UID'] p_gid = dirDict['GID'] @@ -980,14 +979,14 @@ def __getReplicasForIDs(self, fileIDLfnDict, allStatus, connection=False): fields = [] if not self.db.lfnPfnConvention or self.db.lfnPfnConvention == "Weak": fields = ['PFN'] - res = self._getFileReplicas(fileIDLfnDict.keys(), fields_input=fields, + res = self._getFileReplicas(list(fileIDLfnDict), fields_input=fields, allStatus=allStatus, connection=connection) if not res['OK']: return res - for fileID, seDict in res['Value'].iteritems(): + for fileID, seDict in res['Value'].items(): lfn = fileIDLfnDict[fileID] replicas[lfn] = {} - for se, repDict in seDict.iteritems(): + for se, repDict in seDict.items(): pfn = repDict.get('PFN', '') replicas[lfn][se] = pfn @@ -1004,7 +1003,7 @@ def getReplicas(self, lfns, allStatus, connection=False): return res failed = res['Value']['Failed'] fileIDLFNs = {} - for lfn, fileID in res['Value']['Successful'].iteritems(): + for lfn, fileID in res['Value']['Successful'].items(): fileIDLFNs[fileID] = lfn result = self.__getReplicasForIDs(fileIDLFNs, allStatus, connection) @@ -1040,15 +1039,15 @@ def getReplicaStatus(self, lfns, connection=False): return res failed = res['Value']['Failed'] fileIDLFNs = {} - for lfn, fileDict in res['Value']['Successful'].iteritems(): + for lfn, fileDict in res['Value']['Successful'].items(): fileID = fileDict['FileID'] fileIDLFNs[fileID] = lfn successful = {} if fileIDLFNs: - res = self._getFileReplicas(fileIDLFNs.keys(), allStatus=True, connection=connection) + res = self._getFileReplicas(list(fileIDLFNs), allStatus=True, connection=connection) if not res['OK']: return res - for fileID, seDict in res['Value'].iteritems(): + for fileID, seDict in res['Value'].items(): lfn = fileIDLFNs[fileID] requestedSE = lfns[lfn] if not requestedSE: @@ -1118,16 +1117,16 @@ def getFilesInDirectory(self, dirID, verbose=False, connection=False): if not res['Value']: return S_OK(files) fileIDNames = {} - for fileName, fileDict in res['Value'].iteritems(): + for fileName, fileDict in res['Value'].items(): files[fileName] = {} files[fileName]['MetaData'] = fileDict fileIDNames[fileDict['FileID']] = fileName if verbose: - result = self._getFileReplicas(fileIDNames.keys(), connection=connection) + result = self._getFileReplicas(list(fileIDNames), connection=connection) if not result['OK']: return result - for fileID, seDict in result['Value'].iteritems(): + for fileID, seDict in result['Value'].items(): fileName = fileIDNames[fileID] files[fileName]['Replicas'] = seDict diff --git a/DataManagementSystem/DB/FileCatalogComponents/FileManager/FileManagerFlat.py b/DataManagementSystem/DB/FileCatalogComponents/FileManager/FileManagerFlat.py index 6b802f8726a..ad9bb135a1a 100755 --- a/DataManagementSystem/DB/FileCatalogComponents/FileManager/FileManagerFlat.py +++ b/DataManagementSystem/DB/FileCatalogComponents/FileManager/FileManagerFlat.py @@ -41,7 +41,7 @@ def _findFiles(self, lfns, metadata=['FileID'], connection=False): for fileName in fileNames: failed['%s/%s' % (dirPath, fileName)] = error else: - for fileName, fileDict in res['Value'].iteritems(): + for fileName, fileDict in res['Value'].items(): successful["%s/%s" % (dirPath, fileName)] = fileDict return S_OK({"Successful": successful, "Failed": failed}) @@ -103,14 +103,14 @@ def _insertFiles(self, lfns, uid, gid, connection=False): if not res['OK']: return res # Get the fileIDs for the inserted files - res = self._findFiles(lfns.keys(), ['FileID'], connection=connection) + res = self._findFiles(list(lfns), ['FileID'], connection=connection) if not res['OK']: - for lfn in lfns.keys(): + for lfn in list(lfns): failed[lfn] = 'Failed post insert check' lfns.pop(lfn) else: failed.update(res['Value']['Failed']) - for lfn, fileDict in res['Value']['Successful'].iteritems(): + for lfn, fileDict in res['Value']['Successful'].items(): lfns[lfn]['FileID'] = fileDict['FileID'] return S_OK({'Successful': lfns, 'Failed': failed}) @@ -245,12 +245,12 @@ def __existsReplica(self, fileID, seID, connection=False): def _deleteReplicas(self, lfns, connection=False): connection = self._getConnection(connection) successful = {} - res = self._findFiles(lfns.keys(), ['DirID', 'FileID', 'Size'], connection=connection) + res = self._findFiles(list(lfns), ['DirID', 'FileID', 'Size'], connection=connection) failed = res['Value']['Failed'] lfnFileIDDict = res['Value']['Successful'] toRemove = [] directorySESizeDict = {} - for lfn, fileDict in lfnFileIDDict.iteritems(): + for lfn, fileDict in lfnFileIDDict.items(): fileID = fileDict['FileID'] se = lfns[lfn]['SE'] toRemove.append((fileID, se)) diff --git a/DataManagementSystem/DB/FileCatalogComponents/FileManager/FileManagerPs.py b/DataManagementSystem/DB/FileCatalogComponents/FileManager/FileManagerPs.py index 0d18eb1542b..5ae9778fab6 100755 --- a/DataManagementSystem/DB/FileCatalogComponents/FileManager/FileManagerPs.py +++ b/DataManagementSystem/DB/FileCatalogComponents/FileManager/FileManagerPs.py @@ -43,7 +43,7 @@ def _findFiles(self, lfns, metadata=['FileID'], allStatus=False, connection=Fals connection = self._getConnection(connection) dirDict = self._getFileDirectories(lfns) - result = self.db.dtree.findDirs(dirDict.keys()) + result = self.db.dtree.findDirs(list(dirDict)) if not result['OK']: return result @@ -56,7 +56,7 @@ def _findFiles(self, lfns, metadata=['FileID'], allStatus=False, connection=Fals res = self._getDirectoryFiles(directoryIDs[dirPath], fileNames, metadata, allStatus=allStatus, connection=connection) - for fileName, fileDict in res.get('Value', {}).iteritems(): + for fileName, fileDict in res.get('Value', {}).items(): fname = os.path.join(dirPath, fileName) successful[fname] = fileDict @@ -95,7 +95,7 @@ def _findFileIDs(self, lfns, connection=False): filesInDirDict = self._getFileDirectories(lfns) # We get the directory ids - result = self.db.dtree.findDirs(filesInDirDict.keys()) + result = self.db.dtree.findDirs(list(filesInDirDict)) if not result['OK']: return result directoryPathToIds = result['Value'] @@ -234,7 +234,7 @@ def __insertMultipleFiles(self, allFileValues, wantedLfns): def __chunks(self, l, n): """ Yield successive n-sized chunks from l. """ - for i in xrange(0, len(l), n): + for i in range(0, len(l), n): yield l[i:i + n] def _insertFiles(self, lfns, uid, gid, connection=False): @@ -617,19 +617,19 @@ def _deleteReplicas(self, lfns, connection=False): failed = {} successful = {} # First we get the fileIds from our lfns - res = self._findFiles(lfns.keys(), ['FileID'], connection=connection) + res = self._findFiles(list(lfns), ['FileID'], connection=connection) if not res['OK']: return res # If the file does not exist we consider the deletion successful - for lfn, error in res['Value']['Failed'].iteritems(): + for lfn, error in res['Value']['Failed'].items(): if error == 'No such file or directory': successful[lfn] = True else: failed[lfn] = error lfnFileIDDict = res['Value']['Successful'] - for lfn, fileDict in lfnFileIDDict.iteritems(): + for lfn, fileDict in lfnFileIDDict.items(): fileID = fileDict['FileID'] # Then we get our StorageElement Id (cached in seManager) diff --git a/DataManagementSystem/DB/FileCatalogComponents/FileMetadata/FileMetadata.py b/DataManagementSystem/DB/FileCatalogComponents/FileMetadata/FileMetadata.py index 102c4cd5605..5b37c87637f 100644 --- a/DataManagementSystem/DB/FileCatalogComponents/FileMetadata/FileMetadata.py +++ b/DataManagementSystem/DB/FileCatalogComponents/FileMetadata/FileMetadata.py @@ -150,7 +150,7 @@ def setMetadata(self, path, metaDict, credDict): else: return S_ERROR('File %s not found' % path) - for metaName, metaValue in metaDict.iteritems(): + for metaName, metaValue in metaDict.items(): if metaName not in metaFields: result = self.__setFileMetaParameter(fileID, metaName, metaValue, credDict) else: @@ -457,7 +457,7 @@ def __createMetaSelection(self, value): query = '( $s )' % ', '.join(result['Value']) queryList.append(('IN', query)) elif isinstance(value, dict): - for operation, operand in value.iteritems(): + for operation, operand in value.items(): # Prepare the escaped operand first if isinstance(operand, list): @@ -526,7 +526,7 @@ def __buildUserMetaQuery(self, userMetaDict): return S_OK([]) resultList = [] leftJoinTables = [] - for meta, value in userMetaDict.iteritems(): + for meta, value in userMetaDict.items(): table = 'FC_FileMeta_%s' % meta result = self.__createMetaSelection(value) @@ -552,7 +552,7 @@ def __buildStandardMetaQuery(self, standardMetaDict): table = 'FC_Files' queriesFiles = [] queriesFileInfo = [] - for infield, invalue in standardMetaDict.iteritems(): + for infield, invalue in standardMetaDict.items(): value = invalue if infield in FILES_TABLE_METAKEYS: if infield == 'User': @@ -608,7 +608,7 @@ def __findFilesByMetadata(self, metaDict, dirList, credDict): standardMetaDict = {} userMetaDict = {} leftJoinTables = [] - for meta, value in metaDict.iteritems(): + for meta, value in metaDict.items(): if meta == "SE": if isinstance(value, dict): storageElements = value.get('in', []) @@ -710,7 +710,7 @@ def findFilesByMetadata(self, metaDict, path, credDict): if not result['OK']: return result fileMetaKeys = list(result['Value']) + list(FILE_STANDARD_METAKEYS) - fileMetaDict = dict(item for item in metaDict.iteritems() if item[0] in fileMetaKeys) + fileMetaDict = dict(item for item in metaDict.items() if item[0] in fileMetaKeys) fileList = [] idLfnDict = {} diff --git a/DataManagementSystem/DB/FileCatalogComponents/SEManager/SEManagerDB.py b/DataManagementSystem/DB/FileCatalogComponents/SEManager/SEManagerDB.py index 9531ee44ae6..e63990b1be1 100644 --- a/DataManagementSystem/DB/FileCatalogComponents/SEManager/SEManagerDB.py +++ b/DataManagementSystem/DB/FileCatalogComponents/SEManager/SEManagerDB.py @@ -39,7 +39,7 @@ def _refreshSEs(self, connection=False): self.lock.release() return S_OK() - for seName, seId in list(self.db.seNames.iteritems()): + for seName, seId in list(self.db.seNames.items()): if seName not in seNames: del self.db.seNames[seName] del self.db.seids[seId] diff --git a/DataManagementSystem/DB/FileCatalogComponents/SecurityManager/DirectorySecurityManager.py b/DataManagementSystem/DB/FileCatalogComponents/SecurityManager/DirectorySecurityManager.py index 6c1e5e9fa48..314159c7e89 100644 --- a/DataManagementSystem/DB/FileCatalogComponents/SecurityManager/DirectorySecurityManager.py +++ b/DataManagementSystem/DB/FileCatalogComponents/SecurityManager/DirectorySecurityManager.py @@ -22,19 +22,19 @@ def getPathPermissions(self, paths, credDict): permissions = {} failed = {} while toGet: - res = self.db.dtree.getPathPermissions(toGet.keys(), credDict) + res = self.db.dtree.getPathPermissions(list(toGet), credDict) if not res['OK']: return res - for path, mode in list(res['Value']['Successful'].iteritems()): + for path, mode in list(res['Value']['Successful'].items()): for resolvedPath in toGet[path]: permissions[resolvedPath] = mode toGet.pop(path) - for path, error in list(res['Value']['Failed'].iteritems()): + for path, error in list(res['Value']['Failed'].items()): if error != 'No such file or directory': for resolvedPath in toGet[path]: failed[resolvedPath] = error toGet.pop(path) - for path, resolvedPaths in list(toGet.iteritems()): + for path, resolvedPaths in list(toGet.items()): if path == '/': for resolvedPath in resolvedPaths: permissions[path] = {'Read': True, 'Write': True, 'Execute': True} diff --git a/DataManagementSystem/DB/FileCatalogComponents/SecurityManager/FullSecurityManager.py b/DataManagementSystem/DB/FileCatalogComponents/SecurityManager/FullSecurityManager.py index 13e60224466..9e648228861 100644 --- a/DataManagementSystem/DB/FileCatalogComponents/SecurityManager/FullSecurityManager.py +++ b/DataManagementSystem/DB/FileCatalogComponents/SecurityManager/FullSecurityManager.py @@ -24,12 +24,12 @@ def getPathPermissions(self, paths, credDict): res = self.db.fileManager.getPathPermissions(paths, credDict) if not res['OK']: return res - for path, mode in res['Value']['Successful'].iteritems(): + for path, mode in res['Value']['Successful'].items(): for resolvedPath in toGet[path]: permissions[resolvedPath] = mode toGet.pop(path) # Copying items because toGet is changed in the cycle - for path, resolvedPaths in list(toGet.iteritems()): + for path, resolvedPaths in list(toGet.items()): if path == '/': for resolvedPath in resolvedPaths: permissions[path] = {'Read': True, 'Write': True, 'Execute': True} @@ -38,21 +38,20 @@ def getPathPermissions(self, paths, credDict): toGet[os.path.dirname(path)] += resolvedPaths toGet.pop(path) while toGet: - paths = toGet.keys() - res = self.db.dtree.getPathPermissions(paths, credDict) + res = self.db.dtree.getPathPermissions(list(toGet), credDict) if not res['OK']: return res - for path, mode in res['Value']['Successful'].iteritems(): + for path, mode in res['Value']['Successful'].items(): for resolvedPath in toGet[path]: permissions[resolvedPath] = mode toGet.pop(path) - for path, error in res['Value']['Failed'].iteritems(): + for path, error in res['Value']['Failed'].items(): if error != 'No such file or directory': for resolvedPath in toGet[path]: failed[resolvedPath] = error toGet.pop(path) # Copying items because toGet is changed in the cycle - for path, resolvedPaths in list(toGet.iteritems()): + for path, resolvedPaths in list(toGet.items()): if path == '/': for resolvedPath in resolvedPaths: permissions[path] = {'Read': True, 'Write': True, 'Execute': True} diff --git a/DataManagementSystem/DB/FileCatalogComponents/SecurityManager/SecurityManagerBase.py b/DataManagementSystem/DB/FileCatalogComponents/SecurityManager/SecurityManagerBase.py index 3a7cea22cf4..47b4d988d37 100644 --- a/DataManagementSystem/DB/FileCatalogComponents/SecurityManager/SecurityManagerBase.py +++ b/DataManagementSystem/DB/FileCatalogComponents/SecurityManager/SecurityManagerBase.py @@ -67,7 +67,7 @@ def hasAccess(self, opType, paths, credDict): return result permissions = result['Value']['Successful'] - for path, permDict in permissions.iteritems(): + for path, permDict in permissions.items(): if permDict[opType]: successful[path] = True else: diff --git a/DataManagementSystem/DB/FileCatalogComponents/SecurityManager/test/Test_VOMSSecurityManager.py b/DataManagementSystem/DB/FileCatalogComponents/SecurityManager/test/Test_VOMSSecurityManager.py index 336106f9dff..34f5e78c719 100644 --- a/DataManagementSystem/DB/FileCatalogComponents/SecurityManager/test/Test_VOMSSecurityManager.py +++ b/DataManagementSystem/DB/FileCatalogComponents/SecurityManager/test/Test_VOMSSecurityManager.py @@ -6,7 +6,6 @@ # pylint: disable=protected-access,missing-docstring,invalid-name,too-many-lines -from types import ListType import unittest import stat @@ -146,7 +145,7 @@ def exists(self, lfns): return S_OK({'Successful': dict((lfn, lfn in fileTree) for lfn in lfns), 'Failed': {}}) def getFileMetadata(self, lfns): - if not isinstance(lfns, ListType): + if not isinstance(lfns, list): lfns = [lfns] retParam = ['Size', 'Checksum', 'ChecksumType', 'UID', @@ -170,7 +169,7 @@ def getFileMetadata(self, lfns): return S_OK({'Successful': successful, 'Failed': failed}) def getPathPermissions(self, lfns, credDict): - if not isinstance(lfns, ListType): + if not isinstance(lfns, list): lfns = [lfns] successful = {} @@ -282,7 +281,7 @@ def callForDirectories(self, methodName): against all the existing and non existing directories """ - self.existingRet = self.securityManager.hasAccess(methodName, directoryTree.keys(), self.credDict) + self.existingRet = self.securityManager.hasAccess(methodName, list(directoryTree), self.credDict) self.nonExistingRet = self.securityManager.hasAccess(methodName, nonExistingDirectories, self.credDict) def callForFiles(self, methodName): @@ -290,7 +289,7 @@ def callForFiles(self, methodName): against all the existing and non existing files """ - self.existingRet = self.securityManager.hasAccess(methodName, fileTree.keys(), self.credDict) + self.existingRet = self.securityManager.hasAccess(methodName, list(fileTree), self.credDict) self.nonExistingRet = self.securityManager.hasAccess(methodName, nonExistingFiles, self.credDict) def compareResult(self): diff --git a/DataManagementSystem/DB/FileCatalogDB.py b/DataManagementSystem/DB/FileCatalogDB.py index b0af46660c9..98f1965cfd4 100755 --- a/DataManagementSystem/DB/FileCatalogDB.py +++ b/DataManagementSystem/DB/FileCatalogDB.py @@ -791,13 +791,13 @@ def getFileDetails(self, lfnList, credDict): for lfn in lfnDict: fileIDDict[lfnDict[lfn]['FileID']] = lfn - result = self.fileManager._getFileMetadataByID(fileIDDict.keys(), connection=connection) + result = self.fileManager._getFileMetadataByID(list(fileIDDict), connection=connection) if not result['OK']: return result for fileID in result['Value']: resultDict[fileIDDict[fileID]] = result['Value'][fileID] - result = self.fmeta._getFileUserMetadataByID(fileIDDict.keys(), credDict, connection=connection) + result = self.fmeta._getFileUserMetadataByID(list(fileIDDict), credDict, connection=connection) if not result['OK']: return result for fileID in fileIDDict: @@ -1068,7 +1068,7 @@ def setMetadataBulk(self, pathMetadataDict, credDict): """ successful = {} failed = {} - for path, metadataDict in pathMetadataDict.iteritems(): + for path, metadataDict in pathMetadataDict.items(): result = self.setMetadata(path, metadataDict, credDict) if result['OK']: successful[path] = True @@ -1082,7 +1082,7 @@ def removeMetadata(self, pathMetadataDict, credDict): """ successful = {} failed = {} - for path, metadataDict in pathMetadataDict.iteritems(): + for path, metadataDict in pathMetadataDict.items(): result = self.__removeMetadata(path, metadataDict, credDict) if result['OK']: successful[path] = True @@ -1159,16 +1159,16 @@ def _checkPathPermissions(self, operation, lfns, credDict): return res lfns = res['Value'] - res = self.securityManager.hasAccess(operation, lfns.keys(), credDict) + res = self.securityManager.hasAccess(operation, list(lfns), credDict) if not res['OK']: return res # Do not consider those paths for which we failed to determine access failed = res['Value']['Failed'] - for lfn in failed.keys(): + for lfn in failed: lfns.pop(lfn) # Do not consider those paths for which access is denied successful = {} - for lfn, access in res['Value']['Successful'].iteritems(): + for lfn, access in res['Value']['Successful'].items(): if not access: failed[lfn] = 'Permission denied' else: diff --git a/DataManagementSystem/Service/FileCatalogHandler.py b/DataManagementSystem/Service/FileCatalogHandler.py index ed15f22e31c..0d1123e7983 100644 --- a/DataManagementSystem/Service/FileCatalogHandler.py +++ b/DataManagementSystem/Service/FileCatalogHandler.py @@ -16,7 +16,7 @@ # imports import six -import cStringIO +from six import StringIO import csv import os from types import IntType, LongType, DictType, StringTypes, BooleanType, ListType @@ -759,7 +759,7 @@ def transfer_toClient(self, seName, token, fileHelper): retVal = self.getSEDump(seName) try: - csvOutput = cStringIO.StringIO() + csvOutput = StringIO() writer = csv.writer(csvOutput, delimiter='|') for lfn in retVal: writer.writerow(lfn) diff --git a/DataManagementSystem/Service/StorageElementHandler.py b/DataManagementSystem/Service/StorageElementHandler.py index 81790b9141e..fee5d75bbaa 100755 --- a/DataManagementSystem/Service/StorageElementHandler.py +++ b/DataManagementSystem/Service/StorageElementHandler.py @@ -34,6 +34,7 @@ import re import errno import shlex +import six # from DIRAC from DIRAC import gLogger, S_OK, S_ERROR @@ -234,7 +235,7 @@ def __getFileStat(path): return S_OK(resultDict) - types_exists = [basestring] + types_exists = [six.string_types] def export_exists(self, fileID): """ Check existence of the fileID """ @@ -242,7 +243,7 @@ def export_exists(self, fileID): return S_OK(True) return S_OK(False) - types_getMetadata = [basestring] + types_getMetadata = [six.string_types] def export_getMetadata(self, fileID): """ Get metadata for the file or directory specified by fileID @@ -269,7 +270,7 @@ def export_getTotalDiskSpace(): """ return getTotalDiskSpace() - types_createDirectory = [basestring] + types_createDirectory = [six.string_types] def export_createDirectory(self, dir_path): """ Creates the directory on the storage @@ -292,7 +293,7 @@ def export_createDirectory(self, dir_path): gLogger.error("StorageElementHandler.createDirectory: %s" % errStr, repr(x)) return S_ERROR(errStr) - types_listDirectory = [basestring, basestring] + types_listDirectory = [six.string_types, six.string_types] def export_listDirectory(self, dir_path, mode): """ Return the dir_path directory listing @@ -430,7 +431,7 @@ def transfer_bulkToClient(self, fileId, token, fileHelper): gLogger.error('Failed to send bulk to network', res['Message']) return res - types_remove = [basestring, basestring] + types_remove = [six.string_types, six.string_types] def export_remove(self, fileID, token): """ Remove fileID from the storage. token is used for access rights confirmation. """ @@ -452,7 +453,7 @@ def __removeFile(self, fileID, token): else: return S_ERROR('File removal %s not authorized' % fileID) - types_getDirectorySize = [basestring] + types_getDirectorySize = [six.string_types] def export_getDirectorySize(self, fileID): """ Get the size occupied by the given directory @@ -468,7 +469,7 @@ def export_getDirectorySize(self, fileID): else: return S_ERROR("Directory does not exists") - types_removeDirectory = [basestring, basestring] + types_removeDirectory = [six.string_types, six.string_types] def export_removeDirectory(self, fileID, token): """ Remove the given directory from the storage @@ -489,7 +490,7 @@ def export_removeDirectory(self, fileID, token): gLogger.error(str(error)) return S_ERROR("Failed to remove directory %s" % dir_path) - types_removeFileList = [list, basestring] + types_removeFileList = [list, six.string_types] def export_removeFileList(self, fileList, token): """ Remove files in the given list diff --git a/DataManagementSystem/Utilities/lfc_dfc_copy.py b/DataManagementSystem/Utilities/lfc_dfc_copy.py index 824cc3ed274..862480694f4 100644 --- a/DataManagementSystem/Utilities/lfc_dfc_copy.py +++ b/DataManagementSystem/Utilities/lfc_dfc_copy.py @@ -152,7 +152,7 @@ def processDir(initPath, writerQueue, recursive=False, host=None, fcInit=None, d lfns[lfn]['Owner'] = owner if info['Replicas']: - seList = info['Replicas'].keys() + seList = list(info['Replicas']) lfns[lfn]['SE'] = seList p_files = time.time() - s @@ -226,7 +226,7 @@ def processDir(initPath, writerQueue, recursive=False, host=None, fcInit=None, d resultDict['NumberOfDirectories'] = nDir resultDict['NumberOfReplicas'] = nRep resultDict['Path'] = initPath - resultDict['Directories'] = dirDict.keys() + resultDict['Directories'] = list(dirDict) # print "AT >>> processDir",initPath,"done %.2f" % (time.time()-start) diff --git a/DataManagementSystem/private/FTS3Utilities.py b/DataManagementSystem/private/FTS3Utilities.py index 64d9ee28828..57fc339d349 100644 --- a/DataManagementSystem/private/FTS3Utilities.py +++ b/DataManagementSystem/private/FTS3Utilities.py @@ -121,7 +121,7 @@ def __init__(self, serverDict, serverPolicy="Random"): self.log = gLogger.getSubLogger("FTS3ServerPolicy") self._serverDict = serverDict - self._serverList = serverDict.keys() + self._serverList = list(serverDict) self._maxAttempts = len(self._serverList) self._nextServerID = 0 self._resourceStatus = ResourceStatus() diff --git a/DataManagementSystem/private/test/Test_FTS3Utilities.py b/DataManagementSystem/private/test/Test_FTS3Utilities.py index c167750e538..33ced557a44 100644 --- a/DataManagementSystem/private/test/Test_FTS3Utilities.py +++ b/DataManagementSystem/private/test/Test_FTS3Utilities.py @@ -88,7 +88,7 @@ def test_04_selectUniqueRandomSource(self, _mk_checkSourceReplicas): # There should be only f1,f2 and f3 allReturnedFiles = [] existingFiles = [self.f1, self.f2, self.f3] - for _srcSe, ftsFiles in uniqueSources.iteritems(): + for _srcSe, ftsFiles in uniqueSources.items(): allReturnedFiles.extend(ftsFiles) # No files should be duplicated and all files should be there, except the non existing one diff --git a/DataManagementSystem/scripts/dirac-admin-allow-se.py b/DataManagementSystem/scripts/dirac-admin-allow-se.py index f2d647a5ec0..2aa141a683a 100755 --- a/DataManagementSystem/scripts/dirac-admin-allow-se.py +++ b/DataManagementSystem/scripts/dirac-admin-allow-se.py @@ -121,7 +121,7 @@ reason = 'Forced with dirac-admin-allow-se by %s' % userName -for se, seOptions in res['Value'].iteritems(): +for se, seOptions in res['Value'].items(): # InActive is used on the CS model, Banned is the equivalent in RSS for statusType in STATUS_TYPES: diff --git a/DataManagementSystem/scripts/dirac-dms-create-archive-request.py b/DataManagementSystem/scripts/dirac-dms-create-archive-request.py index 38a6d72a3c5..85f97755344 100644 --- a/DataManagementSystem/scripts/dirac-dms-create-archive-request.py +++ b/DataManagementSystem/scripts/dirac-dms-create-archive-request.py @@ -293,7 +293,7 @@ def splitLFNsBySize(self): sLog.notice('Splitting files by Size') lfnChunk = [] totalSize = 0 - for lfn, info in self.metaData['Successful'].iteritems(): + for lfn, info in self.metaData['Successful'].items(): if (totalSize > self.switches['MaxSize'] or len(lfnChunk) >= self.switches['MaxFiles']): self.lfnChunks.append(lfnChunk) sLog.notice('Created Chunk of %s lfns with %s bytes' % (len(lfnChunk), totalSize)) @@ -486,14 +486,14 @@ def _checkReplicaSites(self, request, lfnChunk): notAt = [] failed = [] sourceSE = self.sourceSEs[0] - for lfn, replInfo in resReplica['Value']['Successful'].iteritems(): + for lfn, replInfo in resReplica['Value']['Successful'].items(): if sourceSE in replInfo: atSource.append(lfn) else: sLog.notice('WARN: LFN %r not found at source, only at: %s' % (lfn, ','.join(replInfo.keys()))) notAt.append(lfn) - for lfn, errorMessage in resReplica['Value']['Failed'].iteritems(): + for lfn, errorMessage in resReplica['Value']['Failed'].items(): sLog.error('Failed to get replica info', '%s: %s' % (lfn, errorMessage)) failed.append(lfn) diff --git a/DataManagementSystem/scripts/dirac-dms-create-removal-request.py b/DataManagementSystem/scripts/dirac-dms-create-removal-request.py index ba864656056..f85b8e8f6f2 100755 --- a/DataManagementSystem/scripts/dirac-dms-create-removal-request.py +++ b/DataManagementSystem/scripts/dirac-dms-create-removal-request.py @@ -65,7 +65,10 @@ for lfnList in breakListIntoChunks(lfns, 100): oRequest = Request() - requestName = "%s_%s" % (md5(repr(time.time())).hexdigest()[:16], md5(repr(time.time())).hexdigest()[:16]) + requestName = "%s_%s" % ( + md5(repr(time.time()).encode()).hexdigest()[:16], + md5(repr(time.time()).encode()).hexdigest()[:16], + ) oRequest.RequestName = requestName oOperation = Operation() diff --git a/DataManagementSystem/scripts/dirac-dms-move-replica-request.py b/DataManagementSystem/scripts/dirac-dms-move-replica-request.py index 129d86527af..9fa7de43fe2 100755 --- a/DataManagementSystem/scripts/dirac-dms-move-replica-request.py +++ b/DataManagementSystem/scripts/dirac-dms-move-replica-request.py @@ -90,7 +90,10 @@ def getLFNList(arg): count += 1 request = Request() - request.RequestName = "%s_%s" % (md5(repr(time.time())).hexdigest()[:16], md5(repr(time.time())).hexdigest()[:16]) + request.RequestName = "%s_%s" % ( + md5(repr(time.time()).encode()).hexdigest()[:16], + md5(repr(time.time()).encode()).hexdigest()[:16], + ) moveReplica = Operation() moveReplica.Type = 'MoveReplica' diff --git a/DataManagementSystem/scripts/dirac-dms-remove-files.py b/DataManagementSystem/scripts/dirac-dms-remove-files.py index b6bdc9262bf..c0edd1a9aa6 100755 --- a/DataManagementSystem/scripts/dirac-dms-remove-files.py +++ b/DataManagementSystem/scripts/dirac-dms-remove-files.py @@ -47,10 +47,10 @@ DIRAC.exit(-2) for lfn, r in res['Value']['Failed'].items(): reason = str(r) - if reason not in errorReasons.keys(): + if reason not in errorReasons: errorReasons[reason] = [] errorReasons[reason].append(lfn) - successfullyRemoved += len(res['Value']['Successful'].keys()) + successfullyRemoved += len(res['Value']['Successful']) for reason, lfns in errorReasons.items(): gLogger.notice("Failed to remove %d files with error: %s" % (len(lfns), reason)) diff --git a/FrameworkSystem/Client/BundleDeliveryClient.py b/FrameworkSystem/Client/BundleDeliveryClient.py index 4fb1d6d26f8..34b1ae103cb 100644 --- a/FrameworkSystem/Client/BundleDeliveryClient.py +++ b/FrameworkSystem/Client/BundleDeliveryClient.py @@ -7,7 +7,7 @@ import os import io import tarfile -import cStringIO +from six import BytesIO from DIRAC import S_OK, gLogger from DIRAC.Core.Base.Client import Client, createClient @@ -59,7 +59,7 @@ def syncDir(self, bundleID, dirToSyncTo): dirCreated = True currentHash = self.__getHash(bundleID, dirToSyncTo) self.log.info("Current hash for bundle %s in dir %s is '%s'" % (bundleID, dirToSyncTo, currentHash)) - buff = cStringIO.StringIO() + buff = BytesIO() transferClient = self.__getTransferClient() result = transferClient.receiveFile(buff, [bundleID, currentHash]) if not result['OK']: diff --git a/FrameworkSystem/Client/MonitoringClient.py b/FrameworkSystem/Client/MonitoringClient.py index d3e3f5885fc..857e40b5ad7 100755 --- a/FrameworkSystem/Client/MonitoringClient.py +++ b/FrameworkSystem/Client/MonitoringClient.py @@ -397,7 +397,7 @@ def __pruneMarksData(self): for acName in self.marksToSend: maxBuckets = 86400 / self.activitiesDefinitions[acName]['bucketLength'] if len(self.marksToSend[acName]) > maxBuckets: - timeSlots = sorted(self.marksToSend[acName].keys()) + timeSlots = sorted(self.marksToSend[acName]) while len(self.marksToSend[acName]) > maxBuckets: del(self.marksToSend[acName][timeSlots.pop(0)]) @@ -472,7 +472,7 @@ def __filterComponent(self, component, condDict): return False condVal = condDict[key] componentVal = component[key] - if type(condVal) in (list, tuple): + if isinstance(condVal, (list, tuple)): if componentVal not in condVal: return False else: diff --git a/FrameworkSystem/Client/ProxyManagerClient.py b/FrameworkSystem/Client/ProxyManagerClient.py index fa881f6e994..a967083f784 100755 --- a/FrameworkSystem/Client/ProxyManagerClient.py +++ b/FrameworkSystem/Client/ProxyManagerClient.py @@ -25,9 +25,8 @@ gVOMSProxiesSync = ThreadSafe.Synchronizer() +@six.add_metaclass(DIRACSingleton.DIRACSingleton) class ProxyManagerClient(object): - __metaclass__ = DIRACSingleton.DIRACSingleton - def __init__(self): self.__usersCache = DictCache() self.__proxiesCache = DictCache() diff --git a/FrameworkSystem/Client/SystemAdministratorClientCLI.py b/FrameworkSystem/Client/SystemAdministratorClientCLI.py index 7509dff1780..606d3f33dcb 100644 --- a/FrameworkSystem/Client/SystemAdministratorClientCLI.py +++ b/FrameworkSystem/Client/SystemAdministratorClientCLI.py @@ -202,7 +202,7 @@ def do_show(self, args): rDict = result['Value'] for compType in rDict: for system in rDict[compType]: - components = sorted(rDict[compType][system].keys()) + components = sorted(rDict[compType][system]) for component in components: record = [] if rDict[compType][system][component]['Installed']: @@ -283,7 +283,7 @@ def do_show(self, args): fields = ['Parameter', 'Value'] records = [] - for parameter in result['Value'].iteritems(): + for parameter in result['Value'].items(): if parameter[0] == 'Extension': extensions = parameter[1].split(',') for extension in extensions: @@ -399,12 +399,12 @@ def do_show(self, args): if result['OK']: text = '' - headers = [result['Value'][0].keys()] + headers = [list(result['Value'][0])] for header in headers: text += str(header).ljust(15) gLogger.notice(text) for record in result['Value']: - for metric in record.itervalues(): + for metric in record.values(): text += str(metric).ljust(15) gLogger.notice(text) else: @@ -1274,7 +1274,7 @@ def do_showall(self, args): if componentType and componentType != compType: continue for system in rDict[compType]: - components = sorted(rDict[compType][system].keys()) + components = sorted(rDict[compType][system]) for component in components: if componentName and componentName not in component: continue diff --git a/FrameworkSystem/Client/UserProfileClient.py b/FrameworkSystem/Client/UserProfileClient.py index 967321b0256..9a010a9b621 100644 --- a/FrameworkSystem/Client/UserProfileClient.py +++ b/FrameworkSystem/Client/UserProfileClient.py @@ -31,7 +31,7 @@ def __generateTypeDest(self, dataObj): if isinstance(dataObj, six.string_types): return "s" # Not even trying here... - if type(dataObj) in Time._allTypes: + if isinstance(dataObj, Time._allTypes): return "t" if isinstance(dataObj, (list, tuple)): return "l%se" % "".join([self.__generateTypeDest(so) for so in dataObj]) diff --git a/FrameworkSystem/DB/ComponentMonitoringDB.py b/FrameworkSystem/DB/ComponentMonitoringDB.py index ecc57f0cd5a..cd803709489 100755 --- a/FrameworkSystem/DB/ComponentMonitoringDB.py +++ b/FrameworkSystem/DB/ComponentMonitoringDB.py @@ -125,7 +125,7 @@ def __registerIfNotThere(self, compDict): if 'startTime' in compDict: sqlInsertFields.append("StartTime") val = compDict['startTime'] - if type(val) in Time._allDateTypes: + if isinstance(val, Time._allDateTypes): val = self.__datetime2str(val) sqlInsertValues.append("'%s'" % val) for field in ('cycles', 'queries'): diff --git a/FrameworkSystem/DB/InstalledComponentsDB.py b/FrameworkSystem/DB/InstalledComponentsDB.py index 85120622e39..7e7161b97aa 100644 --- a/FrameworkSystem/DB/InstalledComponentsDB.py +++ b/FrameworkSystem/DB/InstalledComponentsDB.py @@ -271,7 +271,7 @@ def __init__(self, host=null(), **kwargs): self.hostName = host fields = dir(self) - for key, value in kwargs.iteritems(): + for key, value in kwargs.items(): if key in fields and not re.match('_.*', key): setattr(self, key, value) @@ -285,7 +285,7 @@ def fromDict(self, dictionary): dictionary['DIRAC'] = dictionary.get('DIRACVersion') try: - for key, value in dictionary.iteritems(): + for key, value in dictionary.items(): if key in fields and not re.match('_.*', key): setattr(self, key, value) except Exception as e: @@ -489,17 +489,17 @@ def __filterInstalledComponentsFields(self, session, matchFields=None): matchFields = {} componentKeys = {} - for (key, val) in matchFields.iteritems(): + for (key, val) in matchFields.items(): if 'Component.' in key: componentKeys[key.replace('Component.', '')] = val hostKeys = {} - for (key, val) in matchFields.iteritems(): + for (key, val) in matchFields.items(): if 'Host.' in key: hostKeys[key.replace('Host.', '')] = val selfKeys = {} - for (key, val) in matchFields.iteritems(): + for (key, val) in matchFields.items(): if 'Component.' not in key and 'Host.' not in key: selfKeys[key] = val diff --git a/FrameworkSystem/DB/NotificationDB.py b/FrameworkSystem/DB/NotificationDB.py index f5bf57498e1..e602c5ac305 100644 --- a/FrameworkSystem/DB/NotificationDB.py +++ b/FrameworkSystem/DB/NotificationDB.py @@ -411,7 +411,7 @@ def __generateAlarmLogMessage(self, alarmLog, showOnlyLast=False): msg.append(" On : %s" % data['timestamp'].strftime("%Y/%m/%d %H:%M:%S")) if 'modifications' in data: mods = data['modifications'] - keys = sorted(mods.keys()) + keys = sorted(mods) msg.append(" Modificaitons:") for key in keys: msg.append(" %s -> %s" % (key, mods[key])) diff --git a/FrameworkSystem/DB/ProxyDB.py b/FrameworkSystem/DB/ProxyDB.py index a3f7a506926..d4e1c727eb2 100755 --- a/FrameworkSystem/DB/ProxyDB.py +++ b/FrameworkSystem/DB/ProxyDB.py @@ -1232,7 +1232,7 @@ def generateToken(self, requesterDN, requesterGroup, numUses=1, lifeTime=0, retr numUses = max(1, min(numUses, maxUses)) m = hashlib.md5() rndData = "%s.%s.%s.%s" % (time.time(), random.random(), numUses, lifeTime) - m.update(rndData) + m.update(rndData.encode()) token = m.hexdigest() fieldsSQL = ", ".join(("Token", "RequesterDN", "RequesterGroup", "ExpirationTime", "UsesLeft")) valuesSQL = ", ".join((self._escapeString(token)['Value'], diff --git a/FrameworkSystem/DB/SystemLoggingDB.py b/FrameworkSystem/DB/SystemLoggingDB.py index 3d05ddd6d71..04cb5810208 100755 --- a/FrameworkSystem/DB/SystemLoggingDB.py +++ b/FrameworkSystem/DB/SystemLoggingDB.py @@ -189,7 +189,7 @@ def _checkTable(self): tablesInDB = [t[0] for t in retVal['Value']] tablesToCreate = {} - for tableName, tableDef in self.tableDict.iteritems(): + for tableName, tableDef in self.tableDict.items(): if tableName not in tablesInDB: tablesToCreate[tableName] = tableDef @@ -211,7 +211,7 @@ def __buildTableList(self, showFieldList): 'OwnerDN': 'UserDNs', 'OwnerGroup': 'UserDNs', 'ClientIPNumberString': 'ClientIPs', 'ClientFQDN': 'ClientIPs', 'SiteName': 'Sites'} - tableDictKeys = tableDict.keys() + tableDictKeys = list(tableDict) tableList = [] conjunction = ' NATURAL JOIN ' diff --git a/FrameworkSystem/DB/UserProfileDB.py b/FrameworkSystem/DB/UserProfileDB.py index 8a039b95ae6..39fb8d9e6ea 100755 --- a/FrameworkSystem/DB/UserProfileDB.py +++ b/FrameworkSystem/DB/UserProfileDB.py @@ -546,7 +546,7 @@ def storeHashTagById(self, userIds, tagName, hashTag=False): """ if not hashTag: hashTag = hashlib.md5() - hashTag.update("%s;%s;%s" % (Time.dateTime(), userIds, tagName)) + hashTag.update(("%s;%s;%s" % (Time.dateTime(), userIds, tagName)).encode()) hashTag = hashTag.hexdigest() result = self.insertFields('up_HashTags', ['UserId', 'GroupId', 'VOId', 'TagName', 'HashTag'], diff --git a/FrameworkSystem/Service/BundleDeliveryHandler.py b/FrameworkSystem/Service/BundleDeliveryHandler.py index 0f7be9b40ab..a3b02b84a90 100644 --- a/FrameworkSystem/Service/BundleDeliveryHandler.py +++ b/FrameworkSystem/Service/BundleDeliveryHandler.py @@ -8,7 +8,7 @@ __RCSID__ = "$Id$" import six -import cStringIO +from six import BytesIO import tarfile import os from DIRAC.Core.DISET.RequestHandler import RequestHandler @@ -72,7 +72,7 @@ def updateBundles(self): for bId in dirsToBundle: bundlePaths = dirsToBundle[bId] gLogger.info("Updating %s bundle %s" % (bId, bundlePaths)) - buffer_ = cStringIO.StringIO() + buffer_ = BytesIO() filesToBundle = sorted(File.getGlobbedFiles(bundlePaths)) if filesToBundle: commonPath = File.getCommonPath(filesToBundle) @@ -141,7 +141,7 @@ def transfer_toClient(self, fileId, token, fileHelper): fileHelper.markAsTransferred() return S_OK(bundleVersion) - buffer_ = cStringIO.StringIO(gBundleManager.getBundleData(bId)) + buffer_ = BytesIO(gBundleManager.getBundleData(bId)) result = fileHelper.DataSourceToNetwork(buffer_) buffer_.close() if not result['OK']: diff --git a/FrameworkSystem/Service/MonitoringHandler.py b/FrameworkSystem/Service/MonitoringHandler.py index 5ae7ffe28c3..ccca1fc117b 100755 --- a/FrameworkSystem/Service/MonitoringHandler.py +++ b/FrameworkSystem/Service/MonitoringHandler.py @@ -89,7 +89,7 @@ def export_commitMarks(self, sourceId, activitiesDict, componentExtraInfo={}): """ # we have to cast the dictionary keys, because they are timestamps - activitiesDict = {metric: strToIntDict(value) for metric, value in activitiesDict.iteritems()} + activitiesDict = {metric: strToIntDict(value) for metric, value in activitiesDict.items()} nowEpoch = Time.toEpoch() maxEpoch = nowEpoch + 7200 minEpoch = nowEpoch - 86400 diff --git a/FrameworkSystem/Service/PlottingHandler.py b/FrameworkSystem/Service/PlottingHandler.py index c4fea5a5df8..81657a4c02e 100644 --- a/FrameworkSystem/Service/PlottingHandler.py +++ b/FrameworkSystem/Service/PlottingHandler.py @@ -48,7 +48,11 @@ class PlottingHandler(RequestHandler): def __calculatePlotHash(self, data, metadata, subplotMetadata): m = hashlib.md5() - m.update(repr({'Data': data, 'PlotMetadata': metadata, 'SubplotMetadata': subplotMetadata})) + m.update(repr({ + 'Data': data, + 'PlotMetadata': metadata, + 'SubplotMetadata': subplotMetadata + }).encode()) return m.hexdigest() types_generatePlot = [[dict, list], dict] diff --git a/FrameworkSystem/Service/SystemAdministratorHandler.py b/FrameworkSystem/Service/SystemAdministratorHandler.py index 73e0f59b550..22fac5e7f33 100644 --- a/FrameworkSystem/Service/SystemAdministratorHandler.py +++ b/FrameworkSystem/Service/SystemAdministratorHandler.py @@ -11,7 +11,12 @@ import os import re import time -import commands +# TODO: This should be moderised to use subprocess(32) +try: + import commands +except ImportError: + # Python 3's subprocess module contains a compatability layer + import subprocess as commands import getpass import importlib import shutil @@ -528,7 +533,7 @@ def __readHostInfo(): summary = '' _status, output = commands.getstatusoutput('df') lines = output.split('\n') - for i in xrange(len(lines)): + for i in range(len(lines)): if lines[i].startswith('/dev'): fields = lines[i].split() if len(fields) == 1: @@ -566,7 +571,7 @@ def __readHostInfo(): result.update(infoResult['Value']) # the infoResult value is {"Extensions":{'a1':'v1',a2:'v2'}; we convert to a string result.update({"Extensions": ";".join(["%s:%s" % (key, value) - for (key, value) in infoResult["Value"].get('Extensions').iteritems()])}) + for (key, value) in infoResult["Value"].get('Extensions').items()])}) # Host certificate properties certFile, _keyFile = getHostCertificateAndKeyLocation() diff --git a/FrameworkSystem/Service/SystemLoggingReportHandler.py b/FrameworkSystem/Service/SystemLoggingReportHandler.py index 83847fd8fdb..b4aa4ffac01 100644 --- a/FrameworkSystem/Service/SystemLoggingReportHandler.py +++ b/FrameworkSystem/Service/SystemLoggingReportHandler.py @@ -76,11 +76,11 @@ def __getMessages(self, selectionDict={}, sortList=[], if 'groupField' in selectionDict: groupField = selectionDict['groupField'] if groupField not in selectionDict: - groupField = selectionDict.keys()[0] + groupField = list(selectionDict)[0] del selectionDict['groupField'] elif countMessages: if selectionDict: - groupField = selectionDict.keys()[0] + groupField = list(selectionDict)[0] elif sortList: groupField = sortList[0][0] else: @@ -89,7 +89,7 @@ def __getMessages(self, selectionDict={}, sortList=[], groupField = None if selectionDict: - fieldList = selectionDict.keys() + fieldList = list(selectionDict) fieldList.append(dateField) if not ('LogLevel' in selectionDict and selectionDict['LogLevel']): diff --git a/FrameworkSystem/private/SecurityFileLog.py b/FrameworkSystem/private/SecurityFileLog.py index 9dbcecc04c8..4af0945ca79 100644 --- a/FrameworkSystem/private/SecurityFileLog.py +++ b/FrameworkSystem/private/SecurityFileLog.py @@ -6,7 +6,7 @@ import re import time import gzip -import Queue +from six.moves import queue as Queue import threading from DIRAC import gLogger, S_OK, S_ERROR from DIRAC.Core.Utilities.ThreadScheduler import gThreadScheduler diff --git a/FrameworkSystem/private/monitoring/MonitoringCatalog.py b/FrameworkSystem/private/monitoring/MonitoringCatalog.py index 6942d1ea45d..51102ee5cd3 100755 --- a/FrameworkSystem/private/monitoring/MonitoringCatalog.py +++ b/FrameworkSystem/private/monitoring/MonitoringCatalog.py @@ -278,7 +278,7 @@ def registerActivity(self, sourceId, acName, acDict): m = hashlib.md5() acDict['name'] = acName acDict['sourceId'] = sourceId - m.update(str(acDict)) + m.update(str(acDict).encode()) retList = self.__select("filename", "activities", acDict) if len(retList) > 0: return retList[0][0] diff --git a/FrameworkSystem/private/monitoring/PlotCache.py b/FrameworkSystem/private/monitoring/PlotCache.py index 7616711468d..df5eaf04cf1 100755 --- a/FrameworkSystem/private/monitoring/PlotCache.py +++ b/FrameworkSystem/private/monitoring/PlotCache.py @@ -46,8 +46,8 @@ def __init__(self, rrdManager=None): def __generateName(self, *args, **kwargs): m = hashlib.md5() - m.update(repr(args)) - m.update(repr(kwargs)) + m.update(repr(args).encode()) + m.update(repr(kwargs).encode()) return m.hexdigest() def __isCurrentTime(self, toSecs): diff --git a/FrameworkSystem/private/monitoring/RRDManager.py b/FrameworkSystem/private/monitoring/RRDManager.py index c0c812488c7..ed53226f2ee 100755 --- a/FrameworkSystem/private/monitoring/RRDManager.py +++ b/FrameworkSystem/private/monitoring/RRDManager.py @@ -200,8 +200,8 @@ def __generateName(self, *args, **kwargs): Generates a random name """ m = hashlib.md5() - m.update(str(args)) - m.update(str(kwargs)) + m.update(str(args).encode()) + m.update(str(kwargs).encode()) return m.hexdigest() def __generateRRDGraphVar(self, entryName, activity, timeSpan, plotWidth): diff --git a/FrameworkSystem/private/monitoring/ServiceInterface.py b/FrameworkSystem/private/monitoring/ServiceInterface.py index 9a00bb15036..266102b82a5 100755 --- a/FrameworkSystem/private/monitoring/ServiceInterface.py +++ b/FrameworkSystem/private/monitoring/ServiceInterface.py @@ -186,7 +186,7 @@ def commitMarks(self, sourceId, activitiesDict, componentExtraInfo): unregisteredActivities.append(acName) continue gLogger.info("Updating activity", "%s -> %s" % (acName, rrdFile)) - timeList = sorted(acData.keys()) + timeList = sorted(acData) entries = [] for instant in timeList: entries.append((instant, acData[instant])) diff --git a/FrameworkSystem/private/standardLogging/Handler/ServerHandler.py b/FrameworkSystem/private/standardLogging/Handler/ServerHandler.py index 97f20f966ad..9fa739b6291 100644 --- a/FrameworkSystem/private/standardLogging/Handler/ServerHandler.py +++ b/FrameworkSystem/private/standardLogging/Handler/ServerHandler.py @@ -8,7 +8,7 @@ __RCSID__ = "$Id$" import logging -import Queue +from six.moves import queue as Queue import threading from DIRAC.Core.Utilities import Network diff --git a/FrameworkSystem/private/standardLogging/Logging.py b/FrameworkSystem/private/standardLogging/Logging.py index 65f7f8f5a5c..3da7f47f978 100644 --- a/FrameworkSystem/private/standardLogging/Logging.py +++ b/FrameworkSystem/private/standardLogging/Logging.py @@ -151,7 +151,7 @@ def _setOption(self, optionName, value, directCall=True): self._options[optionName] = value # propagate in the children - for child in self._children.itervalues(): + for child in self._children.values(): child._setOption(optionName, value, directCall=False) # pylint: disable=protected-access finally: self._lockOptions.release() diff --git a/FrameworkSystem/private/standardLogging/LoggingRoot.py b/FrameworkSystem/private/standardLogging/LoggingRoot.py index ab5dbb7c661..22eb3b720ed 100644 --- a/FrameworkSystem/private/standardLogging/LoggingRoot.py +++ b/FrameworkSystem/private/standardLogging/LoggingRoot.py @@ -10,13 +10,14 @@ import logging import time import sys - +import six from DIRAC.FrameworkSystem.private.standardLogging.LogLevels import LogLevels from DIRAC.FrameworkSystem.private.standardLogging.Logging import Logging from DIRAC.Resources.LogBackends.StdoutBackend import StdoutBackend from DIRAC.Core.Utilities import DIRACSingleton +@six.add_metaclass(DIRACSingleton.DIRACSingleton) class LoggingRoot(Logging): """ LoggingRoot is a particular Logging object: the first parent of the chain. @@ -26,8 +27,6 @@ class LoggingRoot(Logging): - It is unique, there is one and only one parent at the top of the chain: this justifies the usage of a Singleton """ - __metaclass__ = DIRACSingleton.DIRACSingleton - # Boolean preventing the LoggingRoot to be configured once more __configuredLogging = False diff --git a/FrameworkSystem/private/standardLogging/Message.py b/FrameworkSystem/private/standardLogging/Message.py index efa70a7b53b..c93a6ba05e7 100644 --- a/FrameworkSystem/private/standardLogging/Message.py +++ b/FrameworkSystem/private/standardLogging/Message.py @@ -21,7 +21,7 @@ def tupleToMessage(varTuple): class Message: def __init__(self, systemName, level, time, msgText, variableText, frameInfo, subSystemName=''): - import thread + from six.moves import _thread as thread self.systemName = systemName self.level = level self.time = time diff --git a/FrameworkSystem/private/standardLogging/test/TestLogUtilities.py b/FrameworkSystem/private/standardLogging/test/TestLogUtilities.py index e50449a811d..625bdf62678 100644 --- a/FrameworkSystem/private/standardLogging/test/TestLogUtilities.py +++ b/FrameworkSystem/private/standardLogging/test/TestLogUtilities.py @@ -5,7 +5,7 @@ __RCSID__ = "$Id$" import logging -from StringIO import StringIO +from six import StringIO from DIRAC.FrameworkSystem.private.standardLogging.LoggingRoot import LoggingRoot from DIRAC.FrameworkSystem.private.standardLogging.Logging import Logging diff --git a/FrameworkSystem/private/standardLogging/test/Test_LoggingRoot_ConfigForExternalLibs.py b/FrameworkSystem/private/standardLogging/test/Test_LoggingRoot_ConfigForExternalLibs.py index 5f2e2ab7f56..30388f894b4 100644 --- a/FrameworkSystem/private/standardLogging/test/Test_LoggingRoot_ConfigForExternalLibs.py +++ b/FrameworkSystem/private/standardLogging/test/Test_LoggingRoot_ConfigForExternalLibs.py @@ -9,7 +9,7 @@ __RCSID__ = "$Id$" import logging -from StringIO import StringIO +from six import StringIO import pytest from DIRAC.FrameworkSystem.private.standardLogging.test.TestLogUtilities import gLogger, cleaningLog, gLoggerReset diff --git a/FrameworkSystem/private/standardLogging/test/Test_Logging_CreationLogRecord.py b/FrameworkSystem/private/standardLogging/test/Test_Logging_CreationLogRecord.py index 9cb11389719..fec358e6eae 100644 --- a/FrameworkSystem/private/standardLogging/test/Test_Logging_CreationLogRecord.py +++ b/FrameworkSystem/private/standardLogging/test/Test_Logging_CreationLogRecord.py @@ -47,6 +47,7 @@ def test_createLogRecord(sMsg, sVarMsg, exc_info, expected): logExpected = "Framework%s %s: %s\n" % (logInfo, level, expected) assert logExpected == logstring capturedBackend.truncate(0) + capturedBackend.seek(0) def test_showStack(): @@ -65,6 +66,7 @@ def test_showStack(): logstring = cleaningLog(capturedBackend.getvalue()) assert logstring == "" capturedBackend.truncate(0) + capturedBackend.seek(0) # Set level to debug gLogger.setLevel('debug') @@ -77,3 +79,4 @@ def test_showStack(): logstring = cleaningLog(capturedBackend.getvalue()) assert logstring == "Framework%s DEBUG: \n" % logInfo capturedBackend.truncate(0) + capturedBackend.seek(0) diff --git a/FrameworkSystem/private/standardLogging/test/Test_Logging_FormatOptions.py b/FrameworkSystem/private/standardLogging/test/Test_Logging_FormatOptions.py index 60203c2a067..8f5b48b2a81 100644 --- a/FrameworkSystem/private/standardLogging/test/Test_Logging_FormatOptions.py +++ b/FrameworkSystem/private/standardLogging/test/Test_Logging_FormatOptions.py @@ -4,7 +4,7 @@ __RCSID__ = "$Id$" -import thread +from six.moves import _thread as thread import pytest from DIRAC.FrameworkSystem.private.standardLogging.test.TestLogUtilities import gLogger, gLoggerReset @@ -35,14 +35,17 @@ def test_showFormatOptionsInit(): gLogger.notice("me") logstring = cleaningLog(capturedBackend.getvalue()) capturedBackend.truncate(0) + capturedBackend.seek(0) log.notice("ss") logstring += cleaningLog(capturedBackend.getvalue()) capturedBackend.truncate(0) + capturedBackend.seek(0) sublog.notice("age") logstring += cleaningLog(capturedBackend.getvalue()) capturedBackend.truncate(0) + capturedBackend.seek(0) assert logstring == "Framework NOTICE: me\nFramework/log NOTICE: ss\nFramework/log/sublog NOTICE: age\n" @@ -131,18 +134,21 @@ def test_showFormatOptionsgLogger(header, threadID, timeStamp, context, msg, assert ('UTC' in logValue) == isTimeStampAvailable logstring = cleaningLog(logValue) capturedBackend.truncate(0) + capturedBackend.seek(0) log.notice(msg) logValue = capturedBackend.getvalue() assert ('UTC' in logValue) == isTimeStampAvailable logstring += cleaningLog(logValue) capturedBackend.truncate(0) + capturedBackend.seek(0) sublog.notice(msg) logValue = capturedBackend.getvalue() assert ('UTC' in logValue) == isTimeStampAvailable logstring += cleaningLog(logValue) capturedBackend.truncate(0) + capturedBackend.seek(0) # check that threadID is present in the log when it should be present threadIDValue = str(thread.get_ident()) @@ -185,14 +191,17 @@ def test_showFormatOptionsLog(header, threadID, msg, expectedLog, isThreadIDAvai gLogger.notice(msg) logstring = cleaningLog(capturedBackend.getvalue()) capturedBackend.truncate(0) + capturedBackend.seek(0) log.notice(msg) logstring += cleaningLog(capturedBackend.getvalue()) capturedBackend.truncate(0) + capturedBackend.seek(0) sublog.notice(msg) logstring += cleaningLog(capturedBackend.getvalue()) capturedBackend.truncate(0) + capturedBackend.seek(0) expectedLog = "Framework NOTICE: message\n" + expectedLog @@ -238,14 +247,17 @@ def test_showFormatOptionsSubLog(header, threadID, msg, expectedLog, isThreadIDA gLogger.notice(msg) logstring = cleaningLog(capturedBackend.getvalue()) capturedBackend.truncate(0) + capturedBackend.seek(0) log.notice(msg) logstring += cleaningLog(capturedBackend.getvalue()) capturedBackend.truncate(0) + capturedBackend.seek(0) sublog.notice(msg) logstring += cleaningLog(capturedBackend.getvalue()) capturedBackend.truncate(0) + capturedBackend.seek(0) expectedLog = 'Framework NOTICE: message\nmessage\n' + expectedLog @@ -284,14 +296,17 @@ def test_showFormatOptionsPropag(): gLogger.notice("me") logstring = cleaningLog(capturedBackend.getvalue()) capturedBackend.truncate(0) + capturedBackend.seek(0) log.notice("ss") logstring += cleaningLog(capturedBackend.getvalue()) capturedBackend.truncate(0) + capturedBackend.seek(0) sublog.notice("age") logstring += cleaningLog(capturedBackend.getvalue()) capturedBackend.truncate(0) + capturedBackend.seek(0) threadID = str(thread.get_ident()) expectedLog = "Framework [%s] NOTICE: me\nss\nage\n" % threadID diff --git a/FrameworkSystem/private/standardLogging/test/Test_Logging_GetSubLogger.py b/FrameworkSystem/private/standardLogging/test/Test_Logging_GetSubLogger.py index 92368e9b832..ed4203f9d0a 100644 --- a/FrameworkSystem/private/standardLogging/test/Test_Logging_GetSubLogger.py +++ b/FrameworkSystem/private/standardLogging/test/Test_Logging_GetSubLogger.py @@ -21,12 +21,14 @@ def test_getSubLoggerLogRecord(): # Reinitialize the buffer and send a log with a child of the sublogger capturedBackend.truncate(0) + capturedBackend.seek(0) sublog.always('message') assert " Framework/log/sublog " in capturedBackend.getvalue() # Generate a new sublogger from sublog capturedBackend.truncate(0) + capturedBackend.seek(0) subsublog = sublog.getSubLogger('subsublog') subsublog.always('message') diff --git a/FrameworkSystem/private/standardLogging/test/Test_Logging_Levels.py b/FrameworkSystem/private/standardLogging/test/Test_Logging_Levels.py index df4a3039b3e..0f0abe6f0da 100644 --- a/FrameworkSystem/private/standardLogging/test/Test_Logging_Levels.py +++ b/FrameworkSystem/private/standardLogging/test/Test_Logging_Levels.py @@ -143,6 +143,7 @@ def test_setLevelShowngLogger(loggerLevel, isSuperiorTo, logRecordLevel): assert res assert logstring == "Framework %s: message\n" % logRecordLevel.upper() capturedBackend.truncate(0) + capturedBackend.seek(0) @pytest.mark.parametrize("loggerLevel, isSuperiorTo, logRecordLevel,", [ @@ -183,6 +184,7 @@ def test_setLevelShownLog(loggerLevel, isSuperiorTo, logRecordLevel): assert res assert logstring == "Framework/log %s: message\n" % logRecordLevel.upper() capturedBackend.truncate(0) + capturedBackend.seek(0) @pytest.mark.parametrize("loggerLevel, isSuperiorTo, logRecordLevel,", [ @@ -224,3 +226,4 @@ def test_setLevelShownSubLog(loggerLevel, isSuperiorTo, logRecordLevel): assert res assert logstring == "Framework/log/sublog %s: message\n" % logRecordLevel.upper() capturedBackend.truncate(0) + capturedBackend.seek(0) diff --git a/FrameworkSystem/scripts/dirac-populate-component-db.py b/FrameworkSystem/scripts/dirac-populate-component-db.py index 34994f274fc..5a32fcefaeb 100755 --- a/FrameworkSystem/scripts/dirac-populate-component-db.py +++ b/FrameworkSystem/scripts/dirac-populate-component-db.py @@ -150,7 +150,7 @@ def setExcludedHosts(value): if componentType and componentType != compType: continue for system in rDict[compType]: - components = sorted(rDict[compType][system].keys()) + components = sorted(rDict[compType][system]) for component in components: record = {'Installation': {}, 'Component': {}, 'Host': {}} if rDict[compType][system][component]['Installed'] and \ diff --git a/Interfaces/API/Dirac.py b/Interfaces/API/Dirac.py index d64b4819607..64197bc69fb 100755 --- a/Interfaces/API/Dirac.py +++ b/Interfaces/API/Dirac.py @@ -31,7 +31,7 @@ import tarfile import urllib import shlex -import StringIO +from six import StringIO import DIRAC from DIRAC import gConfig, gLogger, S_OK, S_ERROR @@ -149,7 +149,7 @@ def getRepositoryJobs(self, printOutput=False): if not self.jobRepo: gLogger.warn("No repository is initialised") return S_OK() - jobIDs = self.jobRepo.readRepository()['Value'].keys() + jobIDs = list(self.jobRepo.readRepository()['Value']) if printOutput: print(self.pPrint.pformat(jobIDs)) return S_OK(jobIDs) @@ -168,7 +168,7 @@ def monitorRepository(self, printOutput=False): gLogger.warn("No repository is initialised") return S_OK() jobs = self.jobRepo.readRepository()['Value'] - jobIDs = jobs.keys() + jobIDs = list(jobs) res = self.getJobStatus(jobIDs) if not res['OK']: return self._errorReport(res['Message'], 'Failed to get status of jobs from WMS') @@ -353,7 +353,7 @@ def submitJob(self, job, mode='wms'): self.log.error(msg) return S_ERROR(msg) - jobDescriptionObject = StringIO.StringIO(job._toXML()) # pylint: disable=protected-access + jobDescriptionObject = StringIO(job._toXML()) # pylint: disable=protected-access jdlAsString = job._toJDL(jobDescriptionObject=jobDescriptionObject) # pylint: disable=protected-access if mode.lower() == 'local': @@ -1212,7 +1212,7 @@ def splitInputData(self, lfns, maxFilesPerJob=20, printOutput=False): if not replicaDict['OK']: return replicaDict if not replicaDict['Value']['Successful']: - return self._errorReport(list(replicaDict['Value']['Failed'].iteritems())[0], + return self._errorReport(list(replicaDict['Value']['Failed'].items())[0], 'Failed to get replica information') siteLfns = {} for lfn, reps in replicaDict['Value']['Successful'].items(): # can be an iterator @@ -1269,7 +1269,7 @@ def getLfnMetadata(self, lfns, printOutput=False): repsResult = fileResult['Value'] if repsResult['Failed']: # Some entries can be directories - dirs = repsResult['Failed'].keys() + dirs = list(repsResult['Failed']) dirResult = fc.getDirectoryMetadata(dirs) if not dirResult['OK']: self.log.warn('Failed to retrieve directory metadata from the catalogue') @@ -2075,7 +2075,7 @@ def selectJobs(self, status=None, minorStatus=None, applicationStatus=None, """ options = {'Status': status, 'MinorStatus': minorStatus, 'ApplicationStatus': applicationStatus, 'Owner': owner, 'Site': site, 'JobGroup': jobGroup, 'OwnerGroup': ownerGroup} - conditions = dict((key, str(value)) for key, value in options.iteritems() if value) + conditions = dict((key, str(value)) for key, value in options.items() if value) if date: try: diff --git a/Interfaces/API/DiracAdmin.py b/Interfaces/API/DiracAdmin.py index 42cac75909f..a2d2f9599c3 100755 --- a/Interfaces/API/DiracAdmin.py +++ b/Interfaces/API/DiracAdmin.py @@ -657,16 +657,16 @@ def getPilotSummary(self, startDate='', endDate=''): ceDict = result['Value'] headers = 'CE'.ljust(28) i = 0 - for ce, summary in ceDict.iteritems(): - states = summary.keys() + for ce, summary in ceDict.items(): + states = list(summary) if len(states) > i: i = len(states) - for i in xrange(i): + for i in range(i): headers += 'Status'.ljust(12) + 'Count'.ljust(12) gLogger.notice(headers) - for ce, summary in ceDict.iteritems(): + for ce, summary in ceDict.items(): line = ce.ljust(28) states = sorted(summary) for state in states: diff --git a/Interfaces/API/Job.py b/Interfaces/API/Job.py index 7f1a1ccfce2..b1b3027df69 100755 --- a/Interfaces/API/Job.py +++ b/Interfaces/API/Job.py @@ -33,7 +33,7 @@ import os import urllib import shlex -import StringIO +from six import StringIO from DIRAC import S_OK, gLogger from DIRAC.Core.Base.API import API @@ -141,7 +141,7 @@ def setExecutable(self, executable, arguments='', logFile='', :type parameters: python:list of tuples """ kwargs = {'executable': executable, 'arguments': arguments, 'logFile': logFile} - if not isinstance(executable, basestring) or not isinstance(arguments, six.string_types) or \ + if not isinstance(executable, six.string_types) or not isinstance(arguments, six.string_types) or \ not isinstance(logFile, six.string_types): return self._reportError('Expected strings for executable and arguments', **kwargs) @@ -302,7 +302,7 @@ def setInputData(self, lfns): :type lfns: Single LFN string or list of LFNs """ if isinstance(lfns, list) and lfns: - for i in xrange(len(lfns)): + for i in range(len(lfns)): lfns[i] = lfns[i].replace('LFN:', '') inputData = ['LFN:' + x for x in lfns] inputDataStr = ';'.join(inputData) @@ -866,7 +866,7 @@ def setExecutionEnv(self, environmentDict): if environmentDict: environment = [] - for var, val in environmentDict.iteritems(): + for var, val in environmentDict.items(): try: environment.append('='.join([str(var), urllib.quote(str(val))])) except Exception: @@ -909,7 +909,7 @@ def _dumpParameters(self, showType=None): self.log.info('--------------------------------------') # print self.workflow.parameters # print params.getParametersNames() - for name, _props in paramsDict.iteritems(): + for name, _props in paramsDict.items(): ptype = paramsDict[name]['type'] value = paramsDict[name]['value'] if showType: @@ -1127,7 +1127,7 @@ def _toJDL(self, xmlFile='', jobDescriptionObject=None): self.log.warn("Job description XML file not found") self.addToInputSandbox.append(scriptName) - elif isinstance(jobDescriptionObject, StringIO.StringIO): + elif isinstance(jobDescriptionObject, StringIO): self.log.verbose("jobDescription is passed in as a StringIO object") else: @@ -1155,7 +1155,7 @@ def _toJDL(self, xmlFile='', jobDescriptionObject=None): else: self.log.warn('JobConfigArgs defined with null value') if self.parametricWFArguments: - for name, value in self.parametricWFArguments.iteritems(): + for name, value in self.parametricWFArguments.items(): arguments.append("-p %s='%s'" % (name, value)) classadJob.insertAttributeString('Executable', self.executable) @@ -1214,7 +1214,7 @@ def _toJDL(self, xmlFile='', jobDescriptionObject=None): classadJob.insertAttributeString('Arguments', ' '.join(arguments)) # Add any JDL parameters to classad obeying lists with ';' rule - for name, props in paramsDict.iteritems(): + for name, props in paramsDict.items(): ptype = props['type'] value = props['value'] if isinstance(value, six.string_types) and re.search(';', value): diff --git a/Interfaces/API/JobRepository.py b/Interfaces/API/JobRepository.py index 56ee53f33d6..db247d8c25a 100644 --- a/Interfaces/API/JobRepository.py +++ b/Interfaces/API/JobRepository.py @@ -54,7 +54,7 @@ def writeRepository(self, alternativePath=None): def resetRepository(self, jobIDs=[]): if not jobIDs: jobs = self.readRepository()['Value'] - jobIDs = jobs.keys() + jobIDs = list(jobs) paramDict = {'State': 'Submitted', 'Retrieved': 0, 'OutputData': 0} diff --git a/Interfaces/API/test/Test_DIRAC.py b/Interfaces/API/test/Test_DIRAC.py index b1233d3d2ba..38dd7118f27 100644 --- a/Interfaces/API/test/Test_DIRAC.py +++ b/Interfaces/API/test/Test_DIRAC.py @@ -6,6 +6,7 @@ from __future__ import division import logging +import six from pprint import pformat import pytest from mock import MagicMock, call @@ -112,7 +113,10 @@ def test_JobJob(dirac, job): def test_runLocal(dirac, job, mocker, osmock, confMock): mocker.patch('DIRAC.Interfaces.API.Dirac.os', new=osmock) mocker.patch('DIRAC.Interfaces.API.Dirac.tarfile', new=MagicMock(return_value=False)) - mocker.patch('__builtin__.open') + if six.PY2: + mocker.patch('__builtin__.open') + else: + mocker.patch('builtins.open') mocker.patch('DIRAC.Interfaces.API.Dirac.gConfig', new=confMock) tempMock = mocker.patch('DIRAC.Interfaces.API.Dirac.tempfile') tempMock.mkdtemp.return_value = 'tempFolder' diff --git a/Interfaces/API/test/Test_JobAPI.py b/Interfaces/API/test/Test_JobAPI.py index 4543d7b0be0..8107d4c0bc8 100644 --- a/Interfaces/API/test/Test_JobAPI.py +++ b/Interfaces/API/test/Test_JobAPI.py @@ -7,7 +7,7 @@ __RCSID__ = "$Id$" import pytest -import StringIO +from six import StringIO from DIRAC.Interfaces.API.Job import Job from DIRAC.Core.Utilities.ClassAd.ClassAdLight import ClassAd @@ -42,7 +42,7 @@ def test_basicJob(): with open('./Interfaces/API/test/testWFSIO.jdl') as fd: expected = fd.read() - jdlSIO = job._toJDL(jobDescriptionObject=StringIO.StringIO(job._toXML())) + jdlSIO = job._toJDL(jobDescriptionObject=StringIO(job._toXML())) assert jdlSIO == expected diff --git a/Interfaces/API/test/testWF.jdl b/Interfaces/API/test/testWF.jdl index e2322785bb0..f6f36b0ac8b 100644 --- a/Interfaces/API/test/testWF.jdl +++ b/Interfaces/API/test/testWF.jdl @@ -1,5 +1,20 @@ + Arguments = "jobDescription.xml -o LogLevel=DEBUG -p JOB_ID=%(JOB_ID)s -p InputData=%(InputData)s"; + Executable = "$DIRACROOT/scripts/dirac-jobexec"; + InputData = %(InputData)s; + InputSandbox = jobDescription.xml; + JOB_ID = %(JOB_ID)s; + JobName = Name; + JobType = User; + LogLevel = DEBUG; Origin = DIRAC; + OutputSandbox = + { + Script1_CodeOutput.log, + std.err, + std.out + }; + Parameters = 3; Parameters.InputData = { {/lhcb/data/data1, @@ -9,27 +24,12 @@ {/lhcb/data/data5, /lhcb/data/data6} }; - JOB_ID = %(JOB_ID)s; - Parameters = 3; - StdError = std.err; - LogLevel = DEBUG; - Executable = "$DIRACROOT/scripts/dirac-jobexec"; - JobName = Name; - Priority = 1; - InputSandbox = jobDescription.xml; - Arguments = "jobDescription.xml -o LogLevel=DEBUG -p JOB_ID=%(JOB_ID)s -p InputData=%(InputData)s"; - StdOutput = std.out; - OutputSandbox = - { - Script1_CodeOutput.log, - std.err, - std.out - }; Parameters.JOB_ID = { 1, 2, 3 }; - InputData = %(InputData)s; - JobType = User; \ No newline at end of file + Priority = 1; + StdError = std.err; + StdOutput = std.out; \ No newline at end of file diff --git a/Interfaces/API/test/testWF.xml b/Interfaces/API/test/testWF.xml index 75a9e5e013c..73fc72a7d21 100644 --- a/Interfaces/API/test/testWF.xml +++ b/Interfaces/API/test/testWF.xml @@ -1,10 +1,10 @@ - - -0.0 - + jobName + + +0.0 @@ -24,34 +24,34 @@ - + - + -0.0 Script +0.0 + + -0.0 ScriptStep1 - - +0.0 -Script -Script +Script +Script -ScriptStep1 -RunScriptStep1 +RunScriptStep1 +ScriptStep1 diff --git a/Interfaces/API/test/testWFSIO.jdl b/Interfaces/API/test/testWFSIO.jdl index 868cb8c1de6..cd18e455030 100644 --- a/Interfaces/API/test/testWFSIO.jdl +++ b/Interfaces/API/test/testWFSIO.jdl @@ -1,20 +1,20 @@ - Origin = DIRAC; - Priority = 1; - Executable = "$DIRACROOT/scripts/dirac-jobexec"; - StdError = std.err; - LogLevel = info; - Site = ANY; - JobName = jobName; - OwnerGroup = ownerGroup; Arguments = "jobDescription.xml -o LogLevel=info"; + Executable = "$DIRACROOT/scripts/dirac-jobexec"; JobGroup = jobGroup; - StdOutput = std.out; - Owner = ownerName; + JobName = jobName; + JobType = jobType; + LogLevel = info; + Origin = DIRAC; OutputSandbox = { Script1_CodeOutput.log, std.err, std.out }; - JobType = jobType; \ No newline at end of file + Owner = ownerName; + OwnerGroup = ownerGroup; + Priority = 1; + Site = ANY; + StdError = std.err; + StdOutput = std.out; \ No newline at end of file diff --git a/Interfaces/scripts/dirac-wms-jobs-select-output-search.py b/Interfaces/scripts/dirac-wms-jobs-select-output-search.py index 372c15347d7..9b6daa53357 100755 --- a/Interfaces/scripts/dirac-wms-jobs-select-output-search.py +++ b/Interfaces/scripts/dirac-wms-jobs-select-output-search.py @@ -106,7 +106,7 @@ errorList.append((job, result['Message'])) exitCode = 2 -for result in resultDict.iteritems(): +for result in resultDict.items(): print(result) diff --git a/Interfaces/scripts/dirac-wms-select-jobs.py b/Interfaces/scripts/dirac-wms-select-jobs.py index 30fdee4a3f7..145ea6c7cd5 100755 --- a/Interfaces/scripts/dirac-wms-select-jobs.py +++ b/Interfaces/scripts/dirac-wms-select-jobs.py @@ -115,7 +115,7 @@ gLogger.error("Can't select jobs: ", res['Message']) -conds = ['%s = %s' % (n, v) for n, v in conditions.iteritems() if v] +conds = ['%s = %s' % (n, v) for n, v in conditions.items() if v] if maxJobs and len(jobs) > maxJobs: jobs = jobs[:maxJobs] constrained = ' (first %d shown) ' % maxJobs diff --git a/MonitoringSystem/private/DBUtils.py b/MonitoringSystem/private/DBUtils.py index 69e366ef1a8..8d5acfa288d 100644 --- a/MonitoringSystem/private/DBUtils.py +++ b/MonitoringSystem/private/DBUtils.py @@ -116,7 +116,7 @@ def _determineBucketSize(self, start, end): bucketSeconds = {} try: # Convert intervals into seconds - for interval, binUnit in self.__esbucket.iteritems(): + for interval, binUnit in self.__esbucket.items(): bucketSeconds[_convertToSeconds(interval)] = (binUnit, _convertToSeconds(binUnit)) # Determine bin size according to time span for interval in sorted(bucketSeconds): diff --git a/MonitoringSystem/private/MainReporter.py b/MonitoringSystem/private/MainReporter.py index 27abb5b72f0..1e5a334fc68 100644 --- a/MonitoringSystem/private/MainReporter.py +++ b/MonitoringSystem/private/MainReporter.py @@ -81,8 +81,8 @@ def __calculateReportHash(self, reportRequest): epoch = requestToHash[key] requestToHash[key] = epoch - epoch % granularity md5Hash = hashlib.md5() - md5Hash.update(repr(requestToHash)) - md5Hash.update(self.__setup) + md5Hash.update(repr(requestToHash).encode()) + md5Hash.update(self.__setup.encode()) return md5Hash.hexdigest() def generate(self, reportRequest, credDict): diff --git a/ProductionSystem/Utilities/ProdValidator.py b/ProductionSystem/Utilities/ProdValidator.py index 7979fdb221f..08818f202f5 100644 --- a/ProductionSystem/Utilities/ProdValidator.py +++ b/ProductionSystem/Utilities/ProdValidator.py @@ -122,12 +122,12 @@ def checkformatQuery(self, MetaQueryDict): for meta, value in MetaQueryDict.items(): values = [] if isinstance(value, dict): - operation = value.keys()[0] + operation = list(value)[0] if operation not in ['=', 'in']: msg = 'Operation %s is not supported' % operation return S_ERROR(msg) else: - if not isinstance(value.values()[0], list): + if not isinstance(list(value.values())[0], list): MetaQueryDict[meta] = {"in": value.values()} else: values.append(value) @@ -141,8 +141,5 @@ def compareValues(self, value, parentValue): :param dict value: a dictionary with meta data values to be compared with the parentValues :param dict parentValue: a dictionary with meta data parentValues be compared with values """ - return set( - value.values()[0]).issubset( - set(parentValue.values()[0])) or set( - parentValue.values()[0]).issubset( - set(value.values()[0])) + return set(list(value.values())[0]).issubset(set(list(parentValue.values())[0])) or \ + set(list(parentValue.values())[0]).issubset(set(list(value.values())[0])) diff --git a/RequestManagementSystem/Client/File.py b/RequestManagementSystem/Client/File.py index 90dfb675b5e..90a2cdf5d34 100644 --- a/RequestManagementSystem/Client/File.py +++ b/RequestManagementSystem/Client/File.py @@ -79,11 +79,12 @@ def __init__(self, fromDict=None): else json.loads(fromDict) if isinstance(fromDict, six.string_types)\ else {} - for attrName, attrValue in fromDict.iteritems(): + for attrName, attrValue in fromDict.items(): # The JSON module forces the use of UTF-8, which is not properly # taken into account in DIRAC. # One would need to replace all the '== str' with 'in six.string_types' - if isinstance(attrValue, six.string_types): + # This is converting `unicode` to `str` and doesn't make sense in Python 3 + if six.PY2 and isinstance(attrValue, six.string_types): attrValue = attrValue.encode() if attrValue: setattr(self, attrName, attrValue) diff --git a/RequestManagementSystem/Client/Operation.py b/RequestManagementSystem/Client/Operation.py index ce14a2a6b4d..6935ca91886 100644 --- a/RequestManagementSystem/Client/Operation.py +++ b/RequestManagementSystem/Client/Operation.py @@ -100,7 +100,8 @@ def __init__(self, fromDict=None): # The JSON module forces the use of UTF-8, which is not properly # taken into account in DIRAC. # One would need to replace all the '== str' with 'in six.string_types' - if isinstance(value, six.string_types): + # This is converting `unicode` to `str` and doesn't make sense in Python 3 + if six.PY2 and isinstance(value, six.string_types): value = value.encode() if value: setattr(self, key, value) @@ -191,11 +192,14 @@ def fileStatusList(self): """ get list of files statuses """ return [subFile.Status for subFile in self] - def __nonzero__(self): + def __bool__(self): """ for comparisons """ return True + # For Python 2 compatibility + __nonzero__ = __bool__ + def __len__(self): """ nb of subFiles """ return len(self.__files__) @@ -218,16 +222,16 @@ def Catalog(self): @Catalog.setter def Catalog(self, value): """ catalog setter """ - if type(value) not in (str, unicode, list): + if not isinstance(value, six.string_types + (list,)): raise TypeError("wrong type for value") - if type(value) in (str, unicode): + if isinstance(value, six.string_types): value = value.split(',') value = ",".join(list(set([str(item).strip() for item in value if str(item).strip()]))) if len(value) > 255: raise ValueError("Catalog list too long") - self._Catalog = value.encode() if value else "" + self._Catalog = value if value else "" @property def catalogList(self): diff --git a/RequestManagementSystem/Client/ReqClient.py b/RequestManagementSystem/Client/ReqClient.py index 293e7a63e48..e7f08b66759 100755 --- a/RequestManagementSystem/Client/ReqClient.py +++ b/RequestManagementSystem/Client/ReqClient.py @@ -405,7 +405,7 @@ def readRequestsForJobs(self, jobIDs): ret = readReqsForJobs["Value"] # # create Requests out of JSONs for successful reads # Do not forget to cast back str keys to int - successful = {int(jobID): Request(jsonReq) for jobID, jsonReq in ret['Successful'].iteritems()} + successful = {int(jobID): Request(jsonReq) for jobID, jsonReq in ret['Successful'].items()} failed = strToIntDict(ret['Failed']) return S_OK({'Successful': successful, 'Failed': failed}) diff --git a/RequestManagementSystem/Client/Request.py b/RequestManagementSystem/Client/Request.py index 80a6255b73f..339776bbdac 100644 --- a/RequestManagementSystem/Client/Request.py +++ b/RequestManagementSystem/Client/Request.py @@ -107,7 +107,8 @@ def __init__(self, fromDict=None): # The JSON module forces the use of UTF-8, which is not properly # taken into account in DIRAC. # One would need to replace all the '== str' with 'in six.string_types' - if isinstance(value, six.string_types): + # This is converting `unicode` to `str` and doesn't make sense in Python 3 + if six.PY2 and isinstance(value, six.string_types): value = value.encode() if value: @@ -258,11 +259,14 @@ def indexOf(self, subReq): """ return index of subReq (execution order) """ return self.__operations__.index(subReq) if subReq in self else -1 - def __nonzero__(self): + def __bool__(self): """ for comparisons """ return True + # For Python 2 compatibility + __nonzero__ = __bool__ + def __len__(self): """ nb of subRequests """ return len(self.__operations__) @@ -479,9 +483,9 @@ def optimize(self): optimized = True insertBefore = self.__operations__[i] if i < len(self.__operations__) else None # print 'Insert new operations before', insertBefore - for op in \ - [op for _targetSE, op in sorted(repAndRegList)] + \ - [op for _targetSE, op in sorted(removeRepList)]: + ops = [op for _, op in sorted(repAndRegList, key=lambda x: x[0])] + ops += [op for _, op in sorted(removeRepList, key=lambda x: x[0])] + for op in ops: _res = self.insertBefore(op, insertBefore) if insertBefore else self.addOperation(op) # Skip the newly inserted operation i += 1 diff --git a/RequestManagementSystem/Service/ReqProxyHandler.py b/RequestManagementSystem/Service/ReqProxyHandler.py index 00db867589a..58f9de96c67 100644 --- a/RequestManagementSystem/Service/ReqProxyHandler.py +++ b/RequestManagementSystem/Service/ReqProxyHandler.py @@ -152,7 +152,7 @@ def __saveRequest(self, requestName, requestJSON): :param str requestJSON: request serialized to JSON format """ try: - requestFile = os.path.join(self.cacheDir(), md5(requestJSON).hexdigest()) + requestFile = os.path.join(self.cacheDir(), md5(requestJSON.encode()).hexdigest()) with open(requestFile, "w+") as request: request.write(requestJSON) return S_OK(requestFile) diff --git a/RequestManagementSystem/private/OperationHandlerBase.py b/RequestManagementSystem/private/OperationHandlerBase.py index da134185f05..0dae0d9447c 100644 --- a/RequestManagementSystem/private/OperationHandlerBase.py +++ b/RequestManagementSystem/private/OperationHandlerBase.py @@ -53,6 +53,7 @@ # # imports import os +import six # # from DIRAC from DIRAC import gLogger, gConfig, S_ERROR, S_OK from DIRAC.Core.Utilities.Graph import DynamicProps @@ -64,17 +65,14 @@ from DIRAC.Resources.Catalog.FileCatalog import FileCatalog from DIRAC.Core.Utilities import Time, Network -######################################################################## - +@six.add_metaclass(DynamicProps) class OperationHandlerBase(object): """ .. class:: OperationHandlerBase request operation handler base class """ - __metaclass__ = DynamicProps - # # private data logging client # __dataLoggingClient = None # # private ResourceStatusClient @@ -104,7 +102,7 @@ def __init__(self, operation=None, csPath=None): csOptionsDict = gConfig.getOptionsDict(self.csPath) csOptionsDict = csOptionsDict.get("Value", {}) - for option, value in csOptionsDict.iteritems(): + for option, value in csOptionsDict.items(): # # hack to set proper types try: value = eval(value) diff --git a/RequestManagementSystem/private/RequestValidator.py b/RequestManagementSystem/private/RequestValidator.py index 498f0649571..4d7321286b5 100644 --- a/RequestManagementSystem/private/RequestValidator.py +++ b/RequestManagementSystem/private/RequestValidator.py @@ -62,6 +62,7 @@ def hasBar( request ): # @brief Definition of RequestValidator class. # # import import inspect +import six # # from DIRAC from DIRAC import S_OK, S_ERROR, gConfig, gLogger from DIRAC.Core.Security.Properties import FULL_DELEGATION, LIMITED_DELEGATION @@ -70,6 +71,7 @@ def hasBar( request ): ######################################################################## +@six.add_metaclass(DIRACSingleton) class RequestValidator(object): """ .. class:: RequestValidator @@ -77,9 +79,6 @@ class RequestValidator(object): This class validates newly created requests (before saving them in RequestDB) for required attributes. """ - # # one to rule them all - __metaclass__ = DIRACSingleton - # # dict with required attrs reqAttrs = { "ForwardDISET": { diff --git a/RequestManagementSystem/scripts/dirac-rms-list-req-cache.py b/RequestManagementSystem/scripts/dirac-rms-list-req-cache.py index ff02bcf0238..8a2a1fb8372 100755 --- a/RequestManagementSystem/scripts/dirac-rms-list-req-cache.py +++ b/RequestManagementSystem/scripts/dirac-rms-list-req-cache.py @@ -25,7 +25,7 @@ reqClient = ReqClient() - for server, rpcClient in reqClient.requestProxies().iteritems(): + for server, rpcClient in reqClient.requestProxies().items(): DIRAC.gLogger.always("Checking request cache at %s" % server) reqCache = rpcClient.listCacheDir() if not reqCache['OK']: diff --git a/ResourceStatusSystem/Agent/CacheFeederAgent.py b/ResourceStatusSystem/Agent/CacheFeederAgent.py index a10bd29431f..2b431d3e0de 100644 --- a/ResourceStatusSystem/Agent/CacheFeederAgent.py +++ b/ResourceStatusSystem/Agent/CacheFeederAgent.py @@ -112,7 +112,7 @@ def loadCommand(self, commandModule, commandDict): :type commandDict: dict """ - commandName = commandDict.keys()[0] + commandName = list(commandDict)[0] commandArgs = commandDict[commandName] commandTuple = ('%sCommand' % commandModule, '%sCommand' % commandName) @@ -135,7 +135,7 @@ def execute(self): """ Just executes, via `loadCommand`, the commands in self.commands one after the other """ - for commandModule, commandList in self.commands.iteritems(): + for commandModule, commandList in self.commands.items(): self.log.info('%s module initialization' % commandModule) diff --git a/ResourceStatusSystem/Agent/ElementInspectorAgent.py b/ResourceStatusSystem/Agent/ElementInspectorAgent.py index 56569c9a2d9..69e395633e8 100644 --- a/ResourceStatusSystem/Agent/ElementInspectorAgent.py +++ b/ResourceStatusSystem/Agent/ElementInspectorAgent.py @@ -19,7 +19,7 @@ import datetime import math -import Queue +from six.moves import queue as Queue from DIRAC import S_ERROR, S_OK from DIRAC.Core.Base.AgentModule import AgentModule @@ -128,7 +128,7 @@ def execute(self): self.log.info('Needed %d threads to process %d elements' % (numberOfThreads, queueSize)) - for _x in xrange(numberOfThreads): + for _x in range(numberOfThreads): jobUp = self.threadPool.generateJobAndQueueIt(self._execute) if not jobUp['OK']: self.log.error(jobUp['Message']) @@ -182,7 +182,7 @@ def getElementsToBeChecked(self): # be there, but in any case, it is not a big problem. lowerElementDict = {'element': self.elementType} - for key, value in elemDict.iteritems(): + for key, value in elemDict.items(): lowerElementDict[key[0].lower() + key[1:]] = value # We add lowerElementDict to the queue diff --git a/ResourceStatusSystem/Agent/EmailAgent.py b/ResourceStatusSystem/Agent/EmailAgent.py index a14ef983d58..6ec859dd413 100644 --- a/ResourceStatusSystem/Agent/EmailAgent.py +++ b/ResourceStatusSystem/Agent/EmailAgent.py @@ -69,7 +69,7 @@ def execute(self): columnNames = result['Columns'] result = self._groupBySiteName(result) - for site, records in result.iteritems(): + for site, records in result.items(): email = "" html_body = "" diff --git a/ResourceStatusSystem/Agent/SiteInspectorAgent.py b/ResourceStatusSystem/Agent/SiteInspectorAgent.py index 90f02cec2f7..df426e8e678 100644 --- a/ResourceStatusSystem/Agent/SiteInspectorAgent.py +++ b/ResourceStatusSystem/Agent/SiteInspectorAgent.py @@ -17,7 +17,7 @@ __RCSID__ = '$Id$' import math -import Queue +from six.moves import queue as Queue from DIRAC import S_OK from DIRAC.Core.Base.AgentModule import AgentModule @@ -114,7 +114,7 @@ def execute(self): self.log.info('Needed %d threads to process %d elements' % (numberOfThreads, queueSize)) - for _x in xrange(numberOfThreads): + for _x in range(numberOfThreads): jobUp = self.threadPool.generateJobAndQueueIt(self._execute) if not jobUp['OK']: self.log.error(jobUp['Message']) diff --git a/ResourceStatusSystem/Agent/SummarizeLogsAgent.py b/ResourceStatusSystem/Agent/SummarizeLogsAgent.py index 478423b21c7..f5d9478ba53 100644 --- a/ResourceStatusSystem/Agent/SummarizeLogsAgent.py +++ b/ResourceStatusSystem/Agent/SummarizeLogsAgent.py @@ -81,7 +81,7 @@ def execute(self): # logElements is a dictionary of key-value pairs as follows: # ( name, statusType ) : list( logs ) - for key, logs in logElements.iteritems(): + for key, logs in logElements.items(): sumResult = self._registerLogs(element, key, logs) if not sumResult['OK']: diff --git a/ResourceStatusSystem/Agent/TokenAgent.py b/ResourceStatusSystem/Agent/TokenAgent.py index a4bae0e34d3..69790c60f78 100644 --- a/ResourceStatusSystem/Agent/TokenAgent.py +++ b/ResourceStatusSystem/Agent/TokenAgent.py @@ -172,7 +172,7 @@ def _notifyOfTokens(self): adminExpired = [] adminExpiring = [] - for tokenOwner, tokenLists in self.tokenDict.iteritems(): + for tokenOwner, tokenLists in self.tokenDict.items(): expired = [] expiring = [] diff --git a/ResourceStatusSystem/Agent/test/Test_Agent_ElementInspectorAgent.py b/ResourceStatusSystem/Agent/test/Test_Agent_ElementInspectorAgent.py index 36f84f9450d..91bc90685b5 100644 --- a/ResourceStatusSystem/Agent/test/Test_Agent_ElementInspectorAgent.py +++ b/ResourceStatusSystem/Agent/test/Test_Agent_ElementInspectorAgent.py @@ -3,7 +3,7 @@ from __future__ import absolute_import from __future__ import division from __future__ import print_function -import Queue +from six.moves import queue as Queue # imports import pytest @@ -40,7 +40,10 @@ def test__execute(mocker, elementsToBeCheckedValue): """ mocker.patch("DIRAC.ResourceStatusSystem.Agent.ElementInspectorAgent.AgentModule.__init__") - mocker.patch("DIRAC.ResourceStatusSystem.Agent.ElementInspectorAgent.AgentModule.am_getOption", side_effect=mockAM) + mocker.patch( + "DIRAC.ResourceStatusSystem.Agent.ElementInspectorAgent.AgentModule._AgentModule__moduleProperties", + side_effect=lambda x, y=None: y, create=True + ) elementInspectorAgent = ElementInspectorAgent() elementInspectorAgent.log = gLogger diff --git a/ResourceStatusSystem/Agent/test/Test_Agent_SiteInspectorAgent.py b/ResourceStatusSystem/Agent/test/Test_Agent_SiteInspectorAgent.py index f2317e1f25b..b649d389e81 100644 --- a/ResourceStatusSystem/Agent/test/Test_Agent_SiteInspectorAgent.py +++ b/ResourceStatusSystem/Agent/test/Test_Agent_SiteInspectorAgent.py @@ -3,7 +3,7 @@ from __future__ import absolute_import from __future__ import division from __future__ import print_function -import Queue +from six.moves import queue as Queue # imports import pytest @@ -40,7 +40,10 @@ def test__execute(mocker, sitesToBeCheckedValue): """ mocker.patch("DIRAC.ResourceStatusSystem.Agent.SiteInspectorAgent.AgentModule.__init__") - mocker.patch("DIRAC.ResourceStatusSystem.Agent.SiteInspectorAgent.AgentModule.am_getOption", side_effect=mockAM) + mocker.patch( + "DIRAC.ResourceStatusSystem.Agent.SiteInspectorAgent.AgentModule._AgentModule__moduleProperties", + side_effect=lambda x, y=None: y, create=True + ) siteInspectorAgent = SiteInspectorAgent() siteInspectorAgent.log = gLogger diff --git a/ResourceStatusSystem/Client/ResourceStatus.py b/ResourceStatusSystem/Client/ResourceStatus.py index 115ebfdcc53..0a09d1dacaf 100644 --- a/ResourceStatusSystem/Client/ResourceStatus.py +++ b/ResourceStatusSystem/Client/ResourceStatus.py @@ -10,6 +10,7 @@ from datetime import datetime, timedelta import math from time import sleep +import six from DIRAC import gConfig, gLogger, S_OK, S_ERROR from DIRAC.Core.Utilities.DIRACSingleton import DIRACSingleton @@ -22,15 +23,13 @@ from DIRAC.Core.Utilities import DErrno +@six.add_metaclass(DIRACSingleton) class ResourceStatus(object): """ ResourceStatus helper that connects to CS if RSS flag is not Active. It keeps the connection to the db / server as an object member, to avoid creating a new one massively. """ - - __metaclass__ = DIRACSingleton - def __init__(self, rssFlag=None): """ Constructor, initializes the rssClient. diff --git a/ResourceStatusSystem/Client/SiteStatus.py b/ResourceStatusSystem/Client/SiteStatus.py index cb041cf8100..d368a333fe9 100644 --- a/ResourceStatusSystem/Client/SiteStatus.py +++ b/ResourceStatusSystem/Client/SiteStatus.py @@ -27,6 +27,7 @@ from DIRAC.ResourceStatusSystem.Utilities.RssConfiguration import RssConfiguration +@six.add_metaclass(DIRACSingleton) class SiteStatus(object): """ RSS helper to interact with the 'Site' family on the DB. It provides the most @@ -38,9 +39,6 @@ class SiteStatus(object): * getUsableSites * getSites """ - - __metaclass__ = DIRACSingleton - def __init__(self): """ Constructor, initializes the rssClient. @@ -65,7 +63,7 @@ def __updateRssCache(self): meta = {'columns': ['Name', 'Status']} - for ti in xrange(5): + for ti in range(5): rawCache = self.rsClient.selectStatusElement('Site', 'Status', meta=meta) if rawCache['OK']: break @@ -177,7 +175,7 @@ def getUsableSites(self, siteNames=None): siteStatusDictRes = self.getSiteStatuses(siteNames) if not siteStatusDictRes['OK']: return siteStatusDictRes - siteStatusList = [x[0] for x in siteStatusDictRes['Value'].iteritems() if x[1] in ['Active', 'Degraded']] + siteStatusList = [x[0] for x in siteStatusDictRes['Value'].items() if x[1] in ['Active', 'Degraded']] return S_OK(siteStatusList) @@ -222,7 +220,7 @@ def getSites(self, siteState='Active'): if siteState not in allowedStateList: return S_ERROR(errno.EINVAL, 'Not a valid status, parameter rejected') - siteList = [x[0] for x in siteStatusDictRes['Value'].iteritems() if x[1] == siteState] + siteList = [x[0] for x in siteStatusDictRes['Value'].items() if x[1] == siteState] return S_OK(siteList) diff --git a/ResourceStatusSystem/Command/AccountingCacheCommand.py b/ResourceStatusSystem/Command/AccountingCacheCommand.py index b0d83cb660f..9258362fe18 100644 --- a/ResourceStatusSystem/Command/AccountingCacheCommand.py +++ b/ResourceStatusSystem/Command/AccountingCacheCommand.py @@ -85,9 +85,9 @@ def doCommand(self): singlePlots = {} - successfulJobs['data'] = {site: strToIntDict(value) for site, value in successfulJobs['data'].iteritems()} + successfulJobs['data'] = {site: strToIntDict(value) for site, value in successfulJobs['data'].items()} - for site, value in successfulJobs['data'].iteritems(): + for site, value in successfulJobs['data'].items(): if site in sites: plot = {} plot['data'] = {site: value} @@ -160,11 +160,11 @@ def doCommand(self): if 'granularity' not in failedJobs: return S_ERROR('Missing granularity key') - failedJobs['data'] = {site: strToIntDict(value) for site, value in failedJobs['data'].iteritems()} + failedJobs['data'] = {site: strToIntDict(value) for site, value in failedJobs['data'].items()} singlePlots = {} - for site, value in failedJobs['data'].iteritems(): + for site, value in failedJobs['data'].items(): if site in sites: plot = {} plot['data'] = {site: value} @@ -237,11 +237,11 @@ def doCommand(self): if 'granularity' not in succesfulPilots: return S_ERROR('Missing granularity key') - succesfulPilots['data'] = {site: strToIntDict(value) for site, value in succesfulPilots['data'].iteritems()} + succesfulPilots['data'] = {site: strToIntDict(value) for site, value in succesfulPilots['data'].items()} singlePlots = {} - for site, value in succesfulPilots['data'].iteritems(): + for site, value in succesfulPilots['data'].items(): if site in sites: plot = {} plot['data'] = {site: value} @@ -314,11 +314,11 @@ def doCommand(self): if 'granularity' not in failedPilots: return S_ERROR('Missing granularity key') - failedPilots['data'] = {site: strToIntDict(value)for site, value in failedPilots['data'].iteritems()} + failedPilots['data'] = {site: strToIntDict(value)for site, value in failedPilots['data'].items()} singlePlots = {} - for site, value in failedPilots['data'].iteritems(): + for site, value in failedPilots['data'].items(): if site in sites: plot = {} plot['data'] = {site: value} @@ -393,11 +393,11 @@ def doCommand(self): if 'granularity' not in successfulPilots: return S_ERROR('Missing granularity key') - successfulPilots['data'] = {site: strToIntDict(value) for site, value in successfulPilots['data'].iteritems()} + successfulPilots['data'] = {site: strToIntDict(value) for site, value in successfulPilots['data'].items()} singlePlots = {} - for ce, value in successfulPilots['data'].iteritems(): + for ce, value in successfulPilots['data'].items(): if ce in ces: plot = {} plot['data'] = {ce: value} @@ -479,11 +479,11 @@ def doCommand(self): if 'granularity' not in failedPilots: return S_ERROR('Missing granularity key') - failedPilots['data'] = {site: strToIntDict(value) for site, value in failedPilots['data'].iteritems()} + failedPilots['data'] = {site: strToIntDict(value) for site, value in failedPilots['data'].items()} singlePlots = {} - for ce, value in failedPilots['data'].iteritems(): + for ce, value in failedPilots['data'].items(): if ce in ces: plot = {} plot['data'] = {ce: value} @@ -561,11 +561,11 @@ def doCommand(self): if 'granularity' not in runJobs: return S_ERROR('Missing granularity key') - runJobs['data'] = {site: strToIntDict(value) for site, value in runJobs['data'].iteritems()} + runJobs['data'] = {site: strToIntDict(value) for site, value in runJobs['data'].items()} singlePlots = {} - for site, value in runJobs['data'].iteritems(): + for site, value in runJobs['data'].items(): if site in sites: plot = {} plot['data'] = {site: value} diff --git a/ResourceStatusSystem/Command/DIRACAccountingCommand.py b/ResourceStatusSystem/Command/DIRACAccountingCommand.py index c3e9e0bd822..35f32674546 100644 --- a/ResourceStatusSystem/Command/DIRACAccountingCommand.py +++ b/ResourceStatusSystem/Command/DIRACAccountingCommand.py @@ -143,7 +143,7 @@ def doCommand(self): if not results['OK']: return results - pr_q_d = {channel: strToIntDict(value) for channel, value in results['Value']['data'].iteritems()} + pr_q_d = {channel: strToIntDict(value) for channel, value in results['Value']['data'].items()} # FIXME: WHAT the hell is this doing ? values = [] @@ -324,7 +324,7 @@ def doCommand(self): num, den = 0, 0 - se = results['data'].keys()[0] + se = list(results['data'])[0] num = num + len(results['data'][se]) den = den + sum(results['data'][se].values()) diff --git a/ResourceStatusSystem/Command/DowntimeCommand.py b/ResourceStatusSystem/Command/DowntimeCommand.py index 3a29c855f59..083b216f2a8 100644 --- a/ResourceStatusSystem/Command/DowntimeCommand.py +++ b/ResourceStatusSystem/Command/DowntimeCommand.py @@ -9,7 +9,7 @@ __RCSID__ = '$Id$' -import urllib2 +from six.moves.urllib_error import URLError import re from datetime import datetime, timedelta @@ -210,11 +210,11 @@ def doNew(self, masterParams=None): # WARNING: checking all the DT that are ongoing or starting in given from now try: results = self.gClient.getStatus(element, name=elementNames, startingInHours=hours) - except urllib2.URLError: + except URLError: try: # Let's give it a second chance.. results = self.gClient.getStatus(element, name=elementNames, startingInHours=hours) - except urllib2.URLError as e: + except URLError as e: return S_ERROR(e) if not results['OK']: diff --git a/ResourceStatusSystem/Command/GGUSTicketsCommand.py b/ResourceStatusSystem/Command/GGUSTicketsCommand.py index c770b9b9a9e..982a7ef7a03 100644 --- a/ResourceStatusSystem/Command/GGUSTicketsCommand.py +++ b/ResourceStatusSystem/Command/GGUSTicketsCommand.py @@ -10,7 +10,7 @@ __RCSID__ = '$Id$' -import urllib2 +from six.moves.urllib_error import URLError from DIRAC import S_ERROR, S_OK from DIRAC.ConfigurationSystem.Client.Helpers.Resources import getGOCSiteName, getGOCSites @@ -91,7 +91,7 @@ def doNew(self, masterParams=None): try: results = self.gClient.getTicketsList(gocName) - except urllib2.URLError as e: + except URLError as e: return S_ERROR('%s %s' % (gocName, e)) if not results['OK']: diff --git a/ResourceStatusSystem/Command/TransferCommand.py b/ResourceStatusSystem/Command/TransferCommand.py index cf7770abc21..ccddb948389 100644 --- a/ResourceStatusSystem/Command/TransferCommand.py +++ b/ResourceStatusSystem/Command/TransferCommand.py @@ -128,7 +128,7 @@ def doNew(self, masterParams=None): if 'data' not in transferResults: return S_ERROR('Missing data key') - transferResults = {channel: strToIntDict(value) for channel, value in transferResults['data'].iteritems()} + transferResults = {channel: strToIntDict(value) for channel, value in transferResults['data'].items()} uniformResult = [] diff --git a/ResourceStatusSystem/Command/VOBOXAvailabilityCommand.py b/ResourceStatusSystem/Command/VOBOXAvailabilityCommand.py index dec8a6505aa..9ea318bbf11 100644 --- a/ResourceStatusSystem/Command/VOBOXAvailabilityCommand.py +++ b/ResourceStatusSystem/Command/VOBOXAvailabilityCommand.py @@ -7,7 +7,7 @@ # FIXME: NOT Usable ATM # missing doNew, doCache, doMaster -import urlparse +from six.moves.urllib import parse as urlparse from DIRAC import S_OK, S_ERROR from DIRAC.Core.DISET.RPCClient import RPCClient diff --git a/ResourceStatusSystem/Command/test/Test_RSS_Command_VOBOXAvailabilityCommand.py b/ResourceStatusSystem/Command/test/Test_RSS_Command_VOBOXAvailabilityCommand.py index 2aef96d7dbc..8913c2619a8 100644 --- a/ResourceStatusSystem/Command/test/Test_RSS_Command_VOBOXAvailabilityCommand.py +++ b/ResourceStatusSystem/Command/test/Test_RSS_Command_VOBOXAvailabilityCommand.py @@ -12,7 +12,12 @@ __RCSID__ = '$Id: $' -################################################################################ +try: + # Python 2: "reload" is built-in + reload +except NameError: + from importlib import reload # pylint: disable=no-name-in-module + class VOBOXAvailabilityCommand_TestCase( unittest.TestCase ): diff --git a/ResourceStatusSystem/DB/ResourceManagementDB.py b/ResourceStatusSystem/DB/ResourceManagementDB.py index 35c2015ba40..50d80e25bdc 100644 --- a/ResourceStatusSystem/DB/ResourceManagementDB.py +++ b/ResourceStatusSystem/DB/ResourceManagementDB.py @@ -418,7 +418,7 @@ def addOrModify(self, table, params): try: select = Query(table_c, session=session) - for columnName, columnValue in params.iteritems(): + for columnName, columnValue in params.items(): if not columnValue or columnName not in primaryKeys: continue column_a = getattr(table_c, columnName.lower()) @@ -440,7 +440,7 @@ def addOrModify(self, table, params): params['DateEffective'] = None # now we assume we need to modify - for columnName, columnValue in params.iteritems(): + for columnName, columnValue in params.items(): if columnName == 'LastCheckTime' and not columnValue: # we always update lastCheckTime columnValue = datetime.datetime.utcnow().replace(microsecond=0) if columnName == 'DateEffective' and not columnValue: # we always update DateEffective, if there diff --git a/ResourceStatusSystem/DB/ResourceStatusDB.py b/ResourceStatusSystem/DB/ResourceStatusDB.py index 155f1eb1e68..f7f3f0196b0 100644 --- a/ResourceStatusSystem/DB/ResourceStatusDB.py +++ b/ResourceStatusSystem/DB/ResourceStatusDB.py @@ -319,7 +319,7 @@ def addOrModify(self, table, params): try: select = Query(table_c, session=session) - for columnName, columnValue in params.iteritems(): + for columnName, columnValue in params.items(): if not columnValue or columnName not in primaryKeys: continue column_a = getattr(table_c, columnName.lower()) @@ -348,7 +348,7 @@ def addOrModify(self, table, params): if params.get('Status') != res.status: # we update dateEffective iff we change the status changeDE = True - for columnName, columnValue in params.iteritems(): + for columnName, columnValue in params.items(): if columnName == 'LastCheckTime' and not columnValue: # we always update lastCheckTime columnValue = datetime.datetime.utcnow().replace(microsecond=0) if changeDE and columnName == 'DateEffective' and not columnValue: @@ -389,7 +389,7 @@ def addIfNotThere(self, table, params): try: select = Query(table_c, session=session) - for columnName, columnValue in params.iteritems(): + for columnName, columnValue in params.items(): if not columnValue or columnName not in primaryKeys: continue column_a = getattr(table_c, columnName.lower()) diff --git a/ResourceStatusSystem/Policy/test/Test_RSS_Policy_Configurations.py b/ResourceStatusSystem/Policy/test/Test_RSS_Policy_Configurations.py index 826e895478d..477bdff3638 100644 --- a/ResourceStatusSystem/Policy/test/Test_RSS_Policy_Configurations.py +++ b/ResourceStatusSystem/Policy/test/Test_RSS_Policy_Configurations.py @@ -41,7 +41,7 @@ def test_policiesMeta( self ): policiesMeta = self.moduleTested.POLICIESMETA for _policyName, policyMeta in policiesMeta.items(): - self.assertEqual( [ 'args', 'command', 'description', 'module' ], policyMeta.keys() ) + self.assertEqual({'args', 'command', 'description', 'module'}, set(policyMeta)) diff --git a/ResourceStatusSystem/Utilities/InfoGetter.py b/ResourceStatusSystem/Utilities/InfoGetter.py index dae6ad415ab..d58f801f057 100644 --- a/ResourceStatusSystem/Utilities/InfoGetter.py +++ b/ResourceStatusSystem/Utilities/InfoGetter.py @@ -48,7 +48,7 @@ def getPoliciesThatApply(decisionParams): # ... # Get policies that match the given decisionParameters - for policyName, policySetup in policiesConfig.iteritems(): + for policyName, policySetup in policiesConfig.items(): # The parameter policyType replaces policyName, so if it is not present, # we pick policyName @@ -131,7 +131,7 @@ def getPolicyActionsThatApply(decisionParams, singlePolicyResults, policyCombine continue # Get policies that match the given decissionParameters - for policyActionName, policyActionConfig in policyActionsConfig.iteritems(): + for policyActionName, policyActionConfig in policyActionsConfig.items(): # The parameter policyType is mandatory. If not present, we pick policyActionName try: diff --git a/ResourceStatusSystem/Utilities/RSSCache.py b/ResourceStatusSystem/Utilities/RSSCache.py index 057e3514eee..0bf65d5f237 100644 --- a/ResourceStatusSystem/Utilities/RSSCache.py +++ b/ResourceStatusSystem/Utilities/RSSCache.py @@ -216,7 +216,7 @@ def __updateCache(self, newCache): itemsCounter = 0 - for cacheKey, cacheValue in newCache.iteritems(): + for cacheKey, cacheValue in newCache.items(): self.__rssCache.add(cacheKey, self.__lifeTime, value=cacheValue) itemsCounter += 1 diff --git a/ResourceStatusSystem/Utilities/RSSCacheNoThread.py b/ResourceStatusSystem/Utilities/RSSCacheNoThread.py index 099a151d790..5fab289f057 100644 --- a/ResourceStatusSystem/Utilities/RSSCacheNoThread.py +++ b/ResourceStatusSystem/Utilities/RSSCacheNoThread.py @@ -153,7 +153,7 @@ def __updateCache(self, newCache): :return: dictionary. It is newCache argument. """ - for cacheKey, cacheValue in newCache.iteritems(): + for cacheKey, cacheValue in newCache.items(): self.__cache.add(cacheKey, self.__lifeTime, value=cacheValue) # We are assuming nothing will fail while inserting in the cache. There is @@ -322,7 +322,7 @@ def __match(self, validCache, elementNames, elementType, statusTypes): :return: S_OK() || S_ERROR() """ - cacheKeys = validCache.keys() + cacheKeys = list(validCache) if isinstance(elementNames, six.string_types): elementNames = [elementNames] @@ -388,7 +388,7 @@ def __getDictFromCacheMatches(cacheMatches): result = {} - for cacheKey, cacheValue in cacheMatches.iteritems(): + for cacheKey, cacheValue in cacheMatches.items(): elementName, _elementType, statusType = cacheKey result.setdefault(elementName, {})[statusType] = cacheValue diff --git a/ResourceStatusSystem/Utilities/Utils.py b/ResourceStatusSystem/Utilities/Utils.py index a3165da8082..1e6e40cdd4f 100644 --- a/ResourceStatusSystem/Utilities/Utils.py +++ b/ResourceStatusSystem/Utilities/Utils.py @@ -53,7 +53,7 @@ def getCSTreeAsDict(treePath): opts = opts['Value'] - for optKey, optValue in opts.iteritems(): + for optKey, optValue in opts.items(): if optValue.find(',') > -1: optValue = List.fromChar(optValue) else: diff --git a/ResourceStatusSystem/scripts/dirac-rss-list-status.py b/ResourceStatusSystem/scripts/dirac-rss-list-status.py index f9c022c1af7..93405d5af58 100755 --- a/ResourceStatusSystem/scripts/dirac-rss-list-status.py +++ b/ResourceStatusSystem/scripts/dirac-rss-list-status.py @@ -84,7 +84,7 @@ def parseSwitches(): Script.showHelp(exitCode=1) subLogger.debug("The switches used are:") - map(subLogger.debug, switches.iteritems()) + map(subLogger.debug, switches.items()) return switches diff --git a/ResourceStatusSystem/scripts/dirac-rss-query-db.py b/ResourceStatusSystem/scripts/dirac-rss-query-db.py index 3e214cefe41..257679020df 100755 --- a/ResourceStatusSystem/scripts/dirac-rss-query-db.py +++ b/ResourceStatusSystem/scripts/dirac-rss-query-db.py @@ -112,7 +112,7 @@ def parseSwitches(): switches['statusType'] = 'all' subLogger.debug("The switches used are:") - map(subLogger.debug, switches.iteritems()) + map(subLogger.debug, switches.items()) return args, switches @@ -252,11 +252,11 @@ def confirm(query, matches): def tabularPrint(table): - columns_names = table[0].keys() + columns_names = list(table[0]) records = [] for row in table: record = [] - for _k, v in row.iteritems(): + for _k, v in row.items(): if isinstance(v, datetime.datetime): record.append(Time.toString(v)) elif v is None: diff --git a/ResourceStatusSystem/scripts/dirac-rss-query-dtcache.py b/ResourceStatusSystem/scripts/dirac-rss-query-dtcache.py index 28dee6d933e..4ec9ff2f2f2 100755 --- a/ResourceStatusSystem/scripts/dirac-rss-query-dtcache.py +++ b/ResourceStatusSystem/scripts/dirac-rss-query-dtcache.py @@ -96,7 +96,7 @@ def parseSwitches(): error("'ongoing' switch can be used only with 'select'") subLogger.debug("The switches used are:") - map(subLogger.debug, switches.iteritems()) + map(subLogger.debug, switches.items()) return (args, switches) @@ -208,11 +208,11 @@ def confirm(query, matches): def tabularPrint(table): - columns_names = table[0].keys() + columns_names = list(table[0]) records = [] for row in table: record = [] - for k, v in row.iteritems(): + for k, v in row.items(): if isinstance(v, datetime.datetime): record.append(Time.toString(v)) elif v is None: diff --git a/ResourceStatusSystem/scripts/dirac-rss-set-status.py b/ResourceStatusSystem/scripts/dirac-rss-set-status.py index 33f45d7289a..9fba2a9166a 100755 --- a/ResourceStatusSystem/scripts/dirac-rss-set-status.py +++ b/ResourceStatusSystem/scripts/dirac-rss-set-status.py @@ -87,7 +87,7 @@ def parseSwitches(): Script.showHelp(exitCode=1) subLogger.debug("The switches used are:") - map(subLogger.debug, switches.iteritems()) + map(subLogger.debug, switches.items()) return switches diff --git a/ResourceStatusSystem/scripts/dirac-rss-set-token.py b/ResourceStatusSystem/scripts/dirac-rss-set-token.py index b5e48a12a50..cd447543343 100755 --- a/ResourceStatusSystem/scripts/dirac-rss-set-token.py +++ b/ResourceStatusSystem/scripts/dirac-rss-set-token.py @@ -92,7 +92,7 @@ def parseSwitches(): Script.showHelp(exitCode=1) subLogger.debug("The switches used are:") - map(subLogger.debug, switches.iteritems()) + map(subLogger.debug, switches.items()) return switches diff --git a/ResourceStatusSystem/scripts/dirac-rss-sync.py b/ResourceStatusSystem/scripts/dirac-rss-sync.py index 6dddecb57fe..d92f3c0c011 100755 --- a/ResourceStatusSystem/scripts/dirac-rss-sync.py +++ b/ResourceStatusSystem/scripts/dirac-rss-sync.py @@ -69,7 +69,7 @@ def parseSwitches(): Script.showHelp(exitCode=1) subLogger.debug("The switches used are:") - map(subLogger.debug, switches.iteritems()) + map(subLogger.debug, switches.items()) return switches @@ -134,7 +134,7 @@ def initSites(): subLogger.error(sites['Message']) DIRACExit(1) - for site, elements in sites['Value'].iteritems(): + for site, elements in sites['Value'].items(): result = rssClient.addOrModifyStatusElement("Site", "Status", name=site, statusType='all', @@ -183,7 +183,7 @@ def initSEs(): # We copy the list into a new object to remove items INSIDE the loop ! statusTypesList = statusTypes[:] - for statusType, status in opts.iteritems(): + for statusType, status in opts.items(): # Sanity check... if statusType not in statusTypesList: diff --git a/Resources/Catalog/FileCatalog.py b/Resources/Catalog/FileCatalog.py index b925ec21b41..a2b786b9675 100644 --- a/Resources/Catalog/FileCatalog.py +++ b/Resources/Catalog/FileCatalog.py @@ -202,7 +202,7 @@ def w_execute(self, *parms, **kws): fileInfo, lfnMapDict = result['Value'] # No need to check the LFNs again in the clients kws['LFNChecking'] = False - allLfns = fileInfo.keys() + allLfns = list(fileInfo.keys()) parms1 = parms[1:] for catalogName, oCatalog, master in self.writeCatalogs: diff --git a/Resources/Catalog/FileCatalogClientBase.py b/Resources/Catalog/FileCatalogClientBase.py index 6b366ae3c9c..abc9d8ef75f 100644 --- a/Resources/Catalog/FileCatalogClientBase.py +++ b/Resources/Catalog/FileCatalogClientBase.py @@ -91,7 +91,7 @@ def hasAccess(self, paths, _opType): :return: { successful : { lfn : [ ses ] } : failed : { lfn : { se : msg } } } """ - lfns = paths.keys() + lfns = list(paths) return S_OK({'Failed': {}, 'Successful': dict.fromkeys(lfns, True)}) @checkCatalogArguments diff --git a/Resources/Catalog/LcgFileCatalogClient.py b/Resources/Catalog/LcgFileCatalogClient.py index cf6d50c4d56..b2beb885214 100755 --- a/Resources/Catalog/LcgFileCatalogClient.py +++ b/Resources/Catalog/LcgFileCatalogClient.py @@ -1026,7 +1026,7 @@ def removeReplica(self, lfns): successful[lfn] = True else: failed[lfn] = res['Message'] - lfnRemoved = successful.keys() + lfnRemoved = list(successful) if len(lfnRemoved) > 0: res = self.getReplicas(lfnRemoved, True) zeroReplicaFiles = [] @@ -1347,7 +1347,7 @@ def __executeOperation(self, path, method): return S_ERROR("Unable to invoke %s, it isn't a member function of LcgFileCatalogClient" % method) res = fcn(path) if isinstance(path, dict): - path = path.keys()[0] + path = list(path)[0] if not res['OK']: return res elif path not in res['Value']['Successful']: @@ -1732,7 +1732,7 @@ def __removeDataset(self, datasetName): res = self.__getDirectoryContents(datasetName) if not res['OK']: return res - links = res['Value']['Files'].keys() + links = list(res['Value']['Files']) res = self.removeLink(links) if not res['OK']: return res @@ -1779,7 +1779,7 @@ def getUserDirectory(self, usernames): for username in usernames: userDirectory = "/%s/user/%s/%s" % (vo, username[0], username) usernameDict[userDirectory] = username - res = self.exists(usernameDict.keys()) + res = self.exists(list(usernameDict)) if not res['OK']: return res failed = {} @@ -1834,7 +1834,7 @@ def removeUserDirectory(self, username): successful[username] = True else: directoriesToRemove[directory] = username - res = self.removeDirectory(directoriesToRemove.keys()) + res = self.removeDirectory(list(directoriesToRemove)) if not res['OK']: return res for directory, error in res['Value']['Failed'].items(): diff --git a/Resources/Catalog/test/Test_FileCatalog.py b/Resources/Catalog/test/Test_FileCatalog.py index 34e1b688f92..6158fed6ce8 100644 --- a/Resources/Catalog/test/Test_FileCatalog.py +++ b/Resources/Catalog/test/Test_FileCatalog.py @@ -232,7 +232,7 @@ def test_01_Normal(self, mk_getSelectedCatalogs, mk_getEligibleCatalogs): res = fc.write1(lfn) self.assertTrue(res['OK']) self.assertTrue(lfn in res['Value']['Successful']) - self.assertEqual(sorted(['c1', 'c2']), sorted(res['Value']['Successful'][lfn].keys())) + self.assertEqual(sorted(['c1', 'c2']), sorted(res['Value']['Successful'][lfn])) self.assertTrue(not res['Value']['Failed']) # Test a write method that only the master has @@ -240,7 +240,7 @@ def test_01_Normal(self, mk_getSelectedCatalogs, mk_getEligibleCatalogs): res = fc.write2(lfn) self.assertTrue(res['OK']) self.assertTrue(lfn in res['Value']['Successful']) - self.assertEqual(['c1'], res['Value']['Successful'][lfn].keys()) + self.assertEqual(['c1'], sorted(res['Value']['Successful'][lfn])) self.assertTrue(not res['Value']['Failed']) # Test a write method that makes an error for master @@ -255,23 +255,23 @@ def test_01_Normal(self, mk_getSelectedCatalogs, mk_getEligibleCatalogs): res = fc.write1(lfn) self.assertTrue(res['OK']) self.assertTrue(not res['Value']['Successful']) - self.assertEqual(['c1'], res['Value']['Failed'][lfn].keys()) + self.assertEqual(['c1'], sorted(res['Value']['Failed'][lfn])) # Test a write method that makes an error for non master # The lfn should be in failed for non master and successful for the master lfn = '/lhcb/c2/Error' res = fc.write1(lfn) self.assertTrue(res['OK']) - self.assertEqual(['c1'], res['Value']['Successful'][lfn].keys()) - self.assertEqual(['c2'], res['Value']['Failed'][lfn].keys()) + self.assertEqual(['c1'], sorted(res['Value']['Successful'][lfn])) + self.assertEqual(['c2'], sorted(res['Value']['Failed'][lfn])) # Test a write method that fails for non master # The lfn should be in failed for non master and successful for the master lfn = '/lhcb/c2/Failed' res = fc.write1(lfn) self.assertTrue(res['OK']) - self.assertEqual(['c1'], res['Value']['Successful'][lfn].keys()) - self.assertEqual(['c2'], res['Value']['Failed'][lfn].keys()) + self.assertEqual(['c1'], sorted(res['Value']['Successful'][lfn])) + self.assertEqual(['c2'], sorted(res['Value']['Failed'][lfn])) @mock.patch.object( DIRAC.Resources.Catalog.FileCatalog.FileCatalog, @@ -380,7 +380,7 @@ def test_01_oneMasterNormal(self, mk_getSelectedCatalogs, mk_getEligibleCatalogs res = fc.write1(lfn) self.assertTrue(res['OK']) self.assertTrue(lfn in res['Value']['Successful']) - self.assertEqual(sorted(['c1', 'c2']), sorted(res['Value']['Successful'][lfn].keys())) + self.assertEqual(sorted(['c1', 'c2']), sorted(res['Value']['Successful'][lfn])) self.assertTrue(not res['Value']['Failed']) # Test a write method that only the master has @@ -388,7 +388,7 @@ def test_01_oneMasterNormal(self, mk_getSelectedCatalogs, mk_getEligibleCatalogs res = fc.write2(lfn) self.assertTrue(res['OK']) self.assertTrue(lfn in res['Value']['Successful']) - self.assertEqual(['c1'], res['Value']['Successful'][lfn].keys()) + self.assertEqual(['c1'], sorted(res['Value']['Successful'][lfn])) self.assertTrue(not res['Value']['Failed']) # Test a write method that makes an error for master @@ -403,23 +403,23 @@ def test_01_oneMasterNormal(self, mk_getSelectedCatalogs, mk_getEligibleCatalogs res = fc.write1(lfn) self.assertTrue(res['OK']) self.assertTrue(not res['Value']['Successful']) - self.assertEqual(['c1'], res['Value']['Failed'][lfn].keys()) + self.assertEqual(['c1'], sorted(res['Value']['Failed'][lfn])) # Test a write method that makes an error for non master # The lfn should be in failed for non master and successful for the master lfn = '/lhcb/c2/Error' res = fc.write1(lfn) self.assertTrue(res['OK']) - self.assertEqual(['c1'], res['Value']['Successful'][lfn].keys()) - self.assertEqual(['c2'], res['Value']['Failed'][lfn].keys()) + self.assertEqual(['c1'], sorted(res['Value']['Successful'][lfn])) + self.assertEqual(['c2'], sorted(res['Value']['Failed'][lfn])) # Test a write method that fails for non master # The lfn should be in failed for non master and successful for the master lfn = '/lhcb/c2/Failed' res = fc.write1(lfn) self.assertTrue(res['OK']) - self.assertEqual(['c1'], res['Value']['Successful'][lfn].keys()) - self.assertEqual(['c2'], res['Value']['Failed'][lfn].keys()) + self.assertEqual(['c1'], sorted(res['Value']['Successful'][lfn])) + self.assertEqual(['c2'], sorted(res['Value']['Failed'][lfn])) if __name__ == '__main__': diff --git a/Resources/Computing/ARCComputingElement.py b/Resources/Computing/ARCComputingElement.py index f2305020254..8b346008ed5 100755 --- a/Resources/Computing/ARCComputingElement.py +++ b/Resources/Computing/ARCComputingElement.py @@ -163,7 +163,7 @@ def __writeXRSL(self, executableFile, processors=1): nProcessors = processors if processors > 1 else self.ceParameters.get('NumberOfProcessors', processors) xrslMPAdditions = '' - if nProcessors > 1: + if nProcessors and nProcessors > 1: xrslMPAdditions = """ (count = %(processors)u) %(xrslMPExtraString)s diff --git a/Resources/Computing/BOINCComputingElement.py b/Resources/Computing/BOINCComputingElement.py index 69c6a73221a..1cc6496c515 100644 --- a/Resources/Computing/BOINCComputingElement.py +++ b/Resources/Computing/BOINCComputingElement.py @@ -16,7 +16,7 @@ import base64 import tempfile -from urlparse import urlparse +from six.moves.urllib.parse import urlparse from DIRAC import S_OK, S_ERROR from DIRAC.Resources.Computing.ComputingElement import ComputingElement diff --git a/Resources/Computing/BatchSystems/Condor.py b/Resources/Computing/BatchSystems/Condor.py index 142ba9ee5af..a5ca1d6f101 100644 --- a/Resources/Computing/BatchSystems/Condor.py +++ b/Resources/Computing/BatchSystems/Condor.py @@ -14,7 +14,12 @@ from __future__ import division import re import tempfile -import commands +# TODO: This should be moderised to use subprocess(32) +try: + import commands +except ImportError: + # Python 3's subprocess module contains a compatability layer + import subprocess as commands import os __RCSID__ = "$Id$" @@ -144,7 +149,7 @@ def submitJob(self, **kwargs): if submittedJobs > 0 and cluster: resultDict['Status'] = 0 resultDict['Jobs'] = [] - for i in xrange(submittedJobs): + for i in range(submittedJobs): resultDict['Jobs'].append('.'.join([cluster, str(i)])) else: resultDict['Status'] = status diff --git a/Resources/Computing/BatchSystems/GE.py b/Resources/Computing/BatchSystems/GE.py index e3d4c672c33..63c30d2ceb9 100644 --- a/Resources/Computing/BatchSystems/GE.py +++ b/Resources/Computing/BatchSystems/GE.py @@ -18,7 +18,12 @@ from __future__ import absolute_import from __future__ import division import re -import commands +# TODO: This should be moderised to use subprocess(32) +try: + import commands +except ImportError: + # Python 3's subprocess module contains a compatability layer + import subprocess as commands import os __RCSID__ = "$Id$" @@ -43,7 +48,7 @@ def submitJob(self, **kwargs): outputs = [] output = '' - for _i in xrange(int(nJobs)): + for _i in range(int(nJobs)): cmd = '%s; ' % preamble if preamble else '' cmd += "qsub -o %(OutputDir)s -e %(ErrorDir)s -N DIRACPilot %(SubmitOptions)s %(Executable)s" % kwargs status, output = commands.getstatusoutput(cmd) diff --git a/Resources/Computing/BatchSystems/Host.py b/Resources/Computing/BatchSystems/Host.py index 67b72391f7b..1005017753c 100644 --- a/Resources/Computing/BatchSystems/Host.py +++ b/Resources/Computing/BatchSystems/Host.py @@ -11,7 +11,12 @@ from __future__ import print_function from __future__ import absolute_import from __future__ import division -import commands +# TODO: This should be moderised to use subprocess(32) +try: + import commands +except ImportError: + # Python 3's subprocess module contains a compatability layer + import subprocess as commands import os import glob import shutil diff --git a/Resources/Computing/BatchSystems/LSF.py b/Resources/Computing/BatchSystems/LSF.py index a7c4f725415..3eb80815c40 100644 --- a/Resources/Computing/BatchSystems/LSF.py +++ b/Resources/Computing/BatchSystems/LSF.py @@ -13,7 +13,12 @@ from __future__ import absolute_import from __future__ import division import re -import commands +# TODO: This should be moderised to use subprocess(32) +try: + import commands +except ImportError: + # Python 3's subprocess module contains a compatability layer + import subprocess as commands import os __RCSID__ = "$Id$" @@ -47,7 +52,7 @@ def submitJob(self, **kwargs): outputDir = os.path.expandvars(outputDir) errorDir = os.path.expandvars(errorDir) executable = os.path.expandvars(executable) - for _i in xrange(int(nJobs)): + for _i in range(int(nJobs)): cmd = '%s; ' % preamble if preamble else '' cmd += "bsub -o %s -e %s -q %s -J DIRACPilot %s %s" % (outputDir, errorDir, diff --git a/Resources/Computing/BatchSystems/OAR.py b/Resources/Computing/BatchSystems/OAR.py index 822643652b6..e17f8399402 100644 --- a/Resources/Computing/BatchSystems/OAR.py +++ b/Resources/Computing/BatchSystems/OAR.py @@ -12,7 +12,12 @@ from __future__ import print_function from __future__ import absolute_import from __future__ import division -import commands +# TODO: This should be moderised to use subprocess(32) +try: + import commands +except ImportError: + # Python 3's subprocess module contains a compatability layer + import subprocess as commands import os import json @@ -53,7 +58,7 @@ def submitJob(self, **kwargs): executable = os.path.expandvars(executable) jobIDs = [] preamble = kwargs.get("Preamble") - for _i in xrange(nJobs): + for _i in range(nJobs): cmd = '%s; ' % preamble if preamble else '' cmd += "oarsub -O %s.out -E %s.err -q %s -n DIRACPilot %s %s" % (outFile, errFile, diff --git a/Resources/Computing/BatchSystems/SLURM.py b/Resources/Computing/BatchSystems/SLURM.py index 8b0fba0e95d..adc542a05bc 100644 --- a/Resources/Computing/BatchSystems/SLURM.py +++ b/Resources/Computing/BatchSystems/SLURM.py @@ -12,7 +12,12 @@ from __future__ import print_function from __future__ import absolute_import from __future__ import division -import commands +# TODO: This should be moderised to use subprocess(32) +try: + import commands +except ImportError: + # Python 3's subprocess module contains a compatability layer + import subprocess as commands import os import re diff --git a/Resources/Computing/BatchSystems/TimeLeft/LSFResourceUsage.py b/Resources/Computing/BatchSystems/TimeLeft/LSFResourceUsage.py index 0ce1b999fb4..7170a8a07ec 100644 --- a/Resources/Computing/BatchSystems/TimeLeft/LSFResourceUsage.py +++ b/Resources/Computing/BatchSystems/TimeLeft/LSFResourceUsage.py @@ -51,7 +51,7 @@ def __init__(self): lines = str(result['Value']).split('\n') self.log.debug('From %s' % cmd, '\n'.join([line if len(line) <= 128 else line[:128] + ' [...]' for line in lines])) - for i in xrange(len(lines)): + for i in range(len(lines)): if re.search('.*CPULIMIT.*', lines[i]): info = lines[i + 1].split() if len(info) >= 4: @@ -87,7 +87,7 @@ def __init__(self): if len(l1) > len(l2): self.log.error("Failed lshost command", "%s:\n %s\n %s" % (cmd, lines[0], lines[0])) else: - for i in xrange(len(l1)): + for i in range(len(l1)): if l1[i] == 'cpuf': try: self.normRef = float(l2[i]) @@ -175,7 +175,7 @@ def __init__(self): if len(l1) > len(l2): self.log.error("Failed lshost command", "%s:\n %s\n %s" % (cmd, lines[0], lines[0])) else: - for i in xrange(len(l1)): + for i in range(len(l1)): if l1[i] == 'cpuf': try: self.hostNorm = float(l2[i]) @@ -220,7 +220,7 @@ def getResourceUsage(self): sCPU = None sStart = None - for i in xrange(len(l1)): + for i in range(len(l1)): if l1[i] == 'CPU_USED': sCPU = l2[i] lCPU = sCPU.split(':') diff --git a/Resources/Computing/BatchSystems/Torque.py b/Resources/Computing/BatchSystems/Torque.py index e84c69f30b6..f4f1e51e40d 100644 --- a/Resources/Computing/BatchSystems/Torque.py +++ b/Resources/Computing/BatchSystems/Torque.py @@ -12,7 +12,12 @@ from __future__ import print_function from __future__ import absolute_import from __future__ import division -import commands +# TODO: This should be moderised to use subprocess(32) +try: + import commands +except ImportError: + # Python 3's subprocess module contains a compatability layer + import subprocess as commands import os __RCSID__ = "$Id$" @@ -43,7 +48,7 @@ def submitJob(self, **kwargs): status = -1 preamble = kwargs.get("Preamble") - for _i in xrange(int(nJobs)): + for _i in range(int(nJobs)): cmd = '%s; ' % preamble if preamble else '' cmd += "qsub -o %(OutputDir)s " \ "-e %(ErrorDir)s " \ diff --git a/Resources/Computing/CREAMComputingElement.py b/Resources/Computing/CREAMComputingElement.py index 72824bf237c..050476eacd5 100644 --- a/Resources/Computing/CREAMComputingElement.py +++ b/Resources/Computing/CREAMComputingElement.py @@ -226,7 +226,7 @@ def getCEStatus(self, jobIDList=None): running = 0 waiting = 0 statusDict = {} - for ref, status in resultDict.iteritems(): + for ref, status in resultDict.items(): if jobIDList is not None and ref not in jobIDList: continue if status == 'Scheduled': @@ -262,7 +262,7 @@ def getJobStatus(self, jobIDList): if delegationIDs: # Renew proxies in batches to avoid timeouts chunkSize = 10 - for i in xrange(0, len(delegationIDs), chunkSize): + for i in range(0, len(delegationIDs), chunkSize): chunk = delegationIDs[i:i + chunkSize] cmd = ['glite-ce-proxy-renew', '-e', self.ceName] cmd.extend(chunk) diff --git a/Resources/Computing/ComputingElement.py b/Resources/Computing/ComputingElement.py index 30c330ed034..bab96d81995 100755 --- a/Resources/Computing/ComputingElement.py +++ b/Resources/Computing/ComputingElement.py @@ -165,7 +165,7 @@ def initializeParameters(self): if result['OK'] and result['Value']: localSiteParameters = result['Value'] self.log.debug('Local site parameters are: %s' % (localSiteParameters)) - for option, value in localSiteParameters.iteritems(): + for option, value in localSiteParameters.items(): if option == 'Architecture': self.ceParameters['Platform'] = value self.ceParameters['Architecture'] = value @@ -455,7 +455,7 @@ def getDescription(self): """ ceDict = {} - for option, value in self.ceParameters.iteritems(): + for option, value in self.ceParameters.items(): if isinstance(value, list): ceDict[option] = value elif isinstance(value, six.string_types): diff --git a/Resources/Computing/GlobusComputingElement.py b/Resources/Computing/GlobusComputingElement.py index a95e20d2404..e89bfafabd2 100644 --- a/Resources/Computing/GlobusComputingElement.py +++ b/Resources/Computing/GlobusComputingElement.py @@ -69,7 +69,7 @@ def submitJob(self, executableFile, proxy, numberOfJobs=1): batchIDList = [] stampDict = {} - for _i in xrange(numberOfJobs): + for _i in range(numberOfJobs): diracStamp = makeGuid()[:8] queueName = '%s/%s' % (self.ceName, self.queue) cmd = ['globus-job-submit', queueName, "-s", executableFile] diff --git a/Resources/Computing/HTCondorCEComputingElement.py b/Resources/Computing/HTCondorCEComputingElement.py index aec0bd3f1d5..97a5815e8ee 100644 --- a/Resources/Computing/HTCondorCEComputingElement.py +++ b/Resources/Computing/HTCondorCEComputingElement.py @@ -48,7 +48,12 @@ import six import os import tempfile -import commands +# TODO: This should be moderised to use subprocess(32) +try: + import commands +except ImportError: + # Python 3's subprocess module contains a compatability layer + import subprocess as commands import errno from DIRAC import S_OK, S_ERROR, gConfig @@ -253,7 +258,7 @@ def _reset(self): self.outputURL = self.ceParameters.get('OutputURL', 'gsiftp://localhost') self.gridEnv = self.ceParameters.get('GridEnv') self.daysToKeepLogs = self.ceParameters.get('DaysToKeepLogs', DEFAULT_DAYSTOKEEPLOGS) - self.extraSubmitString = self.ceParameters.get('ExtraSubmitString', '').decode('string_escape') + self.extraSubmitString = str(self.ceParameters.get('ExtraSubmitString', '').encode().decode('unicode_escape')) self.useLocalSchedd = self.ceParameters.get('UseLocalSchedd', self.useLocalSchedd) if isinstance(self.useLocalSchedd, six.string_types): if self.useLocalSchedd == "False": @@ -400,7 +405,7 @@ def getJobStatus(self, jobIDList): treatCondorHistory(condorHistCall, qList) - for job, jobID in condorIDs.iteritems(): + for job, jobID in condorIDs.items(): pilotStatus = parseCondorStatus(qList, jobID) if pilotStatus == 'HELD': diff --git a/Resources/Computing/LocalComputingElement.py b/Resources/Computing/LocalComputingElement.py index 65f9e5a9cdb..f4d99ff33dc 100644 --- a/Resources/Computing/LocalComputingElement.py +++ b/Resources/Computing/LocalComputingElement.py @@ -15,7 +15,7 @@ import tempfile import getpass import errno -from urlparse import urlparse +from six.moves.urllib.parse import urlparse from DIRAC import S_OK, S_ERROR from DIRAC import gConfig diff --git a/Resources/Computing/PoolComputingElement.py b/Resources/Computing/PoolComputingElement.py index fd6380fb825..2a181b86ead 100644 --- a/Resources/Computing/PoolComputingElement.py +++ b/Resources/Computing/PoolComputingElement.py @@ -215,7 +215,7 @@ def getCEStatus(self): self.pPool.processResults() result = S_OK() nJobs = 0 - for _j, value in self.processorsPerTask.iteritems(): + for _j, value in self.processorsPerTask.items(): if value > 0: nJobs += 1 result['SubmittedJobs'] = nJobs diff --git a/Resources/Computing/SSHBatchComputingElement.py b/Resources/Computing/SSHBatchComputingElement.py index 244bd73d8b6..2483b5c3d63 100644 --- a/Resources/Computing/SSHBatchComputingElement.py +++ b/Resources/Computing/SSHBatchComputingElement.py @@ -17,7 +17,7 @@ import os import socket import stat -from urlparse import urlparse +from six.moves.urllib.parse import urlparse from DIRAC import S_OK, S_ERROR from DIRAC import rootPath @@ -169,7 +169,7 @@ def killJob(self, jobIDs): hostDict[host].append(job) failed = [] - for host, jobIDList in hostDict.iteritems(): + for host, jobIDList in hostDict.items(): result = self._killJobOnHost(jobIDList, host) if not result['OK']: failed.extend(jobIDList) @@ -213,7 +213,7 @@ def getJobStatus(self, jobIDList): resultDict = {} failed = [] - for host, jobIDList in hostDict.iteritems(): + for host, jobIDList in hostDict.items(): result = self._getJobStatusOnHost(jobIDList, host) if not result['OK']: failed.extend(jobIDList) diff --git a/Resources/Computing/SSHComputingElement.py b/Resources/Computing/SSHComputingElement.py index 7a8f91562e1..af4ba1089cc 100644 --- a/Resources/Computing/SSHComputingElement.py +++ b/Resources/Computing/SSHComputingElement.py @@ -18,7 +18,7 @@ import stat import shutil import errno -from urlparse import urlparse +from six.moves.urllib.parse import urlparse from DIRAC import S_OK, S_ERROR from DIRAC import rootPath @@ -525,7 +525,7 @@ def _submitJobToHost(self, executableFile, numberOfJobs, host=None): return result jobStamps = [] - for _i in xrange(numberOfJobs): + for _i in range(numberOfJobs): jobStamps.append(makeGuid()[:8]) numberOfProcessors = self.ceParameters.get('NumberOfProcessors', 1) diff --git a/Resources/Computing/glexecComputingElement.py b/Resources/Computing/glexecComputingElement.py index 9fa50aadebc..66b5522cf94 100644 --- a/Resources/Computing/glexecComputingElement.py +++ b/Resources/Computing/glexecComputingElement.py @@ -219,7 +219,7 @@ def analyseExitCode(self, resultTuple): self.log.info('glexec stderr:\n%s' % stdError) error = None - for code, msg in codes.iteritems(): + for code, msg in codes.items(): self.log.verbose('Exit code %s => %s' % (code, msg)) if status == code: error = msg diff --git a/Resources/Computing/test/Test_HTCondorCEComputingElement.py b/Resources/Computing/test/Test_HTCondorCEComputingElement.py index 081c1a94e2c..b50d558fad5 100644 --- a/Resources/Computing/test/Test_HTCondorCEComputingElement.py +++ b/Resources/Computing/test/Test_HTCondorCEComputingElement.py @@ -56,7 +56,7 @@ def test_parseCondorStatus(self): "104098.3": "HELD", "104098.4": "Unknown"} - for jobID, expected in expectedResults.iteritems(): + for jobID, expected in expectedResults.items(): self.assertEqual(HTCE.parseCondorStatus(statusLines, jobID), expected) @patch(MODNAME + ".commands.getstatusoutput", new=Mock(side_effect=([(0, "\n".join(STATUS_LINES)), diff --git a/Resources/Computing/test/Test_SSHComputingElement.py b/Resources/Computing/test/Test_SSHComputingElement.py index 19948d21127..97849e498c9 100644 --- a/Resources/Computing/test/Test_SSHComputingElement.py +++ b/Resources/Computing/test/Test_SSHComputingElement.py @@ -71,4 +71,5 @@ def test_generateControlScript(batchSystem): # Delete the control script and the .pyc file associated os.remove(source) os.remove(dest) - os.remove('%sc' % dest) + if os.path.isfile('%sc' % dest): + os.remove('%sc' % dest) diff --git a/Resources/MessageQueue/MQConsumer.py b/Resources/MessageQueue/MQConsumer.py index 0bc80d88ac1..de121c9f06d 100644 --- a/Resources/MessageQueue/MQConsumer.py +++ b/Resources/MessageQueue/MQConsumer.py @@ -4,7 +4,7 @@ from __future__ import division from __future__ import print_function -import Queue +from six.moves import queue as Queue from DIRAC import S_ERROR, S_OK, gLogger from DIRAC.Resources.MessageQueue.Utilities import getDestinationAddress, getMQService, generateDefaultCallback from DIRAC.Core.Utilities.DErrno import EMQNOM diff --git a/Resources/MessageQueue/StompMQConnector.py b/Resources/MessageQueue/StompMQConnector.py index 4049c0558c0..3f51b3095bd 100644 --- a/Resources/MessageQueue/StompMQConnector.py +++ b/Resources/MessageQueue/StompMQConnector.py @@ -186,7 +186,7 @@ def connect(self, parameters=None, serverIP=None): connections = self.connections connected = False - for ip, connection in connections.iteritems(): + for ip, connection in connections.items(): try: listener = connection.get_listener('ReconnectListener') if listener is None: @@ -210,7 +210,7 @@ def disconnect(self, parameters=None): """ log = LOG.getSubLogger('disconnect') fail = False - for ip, connection in self.connections.iteritems(): + for ip, connection in self.connections.items(): try: if connection.get_listener('ReconnectListener'): connection.remove_listener('ReconnectListener') @@ -244,7 +244,7 @@ def subscribe(self, parameters=None): log.error("No callback specified!") fail = False - for connection in self.connections.itervalues(): + for connection in self.connections.values(): try: listener = StompListener(callback, acknowledgement, connection, mId) connection.set_listener('StompListener', listener) @@ -265,7 +265,7 @@ def unsubscribe(self, parameters): dest = parameters.get('destination', '') mId = parameters.get('messengerId', '') fail = False - for ip, connection in self.connections.iteritems(): + for ip, connection in self.connections.items(): try: connection.unsubscribe(destination=dest, id=mId) except Exception as e: diff --git a/Resources/MessageQueue/Utilities.py b/Resources/MessageQueue/Utilities.py index 05e1441970c..a5aaab7e7e2 100644 --- a/Resources/MessageQueue/Utilities.py +++ b/Resources/MessageQueue/Utilities.py @@ -6,7 +6,7 @@ __RCSID__ = "$Id$" -import Queue +from six.moves import queue as Queue from DIRAC import S_OK, S_ERROR, gConfig from DIRAC.ConfigurationSystem.Client.CSAPI import CSAPI @@ -35,7 +35,7 @@ def getMQParamsFromCS(mqURI): if not result['OK'] or not result['Value']: return S_ERROR('Requested destination not found in the CS: %s::%s::%s' % (mqService, mqType, mqName)) mqDestinationPath = None - for path, value in result['Value'].iteritems(): + for path, value in result['Value'].items(): if not value and path.endswith(mqName): mqDestinationPath = path diff --git a/Resources/MessageQueue/test/Test_MQ_Utilities.py b/Resources/MessageQueue/test/Test_MQ_Utilities.py index 85cd8cefe17..56937a440c7 100644 --- a/Resources/MessageQueue/test/Test_MQ_Utilities.py +++ b/Resources/MessageQueue/test/Test_MQ_Utilities.py @@ -7,7 +7,7 @@ import DIRAC.Resources.MessageQueue.Utilities as module import unittest -import Queue +from six.moves import queue as Queue from mock import MagicMock diff --git a/Resources/ProxyProvider/DIRACCAProxyProvider.py b/Resources/ProxyProvider/DIRACCAProxyProvider.py index 512ba0d6e60..26b8e51b860 100644 --- a/Resources/ProxyProvider/DIRACCAProxyProvider.py +++ b/Resources/ProxyProvider/DIRACCAProxyProvider.py @@ -91,7 +91,7 @@ def setParameters(self, parameters): :return: S_OK()/S_ERROR() """ for k, v in parameters.items(): - if not isinstance(v, list) and k in ['Match', 'Supplied', 'Optional', 'DNOrder'] + self.fields2nid.keys(): + if not isinstance(v, list) and k in ['Match', 'Supplied', 'Optional', 'DNOrder'] + list(self.fields2nid): parameters[k] = v.replace(', ', ',').split(',') self.parameters = parameters # If CA configuration file exist @@ -118,7 +118,7 @@ def setParameters(self, parameters): # Set defaults for distridutes names self.nid2defField = {} - for field, value in self.parameters.items(): + for field, value in list(self.parameters.items()): if field in self.fields2nid and self.fields2nid[field] in allFields: self.parameters[self.fields2nid[field]] = value self.nid2defField[self.fields2nid[field]] = field @@ -355,7 +355,7 @@ def __fillX509Name(self, field, values): """ for value in values: if value and m2.x509_name_set_by_nid(self.__X509Name.x509_name, # pylint: disable=no-member - self.fields2nid[field], value) == 0: + self.fields2nid[field], value.encode()) == 0: if not self.__X509Name.add_entry_by_txt(field=field, type=ASN1.MBSTRING_ASC, entry=value, len=-1, loc=-1, set=0) == 1: return S_ERROR('Cannot set "%s" field.' % field) @@ -392,15 +392,15 @@ def __createCertM2Crypto(self): caCert = X509.load_cert_string(caCertStr) userCert.set_issuer(caCert.get_subject()) # Use CA key - with open(self.parameters['KeyFile']) as cf: + with open(self.parameters['KeyFile'], "rb") as cf: caKeyStr = cf.read() pkey = EVP.PKey() pkey.assign_rsa(RSA.load_key_string(caKeyStr, callback=util.no_passphrase_callback)) # Sign userCert.sign(pkey, self.algoritm) - userCertStr = userCert.as_pem() - userPubKeyStr = userPubKey.as_pem(cipher=None, callback=util.no_passphrase_callback) + userCertStr = userCert.as_pem().decode() + userPubKeyStr = userPubKey.as_pem(cipher=None, callback=util.no_passphrase_callback).decode() return S_OK((userCertStr, userPubKeyStr)) def _forceGenerateProxyForDN(self, dn, time, group=None): diff --git a/Resources/ProxyProvider/test/Test_DIRACCAProxyProvider.py b/Resources/ProxyProvider/test/Test_DIRACCAProxyProvider.py index 0fa3428d5ad..727dca04a27 100644 --- a/Resources/ProxyProvider/test/Test_DIRACCAProxyProvider.py +++ b/Resources/ProxyProvider/test/Test_DIRACCAProxyProvider.py @@ -9,7 +9,12 @@ import re import sys import shutil -import commands +# TODO: This should be modernised to use subprocess(32) +try: + import commands +except ImportError: + # Python 3's subprocess module contains a compatability layer + import subprocess as commands import unittest import tempfile diff --git a/Resources/Storage/GFAL2_StorageBase.py b/Resources/Storage/GFAL2_StorageBase.py index b8c7e572f83..809ea453db3 100644 --- a/Resources/Storage/GFAL2_StorageBase.py +++ b/Resources/Storage/GFAL2_StorageBase.py @@ -265,7 +265,7 @@ def putFile(self, path, sourceSize=0): failed = {} successful = {} - for dest_url, src_file in urls.iteritems(): + for dest_url, src_file in urls.items(): if not src_file: errStr = "GFAL2_StorageBase.putFile: Source file not set. Argument must be a dictionary \ (or a list of a dictionary) {url : local path}" @@ -796,7 +796,7 @@ def prestageFileStatus(self, path): failed = {} successful = {} - for path, token in urls.iteritems(): + for path, token in urls.items(): res = self.__prestageSingleFileStatus(path, token) if not res['OK']: @@ -929,7 +929,7 @@ def releaseFile(self, path): failed = {} successful = {} - for path, token in urls.iteritems(): + for path, token in urls.items(): res = self.__releaseSingleFile(path, token) if not res['OK']: @@ -1193,7 +1193,7 @@ def listDirectory(self, path): directories = [] - for url, isDirectory in res['Value']['Successful'].iteritems(): + for url, isDirectory in res['Value']['Successful'].items(): if isDirectory: directories.append(url) else: @@ -1438,7 +1438,7 @@ def putDirectory(self, path): successful = {} failed = {} - for destDir, sourceDir in urls.iteritems(): + for destDir, sourceDir in urls.items(): if not sourceDir: errStr = 'No source directory set, make sure the input format is correct { dest. dir : source dir }' return S_ERROR(errStr) @@ -1518,7 +1518,7 @@ def __putSingleDirectory(self, src_directory, dest_directory): log.debug('Failed to put files to storage.', res['Message']) allSuccessful = False else: - for fileSize in res['Value']['Successful'].itervalues(): + for fileSize in res['Value']['Successful'].values(): filesPut += 1 sizePut += fileSize if res['Value']['Failed']: @@ -1715,7 +1715,7 @@ def __getSingleDirectorySize(self, path): directorySize = 0 directoryFiles = 0 # itervalues returns a list of values of the dictionnary - for fileDict in res['Value']['Files'].itervalues(): + for fileDict in res['Value']['Files'].values(): directorySize += fileDict['Size'] directoryFiles += 1 diff --git a/Resources/Storage/RFIOStorage.py b/Resources/Storage/RFIOStorage.py index f9a1e0cb076..2a7b189e27e 100755 --- a/Resources/Storage/RFIOStorage.py +++ b/Resources/Storage/RFIOStorage.py @@ -1002,7 +1002,7 @@ def __checkArgumentFormat(self, path): elif isinstance(path, list): urls = path elif isinstance(path, dict): - urls = path.keys() + urls = list(path) else: return S_ERROR("RFIOStorage.__checkArgumentFormat: Supplied path is not of the correct format.") return S_OK(urls) diff --git a/Resources/Storage/StorageBase.py b/Resources/Storage/StorageBase.py index 4a0c7a9a859..ec2c63ccef5 100755 --- a/Resources/Storage/StorageBase.py +++ b/Resources/Storage/StorageBase.py @@ -372,7 +372,7 @@ def constructURLFromLFN(self, lfn, withWSUrl=False): urlDict = dict(self.protocolParameters) urlDict['Options'] = '&'.join("%s=%s" % (optionName, urlDict[paramName]) - for paramName, optionName in self.DYNAMIC_OPTIONS.iteritems() + for paramName, optionName in self.DYNAMIC_OPTIONS.items() if urlDict.get(paramName)) if not withWSUrl: urlDict['WSUrl'] = '' diff --git a/Resources/Storage/StorageElement.py b/Resources/Storage/StorageElement.py index 7664b94bb8c..e71470bc342 100755 --- a/Resources/Storage/StorageElement.py +++ b/Resources/Storage/StorageElement.py @@ -685,7 +685,7 @@ def _getAllOutputProtocols(self): return self.__getAllProtocols('OutputProtocols') def generateTransferURLsBetweenSEs(self, lfns, sourceSE, protocols=None): - """ This negociate the URLs to be used for third party copy. + """ This negotiate the URLs to be used for third party copy. This is mostly useful for FTS. If protocols is given, it restricts the list of plugins to use @@ -822,7 +822,7 @@ def negociateProtocolWithOtherSE(self, sourceSE, protocols=None): log = self.log.getSubLogger('negociateProtocolWithOtherSE', child=True) log.debug( - "Negociating protocols between %s and %s (protocols %s)" % + "Negotiating protocols between %s and %s (protocols %s)" % (sourceSE.name, self.name, protocols)) # Take all the protocols the destination can accept as input @@ -844,9 +844,8 @@ def negociateProtocolWithOtherSE(self, sourceSE, protocols=None): protocolList = list(protocols) commonProtocols = sorted( commonProtocols & set(protocolList), - key=lambda x: getIndexInList( - x, - protocolList)) + key=lambda x: getIndexInList(x, protocolList) + ) log.debug("Common protocols %s" % commonProtocols) @@ -1150,7 +1149,7 @@ def __executeMethod(self, lfn, *args, **kwargs): if args: log.verbose("args should be empty!%s" % args) # because there is normally only one kw argument, I can move it from args to kwargs - methDefaultArgs = StorageElementItem.__defaultsArguments.get(self.methodName, {}).keys() + methDefaultArgs = list(StorageElementItem.__defaultsArguments.get(self.methodName, {})) if methDefaultArgs: kwargs[methDefaultArgs[0]] = args[0] args = args[1:] diff --git a/Resources/Storage/StorageFactory.py b/Resources/Storage/StorageFactory.py index aa7788ada05..05dcbb09c39 100755 --- a/Resources/Storage/StorageFactory.py +++ b/Resources/Storage/StorageFactory.py @@ -147,7 +147,7 @@ def getStorages(self, storageName, pluginList=None, hideExceptions=False): requestedProtocolDetails = [] turlProtocols = [] # Generate the protocol specific plug-ins - for protocolSection, protocolDetails in self.protocols.iteritems(): + for protocolSection, protocolDetails in self.protocols.items(): pluginName = protocolDetails.get('PluginName', protocolSection) if pluginList and pluginName not in pluginList: continue @@ -328,7 +328,7 @@ def _getConfigStorageProtocols(self, storageName, derivedStorageName=None, seCon for baseStorageProtocolSection in protocolSections: if protocolSection == baseStorageProtocolSection: inheritanceMatched = True - for key, val in detail.iteritems(): + for key, val in detail.items(): if val: self.protocols[protocolSection][key] = val break diff --git a/Resources/Storage/test/FIXME_Test_StorageElement.py b/Resources/Storage/test/FIXME_Test_StorageElement.py index 3c0af0caf11..fd2a030805f 100755 --- a/Resources/Storage/test/FIXME_Test_StorageElement.py +++ b/Resources/Storage/test/FIXME_Test_StorageElement.py @@ -124,7 +124,7 @@ def test_exists(self): '################\n\n\t\t\tExists test\n') destinationFilePath = '%s/testFile.%s' % (self.destDirectory, time.time()) # pfnForLfnRes = self.storageElement.getURL( destinationFilePath ) - # destinationPfn = pfnForLfnRes['Value']['Successful'].values()[0] + # destinationPfn = list(pfnForLfnRes['Value']['Successful'].values())[0] fileDict = {destinationFilePath: self.localSourceFile} putFileRes = returnSingleResult(self.storageElement.putFile(fileDict)) # File exists diff --git a/Resources/Storage/test/Test_StorageElement.py b/Resources/Storage/test/Test_StorageElement.py index 0c07b5f6b5b..ac46ab8eff4 100644 --- a/Resources/Storage/test/Test_StorageElement.py +++ b/Resources/Storage/test/Test_StorageElement.py @@ -602,49 +602,49 @@ def test_08_multiProtocolFTS(self, _mk_isLocalSE, _mk_addAccounting): This makes it necessary to add gsiftp as third party option to write to ECHO """ - thirdPartyProtocols = ['root', 'gsiftp', 'srm'] + rankedProtocols = ['root', 'gsiftp', 'gsidcap', 'dcap', 'file', 'srm', 'rfio'] lfn = '/lhcb/fake/lfn' # RAL -> GRIDKA # We should read using root and write through srm - res = self.seY.generateTransferURLsBetweenSEs(lfn, self.seX) + res = self.seY.generateTransferURLsBetweenSEs(lfn, self.seX, protocols=rankedProtocols) self.assertTrue(res['OK'], res) urlPair = res['Value']['Successful'].get(lfn) self.assertTupleEqual(urlPair, ('root:%s' % lfn, 'srm:%s' % lfn)) # RAL -> CERN # We should read using root and write directly with it - res = self.seZ.generateTransferURLsBetweenSEs(lfn, self.seX) + res = self.seZ.generateTransferURLsBetweenSEs(lfn, self.seX, protocols=rankedProtocols) self.assertTrue(res['OK'], res) urlPair = res['Value']['Successful'].get(lfn) self.assertTupleEqual(urlPair, ('root:%s' % lfn, 'root:%s' % lfn)) # GRIDKA -> RAL # We should read using gsiftp and write directly with it - res = self.seX.generateTransferURLsBetweenSEs(lfn, self.seY) + res = self.seX.generateTransferURLsBetweenSEs(lfn, self.seY, protocols=rankedProtocols) self.assertTrue(res['OK'], res) urlPair = res['Value']['Successful'].get(lfn) self.assertTupleEqual(urlPair, ('gsiftp:%s' % lfn, 'gsiftp:%s' % lfn)) # GRIDKA -> CERN # We should read using srm and write with root - res = self.seZ.generateTransferURLsBetweenSEs(lfn, self.seY) + res = self.seZ.generateTransferURLsBetweenSEs(lfn, self.seY, protocols=rankedProtocols) self.assertTrue(res['OK'], res) urlPair = res['Value']['Successful'].get(lfn) self.assertTupleEqual(urlPair, ('srm:%s' % lfn, 'root:%s' % lfn)) # CERN -> RAL # We should read using srm and write with gsiftp - res = self.seX.generateTransferURLsBetweenSEs(lfn, self.seZ) + res = self.seX.generateTransferURLsBetweenSEs(lfn, self.seZ, protocols=rankedProtocols) self.assertTrue(res['OK'], res) urlPair = res['Value']['Successful'].get(lfn) self.assertTupleEqual(urlPair, ('srm:%s' % lfn, 'gsiftp:%s' % lfn)) # CERN -> GRIDKA # We should read using root and write directly with srm - res = self.seY.generateTransferURLsBetweenSEs(lfn, self.seZ) + res = self.seY.generateTransferURLsBetweenSEs(lfn, self.seZ, protocols=rankedProtocols) self.assertTrue(res['OK'], res) urlPair = res['Value']['Successful'].get(lfn) diff --git a/Resources/Storage/test/Test_StorageFactory.py b/Resources/Storage/test/Test_StorageFactory.py index dff6c6bf82b..e15f677fb61 100755 --- a/Resources/Storage/test/Test_StorageFactory.py +++ b/Resources/Storage/test/Test_StorageFactory.py @@ -233,7 +233,7 @@ def getOptionsDict(self, path): csSection = self.crawlCS(path) if not csSection: return S_ERROR("Not a valid section") - options = dict((opt, val) for opt, val in csSection.iteritems() if not isinstance(val, dict)) + options = dict((opt, val) for opt, val in csSection.items() if not isinstance(val, dict)) return S_OK(options) def getOptions(self, path): @@ -245,7 +245,7 @@ def getOptions(self, path): csSection = self.crawlCS(path) if not csSection: return S_ERROR("Not a valid section") - options = [opt for opt, val in csSection.iteritems() if not isinstance(val, dict)] + options = [opt for opt, val in csSection.items() if not isinstance(val, dict)] return S_OK(options) def getSections(self, path): @@ -257,7 +257,7 @@ def getSections(self, path): csSection = self.crawlCS(path) if not csSection: return S_ERROR("Not a valid section") - sections = [opt for opt, val in csSection.iteritems() if isinstance(val, dict)] + sections = [opt for opt, val in csSection.items() if isinstance(val, dict)] return S_OK(sections) @@ -319,7 +319,10 @@ def test_standalone(self, _sf_generateStorageObject, _rss_getSEStatus): self.assertEqual(len(storages['ProtocolOptions']), len(allProtocols)) self.assertEqual(len(storages['StorageObjects']), len(allProtocols)) - self.assertListEqual(sorted(allProtocols), sorted(storages['ProtocolOptions'])) + self.assertListEqual( + sorted(allProtocols, key=lambda x: x["Host"]), + sorted(storages['ProtocolOptions'], key=lambda x: x["Host"]) + ) self.assertDictEqual(storages['StorageOptions'], {'BackendType': 'Eos', 'SEType': 'T0D1'}) @@ -612,7 +615,7 @@ def test_more_protocol(self, _sf_generateStorageObject, _rss_getSEStatus): self.assertTrue(storages['OK'], storages) storages = storages['Value'] - self.assertListEqual(storages['RemotePlugins'], ['Extra', 'GFAL2_SRM2']) + self.assertSetEqual(set(storages['RemotePlugins']), set(['Extra', 'GFAL2_SRM2'])) expectedProtocols = [{ 'Access': 'remote', @@ -634,7 +637,10 @@ def test_more_protocol(self, _sf_generateStorageObject, _rss_getSEStatus): 'WSUrl': '/srm/v2/server?SFN:' }] - self.assertListEqual(storages['ProtocolOptions'], expectedProtocols) + self.assertListEqual( + sorted(storages['ProtocolOptions'], key=lambda x: x["PluginName"]), + expectedProtocols, + ) def test_child_inherit_from_base_with_two_same_plugins(self, _sf_generateStorageObject, _rss_getSEStatus): """ In this test, we load a storage element CERN-CHILD-INHERIT-FROM-BASE-WITH-TWO-SAME-PLUGINS that inherits diff --git a/StorageManagementSystem/Agent/RequestFinalizationAgent.py b/StorageManagementSystem/Agent/RequestFinalizationAgent.py index b47cea62542..c2a2536460b 100755 --- a/StorageManagementSystem/Agent/RequestFinalizationAgent.py +++ b/StorageManagementSystem/Agent/RequestFinalizationAgent.py @@ -54,7 +54,7 @@ def clearFailedTasks(self): gLogger.info("RequestFinalization.clearFailedTasks: No tasks to remove.") return S_OK() gLogger.info("RequestFinalization.clearFailedTasks: Removing %s tasks..." % len(failedTasks)) - res = self.stagerClient.removeTasks(failedTasks.keys()) + res = self.stagerClient.removeTasks(list(failedTasks)) if not res['OK']: gLogger.error("RequestFinalization.clearFailedTasks: Failed to remove tasks.", res['Message']) return res @@ -80,7 +80,7 @@ def callbackDoneTasks(self): if not doneTasks: gLogger.info("RequestFinalization.callbackDoneTasks: No tasks to update to Done.") return S_OK() - res = self.stagerClient.removeTasks(doneTasks.keys()) + res = self.stagerClient.removeTasks(list(doneTasks)) if not res['OK']: gLogger.fatal("RequestFinalization.callbackDoneTasks: Failed to remove Done tasks.", res['Message']) return res @@ -110,8 +110,8 @@ def callbackStagedTasks(self): gLogger.info("RequestFinalization.callbackStagedTasks: No tasks to update to Done.") return S_OK() # Daniela: Why is the line below commented out? - # res = self.stagerClient.setTasksDone(stagedTasks.keys()) - res = self.stagerClient.removeTasks(stagedTasks.keys()) + # res = self.stagerClient.setTasksDone(list(stagedTasks)) + res = self.stagerClient.removeTasks(list(stagedTasks)) if not res['OK']: gLogger.fatal("RequestFinalization.callbackStagedTasks: Failed to remove staged Tasks.", res['Message']) return res @@ -152,7 +152,7 @@ def clearReleasedTasks(self): return res stagedTasks = res['Value'] gLogger.info("RequestFinalization.clearReleasedTasks: Removing %s tasks..." % len(stagedTasks)) - res = self.stagerClient.removeTasks(stagedTasks.keys()) + res = self.stagerClient.removeTasks(list(stagedTasks)) if not res['OK']: gLogger.error("RequestFinalization.clearReleasedTasks: Failed to remove tasks.", res['Message']) return res diff --git a/StorageManagementSystem/Agent/RequestPreparationAgent.py b/StorageManagementSystem/Agent/RequestPreparationAgent.py index 7cbee9b4a7e..21506d76fac 100755 --- a/StorageManagementSystem/Agent/RequestPreparationAgent.py +++ b/StorageManagementSystem/Agent/RequestPreparationAgent.py @@ -85,7 +85,7 @@ def execute(self): len(terminal)) # Obtain the replicas from the FileCatalog - res = self.__getFileReplicas(fileSizes.keys()) + res = self.__getFileReplicas(list(fileSizes)) if not res['OK']: return res failed.update(res['Value']['Failed']) diff --git a/StorageManagementSystem/Agent/StageMonitorAgent.py b/StorageManagementSystem/Agent/StageMonitorAgent.py index 7183fe77a9e..92f643d2571 100755 --- a/StorageManagementSystem/Agent/StageMonitorAgent.py +++ b/StorageManagementSystem/Agent/StageMonitorAgent.py @@ -67,7 +67,7 @@ def monitorStageRequests(self): gLogger.info( "StageMonitor.monitorStageRequests: Obtained %s StageSubmitted replicas for monitoring." % len(replicaIDs)) - for storageElement, seReplicaIDs in seReplicas.iteritems(): + for storageElement, seReplicaIDs in seReplicas.items(): self.__monitorStorageElementStageRequests(storageElement, seReplicaIDs, replicaIDs) gDataStoreClient.commit() @@ -105,12 +105,12 @@ def __monitorStorageElementStageRequests(self, storageElement, seReplicaIDs, rep accountingDict = self.__newAccountingDict(storageElement) - for lfn, reason in prestageStatus['Failed'].iteritems(): + for lfn, reason in prestageStatus['Failed'].items(): accountingDict['TransferTotal'] += 1 if re.search('File does not exist', reason): gLogger.error("StageMonitor.__monitorStorageElementStageRequests: LFN did not exist in the StorageElement", lfn) terminalReplicaIDs[lfnRepIDs[lfn]] = 'LFN did not exist in the StorageElement' - for lfn, metadata in prestageStatus['Successful'].iteritems(): + for lfn, metadata in prestageStatus['Successful'].items(): if not metadata: continue staged = metadata.get('Cached', metadata['Accessible']) @@ -203,18 +203,18 @@ def __getStageSubmittedReplicas(self): seReplicas = {} replicaIDs = res['Value'] - for replicaID, info in replicaIDs.iteritems(): + for replicaID, info in replicaIDs.items(): storageElement = info['SE'] seReplicas.setdefault(storageElement, []).append(replicaID) # RequestID was missing from replicaIDs dictionary BUGGY? - res = self.stagerClient.getStageRequests({'ReplicaID': replicaIDs.keys()}) + res = self.stagerClient.getStageRequests({'ReplicaID': list(replicaIDs)}) if not res['OK']: return res if not res['Value']: return S_ERROR('Could not obtain request IDs for replicas %s from StageRequests table' % (replicaIDs.keys())) - for replicaID, info in res['Value'].iteritems(): + for replicaID, info in res['Value'].items(): replicaIDs[replicaID]['RequestID'] = info['RequestID'] return S_OK({'SEReplicas': seReplicas, 'ReplicaIDs': replicaIDs}) diff --git a/StorageManagementSystem/Agent/StageRequestAgent.py b/StorageManagementSystem/Agent/StageRequestAgent.py index 683731dc217..2670c3d7427 100755 --- a/StorageManagementSystem/Agent/StageRequestAgent.py +++ b/StorageManagementSystem/Agent/StageRequestAgent.py @@ -82,7 +82,7 @@ def submitStageRequests(self): if seReplicas: gLogger.info("StageRequest.submitStageRequests: Completing partially Staged Tasks") - for storageElement, seReplicaIDs in seReplicas.iteritems(): + for storageElement, seReplicaIDs in seReplicas.items(): gLogger.debug('Staging at %s:' % storageElement, seReplicaIDs) self._issuePrestageRequests(storageElement, seReplicaIDs, allReplicaInfo) @@ -106,12 +106,12 @@ def submitStageRequests(self): return res # Merge info from both results - for storageElement, seReplicaIDs in res['Value']['SEReplicas'].iteritems(): + for storageElement, seReplicaIDs in res['Value']['SEReplicas'].items(): seReplicas.setdefault(storageElement, []).extend(seReplicaIDs) allReplicaInfo.update(res['Value']['AllReplicaInfo']) gLogger.info("StageRequest.submitStageRequests: Obtained %s replicas for staging." % len(allReplicaInfo)) - for storageElement, seReplicaIDs in seReplicas.iteritems(): + for storageElement, seReplicaIDs in seReplicas.items(): gLogger.debug('Staging at %s:' % storageElement, seReplicaIDs) self._issuePrestageRequests(storageElement, seReplicaIDs, allReplicaInfo) return S_OK() @@ -134,7 +134,7 @@ def _getMissingReplicas(self): allReplicaInfo = res['Value']['AllReplicaInfo'] replicasToStage = [] - for seReplicaIDs in res['Value']['SEReplicas'].itervalues(): + for seReplicaIDs in res['Value']['SEReplicas'].values(): # Consider all SEs replicasToStage += seReplicaIDs @@ -165,7 +165,7 @@ def _getOnlineReplicas(self): return res gLogger.info("StageRequest._getOnlineReplicas: Obtained %s replicas Waiting for staging." % len(allReplicaInfo)) replicasToStage = [] - for storageElement, seReplicaIDs in res['Value']['SEReplicas'].iteritems(): + for storageElement, seReplicaIDs in res['Value']['SEReplicas'].items(): if not self.__usage(storageElement) < self.__cache(storageElement): gLogger.info( 'StageRequest._getOnlineReplicas: Skipping %s, current usage above limit ( %s GB )' % @@ -211,7 +211,7 @@ def _getOfflineReplicas(self): return res gLogger.info("StageRequest._getOfflineReplicas: Obtained %s replicas Offline for staging." % len(allReplicaInfo)) replicasToStage = [] - for storageElement, seReplicaIDs in res['Value']['SEReplicas'].iteritems(): + for storageElement, seReplicaIDs in res['Value']['SEReplicas'].items(): if not self.__usage(storageElement) < self.__cache(storageElement): gLogger.info( 'StageRequest._getOfflineReplicas: Skipping %s, current usage above limit ( %s GB )' % @@ -284,7 +284,7 @@ def _issuePrestageRequests(self, storageElement, seReplicaIDs, allReplicaInfo): "StageRequest._issuePrestageRequests: Completely failed to submit stage requests for replicas.", res['Message']) else: - for lfn, requestID in res['Value']['Successful'].iteritems(): + for lfn, requestID in res['Value']['Successful'].items(): stageRequestMetadata.setdefault(requestID, []).append(lfnRepIDs[lfn]) updatedLfnIDs.append(lfnRepIDs[lfn]) if stageRequestMetadata: @@ -304,7 +304,7 @@ def __sortBySE(self, replicaDict): seReplicas = {} replicaIDs = {} - for replicaID, info in replicaDict.iteritems(): + for replicaID, info in replicaDict.items(): lfn = info['LFN'] storageElement = info['SE'] size = info['Size'] @@ -370,7 +370,7 @@ def __addAssociatedReplicas(self, replicasToStage, seReplicas, allReplicaInfo): return res addReplicas = {'Offline': {}, 'Waiting': {}} replicaIDs = {} - for replicaID, info in res['Value'].iteritems(): + for replicaID, info in res['Value'].items(): lfn = info['LFN'] storageElement = info['SE'] size = info['Size'] @@ -386,7 +386,7 @@ def __addAssociatedReplicas(self, replicasToStage, seReplicas, allReplicaInfo): allReplicaInfo.update(newReplicaInfo) # First handle Waiting Replicas for which metadata is to be checked - for storageElement, seReplicaIDs in waitingReplicas.iteritems(): + for storageElement, seReplicaIDs in waitingReplicas.items(): for replicaID in list(seReplicaIDs): if replicaID in replicasToStage: seReplicaIDs.remove(replicaID) @@ -403,7 +403,7 @@ def __addAssociatedReplicas(self, replicasToStage, seReplicas, allReplicaInfo): replicasToStage.extend(res['Value']['Offline']) # Then handle Offline Replicas for which metadata is already checked - for storageElement, seReplicaIDs in offlineReplicas.iteritems(): + for storageElement, seReplicaIDs in offlineReplicas.items(): for replicaID in sorted(seReplicaIDs): if replicaID in replicasToStage: seReplicaIDs.remove(replicaID) @@ -455,7 +455,7 @@ def __checkIntegrity(self, storageElement, seReplicaIDs, allReplicaInfo): terminalReplicaIDs = {} onlineReplicaIDs = [] offlineReplicaIDs = [] - for lfn, metadata in res['Value']['Successful'].iteritems(): + for lfn, metadata in res['Value']['Successful'].items(): if metadata['Size'] != allReplicaInfo[lfnRepIDs[lfn]]['Size']: gLogger.error("StageRequest.__checkIntegrity: LFN StorageElement size does not match FileCatalog", lfn) @@ -475,7 +475,7 @@ def __checkIntegrity(self, storageElement, seReplicaIDs, allReplicaInfo): else: offlineReplicaIDs.append(lfnRepIDs[lfn]) - for lfn, reason in res['Value']['Failed'].iteritems(): + for lfn, reason in res['Value']['Failed'].items(): if re.search('File does not exist', reason): gLogger.error("StageRequest.__checkIntegrity: LFN does not exist in the StorageElement", lfn) terminalReplicaIDs[lfnRepIDs[lfn]] = 'LFN does not exist in the StorageElement' diff --git a/StorageManagementSystem/Client/StorageManagerClient.py b/StorageManagementSystem/Client/StorageManagerClient.py index 9b1f374a048..ab0d2d2ea46 100644 --- a/StorageManagementSystem/Client/StorageManagerClient.py +++ b/StorageManagementSystem/Client/StorageManagerClient.py @@ -41,7 +41,7 @@ def getFilesToStage(lfnList, jobState=None, checkOnlyTapeSEs=None, jobLog=None): failedLFNs = set() if lfnListReplicas['Value']['Failed']: # Check if files are not existing - for lfn, reason in lfnListReplicas['Value']['Failed'].iteritems(): + for lfn, reason in lfnListReplicas['Value']['Failed'].items(): # FIXME: awful check until FC returns a proper error if cmpError(reason, errno.ENOENT) or 'No such file' in reason: # The file doesn't exist, job must be Failed @@ -58,7 +58,7 @@ def getFilesToStage(lfnList, jobState=None, checkOnlyTapeSEs=None, jobLog=None): # If a file is reported here at a tape SE, it is not at a disk SE as we use disk in priority # We shall check all file anyway in order to make sure they exist seToLFNs = dict() - for lfn, ses in lfnListReplicas.iteritems(): + for lfn, ses in lfnListReplicas.items(): for se in ses: seToLFNs.setdefault(se, list()).append(lfn) @@ -141,7 +141,7 @@ def _checkFilesToStage(seToLFNs, onlineLFNs, offlineLFNs, absentLFNs, checkOnlyTapeSEs = True failed = {} - for se, lfnsInSEList in seToLFNs.iteritems(): + for se, lfnsInSEList in seToLFNs.items(): # If we have found already all files online at another SE, no need to check the others # but still we want to set the SE as Online if not a TapeSE vo = getVOForGroup(proxyUserGroup) @@ -178,7 +178,7 @@ def _checkFilesToStage(seToLFNs, onlineLFNs, offlineLFNs, absentLFNs, if fileMetadata['Value']['Failed']: failed[se] = fileMetadata['Value']['Failed'] # is there at least one replica online? - for lfn, mDict in fileMetadata['Value']['Successful'].iteritems(): + for lfn, mDict in fileMetadata['Value']['Successful'].items(): # SRM returns Cached, but others may only return Accessible if mDict.get('Cached', mDict['Accessible']): onlineLFNs.setdefault(lfn, []).append(se) @@ -194,9 +194,9 @@ def _checkFilesToStage(seToLFNs, onlineLFNs, offlineLFNs, absentLFNs, offlineLFNs.pop(lfn) # If the file was found staged, ignore possible errors, but print out errors - for se, failedLfns in failed.items(): + for se, failedLfns in list(failed.items()): logger.error("Errors when getting files metadata", 'at %s' % se) - for lfn, reason in failedLfns.items(): + for lfn, reason in list(failedLfns.items()): if lfn in onlineLFNs: logger.warn(reason, 'for %s, but there is an online replica' % lfn) failed[se].pop(lfn) @@ -210,12 +210,12 @@ def _checkFilesToStage(seToLFNs, onlineLFNs, offlineLFNs, absentLFNs, # Find the files that do not exist at SE if failed: logger.error("Error getting metadata", "for %d files" % - len(set(lfn for lfnList in failed.itervalues() for lfn in lfnList))) + len(set(lfn for lfnList in failed.values() for lfn in lfnList))) for lfn in absentLFNs: seList = absentLFNs[lfn] # FIXME: it is not possible to return here an S_ERROR(), return the message only - absentLFNs[lfn] = S_ERROR(errno.ENOENT, "File not at %s" % ','.join(seList))['Message'] + absentLFNs[lfn] = S_ERROR(errno.ENOENT, "File not at %s" % ','.join(sorted(seList)))['Message'] # Format the error for absent files return S_OK() diff --git a/StorageManagementSystem/Client/test/Test_Client_StorageManagementSystem.py b/StorageManagementSystem/Client/test/Test_Client_StorageManagementSystem.py index fe9fed6697d..d23c415d0cb 100644 --- a/StorageManagementSystem/Client/test/Test_Client_StorageManagementSystem.py +++ b/StorageManagementSystem/Client/test/Test_Client_StorageManagementSystem.py @@ -14,51 +14,63 @@ from DIRAC.DataManagementSystem.Client.test.mock_DM import dm_mock import errno +# Use side_effect instead of return_value so the dict in the result is copied mockObjectSE1 = MagicMock() -mockObjectSE1.getFileMetadata.return_value = S_OK( {'Successful':{'/a/lfn/1.txt':{'Accessible':False}}, - 'Failed':{}} ) -mockObjectSE1.getStatus.return_value = S_OK( {'DiskSE': False, 'TapeSE':True} ) +mockObjectSE1.getFileMetadata.side_effect = lambda *_: S_OK({ + 'Successful': {'/a/lfn/1.txt': {'Accessible': False}}, + 'Failed': {}, +}) +mockObjectSE1.getStatus.side_effect = lambda *_: S_OK({'DiskSE': False, 'TapeSE': True}) mockObjectSE2 = MagicMock() -mockObjectSE2.getFileMetadata.return_value = S_OK( {'Successful':{'/a/lfn/2.txt':{'Cached':1, 'Accessible':True}}, - 'Failed':{}} ) -mockObjectSE2.getStatus.return_value = S_OK( {'DiskSE': False, 'TapeSE':True} ) +mockObjectSE2.getFileMetadata.side_effect = lambda *_: S_OK({ + 'Successful': {'/a/lfn/2.txt': {'Cached': 1, 'Accessible': True}}, + 'Failed': {}, +}) +mockObjectSE2.getStatus.side_effect = lambda *_: S_OK({'DiskSE': False, 'TapeSE': True}) mockObjectSE3 = MagicMock() -mockObjectSE3.getFileMetadata.return_value = S_OK( {'Successful':{}, - 'Failed':{'/a/lfn/2.txt': 'error'}} ) -mockObjectSE3.getStatus.return_value = S_OK( {'DiskSE': False, 'TapeSE':True} ) +mockObjectSE3.getFileMetadata.side_effect = lambda *_: S_OK({ + 'Successful': {}, + 'Failed': {'/a/lfn/2.txt': 'error'}, +}) +mockObjectSE3.getStatus.side_effect = lambda *_: S_OK({'DiskSE': False, 'TapeSE': True}) mockObjectSE4 = MagicMock() -mockObjectSE4.getFileMetadata.return_value = S_OK( {'Successful':{}, - 'Failed':{'/a/lfn/2.txt': - S_ERROR( errno.ENOENT, '' )['Message']}} ) -mockObjectSE4.getStatus.return_value = S_OK( {'DiskSE': False, 'TapeSE':True} ) +mockObjectSE4.getFileMetadata.side_effect = lambda *_: S_OK({ + 'Successful': {}, + 'Failed': {'/a/lfn/2.txt': S_ERROR(errno.ENOENT, '')['Message']} +}) +mockObjectSE4.getStatus.side_effect = lambda *_: S_OK({'DiskSE': False, 'TapeSE': True}) mockObjectSE5 = MagicMock() -mockObjectSE5.getFileMetadata.return_value = S_OK( {'Successful':{'/a/lfn/1.txt':{'Accessible':False}}, - 'Failed':{}} ) -mockObjectSE5.getStatus.return_value = S_OK( {'DiskSE': True, 'TapeSE':False} ) +mockObjectSE5.getFileMetadata.side_effect = lambda *_: S_OK({ + 'Successful': {'/a/lfn/1.txt': {'Accessible': False}}, + 'Failed': {} +}) +mockObjectSE5.getStatus.side_effect = lambda *_: S_OK({'DiskSE': True, 'TapeSE': False}) mockObjectSE6 = MagicMock() -mockObjectSE6.getFileMetadata.return_value = S_OK( {'Successful':{'/a/lfn/2.txt':{'Cached':0, 'Accessible':False}}, - 'Failed':{}} ) -mockObjectSE6.getStatus.return_value = S_OK( {'DiskSE': False, 'TapeSE':True} ) +mockObjectSE6.getFileMetadata.side_effect = lambda *_: S_OK({ + 'Successful': {'/a/lfn/2.txt': {'Cached': 0, 'Accessible': False}}, + 'Failed': {} +}) +mockObjectSE6.getStatus.side_effect = lambda *_: S_OK({'DiskSE': False, 'TapeSE': True}) mockObjectDMSHelper = MagicMock() -mockObjectDMSHelper.getLocalSiteForSE.return_value = S_OK( 'mySite' ) -mockObjectDMSHelper.getSitesForSE.return_value = S_OK( ['mySite'] ) +mockObjectDMSHelper.getLocalSiteForSE.side_effect = lambda *_: S_OK('mySite') +mockObjectDMSHelper.getSitesForSE.side_effect = lambda *_: S_OK(['mySite']) -class ClientsTestCase( unittest.TestCase ): + +class ClientsTestCase(unittest.TestCase): """ Base class for the clients test cases """ - def setUp( self ): - + def setUp(self): from DIRAC import gLogger - gLogger.setLevel( 'DEBUG' ) + gLogger.setLevel('DEBUG') - def tearDown( self ): + def tearDown(self): pass ############################################################################# @@ -108,11 +120,12 @@ def test_getFilesToStage_noSuchFile( self, _patch, _patched ): """ Test where the StorageElement will return file is absent """ res = getFilesToStage( ['/a/lfn/2.txt'], checkOnlyTapeSEs = False ) - self.assertTrue( res['OK'] ) - self.assertEqual( res['Value']['onlineLFNs'], [] ) - self.assertEqual( res['Value']['offlineLFNs'], {} ) - self.assertEqual( res['Value']['absentLFNs'], {'/a/lfn/2.txt': 'No such file or directory ( 2 : File not at SE2)'} ) - self.assertEqual( res['Value']['failedLFNs'], [] ) + self.assertTrue(res['OK']) + self.assertEqual(res['Value']['onlineLFNs'], []) + self.assertEqual(res['Value']['offlineLFNs'], {}) + self.assertEqual(res['Value']['absentLFNs'], + {'/a/lfn/2.txt': 'No such file or directory ( 2 : File not at SE1,SE2)'}) + self.assertEqual(res['Value']['failedLFNs'], []) @patch( "DIRAC.StorageManagementSystem.Client.StorageManagerClient.DataManager", return_value = dm_mock ) @patch( "DIRAC.StorageManagementSystem.Client.StorageManagerClient.StorageElement", return_value = mockObjectSE5 ) diff --git a/StorageManagementSystem/DB/StorageManagementDB.py b/StorageManagementSystem/DB/StorageManagementDB.py index 5eb9ab8df18..f0592a50e61 100644 --- a/StorageManagementSystem/DB/StorageManagementDB.py +++ b/StorageManagementSystem/DB/StorageManagementDB.py @@ -663,7 +663,7 @@ def setRequest(self, lfnDict, source, callbackMethod, sourceTaskID, connection=F # Get the Replicas which already exist in the CacheReplicas table allReplicaIDs = [] taskStates = [] - for se, lfns in lfnDict.iteritems(): + for se, lfns in lfnDict.items(): if isinstance(lfns, six.string_types): lfns = [lfns] res = self._getExistingReplicas(se, lfns, connection=connection) @@ -882,7 +882,7 @@ def getTasksWithStatus(self, status): def updateReplicaFailure(self, terminalReplicaIDs): """ This method sets the status to Failure with the failure reason for the supplied Replicas. """ - res = self.updateReplicaStatus(terminalReplicaIDs.keys(), 'Failed') + res = self.updateReplicaStatus(list(terminalReplicaIDs), 'Failed') if not res['OK']: return res updated = res['Value'] @@ -995,7 +995,7 @@ def getSubmittedStagePins(self): def insertStageRequest(self, requestDict, pinLifeTime): req = "INSERT INTO StageRequests (ReplicaID,RequestID,StageRequestSubmitTime,PinLength) VALUES " - for requestID, replicaIDs in requestDict.iteritems(): + for requestID, replicaIDs in requestDict.items(): for replicaID in replicaIDs: replicaString = "(%s,'%s',UTC_TIMESTAMP(),%d)," % (replicaID, requestID, pinLifeTime) req = "%s %s" % (req, replicaString) @@ -1007,7 +1007,7 @@ def insertStageRequest(self, requestDict, pinLifeTime): res['Message']) return res - for requestID, replicaIDs in requestDict.iteritems(): + for requestID, replicaIDs in requestDict.items(): for replicaID in replicaIDs: # fix, no individual queries reqSelect = "SELECT * FROM StageRequests WHERE ReplicaID = %s AND RequestID = '%s';" % ( diff --git a/StorageManagementSystem/scripts/dirac-stager-monitor-file.py b/StorageManagementSystem/scripts/dirac-stager-monitor-file.py index ca7834da28f..d692338660a 100755 --- a/StorageManagementSystem/scripts/dirac-stager-monitor-file.py +++ b/StorageManagementSystem/scripts/dirac-stager-monitor-file.py @@ -45,7 +45,7 @@ gLogger.error(res['Message']) cacheReplicaInfo = res['Value'] if cacheReplicaInfo: - replicaID = cacheReplicaInfo.keys()[0] + replicaID = list(cacheReplicaInfo)[0] outStr = "\n--------------------" outStr += "\n%s: %s" % ('LFN'.ljust(8), cacheReplicaInfo[replicaID]['LFN'].ljust(100)) outStr += "\n%s: %s" % ('SE'.ljust(8), cacheReplicaInfo[replicaID]['SE'].ljust(100)) @@ -71,7 +71,7 @@ if resStageRequests['Records']: stageRequests = resStageRequests['Value'] outStr += "\n------SRM staging request info--------------" - for info in stageRequests.itervalues(): + for info in stageRequests.values(): outStr += "\n%s: %s" % ('SRM RequestID'.ljust(8), info['RequestID'].ljust(100)) outStr += "\n%s: %s" % ('SRM StageStatus'.ljust(8), info['StageStatus'].ljust(100)) outStr += "\n%s: %s" % ('SRM StageRequestSubmitTime'.ljust(8), str(info['StageRequestSubmitTime']).ljust(100)) diff --git a/StorageManagementSystem/scripts/dirac-stager-monitor-jobs.py b/StorageManagementSystem/scripts/dirac-stager-monitor-jobs.py index ba7373e40ac..973a19dbd90 100755 --- a/StorageManagementSystem/scripts/dirac-stager-monitor-jobs.py +++ b/StorageManagementSystem/scripts/dirac-stager-monitor-jobs.py @@ -57,7 +57,7 @@ gLogger.notice('No info on files for the job = %s, that is odd' % jobID) continue else: - for lfn, metadata in replicaInfo.iteritems(): + for lfn, metadata in replicaInfo.items(): outStr += "\n\t--------------------" outStr += "\n\t%s: %s" % ('LFN'.ljust(8), lfn.ljust(100)) outStr += "\n\t%s: %s" % ('SE'.ljust(8), metadata['StorageElement'].ljust(100)) diff --git a/StorageManagementSystem/scripts/dirac-stager-monitor-request.py b/StorageManagementSystem/scripts/dirac-stager-monitor-request.py index 3e972427063..4d6c10a8986 100755 --- a/StorageManagementSystem/scripts/dirac-stager-monitor-request.py +++ b/StorageManagementSystem/scripts/dirac-stager-monitor-request.py @@ -50,7 +50,7 @@ outStr += "\n%s: %s" % ('CallBackMethod'.ljust(20), taskInfo[taskID]['CallBackMethod']) outStr += "\n%s: %s" % ('SubmitTime'.ljust(20), taskInfo[taskID]['SubmitTime']) outStr += "\n%s: %s" % ('CompleteTime'.ljust(20), taskInfo[taskID]['CompleteTime']) -for lfn, metadata in replicaInfo.iteritems(): +for lfn, metadata in replicaInfo.items(): outStr += "\n" outStr += "\n\t%s: %s" % ('LFN'.ljust(8), lfn.ljust(100)) outStr += "\n\t%s: %s" % ('SE'.ljust(8), metadata['StorageElement'].ljust(100)) diff --git a/StorageManagementSystem/scripts/dirac-stager-monitor-requests.py b/StorageManagementSystem/scripts/dirac-stager-monitor-requests.py index 16550572dc5..f8a91e5f729 100755 --- a/StorageManagementSystem/scripts/dirac-stager-monitor-requests.py +++ b/StorageManagementSystem/scripts/dirac-stager-monitor-requests.py @@ -72,7 +72,7 @@ def parseSwitches(): Script.showHelp(exitCode=1) subLogger.debug("The switches used are:") - map(subLogger.debug, switches.iteritems()) + map(subLogger.debug, switches.items()) return switches @@ -117,7 +117,7 @@ def run(): outStr += " %s" % ("PinLength(sec)".ljust(15)) outStr += "\n" - for crid, info in replicas.iteritems(): + for crid, info in replicas.items(): outStr += " %s" % (info['Status'].ljust(15)) outStr += " %s" % (str(info['LastUpdate']).ljust(20)) outStr += " %s" % (info['LFN'].ljust(30)) @@ -143,7 +143,7 @@ def run(): gLogger.error(resStageRequests['Message']) if resStageRequests['Records']: stageRequests = resStageRequests['Value'] - for info in stageRequests.itervalues(): + for info in stageRequests.values(): outStr += " %s" % (str(info['PinExpiryTime']).ljust(20)) outStr += " %s" % (str(info['PinLength']).ljust(10)) outStr += "\n" diff --git a/StorageManagementSystem/scripts/dirac-stager-show-stats.py b/StorageManagementSystem/scripts/dirac-stager-show-stats.py index dc539411b0d..76977554c35 100755 --- a/StorageManagementSystem/scripts/dirac-stager-show-stats.py +++ b/StorageManagementSystem/scripts/dirac-stager-show-stats.py @@ -49,7 +49,7 @@ outStr += " %s" % ("Size(GB)".ljust(20)) outStr += " \n--------------------------------------------------------------------------\n" if stagerInfo: - for info in stagerInfo.itervalues(): + for info in stagerInfo.values(): outStr += " %s" % (info['Status'].ljust(20)) outStr += " %s" % (info['SE'].ljust(20)) outStr += " %s" % (str(info['NumFiles']).ljust(20)) diff --git a/TransformationSystem/Agent/DataRecoveryAgent.py b/TransformationSystem/Agent/DataRecoveryAgent.py index 6cc51de7ffa..16c18933f01 100644 --- a/TransformationSystem/Agent/DataRecoveryAgent.py +++ b/TransformationSystem/Agent/DataRecoveryAgent.py @@ -307,7 +307,7 @@ def execute(self): if not transformations['OK']: self.log.error("Failure to get transformations", transformations['Message']) return S_ERROR("Failure to get transformations") - for transID, transInfoDict in transformations['Value'].iteritems(): + for transID, transInfoDict in transformations['Value'].items(): if transID in self.transformationsToIgnore: self.log.notice('Ignoring Transformation: %s' % transID) continue @@ -485,7 +485,7 @@ def printSummary(self): def __resetCounters(self): """ reset counters for modified jobs """ - for _name, checks in self.todo.iteritems(): + for _name, checks in self.todo.items(): for do in checks: do['Counter'] = 0 diff --git a/TransformationSystem/Agent/TaskManagerAgentBase.py b/TransformationSystem/Agent/TaskManagerAgentBase.py index 551f51859fc..22bfb8a6c86 100644 --- a/TransformationSystem/Agent/TaskManagerAgentBase.py +++ b/TransformationSystem/Agent/TaskManagerAgentBase.py @@ -15,7 +15,7 @@ import time import datetime -from Queue import Queue +from six.moves.queue import Queue from DIRAC import S_OK @@ -103,7 +103,7 @@ def initialize(self): threadPool = ThreadPool(maxNumberOfThreads, maxNumberOfThreads) self.log.verbose("Multithreaded with %d threads" % maxNumberOfThreads) - for i in xrange(maxNumberOfThreads): + for i in range(maxNumberOfThreads): threadPool.generateJobAndQueueIt(self._execute, [i]) return S_OK() @@ -240,7 +240,7 @@ def _fillTheQueue(self, operationsOnTransformationsDict): """ Just fill the queue with the operation to be done on a certain transformation """ count = 0 - for transID, bodyAndOps in operationsOnTransformationsDict.iteritems(): + for transID, bodyAndOps in operationsOnTransformationsDict.items(): if transID not in self.transInQueue: count += 1 self.transInQueue.append(transID) @@ -395,7 +395,7 @@ def updateTaskStatus(self, transIDOPBody, clients): method=method, transID=transID) # Set status for tasks that changes - for status, taskIDs in statusDict.iteritems(): + for status, taskIDs in statusDict.items(): self._logVerbose("%4d: Updating %d task(s) to %s" % (nb, len(taskIDs), status), method=method, transID=transID) setTaskStatus = clients['TransformationClient'].setTaskStatus(transID, taskIDs, status) @@ -405,7 +405,7 @@ def updateTaskStatus(self, transIDOPBody, clients): return setTaskStatus updated[status] = updated.setdefault(status, 0) + len(taskIDs) - for status, nb in updated.iteritems(): + for status, nb in updated.items(): self._logInfo("Updated %d tasks to status %s" % (nb, status), method=method, transID=transID) return S_OK() @@ -461,7 +461,7 @@ def updateFileStatus(self, transIDOPBody, clients): # Set the status of files fileReport = FileReport(server=clients['TransformationClient'].getServer()) - for lfn, status in statusDict.iteritems(): + for lfn, status in statusDict.items(): updated[status] = updated.setdefault(status, 0) + 1 setFileStatus = fileReport.setFileStatus(transID, lfn, status) if not setFileStatus['OK']: @@ -475,7 +475,7 @@ def updateFileStatus(self, transIDOPBody, clients): self._logVerbose("%4d: Updated the states of %d files" % (nb, len(commit['Value'])), method=method, transID=transID) - for status, nb in updated.iteritems(): + for status, nb in updated.items(): self._logInfo("Updated %d files to status %s" % (nb, status), method=method, transID=transID) return S_OK() diff --git a/TransformationSystem/Agent/TransformationAgent.py b/TransformationSystem/Agent/TransformationAgent.py index ff074a959d3..822b965ca1b 100644 --- a/TransformationSystem/Agent/TransformationAgent.py +++ b/TransformationSystem/Agent/TransformationAgent.py @@ -14,7 +14,7 @@ from past.builtins import long import time -import Queue +from six.moves import queue as Queue import os import datetime import pickle @@ -118,7 +118,7 @@ def initialize(self): threadPool = ThreadPool(maxNumberOfThreads, maxNumberOfThreads) self.log.info("Multithreaded with %d threads" % maxNumberOfThreads) - for i in xrange(maxNumberOfThreads): + for i in range(maxNumberOfThreads): threadPool.generateJobAndQueueIt(self._execute, [i]) self.log.info("Will treat the following transformation types: %s" % str(self.transformationTypes)) @@ -162,7 +162,7 @@ def execute(self): parentProd, movedFiles = res['Value'] if movedFiles: self._logInfo("Successfully moved files from %d to %d:" % (parentProd, transID), transID=transID) - for status, val in movedFiles.iteritems(): + for status, val in movedFiles.items(): self._logInfo("\t%d files to status %s" % (val, status), transID=transID) if transID not in self.transInQueue: count += 1 @@ -464,7 +464,7 @@ def __getDataReplicas(self, transDict, lfns, clients, forJobs=True): cachedReplicaSets = self.replicaCache.get(transID, {}) cachedReplicas = {} # Merge all sets of replicas - for replicas in cachedReplicaSets.itervalues(): + for replicas in cachedReplicaSets.values(): cachedReplicas.update(replicas) self._logInfo("Number of cached replicas: %d" % len(cachedReplicas), method=method, transID=transID) setCached = set(cachedReplicas) @@ -482,7 +482,7 @@ def __getDataReplicas(self, transDict, lfns, clients, forJobs=True): for chunk in breakListIntoChunks(newLFNs, 10000): res = self._getDataReplicasDM(transID, chunk, clients, forJobs=forJobs) if res['OK']: - reps = dict((lfn, ses) for lfn, ses in res['Value'].iteritems() if ses) + reps = dict((lfn, ses) for lfn, ses in res['Value'].items() if ses) newReplicas.update(reps) self.__updateCache(transID, reps) else: @@ -532,7 +532,7 @@ def _getDataReplicasDM(self, transID, lfns, clients, forJobs=True, ignoreMissing method=method, transID=transID) # Create a dictionary containing all the file replicas failoverLfns = [] - for lfn, replicaDict in replicas['Successful'].iteritems(): + for lfn, replicaDict in replicas['Successful'].items(): for se in replicaDict: # This remains here for backward compatibility in case VOs have not defined SEs not to be used for jobs if forJobs and 'failover' in se.lower(): @@ -545,7 +545,7 @@ def _getDataReplicasDM(self, transID, lfns, clients, forJobs=True, ignoreMissing self._logVerbose("%d files have no replica but possibly in Failover SE" % len(failoverLfns)) # Make sure that file missing from the catalog are marked in the transformation DB. missingLfns = [] - for lfn, reason in replicas['Failed'].iteritems(): + for lfn, reason in replicas['Failed'].items(): if "No such file or directory" in reason: self._logVerbose("%s not found in the catalog." % lfn, method=method, transID=transID) missingLfns.append(lfn) @@ -576,7 +576,7 @@ def __cleanReplicas(self, transID, lfns): """ Remove cached replicas that are not in a list """ cachedReplicas = set() - for replicas in self.replicaCache.get(transID, {}).itervalues(): + for replicas in self.replicaCache.get(transID, {}).values(): cachedReplicas.update(replicas) toRemove = cachedReplicas - set(lfns) if toRemove: @@ -646,7 +646,7 @@ def __readCache(self, transID): def __filesInCache(self, transID): cache = self.replicaCache.get(transID, {}) - return sum(len(lfns) for lfns in cache.itervalues()) + return sum(len(lfns) for lfns in cache.values()) @gSynchro def __writeCache(self, transID=None): diff --git a/TransformationSystem/Agent/test/Test_Plugins.py b/TransformationSystem/Agent/test/Test_Plugins.py index 4f3428c0896..106b8300317 100644 --- a/TransformationSystem/Agent/test/Test_Plugins.py +++ b/TransformationSystem/Agent/test/Test_Plugins.py @@ -90,14 +90,14 @@ def test__Standard_Flush_G10(setup): pluginStandard.setParameters(params) pluginStandard.setInputData(data) res = pluginStandard.run() - sortedData = [('SE1', ['/this/is/at.1']), - ('SE1,SE2', ['/this/is/also/at.12', '/this/is/at.12']), - ('SE1,SE2,SE3', ['/this/is/at_123']), - ('SE2', ['/this/is/als/at.2', '/this/is/at.2']), - ('SE2,SE3', ['/this/is/at_23']), - ('SE4', ['/this/is/at_4'])] + sortedData = [('SE1', {'/this/is/at.1'}), + ('SE1,SE2', {'/this/is/also/at.12', '/this/is/at.12'}), + ('SE1,SE2,SE3', {'/this/is/at_123'}), + ('SE2', {'/this/is/als/at.2', '/this/is/at.2'}), + ('SE2,SE3', {'/this/is/at_23'}), + ('SE4', {'/this/is/at_4'})] assert res['OK'] - assert res['Value'] == sortedData + assert [(a, set(b)) for a, b in res['Value']] == sortedData def test__Standard_G1(setup): @@ -116,7 +116,7 @@ def test__Standard_Data_G1(setup): pluginStandard.setInputData(data) res = pluginStandard.run() assert res['OK'] - sortedData = sorted([(",".join(SEs), [lfn]) for lfn, SEs in data.iteritems()]) + sortedData = sorted([(",".join(SEs), [lfn]) for lfn, SEs in data.items()]) assert res['Value'], sortedData @@ -128,9 +128,9 @@ def test__Standard_Flush_G1(setup): pluginStandard.setParameters(params) pluginStandard.setInputData(data) res = pluginStandard.run() - sortedData = sorted([(",".join(SEs), [lfn]) for lfn, SEs in data.iteritems()]) + sortedData = sorted([(",".join(SEs), [lfn]) for lfn, SEs in data.items()]) assert res['OK'] - assert res['Value'] == sortedData + assert sorted(res['Value']) == sorted(sortedData) def test__Broadcast_Active_G1(setup): @@ -146,7 +146,7 @@ def test__Broadcast_Active_G1(setup): # sort returned values by first lfn in LFNs sortedReturn = [(SE, lfns) for SE, lfns in sorted(res['Value'], key=lambda t: t[1][0])] # sort data by lfn - expected = [('SE2', [lfn]) for lfn, SEs in sorted(data.iteritems(), key=lambda t: t[0]) if 'SE1' in SEs] + expected = [('SE2', [lfn]) for lfn, SEs in sorted(data.items(), key=lambda t: t[0]) if 'SE1' in SEs] assert res['OK'] assert sortedReturn == expected @@ -179,6 +179,6 @@ def test__Broadcast_Active_G1_NoSource(setup): # sort returned values by first lfn in LFNs sortedReturn = [(SE, lfns) for SE, lfns in sorted(res['Value'], key=lambda t: t[1][0])] # sort data by lfn - expected = [('SE2', [lfn]) for lfn, _SEs in sorted(data.iteritems(), key=lambda t: t[0])] + expected = [('SE2', [lfn]) for lfn, _SEs in sorted(data.items(), key=lambda t: t[0])] assert res['OK'] assert sortedReturn == expected diff --git a/TransformationSystem/Client/TaskManager.py b/TransformationSystem/Client/TaskManager.py index f7bdc22d557..086a6b77b45 100644 --- a/TransformationSystem/Client/TaskManager.py +++ b/TransformationSystem/Client/TaskManager.py @@ -11,7 +11,7 @@ import six import time -import StringIO +from six import StringIO import json import copy import os @@ -79,7 +79,7 @@ def updateDBAfterTaskSubmission(self, taskDict): """ updated = 0 startTime = time.time() - for taskID, task in taskDict.iteritems(): + for taskID, task in taskDict.items(): transID = task['TransformationID'] if task['Success']: res = self.transClient.setTaskStatusAndWmsID(transID, taskID, 'Submitted', @@ -192,7 +192,7 @@ def _multiOperationsBody(self, transJson, taskDict, ownerDN, ownerGroup): :returns: None """ failedTasks = [] - for taskID, task in taskDict.items(): + for taskID, task in list(taskDict.items()): transID = task['TransformationID'] if not task.get('InputData'): self._logError("Error creating request for task", "%s, No input data" % taskID, transID=transID) @@ -210,7 +210,7 @@ def _multiOperationsBody(self, transJson, taskDict, ownerDN, ownerGroup): for operationTuple in transJson: op = Operation() op.Type = operationTuple[0] - for parameter, value in operationTuple[1].iteritems(): + for parameter, value in operationTuple[1].items(): setattr(op, parameter, value) for lfn in files: @@ -246,7 +246,7 @@ def _singleOperationsBody(self, transBody, taskDict, ownerDN, ownerGroup): pass failedTasks = [] # Do not remove sorted, we might pop elements in the loop - for taskID, task in taskDict.iteritems(): + for taskID, task in taskDict.items(): transID = task['TransformationID'] @@ -308,7 +308,7 @@ def submitTransformationTasks(self, taskDict): failed = 0 startTime = time.time() method = 'submitTransformationTasks' - for task in taskDict.itervalues(): + for task in taskDict.values(): # transID is the same for all tasks, so pick it up every time here transID = task['TransformationID'] if not task['TaskObject']: @@ -406,14 +406,14 @@ def getSubmittedFileStatus(self, fileDicts): requestFiles[externalID] = taskFiles[taskID] updateDict = {} - for requestID, lfnList in requestFiles.iteritems(): + for requestID, lfnList in requestFiles.items(): statusDict = self.requestClient.getRequestFileStatus(requestID, lfnList) if not statusDict['OK']: log = self._logVerbose if 'not exist' in statusDict['Message'] else self._logWarn log("Failed to get files status for request", statusDict['Message'], transID=transID, method='getSubmittedFileStatus') else: - for lfn, newStatus in statusDict['Value'].iteritems(): + for lfn, newStatus in statusDict['Value'].items(): if newStatus == 'Done': updateDict[lfn] = 'Processed' elif newStatus == 'Failed': @@ -524,7 +524,7 @@ def __prepareTasksBulk(self, transBody, taskDict, owner, ownerGroup, ownerDN): :return: S_OK/S_ERROR with updated taskDict """ if taskDict: - transID = taskDict.values()[0]['TransformationID'] + transID = list(taskDict.values())[0]['TransformationID'] else: return S_OK({}) @@ -607,7 +607,7 @@ def __prepareTasksBulk(self, transBody, taskDict, owner, ownerGroup, ownerDN): self._logError("Invalid mixture of jobs with and without input data") return S_ERROR(ETSDATA, "Invalid mixture of jobs with and without input data") - for paramName, paramValue in paramsDict.iteritems(): + for paramName, paramValue in paramsDict.items(): if paramName not in ('InputData', 'Site', 'TargetSE'): if paramValue: self._logVerbose('Setting %s to %s' % (paramName, paramValue), @@ -623,7 +623,7 @@ def __prepareTasksBulk(self, transBody, taskDict, owner, ownerGroup, ownerDN): self._logError("Failed to generate output data", res['Message'], transID=transID, method=method) continue - for name, output in res['Value'].iteritems(): + for name, output in res['Value'].items(): seqDict[name] = output outputParameterList.append(name) if oJob.workflow.findParameter(name): @@ -635,10 +635,10 @@ def __prepareTasksBulk(self, transBody, taskDict, owner, ownerGroup, ownerDN): "%%(%s)s" % name, name) - for pName, seq in seqDict.iteritems(): + for pName, seq in seqDict.items(): paramSeqDict.setdefault(pName, []).append(seq) - for paramName, paramSeq in paramSeqDict.iteritems(): + for paramName, paramSeq in paramSeqDict.items(): if paramName in ['JOB_ID', 'PRODUCTION_ID', 'InputData'] + outputParameterList: res = oJob.setParameterSequence(paramName, paramSeq, addToWorkflow=paramName) else: @@ -665,7 +665,7 @@ def __prepareTasks(self, transBody, taskDict, owner, ownerGroup, ownerDN): :return: S_OK/S_ERROR with updated taskDict """ if taskDict: - transID = taskDict.values()[0]['TransformationID'] + transID = list(taskDict.values())[0]['TransformationID'] else: return S_OK({}) @@ -685,7 +685,7 @@ def __prepareTasks(self, transBody, taskDict, owner, ownerGroup, ownerDN): templateOK = False getOutputDataTiming = 0. - for taskID, paramsDict in taskDict.iteritems(): + for taskID, paramsDict in taskDict.items(): # Create a job for each task and add it to the taskDict if not templateOK: templateOK = True @@ -760,7 +760,7 @@ def __prepareTasks(self, transBody, taskDict, owner, ownerGroup, ownerDN): self._logError("Failed to generate output data", res['Message'], transID=transID, method=method) continue - for name, output in res['Value'].iteritems(): + for name, output in res['Value'].items(): oJob._addJDLParameter(name, ';'.join(output)) paramsDict['TaskObject'] = oJob if taskDict: @@ -823,7 +823,7 @@ def _handleRest(self, oJob, paramsDict): """ add as JDL parameters all the other parameters that are not for inputs or destination """ transID = paramsDict['TransformationID'] - for paramName, paramValue in paramsDict.iteritems(): + for paramName, paramValue in paramsDict.items(): if paramName not in ('InputData', 'Site', 'TargetSE'): if paramValue: self._logDebug('Setting %s to %s' % (paramName, paramValue), @@ -919,7 +919,7 @@ def __submitTransformationTasksBulk(self, taskDict): oJob = taskDict.pop('BulkJobObject') # we can only do this, once the job has been popped, or we _might_ crash - transID = taskDict.values()[0]['TransformationID'] + transID = list(taskDict.values())[0]['TransformationID'] if oJob is None: self._logError('no bulk Job object found', transID=transID, method=method) return S_ERROR(ETSUKN, 'No bulk job object provided for submission') @@ -952,7 +952,7 @@ def __submitTransformationTasks(self, taskDict): submitted = 0 failed = 0 startTime = time.time() - for task in taskDict.itervalues(): + for task in taskDict.values(): transID = task['TransformationID'] if not task['TaskObject']: task['Success'] = False @@ -991,7 +991,7 @@ def submitTaskToExternal(self, job): self._logError("No valid job description found") return S_ERROR("No valid job description found") - workflowFileObject = StringIO.StringIO(oJob._toXML()) + workflowFileObject = StringIO(oJob._toXML()) jdl = oJob._toJDL(jobDescriptionObject=workflowFileObject) return self.submissionClient.submitJob(jdl, workflowFileObject) @@ -1077,7 +1077,7 @@ def getSubmittedFileStatus(self, fileDicts): updateDict = {} for jobName in noTasks: - for lfn, oldStatus in taskFiles[jobName].iteritems(): + for lfn, oldStatus in taskFiles[jobName].items(): if oldStatus != 'Unused': updateDict[lfn] = 'Unused' @@ -1087,13 +1087,13 @@ def getSubmittedFileStatus(self, fileDicts): transID=transID, method=method) return res statusDict = res['Value'] - for jobName, wmsID in taskNameIDs.iteritems(): + for jobName, wmsID in taskNameIDs.items(): jobStatus = statusDict.get(wmsID, {}).get('Status') newFileStatus = {'Done': 'Processed', 'Completed': 'Processed', 'Failed': 'Unused'}.get(jobStatus) if newFileStatus: - for lfn, oldStatus in taskFiles[jobName].iteritems(): + for lfn, oldStatus in taskFiles[jobName].items(): if newFileStatus != oldStatus: updateDict[lfn] = newFileStatus return S_OK(updateDict) diff --git a/TransformationSystem/Client/TaskManagerPlugin.py b/TransformationSystem/Client/TaskManagerPlugin.py index aa8935dc076..1999ca2de71 100644 --- a/TransformationSystem/Client/TaskManagerPlugin.py +++ b/TransformationSystem/Client/TaskManagerPlugin.py @@ -139,7 +139,7 @@ def _ByJobType(self): gLogger.debug(res['Message']) allowed = {} else: - allowed = dict((site, set(fromChar(fromSites))) for site, fromSites in res['Value'].iteritems()) + allowed = dict((site, set(fromChar(fromSites))) for site, fromSites in res['Value'].items()) autoAddedSites = set(self.opsH.getValue('JobTypeMapping/%s/AutoAddedSites' % jobType, autoAddedSites)) gLogger.debug("Auto-added sites for %s task: %s" % (jobType, ','.join(autoAddedSites))) @@ -151,7 +151,7 @@ def _ByJobType(self): # 6. Allowing sites that should be allowed taskSiteDestination = self._BySE() - for destSite, fromSites in allowed.iteritems(): + for destSite, fromSites in allowed.items(): for fromSite in fromSites: if not taskSiteDestination or fromSite in taskSiteDestination: destSites.add(destSite) diff --git a/TransformationSystem/Client/Transformation.py b/TransformationSystem/Client/Transformation.py index ddf07c15237..fd061c41fdd 100644 --- a/TransformationSystem/Client/Transformation.py +++ b/TransformationSystem/Client/Transformation.py @@ -32,21 +32,21 @@ def __init__(self, transID=0, transClient=None): """ super(Transformation, self).__init__() - self.paramTypes = {'TransformationID': [types.IntType, types.LongType], - 'TransformationName': types.StringTypes, - 'Status': types.StringTypes, - 'Description': types.StringTypes, - 'LongDescription': types.StringTypes, - 'Type': types.StringTypes, - 'Plugin': types.StringTypes, - 'AgentType': types.StringTypes, - 'FileMask': types.StringTypes, - 'TransformationGroup': types.StringTypes, - 'GroupSize': [types.IntType, types.LongType, types.FloatType], - 'InheritedFrom': [types.IntType, types.LongType], - 'Body': types.StringTypes, - 'MaxNumberOfTasks': [types.IntType, types.LongType], - 'EventsPerTask': [types.IntType, types.LongType]} + self.paramTypes = {'TransformationID': six.integer_types, + 'TransformationName': six.string_types, + 'Status': six.string_types, + 'Description': six.string_types, + 'LongDescription': six.string_types, + 'Type': six.string_types, + 'Plugin': six.string_types, + 'AgentType': six.string_types, + 'FileMask': six.string_types, + 'TransformationGroup': six.string_types, + 'GroupSize': six.integer_types + (float,), + 'InheritedFrom': six.integer_types, + 'Body': six.string_types, + 'MaxNumberOfTasks': six.integer_types, + 'EventsPerTask': six.integer_types} self.paramValues = {'TransformationID': 0, 'TransformationName': '', 'Status': 'New', @@ -135,12 +135,12 @@ def setBody(self, body): raise TypeError("Expected string, but first entry in tuple %r is %s" % (tup, type(tup[0]))) if not isinstance(tup[1], dict): raise TypeError("Expected dictionary, but second entry in tuple %r is %s" % (tup, type(tup[0]))) - for par, val in tup[1].iteritems(): + for par, val in tup[1].items(): if not isinstance(par, six.string_types): raise TypeError("Expected string, but key in dictionary %r is %s" % (par, type(par))) if par not in Operation.ATTRIBUTE_NAMES: raise ValueError("Unknown attribute for Operation: %s" % par) - if not isinstance(val, (basestring, int, long, float, list, tuple, dict)): + if not isinstance(val, six.string_types + six.integer_types + (float, list, tuple, dict)): raise TypeError("Cannot encode %r, in json" % (val)) return self.__setParam(json.dumps(body)) @@ -200,7 +200,7 @@ def __setParam(self, value): change = False if self.item_called in self.paramTypes: if self.paramValues[self.item_called] != value: - if type(value) in self.paramTypes[self.item_called]: + if isinstance(value, self.paramTypes[self.item_called]): change = True else: raise TypeError("%s %s %s expected one of %s" % (self.item_called, value, type(value), diff --git a/TransformationSystem/Client/test/Test_Client_WorkflowTasks.py b/TransformationSystem/Client/test/Test_Client_WorkflowTasks.py index 01c2e19297d..e065e850bcd 100644 --- a/TransformationSystem/Client/test/Test_Client_WorkflowTasks.py +++ b/TransformationSystem/Client/test/Test_Client_WorkflowTasks.py @@ -87,10 +87,10 @@ def test_prepareTranformationTasks(taskDictionary, bulkSubmissionFlag, result, e bulkSubmissionFlag=bulkSubmissionFlag) assert res['OK'] == result if res['OK']: - for key, value in res['Value'].iteritems(): + for key, value in res['Value'].items(): if key != 'BulkJobObject': assert key in expectedRes['Value'] - for tKey, tValue in value.iteritems(): + for tKey, tValue in value.items(): assert tKey in expectedRes['Value'][key] if tKey == 'TaskObject' and tValue: assert isinstance(tValue, Job) diff --git a/TransformationSystem/DB/TransformationDB.py b/TransformationSystem/DB/TransformationDB.py index c215f5be1a1..4d8550524a1 100755 --- a/TransformationSystem/DB/TransformationDB.py +++ b/TransformationSystem/DB/TransformationDB.py @@ -524,7 +524,7 @@ def addFilesToTransformation(self, transName, lfns, connection=False): res = self.__addDataFiles(lfns, connection=connection) if not res['OK']: return res - fileIDs = dict((fileID, lfn) for lfn, fileID in res['Value'].iteritems()) + fileIDs = dict((fileID, lfn) for lfn, fileID in res['Value'].items()) # Attach files to transformation successful = {} if fileIDs: @@ -556,7 +556,7 @@ def getTransformationFiles(self, condDict=None, older=None, newer=None, timeStam originalFileIDs = res['Value'][0] condDict['FileID'] = list(originalFileIDs) - for val in condDict.itervalues(): + for val in condDict.values(): if not val: return S_OK([]) @@ -619,10 +619,10 @@ def setFileStatusForTransformation(self, transID, fileStatusDict=None, connectio # Get fileID and status for each case: error and no error statusFileDict = {} - for fileID, (status, error) in fileStatusDict.iteritems(): + for fileID, (status, error) in fileStatusDict.items(): statusFileDict.setdefault(error, []).append((fileID, status)) - for error, fileIDStatusList in statusFileDict.iteritems(): + for error, fileIDStatusList in statusFileDict.items(): req = reqBase + ','.join("(%d, %d, '%s', 0, UTC_TIMESTAMP())" % (transID, fileID, status) for fileID, status in fileIDStatusList) if error: @@ -1234,7 +1234,7 @@ def __getFileIDsForLfns(self, lfns, connection=False): return res lfns = dict(res['Value']) # Reverse dictionary - fids = dict((fileID, lfn) for lfn, fileID in lfns.iteritems()) + fids = dict((fileID, lfn) for lfn, fileID in lfns.items()) return S_OK((fids, lfns)) def __getLfnsForFileIDs(self, fileIDs, connection=False): @@ -1246,7 +1246,7 @@ def __getLfnsForFileIDs(self, fileIDs, connection=False): return res fids = dict(res['Value']) # Reverse dictionary - lfns = dict((fileID, lfn) for lfn, fileID in fids.iteritems()) + lfns = dict((fileID, lfn) for lfn, fileID in fids.items()) return S_OK((fids, lfns)) def __addDataFiles(self, lfns, connection=False): @@ -1511,7 +1511,7 @@ def addFile(self, fileDicts, force=False, connection=False): # Add the files to the transformations gLogger.info('Files to add to transformations:', filesToAdd) if filesToAdd: - for transID, lfns in transFiles.iteritems(): + for transID, lfns in transFiles.items(): res = self.addFilesToTransformation(transID, lfns) if not res['OK']: gLogger.error("Failed to add files to transformation", "%s %s" % (transID, res['Message'])) @@ -1639,7 +1639,7 @@ def setMetadata(self, path, usermetadatadict): # Add the files to the transformations gLogger.info('Files to add to transformations:', filesToAdd) if filesToAdd: - for transID, lfns in transFiles.iteritems(): + for transID, lfns in transFiles.items(): res = self.addFilesToTransformation(transID, lfns) if not res['OK']: gLogger.error("Failed to add files to transformation", "%s %s" % (transID, res['Message'])) diff --git a/TransformationSystem/Service/TransformationManagerHandler.py b/TransformationSystem/Service/TransformationManagerHandler.py index 70b2abfba6d..bbfaa5e8875 100644 --- a/TransformationSystem/Service/TransformationManagerHandler.py +++ b/TransformationSystem/Service/TransformationManagerHandler.py @@ -196,7 +196,7 @@ def export_setFileStatusForTransformation(self, transName, dictOfNewFilesStatus) if not dictOfNewFilesStatus: return S_OK({}) - statusSample = dictOfNewFilesStatus.values()[0] + statusSample = list(dictOfNewFilesStatus.values())[0] if isinstance(statusSample, (list, tuple)) and len(statusSample) == 2: newStatusForFileIDs = dictOfNewFilesStatus else: diff --git a/TransformationSystem/Utilities/JobInfo.py b/TransformationSystem/Utilities/JobInfo.py index fd01948cbe2..c022e8853aa 100644 --- a/TransformationSystem/Utilities/JobInfo.py +++ b/TransformationSystem/Utilities/JobInfo.py @@ -4,7 +4,8 @@ from __future__ import print_function from pprint import pformat -from itertools import izip_longest +import six +from six.moves import zip_longest from DIRAC import gLogger from DIRAC.ConfigurationSystem.Client.Helpers.Operations import Operations @@ -55,14 +56,14 @@ def __str__(self): if self.otherTasks: info += ' (Last task %s)' % self.otherTasks if self.inputFiles: - ifInfo = ['<<< %s (%s, %s, Errors %s)' % _ for _ in izip_longest(self.inputFiles, + ifInfo = ['<<< %s (%s, %s, Errors %s)' % _ for _ in zip_longest(self.inputFiles, self.inputFilesExist, self.transFileStatus, self.errorCounts)] info += '\n'.join(ifInfo) if self.outputFiles: info += "\n::: OutputFiles: " - efInfo = ["%s (%s)" % _ for _ in izip_longest(self.outputFiles, self.outputFileStatus)] + efInfo = ["%s (%s)" % _ for _ in zip_longest(self.outputFiles, self.outputFileStatus)] info += ", ".join(efInfo) if self.pendingRequest: info += "\n PENDING REQUEST IGNORE THIS JOB!!!" @@ -211,14 +212,14 @@ def __getJDL(self, diracAPI): def __getOutputFiles(self, jdlParameters): """get the Production Outputfiles for the given Job""" lfns = jdlParameters.get('ProductionOutputData', []) - if isinstance(lfns, basestring): + if isinstance(lfns, six.string_types): lfns = [lfns] self.outputFiles = lfns def __getInputFile(self, jdlParameters): """get the Inputdata for the given job""" lfn = jdlParameters.get('InputData', None) - if isinstance(lfn, basestring): + if isinstance(lfn, six.string_types): self.inputFiles.append(lfn) return if isinstance(lfn, list): @@ -227,7 +228,7 @@ def __getInputFile(self, jdlParameters): def __getTaskID(self, jdlParameters): """Get the taskID.""" - if isinstance(jdlParameters, basestring): + if isinstance(jdlParameters, six.string_types): self.taskID = int(jdlParameters.split('_')[1]) if 'TaskID' not in jdlParameters: return diff --git a/TransformationSystem/Utilities/TransformationInfo.py b/TransformationSystem/Utilities/TransformationInfo.py index 380e2325aa8..b2ce25ae613 100644 --- a/TransformationSystem/Utilities/TransformationInfo.py +++ b/TransformationSystem/Utilities/TransformationInfo.py @@ -4,7 +4,7 @@ from __future__ import print_function from collections import OrderedDict, defaultdict -from itertools import izip_longest +from six.moves import zip_longest from DIRAC import gLogger, S_OK from DIRAC.Core.Utilities.List import breakListIntoChunks @@ -131,7 +131,7 @@ def cleanOutputs(self, jobInfo): descendants = self.__findAllDescendants(jobInfo.outputFiles) existingOutputFiles = [ lfn for lfn, - status in izip_longest( + status in zip_longest( jobInfo.outputFiles, jobInfo.outputFileStatus) if status == "Exists"] filesToDelete = existingOutputFiles + descendants diff --git a/TransformationSystem/test/Test_DRA.py b/TransformationSystem/test/Test_DRA.py index 27020f8b2ed..f965989aca8 100644 --- a/TransformationSystem/test/Test_DRA.py +++ b/TransformationSystem/test/Test_DRA.py @@ -299,7 +299,7 @@ def test_checkAllJob(self): # test with additional task dicts from DIRAC.TransformationSystem.Utilities.TransformationInfo import TransformationInfo tInfoMock = Mock(name="tInfoMock", spec=TransformationInfo) - mockJobs = dict([(i, self.getTestMock()) for i in xrange(11)]) + mockJobs = dict([(i, self.getTestMock()) for i in range(11)]) mockJobs[2].pendingRequest = True mockJobs[3].getJobInformation = Mock(side_effect=(RuntimeError('ARGJob1'), None)) mockJobs[4].getTaskInfo = Mock(side_effect=(TaskInfoException('ARG1'), None)) @@ -311,7 +311,7 @@ def test_checkAllJob(self): self.dra.log.reset_mock() # test inputFile None - mockJobs = dict([(i, self.getTestMock(nameID=i)) for i in xrange(5)]) + mockJobs = dict([(i, self.getTestMock(nameID=i)) for i in range(5)]) mockJobs[1].inputFiles = [] mockJobs[1].getTaskInfo = Mock(side_effect=(TaskInfoException("NoInputFile"), None)) mockJobs[1].tType = "MCSimulation" @@ -323,7 +323,7 @@ def test_checkAllJob_2(self): """Test where failJobHard fails (via cleanOutputs).""" from DIRAC.TransformationSystem.Utilities.TransformationInfo import TransformationInfo tInfoMock = Mock(name='tInfoMock', spec=TransformationInfo) - mockJobs = dict([(i, self.getTestMock()) for i in xrange(5)]) + mockJobs = dict([(i, self.getTestMock()) for i in range(5)]) mockJobs[2].pendingRequest = True mockJobs[3].getTaskInfo = Mock(side_effect=(TaskInfoException('ARGJob3'), None)) mockJobs[3].inputFiles = [] @@ -412,7 +412,7 @@ def test_printSummary(self): def test_setPendingRequests_1(self): """Check the setPendingRequests function.""" - mockJobs = dict((i, self.getTestMock(jobID=i)) for i in xrange(11)) + mockJobs = dict((i, self.getTestMock(jobID=i)) for i in range(11)) reqMock = Mock() reqMock.Status = "Done" reqClient = Mock(name="reqMock", spec=DIRAC.RequestManagementSystem.Client.ReqClient.ReqClient) @@ -424,7 +424,7 @@ def test_setPendingRequests_1(self): def test_setPendingRequests_2(self): """Check the setPendingRequests function.""" - mockJobs = dict((i, self.getTestMock(jobID=i)) for i in xrange(11)) + mockJobs = dict((i, self.getTestMock(jobID=i)) for i in range(11)) reqMock = Mock() reqMock.RequestID = 666 reqClient = Mock(name="reqMock", spec=DIRAC.RequestManagementSystem.Client.ReqClient.ReqClient) @@ -438,7 +438,7 @@ def test_setPendingRequests_2(self): def test_setPendingRequests_3(self): """Check the setPendingRequests function.""" - mockJobs = dict((i, self.getTestMock(jobID=i)) for i in xrange(11)) + mockJobs = dict((i, self.getTestMock(jobID=i)) for i in range(11)) reqMock = Mock() reqMock.RequestID = 555 reqClient = Mock(name="reqMock", spec=DIRAC.RequestManagementSystem.Client.ReqClient.ReqClient) @@ -455,7 +455,7 @@ def test_setPendingRequests_3(self): def test_setPendingRequests_Fail(self): """Check the setPendingRequests function.""" - mockJobs = dict((i, self.getTestMock(jobID=i)) for i in xrange(11)) + mockJobs = dict((i, self.getTestMock(jobID=i)) for i in range(11)) reqMock = Mock() reqMock.Status = "Done" reqClient = Mock(name="reqMock", spec=DIRAC.RequestManagementSystem.Client.ReqClient.ReqClient) @@ -467,7 +467,7 @@ def test_setPendingRequests_Fail(self): def test_getLFNStatus(self): """Check the getLFNStatus function.""" - mockJobs = dict((i, self.getTestMock(jobID=i)) for i in xrange(11)) + mockJobs = dict((i, self.getTestMock(jobID=i)) for i in range(11)) self.dra.fcClient.exists.return_value = S_OK({'Successful': {'/my/stupid/file.lfn': True, '/my/stupid/file2.lfn': True}}) diff --git a/TransformationSystem/test/Test_JobInfo.py b/TransformationSystem/test/Test_JobInfo.py index 49531ba350f..ca79cf284bd 100644 --- a/TransformationSystem/test/Test_JobInfo.py +++ b/TransformationSystem/test/Test_JobInfo.py @@ -5,7 +5,7 @@ import unittest import sys -from StringIO import StringIO +from six import StringIO from mock import MagicMock as Mock diff --git a/TransformationSystem/test/Test_TransformationInfo.py b/TransformationSystem/test/Test_TransformationInfo.py index e0a13805c0a..0eb731ab64a 100644 --- a/TransformationSystem/test/Test_TransformationInfo.py +++ b/TransformationSystem/test/Test_TransformationInfo.py @@ -241,7 +241,7 @@ def test_findAllDescendants(tiFixture): "lfnD1": ["lfnDD1", "lfnDD2"], }})) descList = tiFixture._TransformationInfo__findAllDescendants(lfnList=[]) - assert descList == ["lfnDD1", "lfnDD2", "lfnD1", "lfnD2"] + assert set(descList) == {"lfnDD1", "lfnDD2", "lfnD1", "lfnD2"} tiFixture.fcClient.getFileDescendents = Mock(return_value=S_ERROR("Cannot get descendants")) descList = tiFixture._TransformationInfo__findAllDescendants(lfnList=[]) diff --git a/Workflow/Modules/ModuleBase.py b/Workflow/Modules/ModuleBase.py index 45e9ca64c22..a6d08ce3d62 100644 --- a/Workflow/Modules/ModuleBase.py +++ b/Workflow/Modules/ModuleBase.py @@ -469,7 +469,7 @@ def _applyMask(self, candidateFilesIn, fileMask, stepMask): stepMask = [stepMask] if fileMask and fileMask != ['']: - for fileName, metadata in candidateFiles.items(): + for fileName, metadata in list(candidateFiles.items()): if metadata['type'].lower() not in fileMask: # and ( fileName.split( '.' )[-1] not in fileMask ) ): del candidateFiles[fileName] self.log.info('Output file %s was produced but will not be treated (fileMask is %s)' % (fileName, @@ -479,7 +479,7 @@ def _applyMask(self, candidateFilesIn, fileMask, stepMask): if stepMask and stepMask != ['']: # This supposes that the LFN contains the step ID - for fileName, metadata in candidateFiles.items(): + for fileName, metadata in list(candidateFiles.items()): if fileName.split('_')[-1].split('.')[0] not in stepMask: del candidateFiles[fileName] self.log.info('Output file %s was produced but will not be treated (stepMask is %s)' % (fileName, diff --git a/Workflow/Modules/test/Test_Modules.py b/Workflow/Modules/test/Test_Modules.py index 02eabfe467f..46e9afabcaf 100644 --- a/Workflow/Modules/test/Test_Modules.py +++ b/Workflow/Modules/test/Test_Modules.py @@ -425,7 +425,7 @@ def test__applyMask(self): } ) - for fileMask, result, stepMask in itertools.izip(fileMasks, results, stepMasks): + for fileMask, result, stepMask in zip(fileMasks, results, stepMasks): res = self.mb._applyMask(candidateFiles, fileMask, stepMask) self.assertEqual(res, result) diff --git a/WorkloadManagementSystem/Agent/MultiProcessorSiteDirector.py b/WorkloadManagementSystem/Agent/MultiProcessorSiteDirector.py index 091c4770e89..21ca29d450f 100644 --- a/WorkloadManagementSystem/Agent/MultiProcessorSiteDirector.py +++ b/WorkloadManagementSystem/Agent/MultiProcessorSiteDirector.py @@ -66,7 +66,7 @@ def submitPilots(self): """ Go through defined computing elements and submit jobs if necessary """ - queues = self.queueDict.keys() + queues = list(self.queueDict) # Check that there is some work at all setup = CSGlobals.getSetup() @@ -120,7 +120,7 @@ def submitPilots(self): testSites.add(site) totalWaitingJobs += result['Value'][tqID]['Jobs'] - tqIDList = result['Value'].keys() + tqIDList = list(result['Value']) self.log.info(tqIDList) result = pilotAgentsDB.countPilots({'TaskQueueID': tqIDList, 'Status': WAITING_PILOT_STATUS}, @@ -140,7 +140,7 @@ def submitPilots(self): return result siteMaskList = result['Value'] - queues = self.queueDict.keys() + queues = list(self.queueDict) random.shuffle(queues) totalSubmittedPilots = 0 matchedQueues = 0 @@ -247,7 +247,7 @@ def submitPilots(self): matchedQueues += 1 totalTQJobs = 0 totalTQJobsByProcessors = {} - tqIDList = taskQueueDict.keys() + tqIDList = list(taskQueueDict) tqIDListByProcessors = {} for tq in taskQueueDict: if 'Tags' not in taskQueueDict[tq]: diff --git a/WorkloadManagementSystem/Agent/PilotStatusAgent.py b/WorkloadManagementSystem/Agent/PilotStatusAgent.py index 1a2c3b7f15c..8bc91a77990 100644 --- a/WorkloadManagementSystem/Agent/PilotStatusAgent.py +++ b/WorkloadManagementSystem/Agent/PilotStatusAgent.py @@ -222,7 +222,7 @@ def accountPilots(self, pilotsToAccount, connection): accountingSent = False if accountingFlag: - retVal = self.pilotDB.getPilotInfo(list(pilotsToAccount.keys()), conn=connection) + retVal = self.pilotDB.getPilotInfo(list(pilotsToAccount), conn=connection) if not retVal['OK']: self.log.error('Fail to retrieve Info for pilots', retVal['Message']) return retVal diff --git a/WorkloadManagementSystem/Agent/SiteDirector.py b/WorkloadManagementSystem/Agent/SiteDirector.py index 7d947f05517..4c99c5bf03e 100644 --- a/WorkloadManagementSystem/Agent/SiteDirector.py +++ b/WorkloadManagementSystem/Agent/SiteDirector.py @@ -290,7 +290,7 @@ def __generateQueueHash(self, queueDict): """ Generate a hash of the queue description """ myMD5 = hashlib.md5() - myMD5.update(str(queueDict)) + myMD5.update(str(queueDict).encode()) hexstring = myMD5.hexdigest() return hexstring @@ -447,13 +447,13 @@ def execute(self): self.siteMaskList = result.get('Value', []) if self.rssFlag: - ceNamesList = [queue['CEName'] for queue in self.queueDict.itervalues()] + ceNamesList = [queue['CEName'] for queue in self.queueDict.values()] result = self.rssClient.getElementStatus(ceNamesList, "ComputingElement") if not result['OK']: self.log.error("Can not get the status of computing elements", " %s" % result['Message']) return result - self.ceMaskList = [ceName for ceName in result['Value'].iterkeys() if result['Value'][ceName] + self.ceMaskList = [ceName for ceName in result['Value'] if result['Value'][ceName] ['all'] in ('Active', 'Degraded')] result = self.submitPilots() @@ -626,7 +626,7 @@ def _ifAndWhereToSubmit(self): testSites = set() anySite = False - for tqDescription in matchingTQs.itervalues(): + for tqDescription in matchingTQs.values(): siteList = tqDescription.get('Sites', []) if siteList: jobSites |= set(siteList) @@ -652,7 +652,7 @@ def monitorJobsQueuesPilots(self, matchingTQs): None) totalWaitingJobs = 0 - for tqDescription in matchingTQs.itervalues(): + for tqDescription in matchingTQs.values(): totalWaitingJobs += tqDescription['Jobs'] if not result['OK']: @@ -814,7 +814,7 @@ def _getPilotsWeMayWantToSubmit(self, ceDict): self.log.verbose('No matching TQs found', 'for %s' % ceDict) - for tq in taskQueueDict.itervalues(): + for tq in taskQueueDict.values(): pilotsWeMayWantToSubmit += tq['Jobs'] return pilotsWeMayWantToSubmit, taskQueueDict @@ -927,7 +927,7 @@ def _addPilotTQReference(self, queue, taskQueueDict, pilotList, stampDict): tqDict[tqID] = [] tqDict[tqID].append(pilotID) - for tqID, pilotsList in tqDict.iteritems(): + for tqID, pilotsList in tqDict.items(): result = pilotAgentsDB.addPilotTQReference(pilotsList, tqID, self.pilotDN, @@ -1013,7 +1013,7 @@ def getQueueSlots(self, queue, manyWaitingPilotsFlag): "for queue %s: \n%s" % (queue, result['Message'])) self.failedQueues[queue] += 1 else: - for _pilotRef, pilotDict in result['Value'].iteritems(): + for _pilotRef, pilotDict in result['Value'].items(): if pilotDict["Status"] in TRANSIENT_PILOT_STATUS: totalJobs += 1 if pilotDict["Status"] in WAITING_PILOT_STATUS: diff --git a/WorkloadManagementSystem/Agent/test/Test_Agent_JobAgent.py b/WorkloadManagementSystem/Agent/test/Test_Agent_JobAgent.py index d75e5c1936e..22e13f1caab 100644 --- a/WorkloadManagementSystem/Agent/test/Test_Agent_JobAgent.py +++ b/WorkloadManagementSystem/Agent/test/Test_Agent_JobAgent.py @@ -228,7 +228,10 @@ def test_submitJob(mocker, mockJWInput, expected): mockJW.return_value = mockJWInput mocker.patch("DIRAC.WorkloadManagementSystem.Agent.JobAgent.AgentModule.__init__") - mocker.patch("DIRAC.WorkloadManagementSystem.Agent.JobAgent.AgentModule.am_getOption", side_effect=mockAM) + mocker.patch( + "DIRAC.WorkloadManagementSystem.Agent.JobAgent.AgentModule._AgentModule__moduleProperties", + side_effect=lambda x, y=None: y, create=True + ) mocker.patch("DIRAC.WorkloadManagementSystem.Agent.JobAgent.createJobWrapper", side_effect=mockJW) jobAgent = JobAgent('Test', 'Test1') diff --git a/WorkloadManagementSystem/Agent/test/Test_Agent_JobCleaningAgent.py b/WorkloadManagementSystem/Agent/test/Test_Agent_JobCleaningAgent.py index f457ba98fd3..659140ac3f0 100644 --- a/WorkloadManagementSystem/Agent/test/Test_Agent_JobCleaningAgent.py +++ b/WorkloadManagementSystem/Agent/test/Test_Agent_JobCleaningAgent.py @@ -34,7 +34,10 @@ def test__getAllowedJobTypes(mocker, mockReplyInput, expected): mockReply.return_value = mockReplyInput mocker.patch("DIRAC.WorkloadManagementSystem.Agent.JobCleaningAgent.AgentModule.__init__") - mocker.patch("DIRAC.WorkloadManagementSystem.Agent.JobCleaningAgent.AgentModule.am_getOption", side_effect=mockAM) + mocker.patch( + "DIRAC.WorkloadManagementSystem.Agent.JobCleaningAgent.AgentModule._AgentModule__moduleProperties", + side_effect=lambda x, y=None: y, create=True + ) mocker.patch( "DIRAC.WorkloadManagementSystem.Agent.JobCleaningAgent.JobDB.getDistinctJobAttributes", side_effect=mockReply) @@ -68,7 +71,10 @@ def test_removeJobsByStatus(mocker, conditions, mockReplyInput, expected): mockReply.return_value = mockReplyInput mocker.patch("DIRAC.WorkloadManagementSystem.Agent.JobCleaningAgent.AgentModule.__init__") - mocker.patch("DIRAC.WorkloadManagementSystem.Agent.JobCleaningAgent.AgentModule.am_getOption", side_effect=mockAM) + mocker.patch( + "DIRAC.WorkloadManagementSystem.Agent.JobCleaningAgent.AgentModule._AgentModule__moduleProperties", + side_effect=lambda x, y=None: y, create=True + ) mocker.patch("DIRAC.WorkloadManagementSystem.Agent.JobCleaningAgent.JobDB.selectJobs", side_effect=mockReply) mocker.patch("DIRAC.WorkloadManagementSystem.Agent.JobCleaningAgent.TaskQueueDB.__init__", side_effect=mockNone) mocker.patch("DIRAC.WorkloadManagementSystem.Agent.JobCleaningAgent.JobDB.__init__", side_effect=mockNone) diff --git a/WorkloadManagementSystem/Agent/test/Test_Agent_PilotStatusAgent.py b/WorkloadManagementSystem/Agent/test/Test_Agent_PilotStatusAgent.py index fe59774139b..0b72d5fdbe2 100644 --- a/WorkloadManagementSystem/Agent/test/Test_Agent_PilotStatusAgent.py +++ b/WorkloadManagementSystem/Agent/test/Test_Agent_PilotStatusAgent.py @@ -28,7 +28,10 @@ def test_clearWaitingPilots(mocker): """ mocker.patch("DIRAC.WorkloadManagementSystem.Agent.PilotStatusAgent.AgentModule.__init__") - mocker.patch("DIRAC.WorkloadManagementSystem.Agent.PilotStatusAgent.AgentModule.am_getOption", side_effect=mockAM) + mocker.patch( + "DIRAC.WorkloadManagementSystem.Agent.PilotStatusAgent.AgentModule._AgentModule__moduleProperties", + side_effect=lambda x, y=None: y, create=True + ) mocker.patch("DIRAC.WorkloadManagementSystem.Agent.PilotStatusAgent.PilotAgentsDB.__init__", side_effect=mockNone) mocker.patch("DIRAC.WorkloadManagementSystem.Agent.PilotStatusAgent.JobDB.__init__", side_effect=mockNone) module_str = "DIRAC.WorkloadManagementSystem.Agent.PilotStatusAgent.PilotAgentsDB.buildCondition" @@ -67,7 +70,10 @@ def test_handleOldPilots(mocker, mockReplyInput, expected): mockReply.return_value = mockReplyInput mocker.patch("DIRAC.WorkloadManagementSystem.Agent.PilotStatusAgent.AgentModule.__init__") - mocker.patch("DIRAC.WorkloadManagementSystem.Agent.PilotStatusAgent.AgentModule.am_getOption", side_effect=mockAM) + mocker.patch( + "DIRAC.WorkloadManagementSystem.Agent.PilotStatusAgent.AgentModule._AgentModule__moduleProperties", + side_effect=lambda x, y=None: y, create=True + ) module_str = "DIRAC.WorkloadManagementSystem.Agent.PilotStatusAgent.PilotAgentsDB.selectPilots" mocker.patch(module_str, side_effect=mockReply) mocker.patch("DIRAC.WorkloadManagementSystem.Agent.PilotStatusAgent.PilotAgentsDB.__init__", side_effect=mockNone) diff --git a/WorkloadManagementSystem/Agent/test/Test_Agent_StalledJobAgent.py b/WorkloadManagementSystem/Agent/test/Test_Agent_StalledJobAgent.py index d96608e3097..b21c8c7d308 100644 --- a/WorkloadManagementSystem/Agent/test/Test_Agent_StalledJobAgent.py +++ b/WorkloadManagementSystem/Agent/test/Test_Agent_StalledJobAgent.py @@ -24,7 +24,10 @@ def test__failSubmittingJobs(mocker): """ mocker.patch("DIRAC.WorkloadManagementSystem.Agent.StalledJobAgent.AgentModule.__init__") - mocker.patch("DIRAC.WorkloadManagementSystem.Agent.StalledJobAgent.AgentModule.am_getOption", side_effect=mockAM) + mocker.patch( + "DIRAC.WorkloadManagementSystem.Agent.StalledJobAgent.AgentModule._AgentModule__moduleProperties", + side_effect=lambda x, y=None: y, create=True + ) mocker.patch("DIRAC.WorkloadManagementSystem.Agent.StalledJobAgent.JobDB.__init__", side_effect=mockNone) mocker.patch("DIRAC.WorkloadManagementSystem.Agent.StalledJobAgent.JobLoggingDB.__init__", side_effect=mockNone) @@ -45,7 +48,10 @@ def test__kickStuckJobs(mocker): """ mocker.patch("DIRAC.WorkloadManagementSystem.Agent.StalledJobAgent.AgentModule.__init__") - mocker.patch("DIRAC.WorkloadManagementSystem.Agent.StalledJobAgent.AgentModule.am_getOption", side_effect=mockAM) + mocker.patch( + "DIRAC.WorkloadManagementSystem.Agent.StalledJobAgent.AgentModule._AgentModule__moduleProperties", + side_effect=lambda x, y=None: y, create=True + ) mocker.patch("DIRAC.WorkloadManagementSystem.Agent.StalledJobAgent.JobDB.__init__", side_effect=mockNone) mocker.patch("DIRAC.WorkloadManagementSystem.Agent.StalledJobAgent.JobLoggingDB.__init__", side_effect=mockNone) @@ -66,7 +72,10 @@ def test__failStalledJobs(mocker): """ mocker.patch("DIRAC.WorkloadManagementSystem.Agent.StalledJobAgent.AgentModule.__init__") - mocker.patch("DIRAC.WorkloadManagementSystem.Agent.StalledJobAgent.AgentModule.am_getOption", side_effect=mockAM) + mocker.patch( + "DIRAC.WorkloadManagementSystem.Agent.StalledJobAgent.AgentModule._AgentModule__moduleProperties", + side_effect=lambda x, y=None: y, create=True + ) mocker.patch("DIRAC.WorkloadManagementSystem.Agent.StalledJobAgent.JobDB.__init__", side_effect=mockNone) mocker.patch("DIRAC.WorkloadManagementSystem.Agent.StalledJobAgent.JobLoggingDB.__init__", side_effect=mockNone) @@ -87,7 +96,10 @@ def test__markStalledJobs(mocker): """ mocker.patch("DIRAC.WorkloadManagementSystem.Agent.StalledJobAgent.AgentModule.__init__") - mocker.patch("DIRAC.WorkloadManagementSystem.Agent.StalledJobAgent.AgentModule.am_getOption", side_effect=mockAM) + mocker.patch( + "DIRAC.WorkloadManagementSystem.Agent.StalledJobAgent.AgentModule._AgentModule__moduleProperties", + side_effect=lambda x, y=None: y, create=True + ) mocker.patch("DIRAC.WorkloadManagementSystem.Agent.StalledJobAgent.JobDB.__init__", side_effect=mockNone) mocker.patch("DIRAC.WorkloadManagementSystem.Agent.StalledJobAgent.JobLoggingDB.__init__", side_effect=mockNone) diff --git a/WorkloadManagementSystem/Client/DIRACbenchmark.py b/WorkloadManagementSystem/Client/DIRACbenchmark.py index f5ef4f71c48..deb9bcfa9ae 100755 --- a/WorkloadManagementSystem/Client/DIRACbenchmark.py +++ b/WorkloadManagementSystem/Client/DIRACbenchmark.py @@ -55,7 +55,7 @@ def singleDiracBenchmark(iterations=1, measuredCopies=None): start = os.times() # Now the iterations - for _j in xrange(n): + for _j in range(n): t = random.normalvariate(10, 1) m += t m2 += t * t diff --git a/WorkloadManagementSystem/Client/DownloadInputData.py b/WorkloadManagementSystem/Client/DownloadInputData.py index 68f2d6c14d1..6ead65565b7 100644 --- a/WorkloadManagementSystem/Client/DownloadInputData.py +++ b/WorkloadManagementSystem/Client/DownloadInputData.py @@ -83,7 +83,7 @@ def execute(self, dataToResolve=None): # disk-based replicas are favoured. downloadReplicas = {} - for lfn, reps in replicas.iteritems(): + for lfn, reps in replicas.items(): if lfn not in self.inputData: self.log.verbose('LFN %s is not in requested input data to download') failedReplicas.add(lfn) @@ -118,7 +118,7 @@ def execute(self, dataToResolve=None): totalSize = 0 verbose = self.log.verbose('Replicas to download are:') - for lfn, reps in downloadReplicas.iteritems(): + for lfn, reps in downloadReplicas.items(): self.log.verbose(lfn) if not reps['SE']: self.log.info('Failed to find data at local SEs, will try to download from anywhere', lfn) @@ -157,7 +157,7 @@ def execute(self, dataToResolve=None): resolvedData = {} localSECount = 0 - for lfn, info in downloadReplicas.iteritems(): + for lfn, info in downloadReplicas.items(): seName = info['SE'] guid = info['GUID'] reps = replicas.get(lfn, {}) diff --git a/WorkloadManagementSystem/Client/InputDataByProtocol.py b/WorkloadManagementSystem/Client/InputDataByProtocol.py index 7f3ae57dee3..16a04b579c4 100644 --- a/WorkloadManagementSystem/Client/InputDataByProtocol.py +++ b/WorkloadManagementSystem/Client/InputDataByProtocol.py @@ -124,7 +124,7 @@ def __resolveReplicas(self, seList, replicas, ignoreTape=False, requestedProtoco # Problematic files will be returned and can be handled by another module failedReplicas = set() newReplicasDict = {} - for lfn, reps in replicas.iteritems(): + for lfn, reps in replicas.items(): if lfn in self.inputData: # Check that all replicas are on a valid local SE if not [se for se in reps if se in diskSEs.union(tapeSEs)]: @@ -150,11 +150,11 @@ def __resolveReplicas(self, seList, replicas, ignoreTape=False, requestedProtoco # IMPORTANT, only add replicas for input data that is requested # since this module could have been executed after another. seFilesDict = {} - for lfn, seList in newReplicasDict.iteritems(): + for lfn, seList in newReplicasDict.items(): for seName in seList: seFilesDict.setdefault(seName, []).append(lfn) - sortedSEs = sorted(((len(lfns), seName) for seName, lfns in seFilesDict.iteritems()), reverse=True) + sortedSEs = sorted(((len(lfns), seName) for seName, lfns in seFilesDict.items()), reverse=True) trackLFNs = {} for _len, seName in sortedSEs: @@ -166,13 +166,13 @@ def __resolveReplicas(self, seList, replicas, ignoreTape=False, requestedProtoco 'guid': replicas[lfn]['GUID']}) self.log.debug('Files grouped by SEs are:\n%s' % str(seFilesDict)) - for seName, lfns in seFilesDict.iteritems(): + for seName, lfns in seFilesDict.items(): self.log.info(' %s LFNs found from catalog at SE %s' % (len(lfns), seName)) self.log.verbose('\n'.join(lfns)) # Can now start to obtain TURLs for files grouped by localSE # for requested input data - for seName, lfns in seFilesDict.iteritems(): + for seName, lfns in seFilesDict.items(): if not lfns: continue failedReps = set() @@ -192,7 +192,7 @@ def __resolveReplicas(self, seList, replicas, ignoreTape=False, requestedProtoco if isinstance(failed, dict): self.log.error(failed[lfn], lfn) failedReps.add(lfn) - for lfn, metadata in result['Value']['Successful'].iteritems(): + for lfn, metadata in result['Value']['Successful'].items(): if metadata.get('Lost', False): error = "File has been Lost by the StorageElement %s" % seName elif metadata.get('Unavailable', False): @@ -226,7 +226,7 @@ def __resolveReplicas(self, seList, replicas, ignoreTape=False, requestedProtoco badTURLs = [] seResult = result['Value'] - for lfn, cause in seResult['Failed'].iteritems(): + for lfn, cause in seResult['Failed'].items(): badTURLCount += 1 badTURLs.append('Failed to obtain TURL for %s: %s' % (lfn, cause)) failedReps.add(lfn) @@ -240,7 +240,7 @@ def __resolveReplicas(self, seList, replicas, ignoreTape=False, requestedProtoco self.log.warn("Error setting job param", result['Message']) failedReplicas.update(failedReps) - for lfn, turl in seResult['Successful'].iteritems(): + for lfn, turl in seResult['Successful'].items(): for track in trackLFNs[lfn]: if track['se'] == seName: track['turl'] = turl diff --git a/WorkloadManagementSystem/Client/JobState/JobState.py b/WorkloadManagementSystem/Client/JobState/JobState.py index dba28f18e9c..d5072a5442a 100644 --- a/WorkloadManagementSystem/Client/JobState/JobState.py +++ b/WorkloadManagementSystem/Client/JobState/JobState.py @@ -103,7 +103,7 @@ def commitCache(self, initialState, cache, jobLog): self.__checkType(jobLog, (list, tuple)) except TypeError as excp: return S_ERROR(str(excp)) - result = self.getAttributes(initialState.keys()) + result = self.getAttributes(list(initialState)) if not result['OK']: return result if not result['Value'] == initialState: diff --git a/WorkloadManagementSystem/Client/Matcher.py b/WorkloadManagementSystem/Client/Matcher.py index f915fbaf4e5..a50e0dae9dd 100644 --- a/WorkloadManagementSystem/Client/Matcher.py +++ b/WorkloadManagementSystem/Client/Matcher.py @@ -208,7 +208,7 @@ def _processResourceDescription(self, resourceDescription): resourceDict.setdefault("Tag", []).extend(paramTags) # Add 'MultiProcessor' to the list of tags - if nProcessors > 1: + if nProcessors and nProcessors > 1: resourceDict.setdefault("Tag", []).append("MultiProcessor") # Add 'WholeNode' to the list of tags diff --git a/WorkloadManagementSystem/Client/SandboxStoreClient.py b/WorkloadManagementSystem/Client/SandboxStoreClient.py index 7e7551339ef..107dd330553 100755 --- a/WorkloadManagementSystem/Client/SandboxStoreClient.py +++ b/WorkloadManagementSystem/Client/SandboxStoreClient.py @@ -13,7 +13,7 @@ import hashlib import tempfile import re -import StringIO +from six import BytesIO from DIRAC import gLogger, S_OK, S_ERROR, gConfig @@ -99,7 +99,7 @@ def uploadFilesAsSandbox(self, fileList, sizeLimit=0, assignTo=None): a fileList item can be: - a string, which is an lfn name - a file name (real), that is supposed to be on disk, in the current directory - - a fileObject that should be a StringIO.StringIO type of object + - a fileObject that should be a BytesIO type of object Parameters: - assignTo : Dict containing { 'Job:' : '', ... } @@ -126,7 +126,7 @@ def uploadFilesAsSandbox(self, fileList, sizeLimit=0, assignTo=None): else: errorFiles.append(sFile) - elif isinstance(sFile, StringIO.StringIO): + elif isinstance(sFile, BytesIO): files2Upload.append(sFile) else: return S_ERROR("Objects of type %s can't be part of InputSandbox" % type(sFile)) @@ -144,9 +144,9 @@ def uploadFilesAsSandbox(self, fileList, sizeLimit=0, assignTo=None): for sFile in files2Upload: if isinstance(sFile, six.string_types): tf.add(os.path.realpath(sFile), os.path.basename(sFile), recursive=True) - elif isinstance(sFile, StringIO.StringIO): + elif isinstance(sFile, BytesIO): tarInfo = tarfile.TarInfo(name='jobDescription.xml') - tarInfo.size = len(sFile.buf) + tarInfo.size = len(sFile.getvalue()) tf.addfile(tarinfo=tarInfo, fileobj=sFile) if sizeLimit > 0: diff --git a/WorkloadManagementSystem/Client/WMSClient.py b/WorkloadManagementSystem/Client/WMSClient.py index 2d8a66b9d26..c692e543597 100755 --- a/WorkloadManagementSystem/Client/WMSClient.py +++ b/WorkloadManagementSystem/Client/WMSClient.py @@ -6,7 +6,7 @@ from __future__ import print_function import os -import StringIO +from six import StringIO import time from DIRAC import S_OK, S_ERROR, gLogger @@ -97,9 +97,9 @@ def __uploadInputSandbox(self, classAdJob, jobDescriptionObject=None): stringIOFiles = [] stringIOFilesSize = 0 if jobDescriptionObject is not None: - if isinstance(jobDescriptionObject, StringIO.StringIO): + if isinstance(jobDescriptionObject, StringIO): stringIOFiles = [jobDescriptionObject] - stringIOFilesSize = len(jobDescriptionObject.buf) + stringIOFilesSize = len(jobDescriptionObject.getvalue()) gLogger.debug("Size of the stringIOFiles: " + str(stringIOFilesSize)) else: return S_ERROR(EWMSJDL, "jobDescriptionObject is not a StringIO object") @@ -188,7 +188,7 @@ def submitJob(self, jdl, jobDescriptionObject=None): if len(jobIDList) == nJobs: # Confirm the submitted jobs confirmed = False - for _attempt in xrange(3): + for _attempt in range(3): result = self.jobManager.confirmBulkSubmission(jobIDList) if result['OK']: confirmed = True diff --git a/WorkloadManagementSystem/Client/test/Test_Client_WorkloadManagementSystem.py b/WorkloadManagementSystem/Client/test/Test_Client_WorkloadManagementSystem.py index 1cbbf06f7c6..9721b0c3cca 100644 --- a/WorkloadManagementSystem/Client/test/Test_Client_WorkloadManagementSystem.py +++ b/WorkloadManagementSystem/Client/test/Test_Client_WorkloadManagementSystem.py @@ -8,7 +8,7 @@ import os import unittest import importlib -import StringIO +from six import BytesIO from mock import MagicMock @@ -99,7 +99,7 @@ def test_uploadFilesAsSandbox(self): ourSSC = importlib.import_module('DIRAC.WorkloadManagementSystem.Client.SandboxStoreClient') ourSSC.TransferClient = MagicMock() ssc = SandboxStoreClient() - fileList = [StringIO.StringIO('try')] + fileList = [BytesIO(b'try')] res = ssc.uploadFilesAsSandbox(fileList) print(res) diff --git a/WorkloadManagementSystem/DB/JobDB.py b/WorkloadManagementSystem/DB/JobDB.py index 338f73c603a..a78e143a877 100755 --- a/WorkloadManagementSystem/DB/JobDB.py +++ b/WorkloadManagementSystem/DB/JobDB.py @@ -1343,7 +1343,7 @@ def rescheduleJob(self, jobID): result = self.getJobParameters(jobID) if result['OK']: parDict = result['Value'] - for key, value in parDict.get(jobID, {}).iteritems(): + for key, value in parDict.get(jobID, {}).items(): result = self.setAtticJobParameter(jobID, key, value, rescheduleCounter - 1) if not result['OK']: break diff --git a/WorkloadManagementSystem/DB/PilotAgentsDB.py b/WorkloadManagementSystem/DB/PilotAgentsDB.py index c8249618d69..a84df7adbd1 100755 --- a/WorkloadManagementSystem/DB/PilotAgentsDB.py +++ b/WorkloadManagementSystem/DB/PilotAgentsDB.py @@ -123,7 +123,7 @@ def setPilotStatus(self, pilotRef, status, destination=None, # # Building the request with "ON DUPLICATE KEY UPDATE" # reqBase = "INSERT INTO PilotAgents (PilotJobReference, Status, StatusReason) VALUES " -# for pilotJobReference, status in pilotRefsStatusDict.iteritems(): +# for pilotJobReference, status in pilotRefsStatusDict.items(): # req = reqBase + ','.join("('%s', '%s', '%s')" % (pilotJobReference, status, statusReason)) # req += " ON DUPLICATE KEY UPDATE Status=VALUES(Status),StatusReason=VALUES(StatusReason)" diff --git a/WorkloadManagementSystem/DB/TaskQueueDB.py b/WorkloadManagementSystem/DB/TaskQueueDB.py index 9cb68ce3417..82b8e5f03ad 100755 --- a/WorkloadManagementSystem/DB/TaskQueueDB.py +++ b/WorkloadManagementSystem/DB/TaskQueueDB.py @@ -472,7 +472,7 @@ def __findAndDisableTaskQueue(self, tqDefDict, skipDefinitionCheck=False, retrie :param dict tqDefDict: dict for TQ definition :returns: S_OK() / S_ERROR """ - for _ in xrange(retries): + for _ in range(retries): result = self.__findSmallestTaskQueue(tqDefDict, skipDefinitionCheck=skipDefinitionCheck, connObj=connObj) if not result['OK']: return result @@ -533,7 +533,7 @@ def matchAndGetJob(self, tqMatchDict, numJobsPerTry=50, numQueuesPerTry=10, nega prioSQL = "SELECT `tq_Jobs`.Priority FROM `tq_Jobs` \ WHERE `tq_Jobs`.TQId = %s ORDER BY RAND() / `tq_Jobs`.RealPriority ASC LIMIT 1" postJobSQL = " ORDER BY `tq_Jobs`.JobId ASC LIMIT %s" % numJobsPerTry - for _ in xrange(self.__maxMatchRetry): + for _ in range(self.__maxMatchRetry): noJobsFound = False if 'JobID' in tqMatchDict: # A certain JobID is required by the resource, so all TQ are to be considered diff --git a/WorkloadManagementSystem/Executor/Base/OptimizerExecutor.py b/WorkloadManagementSystem/Executor/Base/OptimizerExecutor.py index 041cf0485aa..8587ccb4f20 100644 --- a/WorkloadManagementSystem/Executor/Base/OptimizerExecutor.py +++ b/WorkloadManagementSystem/Executor/Base/OptimizerExecutor.py @@ -38,9 +38,12 @@ def __init__(self, log, jid): self.__jid = jid self.__log = log - def __nonzero__(self): + def __bool__(self): return True + # For Python 2 compatibility + __nonzero__ = __bool__ + def __getattr__(self, name): return self.LogWrap(self.__log, self.__jid, name) diff --git a/WorkloadManagementSystem/Executor/JobScheduling.py b/WorkloadManagementSystem/Executor/JobScheduling.py index a1e857db161..bc7c4f5b75b 100755 --- a/WorkloadManagementSystem/Executor/JobScheduling.py +++ b/WorkloadManagementSystem/Executor/JobScheduling.py @@ -185,9 +185,9 @@ def optimizeJob(self, jid, jobState): # Some files do not exist at all... set the job Failed # Reverse errors reasons = {} - for lfn, reason in res['Value']['absentLFNs'].iteritems(): + for lfn, reason in res['Value']['absentLFNs'].items(): reasons.setdefault(reason, []).append(lfn) - for reason, lfns in reasons.iteritems(): + for reason, lfns in reasons.items(): # Some files are missing in the FC or in SEs, fail the job self.jobLog.error(reason, ','.join(lfns)) error = ','.join(reasons) @@ -400,7 +400,7 @@ def _getTagsFromManifest(self, jobManifest): else: maxProcessors = -1 - if nProcessors > 1: + if nProcessors and nProcessors > 1: tagList.append("%dProcessors" % nProcessors) tagList.append("MultiProcessor") if maxProcessors == -1 or maxProcessors > 1: diff --git a/WorkloadManagementSystem/Executor/test/Test_Executor.py b/WorkloadManagementSystem/Executor/test/Test_Executor.py index 407df68f383..2d5b56746c1 100644 --- a/WorkloadManagementSystem/Executor/test/Test_Executor.py +++ b/WorkloadManagementSystem/Executor/test/Test_Executor.py @@ -58,7 +58,7 @@ def test__applySiteFilter(sites, banned, expected): ]) def test__getTagsFromManifest(manifestOptions, expected): manifest = JobManifest() - for varName, varValue in manifestOptions.iteritems(): + for varName, varValue in manifestOptions.items(): manifest.setOption(varName, varValue) js = JobScheduling() @@ -93,7 +93,7 @@ def test__getInputSandbox(mocker, manifestOptions, expected): js.setAttribute('JobType', 'User') manifest = JobManifest() - for varName, varValue in manifestOptions.iteritems(): + for varName, varValue in manifestOptions.items(): manifest.setOption(varName, varValue) js.setManifest(manifest) diff --git a/WorkloadManagementSystem/JobWrapper/Watchdog.py b/WorkloadManagementSystem/JobWrapper/Watchdog.py index 505c6130dea..4aefeb2152b 100755 --- a/WorkloadManagementSystem/JobWrapper/Watchdog.py +++ b/WorkloadManagementSystem/JobWrapper/Watchdog.py @@ -597,7 +597,7 @@ def __convertCPUTime(self, cputime): """ cpuValue = 0 cpuHMS = cputime.split(':') - # for i in xrange( len( cpuHMS ) ): + # for i in range( len( cpuHMS ) ): # cpuHMS[i] = cpuHMS[i].replace( '00', '0' ) try: diff --git a/WorkloadManagementSystem/Service/JobManagerHandler.py b/WorkloadManagementSystem/Service/JobManagerHandler.py index bea8c2b701b..9c61391e430 100755 --- a/WorkloadManagementSystem/Service/JobManagerHandler.py +++ b/WorkloadManagementSystem/Service/JobManagerHandler.py @@ -127,7 +127,7 @@ def export_getMaxParametricJobs(self): """ return S_OK(self.maxParametricJobs) - types_submitJob = [basestring] + types_submitJob = [six.string_types] def export_submitJob(self, jobDesc): """ Submit a job to DIRAC WMS. diff --git a/WorkloadManagementSystem/Service/JobMonitoringHandler.py b/WorkloadManagementSystem/Service/JobMonitoringHandler.py index d22468d13e7..6e1528bf070 100755 --- a/WorkloadManagementSystem/Service/JobMonitoringHandler.py +++ b/WorkloadManagementSystem/Service/JobMonitoringHandler.py @@ -9,6 +9,7 @@ from __future__ import division __RCSID__ = "$Id$" +import six from past.builtins import long from datetime import timedelta @@ -446,8 +447,8 @@ def export_getJobPageSummaryWeb(self, selectDict, sortList, startItem, maxItems, return S_OK(resultDict) # prepare the standard structure now - key = summaryDict.keys()[0] - paramNames = summaryDict[key].keys() + key = list(summaryDict)[0] + paramNames = list(summaryDict[key]) records = [] for jobID, jobDict in summaryDict.items(): @@ -463,7 +464,7 @@ def export_getJobPageSummaryWeb(self, selectDict, sortList, startItem, maxItems, return S_OK(resultDict) ############################################################################## - types_getJobStats = [basestring, dict] + types_getJobStats = [six.string_types, dict] @staticmethod def export_getJobStats(attribute, selectDict): @@ -500,7 +501,7 @@ def export_getJobsPrimarySummary(jobIDs): return gJobDB.getAttributesForJobList(jobIDs, PRIMARY_SUMMARY) ############################################################################## - types_getJobParameter = [[basestring, int, long], basestring] + types_getJobParameter = [six.string_types + (int, long), six.string_types] @staticmethod def export_getJobParameter(jobID, parName): @@ -528,7 +529,7 @@ def export_getJobOptParameters(jobID): return gJobDB.getJobOptParameters(jobID) ############################################################################## - types_getJobParameters = [[basestring, int, long, list]] + types_getJobParameters = [six.string_types + (int, long, list)] @staticmethod def export_getJobParameters(jobIDs, parName=None): @@ -564,18 +565,18 @@ def export_getJobParameters(jobIDs, parName=None): return gJobDB.getJobParameters(jobIDs, parName) ############################################################################## - types_traceJobParameter = [basestring, [basestring, int, long, list], - basestring, [basestring, None], - [basestring, None]] + types_traceJobParameter = [six.string_types, six.string_types + (int, long, list), + six.string_types, six.string_types + (None,), + six.string_types + (None,)] @staticmethod def export_traceJobParameter(site, localID, parameter, date, until): return gJobDB.traceJobParameter(site, localID, parameter, date, until) ############################################################################## - types_traceJobParameters = [basestring, [basestring, int, long, list], + types_traceJobParameters = [six.string_types, six.string_types + (int, long, list), [list, None], [list, None], - [basestring, None], [basestring, None]] + six.string_types + (None,), six.string_types + (None,)] @staticmethod def export_traceJobParameters(site, localID, parameterList, attributeList, date, until): @@ -602,7 +603,7 @@ def export_getJobAttributes(jobID): return gJobDB.getJobAttributes(jobID) ############################################################################## - types_getJobAttribute = [int, basestring] + types_getJobAttribute = [int, six.string_types] @staticmethod def export_getJobAttribute(jobID, attribute): diff --git a/WorkloadManagementSystem/Service/MatcherHandler.py b/WorkloadManagementSystem/Service/MatcherHandler.py index 6cef785d116..5333c1153db 100755 --- a/WorkloadManagementSystem/Service/MatcherHandler.py +++ b/WorkloadManagementSystem/Service/MatcherHandler.py @@ -77,7 +77,7 @@ def initialize(self): self.limiter = Limiter(jobDB=gJobDB) ############################################################################## - types_requestJob = [[basestring, dict]] + types_requestJob = [six.string_types + (dict,)] def export_requestJob(self, resourceDescription): """ Serve a job to the request of an agent which is the highest priority diff --git a/WorkloadManagementSystem/Service/PilotManagerHandler.py b/WorkloadManagementSystem/Service/PilotManagerHandler.py index 1b9f5ebcf17..28a3b105a87 100644 --- a/WorkloadManagementSystem/Service/PilotManagerHandler.py +++ b/WorkloadManagementSystem/Service/PilotManagerHandler.py @@ -86,7 +86,7 @@ def export_getCurrentPilotCounters(cls, attrDict={}): return S_OK(resultDict) ########################################################################################## - types_addPilotTQReference = [list, six.integer_types, basestring, basestring] + types_addPilotTQReference = [list, six.integer_types, six.string_types, six.string_types] @classmethod def export_addPilotTQReference(cls, pilotRef, taskQueueID, ownerDN, ownerGroup, broker='Unknown', @@ -97,7 +97,7 @@ def export_addPilotTQReference(cls, pilotRef, taskQueueID, ownerDN, ownerGroup, broker, gridType, pilotStampDict) ############################################################################## - types_getPilotOutput = [basestring] + types_getPilotOutput = [six.string_types] def export_getPilotOutput(self, pilotReference): """ Get the pilot job standard output and standard error files for the Grid @@ -107,7 +107,7 @@ def export_getPilotOutput(self, pilotReference): return getGridJobOutput(pilotReference) ############################################################################## - types_getPilotInfo = [(list, basestring)] + types_getPilotInfo = [(list,) + six.string_types] @classmethod def export_getPilotInfo(cls, pilotReference): @@ -125,7 +125,7 @@ def export_selectPilots(cls, condDict): return pilotDB.selectPilots(condDict) ############################################################################## - types_storePilotOutput = [basestring, basestring, basestring] + types_storePilotOutput = [six.string_types, six.string_types, six.string_types] @classmethod def export_storePilotOutput(cls, pilotReference, output, error): @@ -134,7 +134,7 @@ def export_storePilotOutput(cls, pilotReference, output, error): return pilotDB.storePilotOutput(pilotReference, output, error) ############################################################################## - types_getPilotLoggingInfo = [basestring] + types_getPilotLoggingInfo = [six.string_types] @classmethod def export_getPilotLoggingInfo(cls, pilotReference): @@ -202,7 +202,7 @@ def export_getPilotSummaryWeb(cls, selectDict, sortList, startItem, maxItems): return result ############################################################################## - types_getPilots = [(basestring, int, long)] + types_getPilots = [six.string_types + (int, long)] @classmethod def export_getPilots(cls, jobID): @@ -236,7 +236,7 @@ def export_getPilots(cls, jobID): return pilotDB.getPilotInfo(pilotID=pilots) ############################################################################## - types_killPilot = [(basestring, list)] + types_killPilot = [six.string_types + (list,)] @classmethod def export_killPilot(cls, pilotRefList): @@ -272,7 +272,7 @@ def export_killPilot(cls, pilotRefList): return S_OK() ############################################################################## - types_setJobForPilot = [(basestring, int, long), basestring] + types_setJobForPilot = [six.string_types + (int, long), six.string_types] @classmethod def export_setJobForPilot(cls, jobID, pilotRef, destination=None): @@ -291,7 +291,7 @@ def export_setJobForPilot(cls, jobID, pilotRef, destination=None): return result ########################################################################################## - types_setPilotBenchmark = [basestring, float] + types_setPilotBenchmark = [six.string_types, float] @classmethod def export_setPilotBenchmark(cls, pilotRef, mark): @@ -300,7 +300,7 @@ def export_setPilotBenchmark(cls, pilotRef, mark): return pilotDB.setPilotBenchmark(pilotRef, mark) ########################################################################################## - types_setAccountingFlag = [basestring] + types_setAccountingFlag = [six.string_types] @classmethod def export_setAccountingFlag(cls, pilotRef, mark='True'): @@ -309,7 +309,7 @@ def export_setAccountingFlag(cls, pilotRef, mark='True'): return pilotDB.setAccountingFlag(pilotRef, mark) ########################################################################################## - types_setPilotStatus = [basestring, basestring] + types_setPilotStatus = [six.string_types, six.string_types] def export_setPilotStatus(self, pilotRef, status, destination=None, reason=None, gridSite=None, queue=None): """ Set the pilot agent status @@ -329,7 +329,7 @@ def export_countPilots(cls, condDict, older=None, newer=None, timeStamp='Submiss return pilotDB.countPilots(condDict, older, newer, timeStamp) ########################################################################################## - types_getCounters = [basestring, list, dict] + types_getCounters = [six.string_types, list, dict] @classmethod def export_getCounters(cls, table, keys, condDict, newer=None, timeStamp='SubmissionTime'): @@ -339,7 +339,7 @@ def export_getCounters(cls, table, keys, condDict, newer=None, timeStamp='Submis return pilotDB.getCounters(table, keys, condDict, newer=newer, timeStamp=timeStamp) ############################################################################## - types_getPilotStatistics = [basestring, dict] + types_getPilotStatistics = [six.string_types, dict] @staticmethod def export_getPilotStatistics(attribute, selectDict): @@ -376,7 +376,7 @@ def export_getPilotStatistics(attribute, selectDict): return S_OK(statistics) ############################################################################## - types_deletePilots = [(list, int, long, basestring)] + types_deletePilots = [(list, int, long) + six.string_types] def export_deletePilots(self, pilotIDs): diff --git a/WorkloadManagementSystem/Service/PilotsLoggingHandler.py b/WorkloadManagementSystem/Service/PilotsLoggingHandler.py index 3e35a26a5a8..8286f2771aa 100644 --- a/WorkloadManagementSystem/Service/PilotsLoggingHandler.py +++ b/WorkloadManagementSystem/Service/PilotsLoggingHandler.py @@ -13,6 +13,7 @@ __RCSID__ = "$Id$" +import six from DIRAC import S_OK, gConfig from DIRAC.Core.DISET.RequestHandler import RequestHandler @@ -57,7 +58,8 @@ def consumingCallback(cls, headers, message): cls.pilotsLoggingDB.addPilotsLogging(message['pilotUUID'], message['timestamp'], message['source'], message['phase'], message['status'], message['messageContent']) - types_addPilotsLogging = [basestring, basestring, basestring, basestring, basestring, basestring] + types_addPilotsLogging = [six.string_types, six.string_types, six.string_types, + six.string_types, six.string_types, six.string_types] def export_addPilotsLogging(self, pilotUUID, timestamp, source, phase, status, messageContent): """ @@ -73,7 +75,7 @@ def export_addPilotsLogging(self, pilotUUID, timestamp, source, phase, status, m return PilotsLoggingHandler.pilotsLoggingDB.addPilotsLogging(pilotUUID, timestamp, source, phase, status, messageContent) - types_getPilotsLogging = [basestring] + types_getPilotsLogging = [six.string_types] def export_getPilotsLogging(self, pilotUUID): """ @@ -84,7 +86,7 @@ def export_getPilotsLogging(self, pilotUUID): return PilotsLoggingHandler.pilotsLoggingDB.getPilotsLogging(pilotUUID) - types_deletePilotsLogging = [(basestring, list)] + types_deletePilotsLogging = [six.string_types + (list,)] def export_deletePilotsLogging(self, pilotUUID): """ diff --git a/WorkloadManagementSystem/Service/SandboxStoreHandler.py b/WorkloadManagementSystem/Service/SandboxStoreHandler.py index be4a3f28085..bd84825d1ab 100755 --- a/WorkloadManagementSystem/Service/SandboxStoreHandler.py +++ b/WorkloadManagementSystem/Service/SandboxStoreHandler.py @@ -12,6 +12,7 @@ import threading import tempfile +import six from DIRAC import gLogger, S_OK, S_ERROR from DIRAC.Core.Utilities.File import mkDir from DIRAC.Core.DISET.RequestHandler import RequestHandler @@ -340,7 +341,7 @@ def export_unassignEntities(self, entitiesList, entitiesSetup=False): ################## # Getting assigned sandboxes - types_getSandboxesAssignedToEntity = [basestring] + types_getSandboxesAssignedToEntity = [six.string_types] def export_getSandboxesAssignedToEntity(self, entityId, entitySetup=False): """ diff --git a/WorkloadManagementSystem/Service/WMSAdministratorHandler.py b/WorkloadManagementSystem/Service/WMSAdministratorHandler.py index c32f7c0f407..90856af0992 100755 --- a/WorkloadManagementSystem/Service/WMSAdministratorHandler.py +++ b/WorkloadManagementSystem/Service/WMSAdministratorHandler.py @@ -95,7 +95,7 @@ def export_getAllSiteMaskStatus(cls): return jobDB.getAllSiteMaskStatus() ############################################################################## - types_banSite = [basestring] + types_banSite = [six.string_types] def export_banSite(self, site, comment='No comment'): """ Ban the given site in the site mask @@ -111,7 +111,7 @@ def export_banSite(self, site, comment='No comment'): return result ############################################################################## - types_allowSite = [basestring] + types_allowSite = [six.string_types] def export_allowSite(self, site, comment='No comment'): """ Allow the given site in the site mask @@ -138,7 +138,7 @@ def export_clearMask(cls): return jobDB.removeSiteFromMask(None) ############################################################################## - types_getSiteMaskLogging = [(basestring, list)] + types_getSiteMaskLogging = [six.string_types + (list,)] @classmethod def export_getSiteMaskLogging(cls, sites): @@ -184,7 +184,7 @@ def export_getSiteMaskSummary(cls): return S_OK(siteDict) ############################################################################## - types_getJobPilotOutput = [(basestring, int, long)] + types_getJobPilotOutput = [six.string_types + (int, long)] def export_getJobPilotOutput(self, jobID): """ Get the pilot job standard output and standard error files for the DIRAC diff --git a/WorkloadManagementSystem/Service/WMSUtilities.py b/WorkloadManagementSystem/Service/WMSUtilities.py index 69a28739342..89704906b2e 100644 --- a/WorkloadManagementSystem/Service/WMSUtilities.py +++ b/WorkloadManagementSystem/Service/WMSUtilities.py @@ -162,7 +162,7 @@ def killPilotsInQueues(pilotRefDict): ceFactory = ComputingElementFactory() failed = [] - for key, pilotDict in pilotRefDict.iteritems(): + for key, pilotDict in pilotRefDict.items(): owner, group, site, ce, queue = key.split('@@@') result = getQueue(site, ce, queue) diff --git a/WorkloadManagementSystem/Utilities/PilotWrapper.py b/WorkloadManagementSystem/Utilities/PilotWrapper.py index 060f9c408b9..eaca1fa1fc5 100644 --- a/WorkloadManagementSystem/Utilities/PilotWrapper.py +++ b/WorkloadManagementSystem/Utilities/PilotWrapper.py @@ -115,7 +115,7 @@ def pilotWrapperScript(pilotFilesCompressedEncodedDict=None, logger.error(x) shutil.rmtree(pilotWorkingDirectory) sys.exit(3) -""" % {'encodedPf': encodedPf.decode(), +""" % {'encodedPf': encodedPf.decode() if hasattr(encodedPf, "decode") else encodedPf, 'pfName': pfName} envVariablesString = "" @@ -125,7 +125,7 @@ def pilotWrapperScript(pilotFilesCompressedEncodedDict=None, """ % {'name': name, 'value': value} - # add X509_USER_PROXY to etablish pilot env in Cluster WNs + # add X509_USER_PROXY to establish pilot env in Cluster WNs if 'proxy' in pilotFilesCompressedEncodedDict: envVariablesString += """ os.environ['X509_USER_PROXY'] = os.path.join(pilotWorkingDirectory, 'proxy') diff --git a/WorkloadManagementSystem/Utilities/Utils.py b/WorkloadManagementSystem/Utilities/Utils.py index 72eae7982d7..7cf3b980777 100644 --- a/WorkloadManagementSystem/Utilities/Utils.py +++ b/WorkloadManagementSystem/Utilities/Utils.py @@ -9,6 +9,8 @@ import sys import json +import six + from DIRAC import gConfig, gLogger, S_OK from DIRAC.Core.Utilities.File import mkDir @@ -57,7 +59,7 @@ def createJobWrapper(jobID, jobParams, resourceParams, optimizerParams, jobWrapperJsonFile = jobWrapperFile + '.json' with io.open(jobWrapperJsonFile, 'w', encoding='utf8') as jsonFile: - json.dump(unicode(arguments), jsonFile, ensure_ascii=False) + json.dump(six.text_type(arguments), jsonFile, ensure_ascii=False) with open(jobWrapperFile, "w") as wrapper: wrapper.write(wrapperTemplate) @@ -108,7 +110,7 @@ def createRelocatedJobWrapper(wrapperPath, rootLocation, jobWrapperJsonFile = jobWrapperFile + '.json' with io.open(jobWrapperJsonFile, 'w', encoding='utf8') as jsonFile: - json.dump(unicode(arguments), jsonFile, ensure_ascii=False) + json.dump(six.text_type(arguments), jsonFile, ensure_ascii=False) with open(jobWrapperFile, "w") as wrapper: wrapper.write(wrapperTemplate) diff --git a/WorkloadManagementSystem/Utilities/test/Test_PilotCStoJSONSynchronizer.py b/WorkloadManagementSystem/Utilities/test/Test_PilotCStoJSONSynchronizer.py index 5c317c7d379..f537312354a 100644 --- a/WorkloadManagementSystem/Utilities/test/Test_PilotCStoJSONSynchronizer.py +++ b/WorkloadManagementSystem/Utilities/test/Test_PilotCStoJSONSynchronizer.py @@ -8,6 +8,7 @@ import os from mock import patch, MagicMock as Mock from diraccfg import CFG +import six from DIRAC.WorkloadManagementSystem.Utilities.PilotCStoJSONSynchronizer import PilotCStoJSONSynchronizer from DIRAC.ConfigurationSystem.private.ConfigurationClient import ConfigurationClient @@ -173,9 +174,19 @@ def test_syncchecksum(self): synchroniser._syncChecksum() assert self.testCfgFileName in synchroniser._checksumDict assert synchroniser._checksumDict[self.testCfgFileName] == expectedHash - assert open('checksums.sha512', 'rb').read().split('\n')[0] == '%s %s' % (expectedPJHash, 'pilot.json'), \ - 'pilot.json content: ' + repr(open('pilot.json', 'rb').read()) - assert open('checksums.sha512', 'rb').read().split('\n')[1] == '%s %s' % (expectedHash, self.testCfgFileName) + with open('checksums.sha512', 'rt') as fp: + checksums = fp.read() + with open('pilot.json', 'rt') as fp: + pilot_json_content = fp.read() + + if not six.PY2: + # JSON is a non-deterministic format so hard-coding the SHA512 is unreliable + import subprocess + expectedPJHash = subprocess.check_output(["sha512sum", "pilot.json"], text=True).split(" ")[0] + expectedHash = subprocess.check_output(["sha512sum", self.testCfgFileName], text=True).split(" ")[0] + + assert checksums.split('\n')[0] == '%s %s' % (expectedPJHash, 'pilot.json'), pilot_json_content + assert checksums.split('\n')[1] == '%s %s' % (expectedHash, self.testCfgFileName) # this tests if the checksums file was also "uploaded" assert 'checksums.sha512' in list(synchroniser._checksumDict) diff --git a/WorkloadManagementSystem/Utilities/test/Test_PilotWrapper.py b/WorkloadManagementSystem/Utilities/test/Test_PilotWrapper.py index aa3a1422175..c5b36fe4e7e 100644 --- a/WorkloadManagementSystem/Utilities/test/Test_PilotWrapper.py +++ b/WorkloadManagementSystem/Utilities/test/Test_PilotWrapper.py @@ -39,22 +39,22 @@ def test_scriptReal(): """ test script creation """ diracInstall = os.path.join(os.getcwd(), 'Core/scripts/dirac-install.py') - with open(diracInstall, "r") as fd: + with open(diracInstall, "rb") as fd: diracInstall = fd.read() diracInstallEncoded = base64.b64encode(bz2.compress(diracInstall, 9)) diracPilot = os.path.join(os.getcwd(), 'Core/scripts/dirac-install.py') # just some random file - with open(diracPilot, "r") as fd: + with open(diracPilot, "rb") as fd: diracPilot = fd.read() diracPilotEncoded = base64.b64encode(bz2.compress(diracPilot, 9)) diracPilotTools = os.path.join(os.getcwd(), 'Core/scripts/dirac-install.py') # just some random file - with open(diracPilotTools, "r") as fd: + with open(diracPilotTools, "rb") as fd: diracPilotTools = fd.read() diracPilotToolsEncoded = base64.b64encode(bz2.compress(diracPilotTools, 9)) diracPilotCommands = os.path.join(os.getcwd(), 'Core/scripts/dirac-install.py') # just some random file - with open(diracPilotCommands, "r") as fd: + with open(diracPilotCommands, "rb") as fd: diracPilotCommands = fd.read() diracPilotCommandsEncoded = base64.b64encode(bz2.compress(diracPilotCommands, 9)) diff --git a/WorkloadManagementSystem/scripts/dirac-admin-pilot-logging-info.py b/WorkloadManagementSystem/scripts/dirac-admin-pilot-logging-info.py index edf7d4825db..7ab7dcb7cd9 100755 --- a/WorkloadManagementSystem/scripts/dirac-admin-pilot-logging-info.py +++ b/WorkloadManagementSystem/scripts/dirac-admin-pilot-logging-info.py @@ -51,7 +51,7 @@ def setJobID(optVal): gLogger.error(result['Message']) DIRAC.exit(1) gLogger.debug(result['Value']) - uuid = result['Value'].keys()[0] + uuid = list(result['Value'])[0] result = PilotManagerClient().getPilotLoggingInfo(uuid) if not result['OK']: diff --git a/WorkloadManagementSystem/scripts/dirac-jobexec.py b/WorkloadManagementSystem/scripts/dirac-jobexec.py index e725630cdb4..6c6d22ce472 100755 --- a/WorkloadManagementSystem/scripts/dirac-jobexec.py +++ b/WorkloadManagementSystem/scripts/dirac-jobexec.py @@ -60,9 +60,9 @@ def jobexec(jobxml, wfParameters): workflow.setValue(pName, pValue) # Propagate the command line parameters to the workflow module instances of each step - for stepdefinition in workflow.step_definitions.itervalues(): + for stepdefinition in workflow.step_definitions.values(): for moduleInstance in stepdefinition.module_instances: - for pName, pValue in wfParameters.iteritems(): + for pName, pValue in wfParameters.items(): if moduleInstance.parameters.find(pName): moduleInstance.parameters.setValue(pName, pValue) diff --git a/WorkloadManagementSystem/scripts/dirac-wms-match.py b/WorkloadManagementSystem/scripts/dirac-wms-match.py index 28c944519d8..faab667f379 100644 --- a/WorkloadManagementSystem/scripts/dirac-wms-match.py +++ b/WorkloadManagementSystem/scripts/dirac-wms-match.py @@ -91,7 +91,7 @@ def setSites(optVal_): fields = ('Site', 'CE', 'Queue', 'Status', 'Match', 'Reason') records = [] - for queue, queueInfo in queueDict.iteritems(): + for queue, queueInfo in queueDict.items(): site = queueInfo['Site'] ce = queueInfo['CEName'] siteStatus = "Active" if site in siteMaskList else "InActive" diff --git a/__init__.py b/__init__.py index 60b9a0d059a..9c56c5fa7e4 100755 --- a/__init__.py +++ b/__init__.py @@ -102,21 +102,6 @@ if preVersion: version = "%s-pre%s" % (version, preVersion) -# Check of python version - -__pythonMajorVersion = ("2", ) -__pythonMinorVersion = ("7") - -pythonVersion = pyPlatform.python_version_tuple() -if str(pythonVersion[0]) not in __pythonMajorVersion or str(pythonVersion[1]) not in __pythonMinorVersion: - print("Python Version %s not supported by DIRAC" % pyPlatform.python_version()) - print("Supported versions are: ") - for major in __pythonMajorVersion: - for minor in __pythonMinorVersion: - print("%s.%s.x" % (major, minor)) - - sys.exit(1) - errorMail = "dirac.alarms@gmail.com" alarmMail = "dirac.alarms@gmail.com" diff --git a/environment-py3.yml b/environment-py3.yml new file mode 100644 index 00000000000..68e8ff05a74 --- /dev/null +++ b/environment-py3.yml @@ -0,0 +1,75 @@ +name: dirac-development + +channels: + - diracgrid + - conda-forge + - defaults + +dependencies: + # runtime + - python =3.8 + - pip + - boto3 + - certifi + - cmreshandler >1.0.0b4 + - docutils + - elasticsearch-dsl + - future + - gitpython >=2.1.0 + - m2crypto >=0.36 + - matplotlib + - numpy + - pexpect >=4.0.1 + - pillow + - psutil >=4.2.0 + - pyasn1 >0.4.1 + - pyasn1-modules + - python-json-logger >=0.1.8 + - pytz >=2015.7 + - recommonmark + - requests >=2.9.1 + - six >=1.10 + - sqlalchemy >=1.0.9 + - stomp.py =4.1.23 + - suds-jurko >=0.6 + - xmltodict + - pycurl + # testing and development + - autopep8 + - caniusepython3 + - coverage + - hypothesis + - ipython + - mock + - parameterized + - pylint >=1.6.5 + - pyparsing >=2.0.6 + - pytest >=3.6 + - pytest-cov >=2.2.0 + - pytest-mock + - shellcheck + - flaky + # docs + - pygments >=1.5 + - sphinx + # unused + - funcsigs + - jinja2 + # - readline >=6.2.4 in the standard library + - simplejson >=3.8.1 + #- tornado >=5.0.0,<6.0.0 + - typing >=3.6.6 + - openssl + - mysqlclient + - pip: + - diraccfg + - subprocess32 + # This is a fork of tornado with a patch to allow for configurable iostream + # It should eventually be part of DIRACGrid + - git+https://github.com/chaen/tornado.git@iostreamConfigurable + # This is an extension of Tornado to use M2Crypto + # It should eventually be part of DIRACGrid + - git+https://github.com/chaen/tornado_m2crypto + # fts-rest doesn't support Python 3, this branch changed has just enough to + # make it importing possible + - git+https://gitlab.cern.ch/cburr/fts-rest.git@python3 diff --git a/environment.yml b/environment.yml index f95611f7df0..6d8679c0771 100644 --- a/environment.yml +++ b/environment.yml @@ -50,6 +50,7 @@ dependencies: - pytest-cov >=2.2.0 - pytest-mock - shellcheck + - flaky # docs - pygments >=1.5 - sphinx diff --git a/requirements.txt b/requirements.txt index 871ad7363be..094429e2cf7 100644 --- a/requirements.txt +++ b/requirements.txt @@ -55,3 +55,4 @@ multi-mechanize>=1.2.0 tornado>=5.0.0 caniusepython3 subprocess32 +flaky diff --git a/tests/Integration/DataManagementSystem/Test_Client_DFC.py b/tests/Integration/DataManagementSystem/Test_Client_DFC.py index dc7cb99e3f7..92c65c3d610 100644 --- a/tests/Integration/DataManagementSystem/Test_Client_DFC.py +++ b/tests/Integration/DataManagementSystem/Test_Client_DFC.py @@ -735,7 +735,7 @@ def test_directoryOperations(self): # We do it two times to make sure that # when updating something to the same value # returns a success if it is allowed - for attempt in xrange(2): + for attempt in range(2): print("Attempt %s" % (attempt + 1)) # Only admin can change path group diff --git a/tests/Integration/DataManagementSystem/Test_Client_FTS3.py b/tests/Integration/DataManagementSystem/Test_Client_FTS3.py index 912edfe4cd7..2cdce0fdbda 100644 --- a/tests/Integration/DataManagementSystem/Test_Client_FTS3.py +++ b/tests/Integration/DataManagementSystem/Test_Client_FTS3.py @@ -42,7 +42,7 @@ def generateOperation(self, opType, nbFiles, dests, sources=None): op.username = proxyInfo['username'] op.userGroup = proxyInfo['group'] op.sourceSEs = sources - for _i in xrange(nbFiles * len(dests)): + for _i in range(nbFiles * len(dests)): self.fileCounter += 1 for dest in dests: ftsFile = FTS3File() @@ -451,7 +451,7 @@ def _perf(self): listOfIds = [] persistStart = time.time() - for i in xrange(1000): + for i in range(1000): op = self.generateOperation('Transfer', i % 20 + 1, ['Dest1']) res = self.db.persistOperation(op) self.assertTrue(res['OK']) diff --git a/tests/Integration/DataManagementSystem/Test_FileCatalogDB.py b/tests/Integration/DataManagementSystem/Test_FileCatalogDB.py index 961f762560b..d22866ce92b 100644 --- a/tests/Integration/DataManagementSystem/Test_FileCatalogDB.py +++ b/tests/Integration/DataManagementSystem/Test_FileCatalogDB.py @@ -798,7 +798,7 @@ def test_directoryOperations(self): # We do it two times to make sure that # when updating something to the same value # returns a success if it is allowed - for attempt in xrange(2): + for attempt in range(2): print("Attempt %s" % (attempt + 1)) # Only admin can change path group @@ -1427,14 +1427,14 @@ def test_directoryUsage(self): if __name__ == '__main__': - managerTypes = MANAGER_TO_TEST.keys() + managerTypes = list(MANAGER_TO_TEST) all_combinations = list(itertools.product(*MANAGER_TO_TEST.values())) numberOfManager = len(managerTypes) for setup in all_combinations: print("Running with:") - print(("".join(["\t %s : %s\n" % (managerTypes[i], setup[i]) for i in xrange(numberOfManager)]))) - for i in xrange(numberOfManager): + print(("".join(["\t %s : %s\n" % (managerTypes[i], setup[i]) for i in range(numberOfManager)]))) + for i in range(numberOfManager): DATABASE_CONFIG[managerTypes[i]] = setup[i] suite = unittest.defaultTestLoader.loadTestsFromTestCase(SECase) diff --git a/tests/Integration/Framework/Test_LoggingDB.py b/tests/Integration/Framework/Test_LoggingDB.py index 97657381ccd..a46688df55e 100644 --- a/tests/Integration/Framework/Test_LoggingDB.py +++ b/tests/Integration/Framework/Test_LoggingDB.py @@ -57,7 +57,7 @@ def test_addAndRemove(self): self.assertTrue(res['OK']) gLogger.info('\n Inserting some records\n') - for k in xrange(records): + for k in range(records): result = db.insertMessage(message, site, nodeFQDN, userDN, userGroup, remoteAddress) self.assertTrue(result['OK']) self.assertEqual(result['lastRowId'], k + 1) diff --git a/tests/Integration/Framework/Test_ProxyDB.py b/tests/Integration/Framework/Test_ProxyDB.py index d2047b90b87..797e5d59c19 100644 --- a/tests/Integration/Framework/Test_ProxyDB.py +++ b/tests/Integration/Framework/Test_ProxyDB.py @@ -12,7 +12,12 @@ import stat import shutil import tempfile -import commands +# TODO: This should be modernised to use subprocess(32) +try: + import commands +except ImportError: + # Python 3's subprocess module contains a compatability layer + import subprocess as commands import unittest from diraccfg import CFG diff --git a/tests/Integration/Monitoring/Test_MonitoringSystem.py b/tests/Integration/Monitoring/Test_MonitoringSystem.py index 5e5d8d3a134..da2f367d837 100644 --- a/tests/Integration/Monitoring/Test_MonitoringSystem.py +++ b/tests/Integration/Monitoring/Test_MonitoringSystem.py @@ -246,7 +246,7 @@ def test_getReport(self): 'grouping': ['Site']}, 'Site', {}) result = self.client.getReport(*params) self.assertTrue(result['OK']) - result['Value']['data'] = {site: strToIntDict(value) for site, value in result['Value']['data'].iteritems()} + result['Value']['data'] = {site: strToIntDict(value) for site, value in result['Value']['data'].items()} self.assertDictEqual(result['Value'], {'data': {u'Multiple': {1458198000: 227.0}, u'LCG.RRCKI.ru': {1458225000: 3.0}, diff --git a/tests/Integration/RequestManagementSystem/FIXME_IntegrationFCT.py b/tests/Integration/RequestManagementSystem/FIXME_IntegrationFCT.py index fa5d90dc452..d66976e2ae1 100644 --- a/tests/Integration/RequestManagementSystem/FIXME_IntegrationFCT.py +++ b/tests/Integration/RequestManagementSystem/FIXME_IntegrationFCT.py @@ -111,14 +111,14 @@ def buildRequest( self, owner, group, sourceSE, targetSE1, targetSE2 ): def files( self, userName, userGroup ): """ get list of files in user domain """ files = [] - for i in xrange( 10 ): + for i in range(10): fname = "/tmp/testUserFile-%s" % i if userGroup == "dteam_user": lfn = "/lhcb/user/%s/%s/%s" % ( userName[0], userName, fname.split( "/" )[-1] ) else: lfn = "/lhcb/certification/test/rmsdms/%s" % fname.split( "/" )[-1] fh = open( fname, "w+" ) - for i in xrange( 100 ): + for i in range(100): fh.write( str( random.randint( 0, i ) ) ) fh.close() size = os.stat( fname ).st_size diff --git a/tests/Integration/RequestManagementSystem/Test_ReqDB.py b/tests/Integration/RequestManagementSystem/Test_ReqDB.py index d74d2040d78..14e25c3c013 100644 --- a/tests/Integration/RequestManagementSystem/Test_ReqDB.py +++ b/tests/Integration/RequestManagementSystem/Test_ReqDB.py @@ -60,7 +60,7 @@ def test01Stress(self): db = RequestDB() reqIDs = [] - for i in xrange(self.stressRequests): + for i in range(self.stressRequests): request = Request({"RequestName": "test-%d" % i}) op = Operation({"Type": "RemoveReplica", "TargetSE": "CERN-USER"}) op += File({"LFN": "/lhcb/user/c/cibak/foo"}) @@ -91,7 +91,7 @@ def test01StressBulk(self): db = RequestDB() reqIDs = [] - for i in xrange(self.stressRequests): + for i in range(self.stressRequests): request = Request({"RequestName": "test-%d" % i}) op = Operation({"Type": "RemoveReplica", "TargetSE": "CERN-USER"}) op += File({"LFN": "/lhcb/user/c/cibak/foo"}) @@ -107,7 +107,7 @@ def test01StressBulk(self): time.sleep(1) startTime = time.time() - for i in xrange(loops): + for i in range(loops): get = db.getBulkRequests(self.bulkRequest, True) if "Message" in get: print(get["Message"]) diff --git a/tests/Integration/Resources/Catalog/FIXME_Test_CatalogPlugin.py b/tests/Integration/Resources/Catalog/FIXME_Test_CatalogPlugin.py index fed0431992d..dfaa635e59e 100755 --- a/tests/Integration/Resources/Catalog/FIXME_Test_CatalogPlugin.py +++ b/tests/Integration/Resources/Catalog/FIXME_Test_CatalogPlugin.py @@ -42,7 +42,7 @@ def setUp(self): # Register some files to work with self.numberOfFiles = 2 self.files = [] - for i in xrange(self.numberOfFiles): + for i in range(self.numberOfFiles): lfn = "%s/testFile_%d" % (self.destDir,i) res = self.registerFile(lfn) self.assertTrue(res) @@ -80,7 +80,7 @@ def cleanDirectory(self): return res = self.catalog.listDirectory(self.destDir) returnValue = self.parseResult(res,self.destDir) - toRemove = returnValue['Files'].keys() + toRemove = list(returnValue['Files']) if toRemove: self.purgeFiles(toRemove) res = self.catalog.removeDirectory(self.destDir) diff --git a/tests/Integration/Resources/MessageQueue/Test_ActiveClose.py b/tests/Integration/Resources/MessageQueue/Test_ActiveClose.py index 0913c320115..26cedbf1a18 100644 --- a/tests/Integration/Resources/MessageQueue/Test_ActiveClose.py +++ b/tests/Integration/Resources/MessageQueue/Test_ActiveClose.py @@ -139,7 +139,7 @@ def tearDown(self): def getFirstConnection(self, mqConnection): result = MQComm.connectionManager.getConnector(mqConnection) connector = result['Value'] - return connector.connections.itervalues().next() + return connector.connections.values().next() class Test_MQProducers_1(Test_MQProducers): @@ -187,7 +187,7 @@ def test_success(self, mock_getMQParamsFromCS): self.assertTrue(conn1.is_connected()) producers = [] - for i in xrange(20): + for i in range(20): result = createProducer(mqURI='mardirac3.in2p3.fr::Queue::test1') self.assertTrue(result['OK']) producer = result['Value'] @@ -217,7 +217,7 @@ def test_success(self, mock_getMQParamsFromCS): conn1 = None # creating and closing - for i in xrange(20): + for i in range(20): result = createProducer(mqURI='mardirac3.in2p3.fr::Queue::test1') if i == 1: conn1 = self.getFirstConnection('mardirac3.in2p3.fr') @@ -270,7 +270,7 @@ def test_success(self, mock_getMQParamsFromCS, mock_createMQConnector, mock_reco self.assertFalse(Test_MQProducers.reconnectWasCalled) result = producer.close() self.assertFalse(Test_MQProducers.reconnectWasCalled) - for _ in xrange(20): + for _ in range(20): result = createProducer(mqURI='mardirac3.in2p3.fr::Queue::test1') self.assertTrue(result['OK']) producer = result['Value'] diff --git a/tests/Integration/Resources/Storage/Test_Resources_Echo.py b/tests/Integration/Resources/Storage/Test_Resources_Echo.py index 230d0a3f02b..b92c11aea61 100644 --- a/tests/Integration/Resources/Storage/Test_Resources_Echo.py +++ b/tests/Integration/Resources/Storage/Test_Resources_Echo.py @@ -181,7 +181,7 @@ def setuptest(request): 'Workflow/File3'): os.path.join(local_path, 'Workflow/File3')} - isFile = putFile.keys() + isFile = list(putFile) listDir = [os.path.join(DESTINATION_PATH, 'Workflow'), os.path.join(DESTINATION_PATH, 'Workflow/FolderA'), @@ -214,7 +214,7 @@ def setuptest(request): fileAdlers = {} fileSizes = {} - for lfn, localFn in putFile.iteritems(): + for lfn, localFn in putFile.items(): fileAdlers[lfn] = fileAdler(localFn) fileSizes[lfn] = getSize(localFn) @@ -240,7 +240,7 @@ def test_uploadDirectory_shouldFail(setuptest): """ uploading directories is not possible with Echo""" res = se.putDirectory(putDir) assert res['OK'] - assert res['Value']['Failed'].keys() == putDir.keys() + assert set(res['Value']['Failed']) == set(putDir) print("CHRIS uploadDir %s" % res) # Need to sleep for echo to update ? diff --git a/tests/Integration/Resources/Storage/Test_Resources_GFAL2StorageBase.py b/tests/Integration/Resources/Storage/Test_Resources_GFAL2StorageBase.py index 85dba8308cf..daa1a209c7d 100644 --- a/tests/Integration/Resources/Storage/Test_Resources_GFAL2StorageBase.py +++ b/tests/Integration/Resources/Storage/Test_Resources_GFAL2StorageBase.py @@ -246,7 +246,7 @@ def testWorkflow(self): fileAdlers = {} fileSizes = {} - for lfn, localFn in isFile.iteritems(): + for lfn, localFn in isFile.items(): fileAdlers[lfn] = fileAdler(localFn) fileSizes[lfn] = getSize(localFn) @@ -274,7 +274,7 @@ def testWorkflow(self): # time.sleep(5) res = self.readSE.isFile(isFile) self.assertEqual(res['OK'], True) - self.assertTrue(all([x for x in res['Value']['Successful'].itervalues()])) + self.assertTrue(all([x for x in res['Value']['Successful'].values()])) # self.assertEqual( res['Value']['Successful'][isFile[0]], True ) # self.assertEqual( res['Value']['Successful'][isFile[1]], True ) diff --git a/tests/Integration/Resources/Storage/Test_Resources_S3.py b/tests/Integration/Resources/Storage/Test_Resources_S3.py index 05ae192a879..5290db67dd9 100644 --- a/tests/Integration/Resources/Storage/Test_Resources_S3.py +++ b/tests/Integration/Resources/Storage/Test_Resources_S3.py @@ -191,7 +191,7 @@ def setuptest(request): 'Workflow/File3'): os.path.join(local_path, 'Workflow/File3')} - isFile = putFile.keys() + isFile = list(putFile) listDir = [os.path.join(DESTINATION_PATH, 'Workflow'), os.path.join(DESTINATION_PATH, 'Workflow/FolderA'), @@ -244,7 +244,7 @@ def test_uploadDirectory_shouldFail(setuptest): """ uploading directories is not possible with Echo""" res = se.putDirectory(putDir) assert res['OK'] - assert res['Value']['Failed'].keys() == putDir.keys() + assert list(res['Value']['Failed']) == list(putDir) # Need to sleep for echo to update ? # time.sleep(1) diff --git a/tests/Integration/WorkloadManagementSystem/Test_GenerateAndExecutePilotWrapper.py b/tests/Integration/WorkloadManagementSystem/Test_GenerateAndExecutePilotWrapper.py index e0b10d70a97..e94b2110d1d 100644 --- a/tests/Integration/WorkloadManagementSystem/Test_GenerateAndExecutePilotWrapper.py +++ b/tests/Integration/WorkloadManagementSystem/Test_GenerateAndExecutePilotWrapper.py @@ -74,12 +74,9 @@ from PilotWrapper import pilotWrapperScript # pylint: disable=import-error diracInstall = os.path.join(os.getcwd(), 'dirac-install.py') -with open(diracInstall, "r") as fd: +with open(diracInstall, "rb") as fd: diracInstall = fd.read() -if sys.version_info >= (3,): - diracInstallEncoded = base64.b64encode(bz2.compress(diracInstall.encode(), 9)) -else: - diracInstallEncoded = base64.b64encode(bz2.compress(diracInstall, 9)) +diracInstallEncoded = base64.b64encode(bz2.compress(diracInstall, 9)).decode() res = pilotWrapperScript( pilotFilesCompressedEncodedDict={'dirac-install.py': diracInstallEncoded}, diff --git a/tests/Integration/WorkloadManagementSystem/Test_TaskQueueDB.py b/tests/Integration/WorkloadManagementSystem/Test_TaskQueueDB.py index 17da37dd4ec..c43ff5a9011 100644 --- a/tests/Integration/WorkloadManagementSystem/Test_TaskQueueDB.py +++ b/tests/Integration/WorkloadManagementSystem/Test_TaskQueueDB.py @@ -74,8 +74,14 @@ def test_chainWithParameter(): assert result['Value'] is False result = tqDB.retrieveTaskQueues() assert result['OK'] is True - assert result['Value'].values()[0] == {'OwnerDN': '/my/DN', 'Jobs': 2, 'OwnerGroup': 'myGroup', - 'Setup': 'aSetup', 'CPUTime': 86400, 'Priority': 1.0} + assert list(result['Value'].values())[0] == { + 'OwnerDN': '/my/DN', + 'Jobs': 2, + 'OwnerGroup': 'myGroup', + 'Setup': 'aSetup', + 'CPUTime': 86400, + 'Priority': 1.0 + } # now we will try to delete result = tqDB.deleteJob(123) @@ -415,7 +421,7 @@ def test_chainWithPlatforms(): # but here it returns only 1 (those for ANY), by construction # so, this should be in theory improved - for jobId in xrange(1, 8): + for jobId in range(1, 8): result = tqDB.deleteJob(jobId) assert result['OK'] is True @@ -764,7 +770,7 @@ def test_chainWithTagsAndPlatforms(): res = set([int(x[0]) for x in result['Value']]) assert res == {tq_job1, tq_job2, tq_job3} - for jobId in xrange(1, 8): + for jobId in range(1, 8): result = tqDB.deleteJob(jobId) assert result['OK'] is True @@ -976,7 +982,7 @@ def test_ComplexMatching(): res = set([int(x[0]) for x in result['Value']]) assert res == {tq_job4, tq_job5} - for jobId in xrange(1, 8): + for jobId in range(1, 8): result = tqDB.deleteJob(jobId) assert result['OK'] is True @@ -997,8 +1003,14 @@ def test_TQ(): assert result['Value'] == 1 result = tqDB.retrieveTaskQueues() assert result['OK'] is True - assert result['Value'].values()[0] == {'OwnerDN': '/my/DN', 'Jobs': 1, 'OwnerGroup': 'myGroup', - 'Setup': 'aSetup', 'CPUTime': 86400, 'Priority': 1.0} + assert list(result['Value'].values())[0] == { + 'OwnerDN': '/my/DN', + 'Jobs': 1, + 'OwnerGroup': 'myGroup', + 'Setup': 'aSetup', + 'CPUTime': 86400, + 'Priority': 1.0, + } result = tqDB.findOrphanJobs() assert result['OK'] is True result = tqDB.recalculateTQSharesForAll() diff --git a/tests/Performance/DFCPerformance/FIXME_Test_FC_scaling.py b/tests/Performance/DFCPerformance/FIXME_Test_FC_scaling.py index 4a23aea663d..7156a79d830 100644 --- a/tests/Performance/DFCPerformance/FIXME_Test_FC_scaling.py +++ b/tests/Performance/DFCPerformance/FIXME_Test_FC_scaling.py @@ -146,7 +146,7 @@ def listDirectory(n_queries): startTotal = time.time() - for i in xrange(n_queries): + for i in range(n_queries): start = time.time() result = fc.listDirectory(testDir) @@ -183,7 +183,7 @@ def getBulkReplicas(n_queries): startTotal = time.time() - for i in xrange(n_queries): + for i in range(n_queries): start = time.time() result = fc.getReplicas(lfnList) @@ -226,7 +226,7 @@ def getDirectoryReplicas(n_queries): startTotal = time.time() - for i in xrange(n_queries): + for i in range(n_queries): start = time.time() result = fc.getDirectoryReplicas(testDir) @@ -283,7 +283,7 @@ def runTest(): testFunction = eval(testType) - for c in xrange(nClients): + for c in range(nClients): pp.createAndQueueTask(testFunction, [nQueries], callback=finalize, exceptionCallback=doException) diff --git a/tests/Performance/DFCPerformance/generateDB/generate_db_data.py b/tests/Performance/DFCPerformance/generateDB/generate_db_data.py index dc01fda8b33..e60c5cd8b2e 100644 --- a/tests/Performance/DFCPerformance/generateDB/generate_db_data.py +++ b/tests/Performance/DFCPerformance/generateDB/generate_db_data.py @@ -89,7 +89,7 @@ def loop ( index, cur, parentId ): if index >= len ( config.hierarchySize ): return - for i in xrange( config.hierarchySize[index] ): + for i in range(config.hierarchySize[index]): uid = random.randint( 1, len( config.users ) ) gid = random.randint( 1, len( config.groups ) ) next = cur + [i] @@ -109,7 +109,7 @@ def loop ( index, cur, parentId ): # generate files if nbFiles: - for f in xrange( nbFiles ): + for f in range(nbFiles): filename = "%s.txt" % ( f ) size = random.randint( 1, 1000 ) statusid = 2 diff --git a/tests/Performance/DFCPerformance/generateDB/generate_db_data_22_73_365.py b/tests/Performance/DFCPerformance/generateDB/generate_db_data_22_73_365.py index 4a15ab706f1..3fe9216d46b 100644 --- a/tests/Performance/DFCPerformance/generateDB/generate_db_data_22_73_365.py +++ b/tests/Performance/DFCPerformance/generateDB/generate_db_data_22_73_365.py @@ -97,7 +97,7 @@ def loop ( index, cur, parentId ): if index >= len (d): return - for i in xrange(d[index]): + for i in range(d[index]): uid = random.randint(1, len(config.users) ) gid = random.randint(1, len(config.groups) ) next = cur + [i] @@ -117,7 +117,7 @@ def loop ( index, cur, parentId ): #generate files if nbFiles: - for f in xrange(nbFiles): + for f in range(nbFiles): filename = "%s.txt"%(f) size = random.randint(1,1000) statusid = 2 diff --git a/tests/Performance/DFCPerformance/generateDB/generate_db_data_6_13_25.py b/tests/Performance/DFCPerformance/generateDB/generate_db_data_6_13_25.py index 0efa869c300..c2e118af8fd 100644 --- a/tests/Performance/DFCPerformance/generateDB/generate_db_data_6_13_25.py +++ b/tests/Performance/DFCPerformance/generateDB/generate_db_data_6_13_25.py @@ -96,7 +96,7 @@ def loop ( index, cur, parentId ): if index >= len (d): return - for i in xrange(d[index]): + for i in range(d[index]): uid = random.randint(1, len(config.users) ) gid = random.randint(1, len(config.groups) ) next = cur + [i] @@ -116,7 +116,7 @@ def loop ( index, cur, parentId ): #generate files if nbFiles: - for f in xrange(nbFiles): + for f in range(nbFiles): filename = "%s.txt"%(f) size = random.randint(1,1000) statusid = 2 diff --git a/tests/Performance/DFCPerformance/mixedPerf.py b/tests/Performance/DFCPerformance/mixedPerf.py index fc10d2e0cff..43a9be1362e 100755 --- a/tests/Performance/DFCPerformance/mixedPerf.py +++ b/tests/Performance/DFCPerformance/mixedPerf.py @@ -63,16 +63,16 @@ done = False - -def generatePath(depth, withLetters = False): +def generatePath(depth, withLetters=False): # Between 0 and 3 because in generate we have 4 subdirs per dir. Adapt :-) - rndTab = [random.randint( 0, 3 ) for _i in xrange( depth ) ] + rndTab = [random.randint(0, 3) for _i in range(depth)] if withLetters: - rndLetters = [random.choice( string.letters ) for _i in xrange( 3 ) ] + rndLetters = [random.choice(string.ascii_letters) for _i in range(3)] rndTab += rndLetters dirPath = '/' + '/'.join(map(str, rndTab)) return dirPath + def doRead(depth): dirPath = generatePath(depth) before = time.time() @@ -105,7 +105,7 @@ def doInsert(depth, maxFile): dirPath = generatePath(depth, True) nbOfFiles = random.randint(1,maxFile) lfnDict = {} - for f in xrange(nbOfFiles): + for f in range(nbOfFiles): filename = "%s.txt"%(f) lfn = "%s/%s"%(dirPath, filename) size = random.randint(1,1000) diff --git a/tests/Performance/DFCPerformance/readPerf.py b/tests/Performance/DFCPerformance/readPerf.py index 4711510c8a7..d370d8b1b6d 100755 --- a/tests/Performance/DFCPerformance/readPerf.py +++ b/tests/Performance/DFCPerformance/readPerf.py @@ -50,7 +50,7 @@ while not done: # Between 0 and 3 because in generate we have 4 subdirs per dir. Adapt :-) - rndTab = [random.randint( 0, 3 ) for i in xrange( readDepth ) ] + rndTab = [random.randint(0, 3) for i in range(readDepth)] dirPath = '/' + '/'.join(map(str,rndTab)) before = time.time() beforeC = time.clock() diff --git a/tests/Performance/DFCPerformance/submitJobs.py b/tests/Performance/DFCPerformance/submitJobs.py index ff16be70851..94cbdd6aadf 100755 --- a/tests/Performance/DFCPerformance/submitJobs.py +++ b/tests/Performance/DFCPerformance/submitJobs.py @@ -37,7 +37,7 @@ f = open("%s/jobIdList.txt"%jobName, 'w') -for i in xrange(nbJobs): +for i in range(nbJobs): j = Job() j.setCPUTime(10000) j.setExecutable(scriptName) diff --git a/tests/Performance/DFCPerformance/writePerf.py b/tests/Performance/DFCPerformance/writePerf.py index ea578473011..8627ada530b 100755 --- a/tests/Performance/DFCPerformance/writePerf.py +++ b/tests/Performance/DFCPerformance/writePerf.py @@ -53,13 +53,13 @@ while not done: # Between 0 and 3 because in generate we have 4 subdirs per dir. Adapt :-) - rndTab = [random.randint( 0, 3 ) for i in xrange( writeDepth ) ] - rndLetters = [random.choice(string.letters) for i in xrange(3) ] + rndTab = [random.randint(0, 3) for i in range(writeDepth)] + rndLetters = [random.choice(string.ascii_letters) for i in range(3)] rndTab += rndLetters dirPath = '/' + '/'.join(map(str, rndTab)) - nbOfFiles = random.randint( 1, maxInsert ) + nbOfFiles = random.randint(1, maxInsert) lfnDict = {} - for f in xrange(nbOfFiles): + for f in range(nbOfFiles): filename = "%s.txt"%(f) lfn = "%s/%s"%(dirPath, filename) size = random.randint(1,1000) diff --git a/tests/Performance/ESJobMonitoring/test_scripts/query.py b/tests/Performance/ESJobMonitoring/test_scripts/query.py index ad3a0a56ab1..eed890bee1b 100644 --- a/tests/Performance/ESJobMonitoring/test_scripts/query.py +++ b/tests/Performance/ESJobMonitoring/test_scripts/query.py @@ -23,7 +23,7 @@ def run(self): start_time = time.time() - for i in xrange(0, random.randint(1000, 3000)): + for i in range(0, random.randint(1000, 3000)): jobID = random.randint(1, 1000) self.ElasticJobDB.getJobParameters(jobID) diff --git a/tests/Performance/ESJobMonitoring/test_scripts/update.py b/tests/Performance/ESJobMonitoring/test_scripts/update.py index 93d35aed209..4d4c8b87f63 100644 --- a/tests/Performance/ESJobMonitoring/test_scripts/update.py +++ b/tests/Performance/ESJobMonitoring/test_scripts/update.py @@ -14,7 +14,7 @@ def random_generator(size=6, chars=string.ascii_letters): - return ''.join(random.choice(chars) for x in xrange(size)) + return ''.join(random.choice(chars) for x in range(size)) class Transaction(object): @@ -28,7 +28,7 @@ def run(self): start_time = time.time() - for i in xrange(0, random.randint(1000, 3000)): + for i in range(0, random.randint(1000, 3000)): key = random_generator() value = random_generator(size=12) diff --git a/tests/Performance/MySQLJobMonitoring/test_scripts/query.py b/tests/Performance/MySQLJobMonitoring/test_scripts/query.py index cb8632fbac5..36367eec1c1 100644 --- a/tests/Performance/MySQLJobMonitoring/test_scripts/query.py +++ b/tests/Performance/MySQLJobMonitoring/test_scripts/query.py @@ -23,7 +23,7 @@ def run(self): start_time = time.time() - for i in xrange(0, random.randint(1000, 3000)): + for i in range(0, random.randint(1000, 3000)): jobID = random.randint(1, 1000) self.JobDB.getJobParameters(jobID) diff --git a/tests/Performance/MySQLJobMonitoring/test_scripts/update.py b/tests/Performance/MySQLJobMonitoring/test_scripts/update.py index 6aca786f3d3..e61bc90fbd0 100644 --- a/tests/Performance/MySQLJobMonitoring/test_scripts/update.py +++ b/tests/Performance/MySQLJobMonitoring/test_scripts/update.py @@ -14,7 +14,7 @@ def random_generator(size=6, chars=string.ascii_letters): - return ''.join(random.choice(chars) for x in xrange(size)) + return ''.join(random.choice(chars) for x in range(size)) class Transaction(object): @@ -28,7 +28,7 @@ def run(self): start_time = time.time() - for i in xrange(0, random.randint(1000, 3000)): + for i in range(0, random.randint(1000, 3000)): key = random_generator() value = random_generator(size=12) diff --git a/tests/Utilities/ProcessesCreator_withChildren.py b/tests/Utilities/ProcessesCreator_withChildren.py index 3aa5ad2b6b4..6631d3e8f34 100644 --- a/tests/Utilities/ProcessesCreator_withChildren.py +++ b/tests/Utilities/ProcessesCreator_withChildren.py @@ -17,10 +17,6 @@ def do_sum(li): sum(li) proc_name = current_process().name print(proc_name) - p = Process(target=f, args=(10, )) - p.daemon = False - p.start() - p.join() my_list = range(150000000) @@ -31,3 +27,10 @@ def do_sum(li): my_list[100000000:]]) pool.close() pool.join() + +# Generate system CPU usage +for i in range(1000): + p = Process(target=f, args=(0.001, )) + p.daemon = False + p.start() + p.join() diff --git a/tests/Utilities/assertingUtils.py b/tests/Utilities/assertingUtils.py index ca3800833e5..567aa71fad3 100644 --- a/tests/Utilities/assertingUtils.py +++ b/tests/Utilities/assertingUtils.py @@ -29,7 +29,7 @@ def _parseOption(outDict, inDict, optionPrefix=''): Handle some special cases. """ LOG.info("Parsing into %s, from %s, prefix %r", outDict, inDict, optionPrefix) - for option, value in inDict.iteritems(): + for option, value in inDict.items(): optionName = "/".join([optionPrefix, option]).strip('/') LOG.info("Parsing %r with %r", optionName, value) if isinstance(value, six.string_types) and value.lower() in ('no', 'false'): @@ -88,7 +88,7 @@ def AgentOptionsTest(agentPath, options, mocker): mocker.patch(agentPath + "." + name, new=Mock()) if specialMocks is not None: - for name, retVal in specialMocks.iteritems(): + for name, retVal in specialMocks.items(): mocker.patch(agentPath + "." + name, new=Mock(return_value=retVal)) def returnDefault(*args): @@ -164,7 +164,7 @@ def checkAgentOptions(getOptionMock, systemName, agentName, LOG.info("Options found in ConfigTemplate: %s ", list(optionsDict.keys())) # check that values in ConfigTemplate are used - for option, value in optionsDict.iteritems(): + for option, value in optionsDict.items(): if any(ignoreOp in option for ignoreOp in ignoreOptions): LOG.info("From Agent: ignoring option %r with value %r, (%s)", option, value, type(value)) continue diff --git a/tests/Utilities/mpTest-flexible.py b/tests/Utilities/mpTest-flexible.py index 9bf4d075139..cb327570b42 100644 --- a/tests/Utilities/mpTest-flexible.py +++ b/tests/Utilities/mpTest-flexible.py @@ -27,7 +27,7 @@ def getChunk(aList, chunkSize): process( chunk ) """ - for i in xrange(0, len(aList), chunkSize): + for i in range(0, len(aList), chunkSize): yield aList[i:i + chunkSize]