From 62b10d7021babfe9eea045067bd9513ecfc706f1 Mon Sep 17 00:00:00 2001 From: Chris Burr Date: Sat, 15 Aug 2020 10:12:05 +0200 Subject: [PATCH 01/20] PY3: Replace "import thread" with "from six.moves import _thread as thread" --- ConfigurationSystem/Client/Helpers/Operations.py | 2 +- ConfigurationSystem/private/ConfigurationData.py | 2 +- ConfigurationSystem/private/Refresher.py | 2 +- Core/DISET/private/BaseClient.py | 2 +- Core/Utilities/LockRing.py | 2 +- FrameworkSystem/private/standardLogging/Message.py | 2 +- .../private/standardLogging/test/Test_Logging_FormatOptions.py | 2 +- 7 files changed, 7 insertions(+), 7 deletions(-) diff --git a/ConfigurationSystem/Client/Helpers/Operations.py b/ConfigurationSystem/Client/Helpers/Operations.py index 4bdc0e03e16..9e8d7da8900 100644 --- a/ConfigurationSystem/Client/Helpers/Operations.py +++ b/ConfigurationSystem/Client/Helpers/Operations.py @@ -74,7 +74,7 @@ from __future__ import print_function import six -import thread +from six.moves import _thread as thread import os from diraccfg import CFG from DIRAC import S_OK, S_ERROR, gConfig diff --git a/ConfigurationSystem/private/ConfigurationData.py b/ConfigurationSystem/private/ConfigurationData.py index 21515de514b..4b9aeaa2ca6 100755 --- a/ConfigurationSystem/private/ConfigurationData.py +++ b/ConfigurationSystem/private/ConfigurationData.py @@ -7,7 +7,7 @@ import os.path import zlib import zipfile -import thread +from six.moves import _thread as thread import time import DIRAC diff --git a/ConfigurationSystem/private/Refresher.py b/ConfigurationSystem/private/Refresher.py index 4af11e81469..afd31d2023a 100755 --- a/ConfigurationSystem/private/Refresher.py +++ b/ConfigurationSystem/private/Refresher.py @@ -8,7 +8,7 @@ __RCSID__ = "$Id$" import threading -import thread +from six.moves import _thread as thread import time import random from DIRAC.ConfigurationSystem.Client.ConfigurationData import gConfigurationData diff --git a/Core/DISET/private/BaseClient.py b/Core/DISET/private/BaseClient.py index d5ac0a2eebc..f007d510af6 100755 --- a/Core/DISET/private/BaseClient.py +++ b/Core/DISET/private/BaseClient.py @@ -9,7 +9,7 @@ import six import time -import thread +from six.moves import _thread as thread import DIRAC from DIRAC.Core.DISET.private.Protocols import gProtocolDict from DIRAC.FrameworkSystem.Client.Logger import gLogger diff --git a/Core/Utilities/LockRing.py b/Core/Utilities/LockRing.py index 9168d0dcc09..4928436a009 100644 --- a/Core/Utilities/LockRing.py +++ b/Core/Utilities/LockRing.py @@ -5,7 +5,7 @@ import random import time import threading -import thread +from six.moves import _thread as thread from hashlib import md5 from DIRAC.Core.Utilities.ReturnValues import S_ERROR, S_OK diff --git a/FrameworkSystem/private/standardLogging/Message.py b/FrameworkSystem/private/standardLogging/Message.py index efa70a7b53b..c93a6ba05e7 100644 --- a/FrameworkSystem/private/standardLogging/Message.py +++ b/FrameworkSystem/private/standardLogging/Message.py @@ -21,7 +21,7 @@ def tupleToMessage(varTuple): class Message: def __init__(self, systemName, level, time, msgText, variableText, frameInfo, subSystemName=''): - import thread + from six.moves import _thread as thread self.systemName = systemName self.level = level self.time = time diff --git a/FrameworkSystem/private/standardLogging/test/Test_Logging_FormatOptions.py b/FrameworkSystem/private/standardLogging/test/Test_Logging_FormatOptions.py index 60203c2a067..677b0132d37 100644 --- a/FrameworkSystem/private/standardLogging/test/Test_Logging_FormatOptions.py +++ b/FrameworkSystem/private/standardLogging/test/Test_Logging_FormatOptions.py @@ -4,7 +4,7 @@ __RCSID__ = "$Id$" -import thread +from six.moves import _thread as thread import pytest from DIRAC.FrameworkSystem.private.standardLogging.test.TestLogUtilities import gLogger, gLoggerReset From b9b62991075d4360a170860fd70a2bea18b8b7ae Mon Sep 17 00:00:00 2001 From: Chris Burr Date: Sat, 15 Aug 2020 10:14:49 +0200 Subject: [PATCH 02/20] PY3: Replace "import urlparse" with "import six.moves.urllib.parse" --- AccountingSystem/scripts/dirac-accounting-decode-fileid.py | 2 +- ConfigurationSystem/Client/Helpers/Resources.py | 2 +- ConfigurationSystem/Client/PathFinder.py | 2 +- ConfigurationSystem/Client/Utilities.py | 2 +- ConfigurationSystem/scripts/dirac-admin-add-resources.py | 2 +- Core/Utilities/Network.py | 2 +- Core/Utilities/Pfn.py | 2 +- ResourceStatusSystem/Command/VOBOXAvailabilityCommand.py | 2 +- Resources/Computing/BOINCComputingElement.py | 2 +- Resources/Computing/LocalComputingElement.py | 2 +- Resources/Computing/SSHBatchComputingElement.py | 2 +- Resources/Computing/SSHComputingElement.py | 2 +- 12 files changed, 12 insertions(+), 12 deletions(-) diff --git a/AccountingSystem/scripts/dirac-accounting-decode-fileid.py b/AccountingSystem/scripts/dirac-accounting-decode-fileid.py index e8aee28809b..0dd71abdacf 100755 --- a/AccountingSystem/scripts/dirac-accounting-decode-fileid.py +++ b/AccountingSystem/scripts/dirac-accounting-decode-fileid.py @@ -13,7 +13,7 @@ import pprint import sys -import urlparse +from six.moves.urllib import parse as urlparse import cgi from DIRAC import gLogger from DIRAC.Core.Base import Script diff --git a/ConfigurationSystem/Client/Helpers/Resources.py b/ConfigurationSystem/Client/Helpers/Resources.py index 07204ede5b5..9ac2559959c 100644 --- a/ConfigurationSystem/Client/Helpers/Resources.py +++ b/ConfigurationSystem/Client/Helpers/Resources.py @@ -6,7 +6,7 @@ __RCSID__ = "$Id$" -import urlparse +from six.moves.urllib import parse as urlparse from distutils.version import LooseVersion # pylint: disable=no-name-in-module,import-error import six diff --git a/ConfigurationSystem/Client/PathFinder.py b/ConfigurationSystem/Client/PathFinder.py index 1b5386378ff..4a748aad28e 100755 --- a/ConfigurationSystem/Client/PathFinder.py +++ b/ConfigurationSystem/Client/PathFinder.py @@ -6,7 +6,7 @@ __RCSID__ = "$Id$" -import urlparse +from six.moves.urllib import parse as urlparse from DIRAC.Core.Utilities import List from DIRAC.ConfigurationSystem.Client.ConfigurationData import gConfigurationData diff --git a/ConfigurationSystem/Client/Utilities.py b/ConfigurationSystem/Client/Utilities.py index a8f788cea1a..962c9859210 100644 --- a/ConfigurationSystem/Client/Utilities.py +++ b/ConfigurationSystem/Client/Utilities.py @@ -17,7 +17,7 @@ import re import socket -from urlparse import urlparse +from six.moves.urllib.parse import urlparse import six diff --git a/ConfigurationSystem/scripts/dirac-admin-add-resources.py b/ConfigurationSystem/scripts/dirac-admin-add-resources.py index 862e9a52b64..fb4c9e6e323 100755 --- a/ConfigurationSystem/scripts/dirac-admin-add-resources.py +++ b/ConfigurationSystem/scripts/dirac-admin-add-resources.py @@ -16,7 +16,7 @@ import re import os import shlex -from urlparse import urlparse +from six.moves.urllib.parse import urlparse from DIRAC.Core.Base import Script from DIRAC import gLogger, exit as DIRACExit, S_OK diff --git a/Core/Utilities/Network.py b/Core/Utilities/Network.py index 3f1671feeab..d9a284fb1c0 100755 --- a/Core/Utilities/Network.py +++ b/Core/Utilities/Network.py @@ -8,7 +8,7 @@ __RCSID__ = "$Id$" import socket -import urlparse +from six.moves.urllib import parse as urlparse import os import struct import array diff --git a/Core/Utilities/Pfn.py b/Core/Utilities/Pfn.py index 713c51e9cae..8a42297eace 100755 --- a/Core/Utilities/Pfn.py +++ b/Core/Utilities/Pfn.py @@ -20,7 +20,7 @@ import os # # from DIRAC from DIRAC import S_OK, S_ERROR, gLogger -import urlparse +from six.moves.urllib import parse as urlparse def pfnunparse(pfnDict, srmSpecific=True): diff --git a/ResourceStatusSystem/Command/VOBOXAvailabilityCommand.py b/ResourceStatusSystem/Command/VOBOXAvailabilityCommand.py index dec8a6505aa..9ea318bbf11 100644 --- a/ResourceStatusSystem/Command/VOBOXAvailabilityCommand.py +++ b/ResourceStatusSystem/Command/VOBOXAvailabilityCommand.py @@ -7,7 +7,7 @@ # FIXME: NOT Usable ATM # missing doNew, doCache, doMaster -import urlparse +from six.moves.urllib import parse as urlparse from DIRAC import S_OK, S_ERROR from DIRAC.Core.DISET.RPCClient import RPCClient diff --git a/Resources/Computing/BOINCComputingElement.py b/Resources/Computing/BOINCComputingElement.py index 69c6a73221a..1cc6496c515 100644 --- a/Resources/Computing/BOINCComputingElement.py +++ b/Resources/Computing/BOINCComputingElement.py @@ -16,7 +16,7 @@ import base64 import tempfile -from urlparse import urlparse +from six.moves.urllib.parse import urlparse from DIRAC import S_OK, S_ERROR from DIRAC.Resources.Computing.ComputingElement import ComputingElement diff --git a/Resources/Computing/LocalComputingElement.py b/Resources/Computing/LocalComputingElement.py index 65f9e5a9cdb..f4d99ff33dc 100644 --- a/Resources/Computing/LocalComputingElement.py +++ b/Resources/Computing/LocalComputingElement.py @@ -15,7 +15,7 @@ import tempfile import getpass import errno -from urlparse import urlparse +from six.moves.urllib.parse import urlparse from DIRAC import S_OK, S_ERROR from DIRAC import gConfig diff --git a/Resources/Computing/SSHBatchComputingElement.py b/Resources/Computing/SSHBatchComputingElement.py index 244bd73d8b6..cc20268a1e6 100644 --- a/Resources/Computing/SSHBatchComputingElement.py +++ b/Resources/Computing/SSHBatchComputingElement.py @@ -17,7 +17,7 @@ import os import socket import stat -from urlparse import urlparse +from six.moves.urllib.parse import urlparse from DIRAC import S_OK, S_ERROR from DIRAC import rootPath diff --git a/Resources/Computing/SSHComputingElement.py b/Resources/Computing/SSHComputingElement.py index 7a8f91562e1..599a27f772b 100644 --- a/Resources/Computing/SSHComputingElement.py +++ b/Resources/Computing/SSHComputingElement.py @@ -18,7 +18,7 @@ import stat import shutil import errno -from urlparse import urlparse +from six.moves.urllib.parse import urlparse from DIRAC import S_OK, S_ERROR from DIRAC import rootPath From 4f3f8399ae1b5fc39ae7e89cfab64ad4b50d15c0 Mon Sep 17 00:00:00 2001 From: Chris Burr Date: Sat, 15 Aug 2020 10:15:18 +0200 Subject: [PATCH 03/20] PY3: Remove Python version check on import --- __init__.py | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/__init__.py b/__init__.py index 60b9a0d059a..9c56c5fa7e4 100755 --- a/__init__.py +++ b/__init__.py @@ -102,21 +102,6 @@ if preVersion: version = "%s-pre%s" % (version, preVersion) -# Check of python version - -__pythonMajorVersion = ("2", ) -__pythonMinorVersion = ("7") - -pythonVersion = pyPlatform.python_version_tuple() -if str(pythonVersion[0]) not in __pythonMajorVersion or str(pythonVersion[1]) not in __pythonMinorVersion: - print("Python Version %s not supported by DIRAC" % pyPlatform.python_version()) - print("Supported versions are: ") - for major in __pythonMajorVersion: - for minor in __pythonMinorVersion: - print("%s.%s.x" % (major, minor)) - - sys.exit(1) - errorMail = "dirac.alarms@gmail.com" alarmMail = "dirac.alarms@gmail.com" From d285b56330b4c1ef9599e30d1786e66963711176 Mon Sep 17 00:00:00 2001 From: Chris Burr Date: Sat, 15 Aug 2020 10:33:18 +0200 Subject: [PATCH 04/20] PY3: Encode strings as UTF-8 before using hashlib.md5 --- AccountingSystem/private/MainReporter.py | 4 ++-- Core/DISET/MessageClient.py | 2 +- Core/DISET/private/Transports/BaseTransport.py | 2 +- Core/Utilities/File.py | 6 +++--- Core/Utilities/LockRing.py | 5 +++-- Core/Utilities/ThreadScheduler.py | 2 +- .../FileCatalogComponents/DatasetManager/DatasetManager.py | 2 +- .../scripts/dirac-dms-create-removal-request.py | 5 ++++- .../scripts/dirac-dms-move-replica-request.py | 5 ++++- FrameworkSystem/DB/ProxyDB.py | 2 +- FrameworkSystem/DB/UserProfileDB.py | 2 +- FrameworkSystem/Service/PlottingHandler.py | 6 +++++- FrameworkSystem/private/monitoring/MonitoringCatalog.py | 2 +- FrameworkSystem/private/monitoring/PlotCache.py | 4 ++-- FrameworkSystem/private/monitoring/RRDManager.py | 4 ++-- MonitoringSystem/private/MainReporter.py | 4 ++-- RequestManagementSystem/Service/ReqProxyHandler.py | 2 +- WorkloadManagementSystem/Agent/SiteDirector.py | 2 +- 18 files changed, 36 insertions(+), 25 deletions(-) diff --git a/AccountingSystem/private/MainReporter.py b/AccountingSystem/private/MainReporter.py index 846592b72a3..fc2d46669aa 100644 --- a/AccountingSystem/private/MainReporter.py +++ b/AccountingSystem/private/MainReporter.py @@ -46,8 +46,8 @@ def __calculateReportHash(self, reportRequest): epoch = requestToHash[key] requestToHash[key] = epoch - epoch % granularity md5Hash = hashlib.md5() - md5Hash.update(repr(requestToHash)) - md5Hash.update(self.setup) + md5Hash.update(repr(requestToHash).encode()) + md5Hash.update(self.setup.encode()) return md5Hash.hexdigest() def generate(self, reportRequest, credDict): diff --git a/Core/DISET/MessageClient.py b/Core/DISET/MessageClient.py index 6bf67eae2f7..afb8460e9d8 100755 --- a/Core/DISET/MessageClient.py +++ b/Core/DISET/MessageClient.py @@ -33,7 +33,7 @@ def _initialize(self): def __generateUniqueClientName(self): hashStr = ":".join((Time.toString(), str(random.random()), Network.getFQDN(), gLogger.getName())) - hexHash = md5(hashStr).hexdigest() + hexHash = md5(hashStr.encode()).hexdigest() return hexHash def setUniqueName(self, uniqueName): diff --git a/Core/DISET/private/Transports/BaseTransport.py b/Core/DISET/private/Transports/BaseTransport.py index a431578e27f..c0835e44111 100755 --- a/Core/DISET/private/Transports/BaseTransport.py +++ b/Core/DISET/private/Transports/BaseTransport.py @@ -55,7 +55,7 @@ def __init__(self, stServerAddress, bServerMode=False, **kwargs): self.remoteAddress = False self.appData = "" self.startedKeepAlives = set() - self.keepAliveId = md5(str(stServerAddress) + str(bServerMode)).hexdigest() + self.keepAliveId = md5((str(stServerAddress) + str(bServerMode)).encode()).hexdigest() self.receivedMessages = [] self.sentKeepAlives = 0 self.waitingForKeepAlivePong = False diff --git a/Core/Utilities/File.py b/Core/Utilities/File.py index 47f8f4ce76b..d5b0b0ea921 100755 --- a/Core/Utilities/File.py +++ b/Core/Utilities/File.py @@ -78,13 +78,13 @@ def makeGuid(fileName=None): myMd5 = hashlib.md5() if fileName: try: - with open(fileName, 'r') as fd: + with open(fileName, 'rb') as fd: data = fd.read(10 * 1024 * 1024) myMd5.update(data) except BaseException: return None else: - myMd5.update(str(random.getrandbits(128))) + myMd5.update(str(random.getrandbits(128)).encode()) md5HexString = myMd5.hexdigest().upper() return generateGuid(md5HexString, "MD5") @@ -112,7 +112,7 @@ def generateGuid(checksum, checksumtype): # Failed to use the check sum, generate a new guid myMd5 = hashlib.md5() - myMd5.update(str(random.getrandbits(128))) + myMd5.update(str(random.getrandbits(128)).encode()) md5HexString = myMd5.hexdigest() guid = "%s-%s-%s-%s-%s" % (md5HexString[0:8], md5HexString[8:12], diff --git a/Core/Utilities/LockRing.py b/Core/Utilities/LockRing.py index 4928436a009..e4f73c07daf 100644 --- a/Core/Utilities/LockRing.py +++ b/Core/Utilities/LockRing.py @@ -21,10 +21,11 @@ def __init__(self): self.__events = {} def __genName(self, container): - name = md5(str(time.time() + random.random())).hexdigest() + # TODO: Shouldn't this be a UUID? + name = md5(str(time.time() + random.random()).encode()).hexdigest() retries = 10 while name in container and retries: - name = md5(str(time.time() + random.random())).hexdigest() + name = md5(str(time.time() + random.random()).encode()).hexdigest() retries -= 1 return name diff --git a/Core/Utilities/ThreadScheduler.py b/Core/Utilities/ThreadScheduler.py index 033f240db3f..709b453150c 100644 --- a/Core/Utilities/ThreadScheduler.py +++ b/Core/Utilities/ThreadScheduler.py @@ -45,7 +45,7 @@ def addPeriodicTask(self, period, taskFunc, taskArgs=(), executions=0, elapsedTi 'func': taskFunc, 'args': taskArgs, } - md.update(str(task)) + md.update(str(task).encode()) taskId = md.hexdigest() if taskId in self.__taskDict: return S_ERROR("Task %s is already added" % taskId) diff --git a/DataManagementSystem/DB/FileCatalogComponents/DatasetManager/DatasetManager.py b/DataManagementSystem/DB/FileCatalogComponents/DatasetManager/DatasetManager.py index c7f6677f530..d4a2d43bc30 100644 --- a/DataManagementSystem/DB/FileCatalogComponents/DatasetManager/DatasetManager.py +++ b/DataManagementSystem/DB/FileCatalogComponents/DatasetManager/DatasetManager.py @@ -332,7 +332,7 @@ def __getMetaQueryParameters(self, metaQuery, credDict): lfnIDList = idLfnDict.keys() lfnList = sorted(idLfnDict.values()) myMd5 = hashlib.md5() - myMd5.update(str(lfnList)) + myMd5.update(str(lfnList).encode()) datasetHash = myMd5.hexdigest().upper() numberOfFiles = len(lfnList) result = self.db.fileManager.getFileSize(lfnList) diff --git a/DataManagementSystem/scripts/dirac-dms-create-removal-request.py b/DataManagementSystem/scripts/dirac-dms-create-removal-request.py index ba864656056..7617f636f08 100755 --- a/DataManagementSystem/scripts/dirac-dms-create-removal-request.py +++ b/DataManagementSystem/scripts/dirac-dms-create-removal-request.py @@ -65,7 +65,10 @@ for lfnList in breakListIntoChunks(lfns, 100): oRequest = Request() - requestName = "%s_%s" % (md5(repr(time.time())).hexdigest()[:16], md5(repr(time.time())).hexdigest()[:16]) + requestName = "%s_%s" % ( + md5(repr(time.time()).encode()).hexdigest()[:16], + md5(repr(time.time()).encode()).hexdigest()[:16], + ) oRequest.RequestName = requestName oOperation = Operation() diff --git a/DataManagementSystem/scripts/dirac-dms-move-replica-request.py b/DataManagementSystem/scripts/dirac-dms-move-replica-request.py index 129d86527af..c861e4a4554 100755 --- a/DataManagementSystem/scripts/dirac-dms-move-replica-request.py +++ b/DataManagementSystem/scripts/dirac-dms-move-replica-request.py @@ -90,7 +90,10 @@ def getLFNList(arg): count += 1 request = Request() - request.RequestName = "%s_%s" % (md5(repr(time.time())).hexdigest()[:16], md5(repr(time.time())).hexdigest()[:16]) + request.RequestName = "%s_%s" % ( + md5(repr(time.time()).encode()).hexdigest()[:16], + md5(repr(time.time()).encode()).hexdigest()[:16], + ) moveReplica = Operation() moveReplica.Type = 'MoveReplica' diff --git a/FrameworkSystem/DB/ProxyDB.py b/FrameworkSystem/DB/ProxyDB.py index a3f7a506926..d4e1c727eb2 100755 --- a/FrameworkSystem/DB/ProxyDB.py +++ b/FrameworkSystem/DB/ProxyDB.py @@ -1232,7 +1232,7 @@ def generateToken(self, requesterDN, requesterGroup, numUses=1, lifeTime=0, retr numUses = max(1, min(numUses, maxUses)) m = hashlib.md5() rndData = "%s.%s.%s.%s" % (time.time(), random.random(), numUses, lifeTime) - m.update(rndData) + m.update(rndData.encode()) token = m.hexdigest() fieldsSQL = ", ".join(("Token", "RequesterDN", "RequesterGroup", "ExpirationTime", "UsesLeft")) valuesSQL = ", ".join((self._escapeString(token)['Value'], diff --git a/FrameworkSystem/DB/UserProfileDB.py b/FrameworkSystem/DB/UserProfileDB.py index 8a039b95ae6..39fb8d9e6ea 100755 --- a/FrameworkSystem/DB/UserProfileDB.py +++ b/FrameworkSystem/DB/UserProfileDB.py @@ -546,7 +546,7 @@ def storeHashTagById(self, userIds, tagName, hashTag=False): """ if not hashTag: hashTag = hashlib.md5() - hashTag.update("%s;%s;%s" % (Time.dateTime(), userIds, tagName)) + hashTag.update(("%s;%s;%s" % (Time.dateTime(), userIds, tagName)).encode()) hashTag = hashTag.hexdigest() result = self.insertFields('up_HashTags', ['UserId', 'GroupId', 'VOId', 'TagName', 'HashTag'], diff --git a/FrameworkSystem/Service/PlottingHandler.py b/FrameworkSystem/Service/PlottingHandler.py index c4fea5a5df8..ad446ed02f4 100644 --- a/FrameworkSystem/Service/PlottingHandler.py +++ b/FrameworkSystem/Service/PlottingHandler.py @@ -48,7 +48,11 @@ class PlottingHandler(RequestHandler): def __calculatePlotHash(self, data, metadata, subplotMetadata): m = hashlib.md5() - m.update(repr({'Data': data, 'PlotMetadata': metadata, 'SubplotMetadata': subplotMetadata})) + m.update(repr({ + 'Data': data, + 'PlotMetadata': metadata, + 'SubplotMetadata': subplotMetadata + }).encode()) return m.hexdigest() types_generatePlot = [[dict, list], dict] diff --git a/FrameworkSystem/private/monitoring/MonitoringCatalog.py b/FrameworkSystem/private/monitoring/MonitoringCatalog.py index 6942d1ea45d..51102ee5cd3 100755 --- a/FrameworkSystem/private/monitoring/MonitoringCatalog.py +++ b/FrameworkSystem/private/monitoring/MonitoringCatalog.py @@ -278,7 +278,7 @@ def registerActivity(self, sourceId, acName, acDict): m = hashlib.md5() acDict['name'] = acName acDict['sourceId'] = sourceId - m.update(str(acDict)) + m.update(str(acDict).encode()) retList = self.__select("filename", "activities", acDict) if len(retList) > 0: return retList[0][0] diff --git a/FrameworkSystem/private/monitoring/PlotCache.py b/FrameworkSystem/private/monitoring/PlotCache.py index 7616711468d..df5eaf04cf1 100755 --- a/FrameworkSystem/private/monitoring/PlotCache.py +++ b/FrameworkSystem/private/monitoring/PlotCache.py @@ -46,8 +46,8 @@ def __init__(self, rrdManager=None): def __generateName(self, *args, **kwargs): m = hashlib.md5() - m.update(repr(args)) - m.update(repr(kwargs)) + m.update(repr(args).encode()) + m.update(repr(kwargs).encode()) return m.hexdigest() def __isCurrentTime(self, toSecs): diff --git a/FrameworkSystem/private/monitoring/RRDManager.py b/FrameworkSystem/private/monitoring/RRDManager.py index c0c812488c7..ed53226f2ee 100755 --- a/FrameworkSystem/private/monitoring/RRDManager.py +++ b/FrameworkSystem/private/monitoring/RRDManager.py @@ -200,8 +200,8 @@ def __generateName(self, *args, **kwargs): Generates a random name """ m = hashlib.md5() - m.update(str(args)) - m.update(str(kwargs)) + m.update(str(args).encode()) + m.update(str(kwargs).encode()) return m.hexdigest() def __generateRRDGraphVar(self, entryName, activity, timeSpan, plotWidth): diff --git a/MonitoringSystem/private/MainReporter.py b/MonitoringSystem/private/MainReporter.py index 27abb5b72f0..1e5a334fc68 100644 --- a/MonitoringSystem/private/MainReporter.py +++ b/MonitoringSystem/private/MainReporter.py @@ -81,8 +81,8 @@ def __calculateReportHash(self, reportRequest): epoch = requestToHash[key] requestToHash[key] = epoch - epoch % granularity md5Hash = hashlib.md5() - md5Hash.update(repr(requestToHash)) - md5Hash.update(self.__setup) + md5Hash.update(repr(requestToHash).encode()) + md5Hash.update(self.__setup.encode()) return md5Hash.hexdigest() def generate(self, reportRequest, credDict): diff --git a/RequestManagementSystem/Service/ReqProxyHandler.py b/RequestManagementSystem/Service/ReqProxyHandler.py index 00db867589a..58f9de96c67 100644 --- a/RequestManagementSystem/Service/ReqProxyHandler.py +++ b/RequestManagementSystem/Service/ReqProxyHandler.py @@ -152,7 +152,7 @@ def __saveRequest(self, requestName, requestJSON): :param str requestJSON: request serialized to JSON format """ try: - requestFile = os.path.join(self.cacheDir(), md5(requestJSON).hexdigest()) + requestFile = os.path.join(self.cacheDir(), md5(requestJSON.encode()).hexdigest()) with open(requestFile, "w+") as request: request.write(requestJSON) return S_OK(requestFile) diff --git a/WorkloadManagementSystem/Agent/SiteDirector.py b/WorkloadManagementSystem/Agent/SiteDirector.py index 7d947f05517..1bcc1da87aa 100644 --- a/WorkloadManagementSystem/Agent/SiteDirector.py +++ b/WorkloadManagementSystem/Agent/SiteDirector.py @@ -290,7 +290,7 @@ def __generateQueueHash(self, queueDict): """ Generate a hash of the queue description """ myMD5 = hashlib.md5() - myMD5.update(str(queueDict)) + myMD5.update(str(queueDict).encode()) hexstring = myMD5.hexdigest() return hexstring From 604de9cb29c25d5bfd3079ff4c6861d3c2f4635b Mon Sep 17 00:00:00 2001 From: Chris Burr Date: Sat, 15 Aug 2020 11:10:32 +0200 Subject: [PATCH 05/20] PY3: Allow importing DEncode in Python 3 without actually porting it --- Core/Utilities/DEncode.py | 17 ++++++++++++++++- Core/Utilities/test/Test_Encode.py | 24 ++++++++++++++++++++++-- 2 files changed, 38 insertions(+), 3 deletions(-) diff --git a/Core/Utilities/DEncode.py b/Core/Utilities/DEncode.py index 7b6018a1ca6..451d071f63a 100755 --- a/Core/Utilities/DEncode.py +++ b/Core/Utilities/DEncode.py @@ -19,7 +19,6 @@ from past.builtins import long import six -import types import datetime import os @@ -31,6 +30,22 @@ from pprint import pprint +# This is a hack for Python 3 to make it possible to import DEncode +# There is not point in porting DEncode to Python 3 as it will be removed as +# part of the HTTPS transition. +class types(object): + IntType = int + LongType = long if six.PY2 else int + FloatType = float + BooleanType = bool + StringType = str + UnicodeType = type(u"") + NoneType = type(None) + ListType = list + TupleType = tuple + DictType = dict + + # Setting this environment variable to any value will enable the dump of the debugging # call stack DIRAC_DEBUG_DENCODE_CALLSTACK = bool(os.environ.get('DIRAC_DEBUG_DENCODE_CALLSTACK', False)) diff --git a/Core/Utilities/test/Test_Encode.py b/Core/Utilities/test/Test_Encode.py index 951a73ef17d..ded7b1a7e99 100644 --- a/Core/Utilities/test/Test_Encode.py +++ b/Core/Utilities/test/Test_Encode.py @@ -23,8 +23,8 @@ from hypothesis import given, settings, HealthCheck from hypothesis.strategies import builds, integers, lists, recursive, floats, text,\ booleans, none, dictionaries, tuples, datetimes - -from pytest import mark, approx, raises, fixture +import six +from pytest import mark, approx, raises, fixture, skip parametrize = mark.parametrize @@ -314,3 +314,23 @@ def test_nestedSerializable(data): subObj = Serializable(instAttr=data) objData = Serializable(instAttr=subObj) agnosticTestFunction(jsonTuple, objData) + + +def test_types(): + """ Ensure that the DEncode types object matches the Python 2 types module + """ + if not six.PY2: + skip("This test only makes sense on Python 2") + import types as pythonTypes + from DIRAC.Core.Utilities.DEncode import types as DIRACTypes + + assert pythonTypes.IntType is pythonTypes.IntType + assert pythonTypes.LongType is pythonTypes.LongType + assert pythonTypes.FloatType is pythonTypes.FloatType + assert pythonTypes.BooleanType is pythonTypes.BooleanType + assert pythonTypes.StringType is pythonTypes.StringType + assert pythonTypes.UnicodeType is pythonTypes.UnicodeType + assert pythonTypes.NoneType is pythonTypes.NoneType + assert pythonTypes.ListType is pythonTypes.ListType + assert pythonTypes.TupleType is pythonTypes.TupleType + assert pythonTypes.DictType is pythonTypes.DictType From 4bdc9b0875b369689e38b05c5dcca04aeaeafd74 Mon Sep 17 00:00:00 2001 From: Chris Burr Date: Sat, 15 Aug 2020 11:13:53 +0200 Subject: [PATCH 06/20] PY3: Replace "import Queue" with "from six.moves import queue as Queue" --- Core/Utilities/ProcessPool.py | 2 +- Core/Utilities/ThreadPool.py | 2 +- FrameworkSystem/private/SecurityFileLog.py | 2 +- .../private/standardLogging/Handler/ServerHandler.py | 2 +- ResourceStatusSystem/Agent/ElementInspectorAgent.py | 2 +- ResourceStatusSystem/Agent/SiteInspectorAgent.py | 2 +- .../Agent/test/Test_Agent_ElementInspectorAgent.py | 2 +- .../Agent/test/Test_Agent_SiteInspectorAgent.py | 2 +- Resources/MessageQueue/MQConsumer.py | 2 +- Resources/MessageQueue/Utilities.py | 2 +- Resources/MessageQueue/test/Test_MQ_Utilities.py | 2 +- TransformationSystem/Agent/TaskManagerAgentBase.py | 2 +- TransformationSystem/Agent/TransformationAgent.py | 2 +- 13 files changed, 13 insertions(+), 13 deletions(-) diff --git a/Core/Utilities/ProcessPool.py b/Core/Utilities/ProcessPool.py index dede0e63414..811963035fa 100644 --- a/Core/Utilities/ProcessPool.py +++ b/Core/Utilities/ProcessPool.py @@ -108,7 +108,7 @@ import threading import os import signal -import Queue +from six.moves import queue as Queue import errno from types import FunctionType, TypeType, ClassType diff --git a/Core/Utilities/ThreadPool.py b/Core/Utilities/ThreadPool.py index af16db5ce85..8fad28b7965 100755 --- a/Core/Utilities/ThreadPool.py +++ b/Core/Utilities/ThreadPool.py @@ -75,7 +75,7 @@ import time import sys -import Queue +from six.moves import queue as Queue import threading try: from DIRAC.FrameworkSystem.Client.Logger import gLogger diff --git a/FrameworkSystem/private/SecurityFileLog.py b/FrameworkSystem/private/SecurityFileLog.py index 9dbcecc04c8..4af0945ca79 100644 --- a/FrameworkSystem/private/SecurityFileLog.py +++ b/FrameworkSystem/private/SecurityFileLog.py @@ -6,7 +6,7 @@ import re import time import gzip -import Queue +from six.moves import queue as Queue import threading from DIRAC import gLogger, S_OK, S_ERROR from DIRAC.Core.Utilities.ThreadScheduler import gThreadScheduler diff --git a/FrameworkSystem/private/standardLogging/Handler/ServerHandler.py b/FrameworkSystem/private/standardLogging/Handler/ServerHandler.py index 97f20f966ad..9fa739b6291 100644 --- a/FrameworkSystem/private/standardLogging/Handler/ServerHandler.py +++ b/FrameworkSystem/private/standardLogging/Handler/ServerHandler.py @@ -8,7 +8,7 @@ __RCSID__ = "$Id$" import logging -import Queue +from six.moves import queue as Queue import threading from DIRAC.Core.Utilities import Network diff --git a/ResourceStatusSystem/Agent/ElementInspectorAgent.py b/ResourceStatusSystem/Agent/ElementInspectorAgent.py index 56569c9a2d9..db92b5550f7 100644 --- a/ResourceStatusSystem/Agent/ElementInspectorAgent.py +++ b/ResourceStatusSystem/Agent/ElementInspectorAgent.py @@ -19,7 +19,7 @@ import datetime import math -import Queue +from six.moves import queue as Queue from DIRAC import S_ERROR, S_OK from DIRAC.Core.Base.AgentModule import AgentModule diff --git a/ResourceStatusSystem/Agent/SiteInspectorAgent.py b/ResourceStatusSystem/Agent/SiteInspectorAgent.py index 90f02cec2f7..245a5c7899f 100644 --- a/ResourceStatusSystem/Agent/SiteInspectorAgent.py +++ b/ResourceStatusSystem/Agent/SiteInspectorAgent.py @@ -17,7 +17,7 @@ __RCSID__ = '$Id$' import math -import Queue +from six.moves import queue as Queue from DIRAC import S_OK from DIRAC.Core.Base.AgentModule import AgentModule diff --git a/ResourceStatusSystem/Agent/test/Test_Agent_ElementInspectorAgent.py b/ResourceStatusSystem/Agent/test/Test_Agent_ElementInspectorAgent.py index 36f84f9450d..7856b58ce43 100644 --- a/ResourceStatusSystem/Agent/test/Test_Agent_ElementInspectorAgent.py +++ b/ResourceStatusSystem/Agent/test/Test_Agent_ElementInspectorAgent.py @@ -3,7 +3,7 @@ from __future__ import absolute_import from __future__ import division from __future__ import print_function -import Queue +from six.moves import queue as Queue # imports import pytest diff --git a/ResourceStatusSystem/Agent/test/Test_Agent_SiteInspectorAgent.py b/ResourceStatusSystem/Agent/test/Test_Agent_SiteInspectorAgent.py index f2317e1f25b..7ae2b66db3d 100644 --- a/ResourceStatusSystem/Agent/test/Test_Agent_SiteInspectorAgent.py +++ b/ResourceStatusSystem/Agent/test/Test_Agent_SiteInspectorAgent.py @@ -3,7 +3,7 @@ from __future__ import absolute_import from __future__ import division from __future__ import print_function -import Queue +from six.moves import queue as Queue # imports import pytest diff --git a/Resources/MessageQueue/MQConsumer.py b/Resources/MessageQueue/MQConsumer.py index 0bc80d88ac1..de121c9f06d 100644 --- a/Resources/MessageQueue/MQConsumer.py +++ b/Resources/MessageQueue/MQConsumer.py @@ -4,7 +4,7 @@ from __future__ import division from __future__ import print_function -import Queue +from six.moves import queue as Queue from DIRAC import S_ERROR, S_OK, gLogger from DIRAC.Resources.MessageQueue.Utilities import getDestinationAddress, getMQService, generateDefaultCallback from DIRAC.Core.Utilities.DErrno import EMQNOM diff --git a/Resources/MessageQueue/Utilities.py b/Resources/MessageQueue/Utilities.py index 05e1441970c..c94ec232d0a 100644 --- a/Resources/MessageQueue/Utilities.py +++ b/Resources/MessageQueue/Utilities.py @@ -6,7 +6,7 @@ __RCSID__ = "$Id$" -import Queue +from six.moves import queue as Queue from DIRAC import S_OK, S_ERROR, gConfig from DIRAC.ConfigurationSystem.Client.CSAPI import CSAPI diff --git a/Resources/MessageQueue/test/Test_MQ_Utilities.py b/Resources/MessageQueue/test/Test_MQ_Utilities.py index 85cd8cefe17..56937a440c7 100644 --- a/Resources/MessageQueue/test/Test_MQ_Utilities.py +++ b/Resources/MessageQueue/test/Test_MQ_Utilities.py @@ -7,7 +7,7 @@ import DIRAC.Resources.MessageQueue.Utilities as module import unittest -import Queue +from six.moves import queue as Queue from mock import MagicMock diff --git a/TransformationSystem/Agent/TaskManagerAgentBase.py b/TransformationSystem/Agent/TaskManagerAgentBase.py index 551f51859fc..f222a74eba1 100644 --- a/TransformationSystem/Agent/TaskManagerAgentBase.py +++ b/TransformationSystem/Agent/TaskManagerAgentBase.py @@ -15,7 +15,7 @@ import time import datetime -from Queue import Queue +from six.moves.queue import Queue from DIRAC import S_OK diff --git a/TransformationSystem/Agent/TransformationAgent.py b/TransformationSystem/Agent/TransformationAgent.py index ff074a959d3..97f61b9c4d9 100644 --- a/TransformationSystem/Agent/TransformationAgent.py +++ b/TransformationSystem/Agent/TransformationAgent.py @@ -14,7 +14,7 @@ from past.builtins import long import time -import Queue +from six.moves import queue as Queue import os import datetime import pickle From 331c7ed943107002849d0ad5b05c2eabd1211fd3 Mon Sep 17 00:00:00 2001 From: Chris Burr Date: Sat, 15 Aug 2020 11:23:35 +0200 Subject: [PATCH 07/20] PY3: Replace "itertools.izip_longest" with "six.moves.zip_longest" --- TransformationSystem/Utilities/JobInfo.py | 6 +++--- TransformationSystem/Utilities/TransformationInfo.py | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/TransformationSystem/Utilities/JobInfo.py b/TransformationSystem/Utilities/JobInfo.py index fd01948cbe2..09c98261b9e 100644 --- a/TransformationSystem/Utilities/JobInfo.py +++ b/TransformationSystem/Utilities/JobInfo.py @@ -4,7 +4,7 @@ from __future__ import print_function from pprint import pformat -from itertools import izip_longest +from six.moves import zip_longest from DIRAC import gLogger from DIRAC.ConfigurationSystem.Client.Helpers.Operations import Operations @@ -55,14 +55,14 @@ def __str__(self): if self.otherTasks: info += ' (Last task %s)' % self.otherTasks if self.inputFiles: - ifInfo = ['<<< %s (%s, %s, Errors %s)' % _ for _ in izip_longest(self.inputFiles, + ifInfo = ['<<< %s (%s, %s, Errors %s)' % _ for _ in zip_longest(self.inputFiles, self.inputFilesExist, self.transFileStatus, self.errorCounts)] info += '\n'.join(ifInfo) if self.outputFiles: info += "\n::: OutputFiles: " - efInfo = ["%s (%s)" % _ for _ in izip_longest(self.outputFiles, self.outputFileStatus)] + efInfo = ["%s (%s)" % _ for _ in zip_longest(self.outputFiles, self.outputFileStatus)] info += ", ".join(efInfo) if self.pendingRequest: info += "\n PENDING REQUEST IGNORE THIS JOB!!!" diff --git a/TransformationSystem/Utilities/TransformationInfo.py b/TransformationSystem/Utilities/TransformationInfo.py index 380e2325aa8..b2ce25ae613 100644 --- a/TransformationSystem/Utilities/TransformationInfo.py +++ b/TransformationSystem/Utilities/TransformationInfo.py @@ -4,7 +4,7 @@ from __future__ import print_function from collections import OrderedDict, defaultdict -from itertools import izip_longest +from six.moves import zip_longest from DIRAC import gLogger, S_OK from DIRAC.Core.Utilities.List import breakListIntoChunks @@ -131,7 +131,7 @@ def cleanOutputs(self, jobInfo): descendants = self.__findAllDescendants(jobInfo.outputFiles) existingOutputFiles = [ lfn for lfn, - status in izip_longest( + status in zip_longest( jobInfo.outputFiles, jobInfo.outputFileStatus) if status == "Exists"] filesToDelete = existingOutputFiles + descendants From 58d240d41730d012f64d5a81370f8d4f4679ef40 Mon Sep 17 00:00:00 2001 From: Chris Burr Date: Sat, 15 Aug 2020 14:11:10 +0200 Subject: [PATCH 08/20] PY3: Get urllib2.URLError from six.moves.urllib_error --- Core/LCG/GGUSTicketsClient.py | 4 ++-- ResourceStatusSystem/Command/DowntimeCommand.py | 6 +++--- ResourceStatusSystem/Command/GGUSTicketsCommand.py | 4 ++-- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/Core/LCG/GGUSTicketsClient.py b/Core/LCG/GGUSTicketsClient.py index 13ecca59e79..099174417cc 100644 --- a/Core/LCG/GGUSTicketsClient.py +++ b/Core/LCG/GGUSTicketsClient.py @@ -4,7 +4,7 @@ from __future__ import division from __future__ import print_function -import urllib2 +from six.moves.urllib_error import URLError from suds import WebFault from suds.client import Client @@ -76,7 +76,7 @@ def getTicketsList(self, siteName=None, startDate=None, endDate=None): ticketList = self.gclient.service.TicketGetList(query) except WebFault as e: return S_ERROR(e) - except urllib2.URLError as e: + except URLError as e: return S_ERROR(e) return self.globalStatistics(ticketList) diff --git a/ResourceStatusSystem/Command/DowntimeCommand.py b/ResourceStatusSystem/Command/DowntimeCommand.py index 3a29c855f59..083b216f2a8 100644 --- a/ResourceStatusSystem/Command/DowntimeCommand.py +++ b/ResourceStatusSystem/Command/DowntimeCommand.py @@ -9,7 +9,7 @@ __RCSID__ = '$Id$' -import urllib2 +from six.moves.urllib_error import URLError import re from datetime import datetime, timedelta @@ -210,11 +210,11 @@ def doNew(self, masterParams=None): # WARNING: checking all the DT that are ongoing or starting in given from now try: results = self.gClient.getStatus(element, name=elementNames, startingInHours=hours) - except urllib2.URLError: + except URLError: try: # Let's give it a second chance.. results = self.gClient.getStatus(element, name=elementNames, startingInHours=hours) - except urllib2.URLError as e: + except URLError as e: return S_ERROR(e) if not results['OK']: diff --git a/ResourceStatusSystem/Command/GGUSTicketsCommand.py b/ResourceStatusSystem/Command/GGUSTicketsCommand.py index c770b9b9a9e..982a7ef7a03 100644 --- a/ResourceStatusSystem/Command/GGUSTicketsCommand.py +++ b/ResourceStatusSystem/Command/GGUSTicketsCommand.py @@ -10,7 +10,7 @@ __RCSID__ = '$Id$' -import urllib2 +from six.moves.urllib_error import URLError from DIRAC import S_ERROR, S_OK from DIRAC.ConfigurationSystem.Client.Helpers.Resources import getGOCSiteName, getGOCSites @@ -91,7 +91,7 @@ def doNew(self, masterParams=None): try: results = self.gClient.getTicketsList(gocName) - except urllib2.URLError as e: + except URLError as e: return S_ERROR('%s %s' % (gocName, e)) if not results['OK']: From 846d8a64144d3d496fdbe7634943b2ce2ae701ec Mon Sep 17 00:00:00 2001 From: Chris Burr Date: Sat, 15 Aug 2020 14:28:15 +0200 Subject: [PATCH 09/20] PY3: Escape backslash in Core/Utilities/Grid.py --- Core/Utilities/Grid.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Core/Utilities/Grid.py b/Core/Utilities/Grid.py index bf4814f6e56..66b4bc4226e 100644 --- a/Core/Utilities/Grid.py +++ b/Core/Utilities/Grid.py @@ -147,7 +147,7 @@ def ldapsearchBDII(filt=None, attr=None, host=None, base=None, selectionString=" def ldapSite(site, attr=None, host=None): """ Site information from bdii. -:param site: Site as it defined in GOCDB or part of it with globing, for example: \UKI-* +:param site: Site as it defined in GOCDB or part of it with globing, for example: \\UKI-* :return: standard DIRAC answer with Value equals to list of sites. Each site is dictionary which contains attributes of site. From 07163161d411a480f3d2881077075f4ad073ebc1 Mon Sep 17 00:00:00 2001 From: Chris Burr Date: Sat, 15 Aug 2020 14:30:52 +0200 Subject: [PATCH 10/20] PY3: Minimal types fixes to make pytest discovery functional --- Core/DISET/RequestHandler.py | 5 +++-- Core/Utilities/ProcessPool.py | 3 ++- .../test/Test_VOMSSecurityManager.py | 5 ++--- .../Service/StorageElementHandler.py | 17 +++++++++-------- 4 files changed, 16 insertions(+), 14 deletions(-) diff --git a/Core/DISET/RequestHandler.py b/Core/DISET/RequestHandler.py index 0d423b92ac1..82b83daa549 100755 --- a/Core/DISET/RequestHandler.py +++ b/Core/DISET/RequestHandler.py @@ -9,6 +9,7 @@ import os import time import psutil +import six import DIRAC @@ -344,7 +345,7 @@ def __checkExpectedArgumentTypes(self, method, args): # #### - __connectionCallbackTypes = {'new': [basestring, dict], + __connectionCallbackTypes = {'new': list(six.string_types) + [dict], 'connected': [], 'drop': []} @@ -499,7 +500,7 @@ def export_ping(self): return S_OK(dInfo) - types_echo = [basestring] + types_echo = [six.string_types] @staticmethod def export_echo(data): diff --git a/Core/Utilities/ProcessPool.py b/Core/Utilities/ProcessPool.py index 811963035fa..2f3887983c4 100644 --- a/Core/Utilities/ProcessPool.py +++ b/Core/Utilities/ProcessPool.py @@ -110,7 +110,6 @@ import signal from six.moves import queue as Queue import errno -from types import FunctionType, TypeType, ClassType try: from DIRAC.FrameworkSystem.Client.Logger import gLogger @@ -518,6 +517,8 @@ def process(self): :param self: self reference """ + from types import FunctionType, ClassType + self.__done = True try: # # it's a function? diff --git a/DataManagementSystem/DB/FileCatalogComponents/SecurityManager/test/Test_VOMSSecurityManager.py b/DataManagementSystem/DB/FileCatalogComponents/SecurityManager/test/Test_VOMSSecurityManager.py index 336106f9dff..a0cac28f19d 100644 --- a/DataManagementSystem/DB/FileCatalogComponents/SecurityManager/test/Test_VOMSSecurityManager.py +++ b/DataManagementSystem/DB/FileCatalogComponents/SecurityManager/test/Test_VOMSSecurityManager.py @@ -6,7 +6,6 @@ # pylint: disable=protected-access,missing-docstring,invalid-name,too-many-lines -from types import ListType import unittest import stat @@ -146,7 +145,7 @@ def exists(self, lfns): return S_OK({'Successful': dict((lfn, lfn in fileTree) for lfn in lfns), 'Failed': {}}) def getFileMetadata(self, lfns): - if not isinstance(lfns, ListType): + if not isinstance(lfns, list): lfns = [lfns] retParam = ['Size', 'Checksum', 'ChecksumType', 'UID', @@ -170,7 +169,7 @@ def getFileMetadata(self, lfns): return S_OK({'Successful': successful, 'Failed': failed}) def getPathPermissions(self, lfns, credDict): - if not isinstance(lfns, ListType): + if not isinstance(lfns, list): lfns = [lfns] successful = {} diff --git a/DataManagementSystem/Service/StorageElementHandler.py b/DataManagementSystem/Service/StorageElementHandler.py index 81790b9141e..fee5d75bbaa 100755 --- a/DataManagementSystem/Service/StorageElementHandler.py +++ b/DataManagementSystem/Service/StorageElementHandler.py @@ -34,6 +34,7 @@ import re import errno import shlex +import six # from DIRAC from DIRAC import gLogger, S_OK, S_ERROR @@ -234,7 +235,7 @@ def __getFileStat(path): return S_OK(resultDict) - types_exists = [basestring] + types_exists = [six.string_types] def export_exists(self, fileID): """ Check existence of the fileID """ @@ -242,7 +243,7 @@ def export_exists(self, fileID): return S_OK(True) return S_OK(False) - types_getMetadata = [basestring] + types_getMetadata = [six.string_types] def export_getMetadata(self, fileID): """ Get metadata for the file or directory specified by fileID @@ -269,7 +270,7 @@ def export_getTotalDiskSpace(): """ return getTotalDiskSpace() - types_createDirectory = [basestring] + types_createDirectory = [six.string_types] def export_createDirectory(self, dir_path): """ Creates the directory on the storage @@ -292,7 +293,7 @@ def export_createDirectory(self, dir_path): gLogger.error("StorageElementHandler.createDirectory: %s" % errStr, repr(x)) return S_ERROR(errStr) - types_listDirectory = [basestring, basestring] + types_listDirectory = [six.string_types, six.string_types] def export_listDirectory(self, dir_path, mode): """ Return the dir_path directory listing @@ -430,7 +431,7 @@ def transfer_bulkToClient(self, fileId, token, fileHelper): gLogger.error('Failed to send bulk to network', res['Message']) return res - types_remove = [basestring, basestring] + types_remove = [six.string_types, six.string_types] def export_remove(self, fileID, token): """ Remove fileID from the storage. token is used for access rights confirmation. """ @@ -452,7 +453,7 @@ def __removeFile(self, fileID, token): else: return S_ERROR('File removal %s not authorized' % fileID) - types_getDirectorySize = [basestring] + types_getDirectorySize = [six.string_types] def export_getDirectorySize(self, fileID): """ Get the size occupied by the given directory @@ -468,7 +469,7 @@ def export_getDirectorySize(self, fileID): else: return S_ERROR("Directory does not exists") - types_removeDirectory = [basestring, basestring] + types_removeDirectory = [six.string_types, six.string_types] def export_removeDirectory(self, fileID, token): """ Remove the given directory from the storage @@ -489,7 +490,7 @@ def export_removeDirectory(self, fileID, token): gLogger.error(str(error)) return S_ERROR("Failed to remove directory %s" % dir_path) - types_removeFileList = [list, basestring] + types_removeFileList = [list, six.string_types] def export_removeFileList(self, fileList, token): """ Remove files in the given list From 5a665093296ae3831c7182dc1ebda5347c833606 Mon Sep 17 00:00:00 2001 From: Chris Burr Date: Sat, 15 Aug 2020 14:36:53 +0200 Subject: [PATCH 11/20] PY3: Remove MySQLdb.server_init if using mysqlclient --- Core/Utilities/MySQL.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/Core/Utilities/MySQL.py b/Core/Utilities/MySQL.py index 4e601370d13..8b14e2ce823 100755 --- a/Core/Utilities/MySQL.py +++ b/Core/Utilities/MySQL.py @@ -161,11 +161,15 @@ from DIRAC.Core.Utilities.Time import fromString from DIRAC.Core.Utilities import DErrno -# This is for proper initialization of embedded server, it should only be called once -try: - MySQLdb.server_init(['--defaults-file=/opt/dirac/etc/my.cnf', '--datadir=/opt/mysql/db'], ['mysqld']) -except MySQLdb.ProgrammingError: - pass +# The mysql_server_init function called here is deprecated and not used by mysqlclient +# https://dev.mysql.com/doc/c-api/8.0/en/mysql-server-init.html +# TODO: Check if any changes are needed +if six.PY2: + # This is for proper initialization of embedded server, it should only be called once + try: + MySQLdb.server_init(['--defaults-file=/opt/dirac/etc/my.cnf', '--datadir=/opt/mysql/db'], ['mysqld']) + except MySQLdb.ProgrammingError: + pass gInstancesCount = 0 From 7ece6c37e1637ff05f1588ddf3c3b82919214846 Mon Sep 17 00:00:00 2001 From: Chris Burr Date: Sun, 16 Aug 2020 15:17:20 +0200 Subject: [PATCH 12/20] PY3: Replace StringIO with six.StringIO and six.BytesIO --- Core/DISET/private/FileHelper.py | 8 ++++---- Core/DISET/private/GatewayService.py | 6 +++--- Core/DISET/private/Transports/BaseTransport.py | 4 ++-- Core/Utilities/Plotting/Plots.py | 7 ++----- Core/Utilities/PrettyPrint.py | 4 ++-- DataManagementSystem/Service/FileCatalogHandler.py | 4 ++-- FrameworkSystem/Client/BundleDeliveryClient.py | 4 ++-- FrameworkSystem/Service/BundleDeliveryHandler.py | 6 +++--- .../private/standardLogging/test/TestLogUtilities.py | 2 +- .../test/Test_LoggingRoot_ConfigForExternalLibs.py | 2 +- Interfaces/API/Dirac.py | 4 ++-- Interfaces/API/Job.py | 4 ++-- Interfaces/API/test/Test_JobAPI.py | 4 ++-- TransformationSystem/Client/TaskManager.py | 4 ++-- TransformationSystem/test/Test_JobInfo.py | 2 +- WorkloadManagementSystem/Client/SandboxStoreClient.py | 8 ++++---- WorkloadManagementSystem/Client/WMSClient.py | 4 ++-- .../Client/test/Test_Client_WorkloadManagementSystem.py | 4 ++-- 18 files changed, 39 insertions(+), 42 deletions(-) diff --git a/Core/DISET/private/FileHelper.py b/Core/DISET/private/FileHelper.py index 55230abaed2..5f29f8647e2 100755 --- a/Core/DISET/private/FileHelper.py +++ b/Core/DISET/private/FileHelper.py @@ -7,7 +7,7 @@ import os import hashlib import threading -import cStringIO +from six import StringIO import tarfile import tempfile @@ -132,7 +132,7 @@ def networkToString(self, maxFileSize=0): """ Receive the input from a DISET client and return it as a string """ - stringIO = cStringIO.StringIO() + stringIO = StringIO() result = self.networkToDataSink(stringIO, maxFileSize=maxFileSize) if not result['OK']: return result @@ -184,7 +184,7 @@ def stringToNetwork(self, stringVal): """ Send a given string to the DISET client over the network """ - stringIO = cStringIO.StringIO(stringVal) + stringIO = StringIO(stringVal) iPacketSize = self.packetSize ioffset = 0 @@ -234,7 +234,7 @@ def FDToNetwork(self, iFD): return S_OK() def BufferToNetwork(self, stringToSend): - sIO = cStringIO.StringIO(stringToSend) + sIO = StringIO(stringToSend) try: return self.DataSourceToNetwork(sIO) finally: diff --git a/Core/DISET/private/GatewayService.py b/Core/DISET/private/GatewayService.py index 809bff37501..bca7c39119a 100644 --- a/Core/DISET/private/GatewayService.py +++ b/Core/DISET/private/GatewayService.py @@ -20,7 +20,7 @@ import sys -import cStringIO +from six import BytesIO import os # TODO: Remove ThreadPool later @@ -312,7 +312,7 @@ def errMsg(self, msg, dynMsg=""): gLogger.error("[%s] %s" % (self.__currentMethod, msg), dynMsg) def getDataFromClient(self, clientFileHelper): - sIO = cStringIO.StringIO() + sIO = BytesIO() self.infoMsg("About to get data from client") result = clientFileHelper.networkToDataSink(sIO, self.__transferBytesLimit) if not result['OK']: @@ -363,7 +363,7 @@ def getDataFromService(self, srvMethod, params): _, srvTransport = result['Value'] srvFileHelper = FileHelper(srvTransport) srvFileHelper.setDirection("receive") - sIO = cStringIO.StringIO() + sIO = BytesIO() result = srvFileHelper.networkToDataSink(sIO, self.__transferBytesLimit) if not result['OK']: self.errMsg("Could not receive data from server", result['Message']) diff --git a/Core/DISET/private/Transports/BaseTransport.py b/Core/DISET/private/Transports/BaseTransport.py index c0835e44111..3b142119d82 100755 --- a/Core/DISET/private/Transports/BaseTransport.py +++ b/Core/DISET/private/Transports/BaseTransport.py @@ -25,7 +25,7 @@ import time import select -import cStringIO +from six import BytesIO from hashlib import md5 from DIRAC.Core.Utilities.ReturnValues import S_ERROR, S_OK @@ -235,7 +235,7 @@ def receiveData(self, maxBufferSize=0, blockAfterKeepAlive=True, idleReceive=Fal self.byteStream = pkgData[pkgSize:] else: # If we still need to read stuff - pkgMem = cStringIO.StringIO() + pkgMem = BytesIO() pkgMem.write(pkgData) # Receive while there's still data to be received while readSize < pkgSize: diff --git a/Core/Utilities/Plotting/Plots.py b/Core/Utilities/Plotting/Plots.py index 7dce87ba74b..7d31517dc48 100644 --- a/Core/Utilities/Plotting/Plots.py +++ b/Core/Utilities/Plotting/Plots.py @@ -5,10 +5,7 @@ from __future__ import division from __future__ import print_function -try: - from StringIO import StringIO -except ImportError: - from io import BytesIO as StringIO +from six import BytesIO import errno from DIRAC.Core.Utilities.Graphs import barGraph, lineGraph, pieGraph, qualityGraph, textGraph, histogram @@ -53,7 +50,7 @@ def generateErrorMessagePlot(msgText): :param str msgText: the text which will appear on the plot. :return: the plot. """ - fn = StringIO() + fn = BytesIO() textGraph(msgText, fn, {}) data = fn.getvalue() fn.close() diff --git a/Core/Utilities/PrettyPrint.py b/Core/Utilities/PrettyPrint.py index 957d129947a..22a3d394d9f 100644 --- a/Core/Utilities/PrettyPrint.py +++ b/Core/Utilities/PrettyPrint.py @@ -11,7 +11,7 @@ __RCSID__ = '$Id$' import six -import StringIO +from six import StringIO def int_with_commas(inputValue): @@ -159,7 +159,7 @@ def __writeField(buffer, value, length, columnSeparator, lastColumn=False): totalLength += (numberWidth + separatorWidth) # Accumulate the table output in the stringBuffer now - stringBuffer = StringIO.StringIO() + stringBuffer = StringIO() topLength = (numberWidth + separatorWidth) if numbering else 0 stringBuffer.write(' ' * (topLength)) diff --git a/DataManagementSystem/Service/FileCatalogHandler.py b/DataManagementSystem/Service/FileCatalogHandler.py index ed15f22e31c..0d1123e7983 100644 --- a/DataManagementSystem/Service/FileCatalogHandler.py +++ b/DataManagementSystem/Service/FileCatalogHandler.py @@ -16,7 +16,7 @@ # imports import six -import cStringIO +from six import StringIO import csv import os from types import IntType, LongType, DictType, StringTypes, BooleanType, ListType @@ -759,7 +759,7 @@ def transfer_toClient(self, seName, token, fileHelper): retVal = self.getSEDump(seName) try: - csvOutput = cStringIO.StringIO() + csvOutput = StringIO() writer = csv.writer(csvOutput, delimiter='|') for lfn in retVal: writer.writerow(lfn) diff --git a/FrameworkSystem/Client/BundleDeliveryClient.py b/FrameworkSystem/Client/BundleDeliveryClient.py index 4fb1d6d26f8..34b1ae103cb 100644 --- a/FrameworkSystem/Client/BundleDeliveryClient.py +++ b/FrameworkSystem/Client/BundleDeliveryClient.py @@ -7,7 +7,7 @@ import os import io import tarfile -import cStringIO +from six import BytesIO from DIRAC import S_OK, gLogger from DIRAC.Core.Base.Client import Client, createClient @@ -59,7 +59,7 @@ def syncDir(self, bundleID, dirToSyncTo): dirCreated = True currentHash = self.__getHash(bundleID, dirToSyncTo) self.log.info("Current hash for bundle %s in dir %s is '%s'" % (bundleID, dirToSyncTo, currentHash)) - buff = cStringIO.StringIO() + buff = BytesIO() transferClient = self.__getTransferClient() result = transferClient.receiveFile(buff, [bundleID, currentHash]) if not result['OK']: diff --git a/FrameworkSystem/Service/BundleDeliveryHandler.py b/FrameworkSystem/Service/BundleDeliveryHandler.py index 0f7be9b40ab..a3b02b84a90 100644 --- a/FrameworkSystem/Service/BundleDeliveryHandler.py +++ b/FrameworkSystem/Service/BundleDeliveryHandler.py @@ -8,7 +8,7 @@ __RCSID__ = "$Id$" import six -import cStringIO +from six import BytesIO import tarfile import os from DIRAC.Core.DISET.RequestHandler import RequestHandler @@ -72,7 +72,7 @@ def updateBundles(self): for bId in dirsToBundle: bundlePaths = dirsToBundle[bId] gLogger.info("Updating %s bundle %s" % (bId, bundlePaths)) - buffer_ = cStringIO.StringIO() + buffer_ = BytesIO() filesToBundle = sorted(File.getGlobbedFiles(bundlePaths)) if filesToBundle: commonPath = File.getCommonPath(filesToBundle) @@ -141,7 +141,7 @@ def transfer_toClient(self, fileId, token, fileHelper): fileHelper.markAsTransferred() return S_OK(bundleVersion) - buffer_ = cStringIO.StringIO(gBundleManager.getBundleData(bId)) + buffer_ = BytesIO(gBundleManager.getBundleData(bId)) result = fileHelper.DataSourceToNetwork(buffer_) buffer_.close() if not result['OK']: diff --git a/FrameworkSystem/private/standardLogging/test/TestLogUtilities.py b/FrameworkSystem/private/standardLogging/test/TestLogUtilities.py index e50449a811d..625bdf62678 100644 --- a/FrameworkSystem/private/standardLogging/test/TestLogUtilities.py +++ b/FrameworkSystem/private/standardLogging/test/TestLogUtilities.py @@ -5,7 +5,7 @@ __RCSID__ = "$Id$" import logging -from StringIO import StringIO +from six import StringIO from DIRAC.FrameworkSystem.private.standardLogging.LoggingRoot import LoggingRoot from DIRAC.FrameworkSystem.private.standardLogging.Logging import Logging diff --git a/FrameworkSystem/private/standardLogging/test/Test_LoggingRoot_ConfigForExternalLibs.py b/FrameworkSystem/private/standardLogging/test/Test_LoggingRoot_ConfigForExternalLibs.py index 5f2e2ab7f56..30388f894b4 100644 --- a/FrameworkSystem/private/standardLogging/test/Test_LoggingRoot_ConfigForExternalLibs.py +++ b/FrameworkSystem/private/standardLogging/test/Test_LoggingRoot_ConfigForExternalLibs.py @@ -9,7 +9,7 @@ __RCSID__ = "$Id$" import logging -from StringIO import StringIO +from six import StringIO import pytest from DIRAC.FrameworkSystem.private.standardLogging.test.TestLogUtilities import gLogger, cleaningLog, gLoggerReset diff --git a/Interfaces/API/Dirac.py b/Interfaces/API/Dirac.py index d64b4819607..c0be4ef4ebd 100755 --- a/Interfaces/API/Dirac.py +++ b/Interfaces/API/Dirac.py @@ -31,7 +31,7 @@ import tarfile import urllib import shlex -import StringIO +from six import StringIO import DIRAC from DIRAC import gConfig, gLogger, S_OK, S_ERROR @@ -353,7 +353,7 @@ def submitJob(self, job, mode='wms'): self.log.error(msg) return S_ERROR(msg) - jobDescriptionObject = StringIO.StringIO(job._toXML()) # pylint: disable=protected-access + jobDescriptionObject = StringIO(job._toXML()) # pylint: disable=protected-access jdlAsString = job._toJDL(jobDescriptionObject=jobDescriptionObject) # pylint: disable=protected-access if mode.lower() == 'local': diff --git a/Interfaces/API/Job.py b/Interfaces/API/Job.py index 7f1a1ccfce2..db1ee72ea5f 100755 --- a/Interfaces/API/Job.py +++ b/Interfaces/API/Job.py @@ -33,7 +33,7 @@ import os import urllib import shlex -import StringIO +from six import StringIO from DIRAC import S_OK, gLogger from DIRAC.Core.Base.API import API @@ -1127,7 +1127,7 @@ def _toJDL(self, xmlFile='', jobDescriptionObject=None): self.log.warn("Job description XML file not found") self.addToInputSandbox.append(scriptName) - elif isinstance(jobDescriptionObject, StringIO.StringIO): + elif isinstance(jobDescriptionObject, StringIO): self.log.verbose("jobDescription is passed in as a StringIO object") else: diff --git a/Interfaces/API/test/Test_JobAPI.py b/Interfaces/API/test/Test_JobAPI.py index 4543d7b0be0..8107d4c0bc8 100644 --- a/Interfaces/API/test/Test_JobAPI.py +++ b/Interfaces/API/test/Test_JobAPI.py @@ -7,7 +7,7 @@ __RCSID__ = "$Id$" import pytest -import StringIO +from six import StringIO from DIRAC.Interfaces.API.Job import Job from DIRAC.Core.Utilities.ClassAd.ClassAdLight import ClassAd @@ -42,7 +42,7 @@ def test_basicJob(): with open('./Interfaces/API/test/testWFSIO.jdl') as fd: expected = fd.read() - jdlSIO = job._toJDL(jobDescriptionObject=StringIO.StringIO(job._toXML())) + jdlSIO = job._toJDL(jobDescriptionObject=StringIO(job._toXML())) assert jdlSIO == expected diff --git a/TransformationSystem/Client/TaskManager.py b/TransformationSystem/Client/TaskManager.py index f7bdc22d557..b7a8439f936 100644 --- a/TransformationSystem/Client/TaskManager.py +++ b/TransformationSystem/Client/TaskManager.py @@ -11,7 +11,7 @@ import six import time -import StringIO +from six import StringIO import json import copy import os @@ -991,7 +991,7 @@ def submitTaskToExternal(self, job): self._logError("No valid job description found") return S_ERROR("No valid job description found") - workflowFileObject = StringIO.StringIO(oJob._toXML()) + workflowFileObject = StringIO(oJob._toXML()) jdl = oJob._toJDL(jobDescriptionObject=workflowFileObject) return self.submissionClient.submitJob(jdl, workflowFileObject) diff --git a/TransformationSystem/test/Test_JobInfo.py b/TransformationSystem/test/Test_JobInfo.py index 49531ba350f..ca79cf284bd 100644 --- a/TransformationSystem/test/Test_JobInfo.py +++ b/TransformationSystem/test/Test_JobInfo.py @@ -5,7 +5,7 @@ import unittest import sys -from StringIO import StringIO +from six import StringIO from mock import MagicMock as Mock diff --git a/WorkloadManagementSystem/Client/SandboxStoreClient.py b/WorkloadManagementSystem/Client/SandboxStoreClient.py index 7e7551339ef..0819b9f2a31 100755 --- a/WorkloadManagementSystem/Client/SandboxStoreClient.py +++ b/WorkloadManagementSystem/Client/SandboxStoreClient.py @@ -13,7 +13,7 @@ import hashlib import tempfile import re -import StringIO +from six import BytesIO from DIRAC import gLogger, S_OK, S_ERROR, gConfig @@ -99,7 +99,7 @@ def uploadFilesAsSandbox(self, fileList, sizeLimit=0, assignTo=None): a fileList item can be: - a string, which is an lfn name - a file name (real), that is supposed to be on disk, in the current directory - - a fileObject that should be a StringIO.StringIO type of object + - a fileObject that should be a BytesIO type of object Parameters: - assignTo : Dict containing { 'Job:' : '', ... } @@ -126,7 +126,7 @@ def uploadFilesAsSandbox(self, fileList, sizeLimit=0, assignTo=None): else: errorFiles.append(sFile) - elif isinstance(sFile, StringIO.StringIO): + elif isinstance(sFile, BytesIO): files2Upload.append(sFile) else: return S_ERROR("Objects of type %s can't be part of InputSandbox" % type(sFile)) @@ -144,7 +144,7 @@ def uploadFilesAsSandbox(self, fileList, sizeLimit=0, assignTo=None): for sFile in files2Upload: if isinstance(sFile, six.string_types): tf.add(os.path.realpath(sFile), os.path.basename(sFile), recursive=True) - elif isinstance(sFile, StringIO.StringIO): + elif isinstance(sFile, BytesIO): tarInfo = tarfile.TarInfo(name='jobDescription.xml') tarInfo.size = len(sFile.buf) tf.addfile(tarinfo=tarInfo, fileobj=sFile) diff --git a/WorkloadManagementSystem/Client/WMSClient.py b/WorkloadManagementSystem/Client/WMSClient.py index 2d8a66b9d26..311e496ce5a 100755 --- a/WorkloadManagementSystem/Client/WMSClient.py +++ b/WorkloadManagementSystem/Client/WMSClient.py @@ -6,7 +6,7 @@ from __future__ import print_function import os -import StringIO +from six import StringIO import time from DIRAC import S_OK, S_ERROR, gLogger @@ -97,7 +97,7 @@ def __uploadInputSandbox(self, classAdJob, jobDescriptionObject=None): stringIOFiles = [] stringIOFilesSize = 0 if jobDescriptionObject is not None: - if isinstance(jobDescriptionObject, StringIO.StringIO): + if isinstance(jobDescriptionObject, StringIO): stringIOFiles = [jobDescriptionObject] stringIOFilesSize = len(jobDescriptionObject.buf) gLogger.debug("Size of the stringIOFiles: " + str(stringIOFilesSize)) diff --git a/WorkloadManagementSystem/Client/test/Test_Client_WorkloadManagementSystem.py b/WorkloadManagementSystem/Client/test/Test_Client_WorkloadManagementSystem.py index 1cbbf06f7c6..427924f80d3 100644 --- a/WorkloadManagementSystem/Client/test/Test_Client_WorkloadManagementSystem.py +++ b/WorkloadManagementSystem/Client/test/Test_Client_WorkloadManagementSystem.py @@ -8,7 +8,7 @@ import os import unittest import importlib -import StringIO +from six import BytesIO from mock import MagicMock @@ -99,7 +99,7 @@ def test_uploadFilesAsSandbox(self): ourSSC = importlib.import_module('DIRAC.WorkloadManagementSystem.Client.SandboxStoreClient') ourSSC.TransferClient = MagicMock() ssc = SandboxStoreClient() - fileList = [StringIO.StringIO('try')] + fileList = [BytesIO('try')] res = ssc.uploadFilesAsSandbox(fileList) print(res) From 69f058b83543141b1fc41178ed9db5d681696367 Mon Sep 17 00:00:00 2001 From: Chris Burr Date: Sun, 16 Aug 2020 15:17:49 +0200 Subject: [PATCH 13/20] PY3: Fix string type checking in Core/Utilities/Time.py --- Core/Utilities/Time.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Core/Utilities/Time.py b/Core/Utilities/Time.py index 634b20252e0..5729692e244 100755 --- a/Core/Utilities/Time.py +++ b/Core/Utilities/Time.py @@ -30,7 +30,7 @@ from __future__ import division import time as nativetime import datetime -from types import StringTypes +import six import sys __RCSID__ = "$Id$" @@ -194,7 +194,7 @@ def fromString(myDate=None): See notice on toString method On Error, return None """ - if StringTypes.__contains__(type(myDate)): + if isinstance(myDate, six.string_types): if myDate.find(' ') > 0: dateTimeTuple = myDate.split(' ') dateTuple = dateTimeTuple[0].split('-') From 8a4eaf1eccc49a1de9f17aaafeb402320f850b7b Mon Sep 17 00:00:00 2001 From: Chris Burr Date: Sun, 16 Aug 2020 17:28:37 +0200 Subject: [PATCH 14/20] PY3: Fallback to subprocess.getstatusoutput for commands.getstatusoutput --- Core/scripts/dirac-externals-requirements.py | 7 ++++++- DataManagementSystem/Client/FileCatalogClientCLI.py | 7 ++++++- FrameworkSystem/Service/SystemAdministratorHandler.py | 7 ++++++- Resources/Computing/BatchSystems/Condor.py | 7 ++++++- Resources/Computing/BatchSystems/GE.py | 7 ++++++- Resources/Computing/BatchSystems/Host.py | 7 ++++++- Resources/Computing/BatchSystems/LSF.py | 7 ++++++- Resources/Computing/BatchSystems/OAR.py | 7 ++++++- Resources/Computing/BatchSystems/SLURM.py | 7 ++++++- Resources/Computing/BatchSystems/Torque.py | 7 ++++++- Resources/Computing/HTCondorCEComputingElement.py | 7 ++++++- Resources/ProxyProvider/test/Test_DIRACCAProxyProvider.py | 7 ++++++- tests/Integration/Framework/Test_ProxyDB.py | 7 ++++++- 13 files changed, 78 insertions(+), 13 deletions(-) diff --git a/Core/scripts/dirac-externals-requirements.py b/Core/scripts/dirac-externals-requirements.py index 61013149cea..61c11f53f34 100755 --- a/Core/scripts/dirac-externals-requirements.py +++ b/Core/scripts/dirac-externals-requirements.py @@ -15,7 +15,12 @@ import os import sys -import commands +# TODO: This should be moderised to use subprocess(32) +try: + import commands +except ImportError: + # Python 3's subprocess module contains a compatability layer + import subprocess as commands from diraccfg import CFG from DIRAC.Core.Base import Script diff --git a/DataManagementSystem/Client/FileCatalogClientCLI.py b/DataManagementSystem/Client/FileCatalogClientCLI.py index d3f1b33653c..9a264bfd78a 100644 --- a/DataManagementSystem/Client/FileCatalogClientCLI.py +++ b/DataManagementSystem/Client/FileCatalogClientCLI.py @@ -6,7 +6,12 @@ from __future__ import division __RCSID__ = "$Id$" -import commands +# TODO: This should be moderised to use subprocess(32) +try: + import commands +except ImportError: + # Python 3's subprocess module contains a compatability layer + import subprocess as commands import os.path import time import sys diff --git a/FrameworkSystem/Service/SystemAdministratorHandler.py b/FrameworkSystem/Service/SystemAdministratorHandler.py index 73e0f59b550..715e8fe43ab 100644 --- a/FrameworkSystem/Service/SystemAdministratorHandler.py +++ b/FrameworkSystem/Service/SystemAdministratorHandler.py @@ -11,7 +11,12 @@ import os import re import time -import commands +# TODO: This should be moderised to use subprocess(32) +try: + import commands +except ImportError: + # Python 3's subprocess module contains a compatability layer + import subprocess as commands import getpass import importlib import shutil diff --git a/Resources/Computing/BatchSystems/Condor.py b/Resources/Computing/BatchSystems/Condor.py index 142ba9ee5af..f5e27c8b4d0 100644 --- a/Resources/Computing/BatchSystems/Condor.py +++ b/Resources/Computing/BatchSystems/Condor.py @@ -14,7 +14,12 @@ from __future__ import division import re import tempfile -import commands +# TODO: This should be moderised to use subprocess(32) +try: + import commands +except ImportError: + # Python 3's subprocess module contains a compatability layer + import subprocess as commands import os __RCSID__ = "$Id$" diff --git a/Resources/Computing/BatchSystems/GE.py b/Resources/Computing/BatchSystems/GE.py index e3d4c672c33..53b3f80c76b 100644 --- a/Resources/Computing/BatchSystems/GE.py +++ b/Resources/Computing/BatchSystems/GE.py @@ -18,7 +18,12 @@ from __future__ import absolute_import from __future__ import division import re -import commands +# TODO: This should be moderised to use subprocess(32) +try: + import commands +except ImportError: + # Python 3's subprocess module contains a compatability layer + import subprocess as commands import os __RCSID__ = "$Id$" diff --git a/Resources/Computing/BatchSystems/Host.py b/Resources/Computing/BatchSystems/Host.py index 67b72391f7b..1005017753c 100644 --- a/Resources/Computing/BatchSystems/Host.py +++ b/Resources/Computing/BatchSystems/Host.py @@ -11,7 +11,12 @@ from __future__ import print_function from __future__ import absolute_import from __future__ import division -import commands +# TODO: This should be moderised to use subprocess(32) +try: + import commands +except ImportError: + # Python 3's subprocess module contains a compatability layer + import subprocess as commands import os import glob import shutil diff --git a/Resources/Computing/BatchSystems/LSF.py b/Resources/Computing/BatchSystems/LSF.py index a7c4f725415..a93cb9e057f 100644 --- a/Resources/Computing/BatchSystems/LSF.py +++ b/Resources/Computing/BatchSystems/LSF.py @@ -13,7 +13,12 @@ from __future__ import absolute_import from __future__ import division import re -import commands +# TODO: This should be moderised to use subprocess(32) +try: + import commands +except ImportError: + # Python 3's subprocess module contains a compatability layer + import subprocess as commands import os __RCSID__ = "$Id$" diff --git a/Resources/Computing/BatchSystems/OAR.py b/Resources/Computing/BatchSystems/OAR.py index 822643652b6..76c0dd6ce6b 100644 --- a/Resources/Computing/BatchSystems/OAR.py +++ b/Resources/Computing/BatchSystems/OAR.py @@ -12,7 +12,12 @@ from __future__ import print_function from __future__ import absolute_import from __future__ import division -import commands +# TODO: This should be moderised to use subprocess(32) +try: + import commands +except ImportError: + # Python 3's subprocess module contains a compatability layer + import subprocess as commands import os import json diff --git a/Resources/Computing/BatchSystems/SLURM.py b/Resources/Computing/BatchSystems/SLURM.py index 8b0fba0e95d..adc542a05bc 100644 --- a/Resources/Computing/BatchSystems/SLURM.py +++ b/Resources/Computing/BatchSystems/SLURM.py @@ -12,7 +12,12 @@ from __future__ import print_function from __future__ import absolute_import from __future__ import division -import commands +# TODO: This should be moderised to use subprocess(32) +try: + import commands +except ImportError: + # Python 3's subprocess module contains a compatability layer + import subprocess as commands import os import re diff --git a/Resources/Computing/BatchSystems/Torque.py b/Resources/Computing/BatchSystems/Torque.py index e84c69f30b6..1f7737b2eba 100644 --- a/Resources/Computing/BatchSystems/Torque.py +++ b/Resources/Computing/BatchSystems/Torque.py @@ -12,7 +12,12 @@ from __future__ import print_function from __future__ import absolute_import from __future__ import division -import commands +# TODO: This should be moderised to use subprocess(32) +try: + import commands +except ImportError: + # Python 3's subprocess module contains a compatability layer + import subprocess as commands import os __RCSID__ = "$Id$" diff --git a/Resources/Computing/HTCondorCEComputingElement.py b/Resources/Computing/HTCondorCEComputingElement.py index aec0bd3f1d5..2b3df5310df 100644 --- a/Resources/Computing/HTCondorCEComputingElement.py +++ b/Resources/Computing/HTCondorCEComputingElement.py @@ -48,7 +48,12 @@ import six import os import tempfile -import commands +# TODO: This should be moderised to use subprocess(32) +try: + import commands +except ImportError: + # Python 3's subprocess module contains a compatability layer + import subprocess as commands import errno from DIRAC import S_OK, S_ERROR, gConfig diff --git a/Resources/ProxyProvider/test/Test_DIRACCAProxyProvider.py b/Resources/ProxyProvider/test/Test_DIRACCAProxyProvider.py index 0fa3428d5ad..6dc307bfbee 100644 --- a/Resources/ProxyProvider/test/Test_DIRACCAProxyProvider.py +++ b/Resources/ProxyProvider/test/Test_DIRACCAProxyProvider.py @@ -9,7 +9,12 @@ import re import sys import shutil -import commands +# TODO: This should be moderised to use subprocess(32) +try: + import commands +except ImportError: + # Python 3's subprocess module contains a compatability layer + import subprocess as commands import unittest import tempfile diff --git a/tests/Integration/Framework/Test_ProxyDB.py b/tests/Integration/Framework/Test_ProxyDB.py index d2047b90b87..a8f557d97f5 100644 --- a/tests/Integration/Framework/Test_ProxyDB.py +++ b/tests/Integration/Framework/Test_ProxyDB.py @@ -12,7 +12,12 @@ import stat import shutil import tempfile -import commands +# TODO: This should be moderised to use subprocess(32) +try: + import commands +except ImportError: + # Python 3's subprocess module contains a compatability layer + import subprocess as commands import unittest from diraccfg import CFG From c8c1f9d69f944052dae801c4df3fc12463ecf565 Mon Sep 17 00:00:00 2001 From: Chris Burr Date: Sat, 15 Aug 2020 19:57:17 +0200 Subject: [PATCH 15/20] Run pytest discovery with Python 3 --- .github/workflows/basic-python3.yml | 33 +++++++++++++ environment-py3.yml | 72 +++++++++++++++++++++++++++++ 2 files changed, 105 insertions(+) create mode 100644 .github/workflows/basic-python3.yml create mode 100644 environment-py3.yml diff --git a/.github/workflows/basic-python3.yml b/.github/workflows/basic-python3.yml new file mode 100644 index 00000000000..ffa298a4c81 --- /dev/null +++ b/.github/workflows/basic-python3.yml @@ -0,0 +1,33 @@ +name: Python 3 tests + +on: [push, pull_request] + +jobs: + check: + runs-on: ubuntu-latest + if: github.event_name != 'push' || github.repository == 'DIRACGrid/DIRAC' + timeout-minutes: 30 + + strategy: + fail-fast: False + matrix: + command: + # Only collect tests for now + - pytest --no-cov --collect-only + + steps: + - uses: actions/checkout@v2 + - name: Fail-fast for outdated pipelines + run: .github/workflows/fail-fast.sh + - name: Prepare environment + run: | + conda env create --name dirac-testing environment-py3.yml + - name: Run tests + run: | + source "${CONDA}/bin/activate" + conda activate dirac-testing + set -euxo pipefail + export PYTHONPATH=${PWD%/*} + ${{ matrix.command }} + env: + REFERENCE_BRANCH: ${{ github['base_ref'] || github['head_ref'] }} diff --git a/environment-py3.yml b/environment-py3.yml new file mode 100644 index 00000000000..f0fb247aa52 --- /dev/null +++ b/environment-py3.yml @@ -0,0 +1,72 @@ +name: dirac-development + +channels: + - diracgrid + - conda-forge + - defaults + +dependencies: + # runtime + - python =3.8 + - pip + - boto3 + - certifi + - cmreshandler >1.0.0b4 + - docutils + - elasticsearch-dsl + - future + - gitpython >=2.1.0 + - m2crypto >=0.36 + - matplotlib + - numpy + - pexpect >=4.0.1 + - pillow + - psutil >=4.2.0 + - pyasn1 >0.4.1 + - pyasn1-modules + - python-json-logger >=0.1.8 + - pytz >=2015.7 + - recommonmark + - requests >=2.9.1 + - six >=1.10 + - sqlalchemy >=1.0.9 + - stomp.py =4.1.23 + - suds-jurko >=0.6 + - xmltodict + # testing and development + - autopep8 + - caniusepython3 + - coverage + - hypothesis + - ipython + - mock + - parameterized + - pylint >=1.6.5 + - pyparsing >=2.0.6 + - pytest >=3.6 + - pytest-cov >=2.2.0 + - pytest-mock + - shellcheck + # docs + - pygments >=1.5 + - sphinx + # unused + - funcsigs + - jinja2 + # - readline >=6.2.4 in the standard library + - simplejson >=3.8.1 + #- tornado >=5.0.0,<6.0.0 + - typing >=3.6.6 + - openssl + - mysqlclient + - pip: + - diraccfg + # This is a fork of tornado with a patch to allow for configurable iostream + # It should eventually be part of DIRACGrid + - git+https://github.com/chaen/tornado.git@iostreamConfigurable + # This is an extension of Tornado to use M2Crypto + # It should eventually be part of DIRACGrid + - git+https://github.com/chaen/tornado_m2crypto + # fts-rest doesn't support Python 3, this branch changed has just enough to + # make it importing possible + - git+https://gitlab.cern.ch/cburr/fts-rest.git@python3 From 80e5466ca0d5441c75d46dceddca31a75975bdba Mon Sep 17 00:00:00 2001 From: Chris Burr Date: Mon, 17 Aug 2020 05:17:53 +0200 Subject: [PATCH 16/20] Fix pycodestyle indentation errors --- .../scripts/dirac-dms-create-removal-request.py | 4 ++-- .../scripts/dirac-dms-move-replica-request.py | 4 ++-- FrameworkSystem/Service/PlottingHandler.py | 6 +++--- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/DataManagementSystem/scripts/dirac-dms-create-removal-request.py b/DataManagementSystem/scripts/dirac-dms-create-removal-request.py index 7617f636f08..f85b8e8f6f2 100755 --- a/DataManagementSystem/scripts/dirac-dms-create-removal-request.py +++ b/DataManagementSystem/scripts/dirac-dms-create-removal-request.py @@ -66,8 +66,8 @@ oRequest = Request() requestName = "%s_%s" % ( - md5(repr(time.time()).encode()).hexdigest()[:16], - md5(repr(time.time()).encode()).hexdigest()[:16], + md5(repr(time.time()).encode()).hexdigest()[:16], + md5(repr(time.time()).encode()).hexdigest()[:16], ) oRequest.RequestName = requestName diff --git a/DataManagementSystem/scripts/dirac-dms-move-replica-request.py b/DataManagementSystem/scripts/dirac-dms-move-replica-request.py index c861e4a4554..9fa7de43fe2 100755 --- a/DataManagementSystem/scripts/dirac-dms-move-replica-request.py +++ b/DataManagementSystem/scripts/dirac-dms-move-replica-request.py @@ -91,8 +91,8 @@ def getLFNList(arg): request = Request() request.RequestName = "%s_%s" % ( - md5(repr(time.time()).encode()).hexdigest()[:16], - md5(repr(time.time()).encode()).hexdigest()[:16], + md5(repr(time.time()).encode()).hexdigest()[:16], + md5(repr(time.time()).encode()).hexdigest()[:16], ) moveReplica = Operation() diff --git a/FrameworkSystem/Service/PlottingHandler.py b/FrameworkSystem/Service/PlottingHandler.py index ad446ed02f4..81657a4c02e 100644 --- a/FrameworkSystem/Service/PlottingHandler.py +++ b/FrameworkSystem/Service/PlottingHandler.py @@ -49,9 +49,9 @@ class PlottingHandler(RequestHandler): def __calculatePlotHash(self, data, metadata, subplotMetadata): m = hashlib.md5() m.update(repr({ - 'Data': data, - 'PlotMetadata': metadata, - 'SubplotMetadata': subplotMetadata + 'Data': data, + 'PlotMetadata': metadata, + 'SubplotMetadata': subplotMetadata }).encode()) return m.hexdigest() From bfa7ae993b2da74b446dc005a1cbd5e1b3581404 Mon Sep 17 00:00:00 2001 From: Chris Burr Date: Mon, 17 Aug 2020 17:13:24 +0200 Subject: [PATCH 17/20] Spelling corrections from code review Co-authored-by: fstagni --- Core/scripts/dirac-externals-requirements.py | 2 +- Resources/ProxyProvider/test/Test_DIRACCAProxyProvider.py | 2 +- tests/Integration/Framework/Test_ProxyDB.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Core/scripts/dirac-externals-requirements.py b/Core/scripts/dirac-externals-requirements.py index 61c11f53f34..c32a6f53f26 100755 --- a/Core/scripts/dirac-externals-requirements.py +++ b/Core/scripts/dirac-externals-requirements.py @@ -19,7 +19,7 @@ try: import commands except ImportError: - # Python 3's subprocess module contains a compatability layer + # Python 3's subprocess module contains a compatibility layer import subprocess as commands from diraccfg import CFG diff --git a/Resources/ProxyProvider/test/Test_DIRACCAProxyProvider.py b/Resources/ProxyProvider/test/Test_DIRACCAProxyProvider.py index 6dc307bfbee..727dca04a27 100644 --- a/Resources/ProxyProvider/test/Test_DIRACCAProxyProvider.py +++ b/Resources/ProxyProvider/test/Test_DIRACCAProxyProvider.py @@ -9,7 +9,7 @@ import re import sys import shutil -# TODO: This should be moderised to use subprocess(32) +# TODO: This should be modernised to use subprocess(32) try: import commands except ImportError: diff --git a/tests/Integration/Framework/Test_ProxyDB.py b/tests/Integration/Framework/Test_ProxyDB.py index a8f557d97f5..797e5d59c19 100644 --- a/tests/Integration/Framework/Test_ProxyDB.py +++ b/tests/Integration/Framework/Test_ProxyDB.py @@ -12,7 +12,7 @@ import stat import shutil import tempfile -# TODO: This should be moderised to use subprocess(32) +# TODO: This should be modernised to use subprocess(32) try: import commands except ImportError: From fd23b1636bcf49dbc1a6d841a64c17d992b67f36 Mon Sep 17 00:00:00 2001 From: Chris Burr Date: Fri, 21 Aug 2020 04:54:26 +0200 Subject: [PATCH 18/20] Fix typo for test_types in Test_Encode.py --- Core/Utilities/test/Test_Encode.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/Core/Utilities/test/Test_Encode.py b/Core/Utilities/test/Test_Encode.py index ded7b1a7e99..5285be30d8b 100644 --- a/Core/Utilities/test/Test_Encode.py +++ b/Core/Utilities/test/Test_Encode.py @@ -324,13 +324,13 @@ def test_types(): import types as pythonTypes from DIRAC.Core.Utilities.DEncode import types as DIRACTypes - assert pythonTypes.IntType is pythonTypes.IntType - assert pythonTypes.LongType is pythonTypes.LongType - assert pythonTypes.FloatType is pythonTypes.FloatType - assert pythonTypes.BooleanType is pythonTypes.BooleanType - assert pythonTypes.StringType is pythonTypes.StringType - assert pythonTypes.UnicodeType is pythonTypes.UnicodeType - assert pythonTypes.NoneType is pythonTypes.NoneType - assert pythonTypes.ListType is pythonTypes.ListType - assert pythonTypes.TupleType is pythonTypes.TupleType - assert pythonTypes.DictType is pythonTypes.DictType + assert DIRACTypes.IntType is pythonTypes.IntType + assert DIRACTypes.LongType is pythonTypes.LongType + assert DIRACTypes.FloatType is pythonTypes.FloatType + assert DIRACTypes.BooleanType is pythonTypes.BooleanType + assert DIRACTypes.StringType is pythonTypes.StringType + assert DIRACTypes.UnicodeType is pythonTypes.UnicodeType + assert DIRACTypes.NoneType is pythonTypes.NoneType + assert DIRACTypes.ListType is pythonTypes.ListType + assert DIRACTypes.TupleType is pythonTypes.TupleType + assert DIRACTypes.DictType is pythonTypes.DictType From 5884f201b978df57ad803f1ed294594d1e0b5ef4 Mon Sep 17 00:00:00 2001 From: Chris Burr Date: Tue, 25 Aug 2020 18:33:53 +0200 Subject: [PATCH 19/20] Fix typo in .github/workflows/basic-python3.yml --- .github/workflows/basic-python3.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/basic-python3.yml b/.github/workflows/basic-python3.yml index ffa298a4c81..942ede068ce 100644 --- a/.github/workflows/basic-python3.yml +++ b/.github/workflows/basic-python3.yml @@ -21,7 +21,7 @@ jobs: run: .github/workflows/fail-fast.sh - name: Prepare environment run: | - conda env create --name dirac-testing environment-py3.yml + conda env create --name dirac-testing --file environment-py3.yml - name: Run tests run: | source "${CONDA}/bin/activate" From 305202c50a1cc2d4f76069048b89d5b6e2235180 Mon Sep 17 00:00:00 2001 From: Chris Burr Date: Tue, 25 Aug 2020 19:19:50 +0200 Subject: [PATCH 20/20] Add pycurl and subprocess32 to environment-py3.yml --- environment-py3.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/environment-py3.yml b/environment-py3.yml index f0fb247aa52..816b3fdfad7 100644 --- a/environment-py3.yml +++ b/environment-py3.yml @@ -33,6 +33,7 @@ dependencies: - stomp.py =4.1.23 - suds-jurko >=0.6 - xmltodict + - pycurl # testing and development - autopep8 - caniusepython3 @@ -61,6 +62,7 @@ dependencies: - mysqlclient - pip: - diraccfg + - subprocess32 # This is a fork of tornado with a patch to allow for configurable iostream # It should eventually be part of DIRACGrid - git+https://github.com/chaen/tornado.git@iostreamConfigurable