[v7r2] Run pytest test collection for Python 3

.github/workflows/basic-python3.yml

@@ -0,0 +1,33 @@
+name: Python 3 tests
+
+on: [push, pull_request]
+
+jobs:
+  check:
+    runs-on: ubuntu-latest
+    if: github.event_name != 'push' || github.repository == 'DIRACGrid/DIRAC'
+    timeout-minutes: 30
+
+    strategy:
+      fail-fast: False
+      matrix:
+        command:
+          # Only collect tests for now
+          - pytest --no-cov --collect-only
+
+    steps:
+    - uses: actions/checkout@v2
+    - name: Fail-fast for outdated pipelines
+      run: .github/workflows/fail-fast.sh
+    - name: Prepare environment
+      run: |
+        conda env create --name dirac-testing --file environment-py3.yml
+    - name: Run tests
+      run: |
+        source "${CONDA}/bin/activate"
+        conda activate dirac-testing
+        set -euxo pipefail
+        export PYTHONPATH=${PWD%/*}
+        ${{ matrix.command }}
+      env:
+        REFERENCE_BRANCH: ${{ github['base_ref'] || github['head_ref'] }}

AccountingSystem/private/MainReporter.py

@@ -46,8 +46,8 @@ def __calculateReportHash(self, reportRequest):
       epoch = requestToHash[key]
       requestToHash[key] = epoch - epoch % granularity
     md5Hash = hashlib.md5()
-    md5Hash.update(repr(requestToHash))
-    md5Hash.update(self.setup)
+    md5Hash.update(repr(requestToHash).encode())
+    md5Hash.update(self.setup.encode())
     return md5Hash.hexdigest()
 
   def generate(self, reportRequest, credDict):

AccountingSystem/scripts/dirac-accounting-decode-fileid.py

@@ -13,7 +13,7 @@
 
 import pprint
 import sys
-import urlparse
+from six.moves.urllib import parse as urlparse
 import cgi
 from DIRAC import gLogger
 from DIRAC.Core.Base import Script

ConfigurationSystem/Client/Helpers/Operations.py

@@ -74,7 +74,7 @@
 from __future__ import print_function
 
 import six
-import thread
+from six.moves import _thread as thread
 import os
 from diraccfg import CFG
 from DIRAC import S_OK, S_ERROR, gConfig

ConfigurationSystem/Client/Helpers/Resources.py

@@ -6,7 +6,7 @@
 
 __RCSID__ = "$Id$"
 
-import urlparse
+from six.moves.urllib import parse as urlparse
 from distutils.version import LooseVersion  # pylint: disable=no-name-in-module,import-error
 
 import six

ConfigurationSystem/Client/PathFinder.py

@@ -6,7 +6,7 @@
 
 __RCSID__ = "$Id$"
 
-import urlparse
+from six.moves.urllib import parse as urlparse
 
 from DIRAC.Core.Utilities import List
 from DIRAC.ConfigurationSystem.Client.ConfigurationData import gConfigurationData

ConfigurationSystem/Client/Utilities.py

@@ -17,7 +17,7 @@
 
 import re
 import socket
-from urlparse import urlparse
+from six.moves.urllib.parse import urlparse
 
 import six
 

ConfigurationSystem/private/ConfigurationData.py

@@ -7,7 +7,7 @@
 import os.path
 import zlib
 import zipfile
-import thread
+from six.moves import _thread as thread
 import time
 import DIRAC
 

ConfigurationSystem/private/Refresher.py

@@ -8,7 +8,7 @@
 __RCSID__ = "$Id$"
 
 import threading
-import thread
+from six.moves import _thread as thread
 import time
 import random
 from DIRAC.ConfigurationSystem.Client.ConfigurationData import gConfigurationData

ConfigurationSystem/scripts/dirac-admin-add-resources.py

@@ -16,7 +16,7 @@
 import re
 import os
 import shlex
-from urlparse import urlparse
+from six.moves.urllib.parse import urlparse
 
 from DIRAC.Core.Base import Script
 from DIRAC import gLogger, exit as DIRACExit, S_OK

Core/DISET/MessageClient.py

@@ -33,7 +33,7 @@ def _initialize(self):
 
   def __generateUniqueClientName(self):
     hashStr = ":".join((Time.toString(), str(random.random()), Network.getFQDN(), gLogger.getName()))
-    hexHash = md5(hashStr).hexdigest()
+    hexHash = md5(hashStr.encode()).hexdigest()
     return hexHash
 
   def setUniqueName(self, uniqueName):

Core/DISET/RequestHandler.py

@@ -9,6 +9,7 @@
 import os
 import time
 import psutil
+import six
 
 import DIRAC
 
@@ -344,7 +345,7 @@ def __checkExpectedArgumentTypes(self, method, args):
 #
 ####
 
-  __connectionCallbackTypes = {'new': [basestring, dict],
+  __connectionCallbackTypes = {'new': list(six.string_types) + [dict],
                                'connected': [],
                                'drop': []}
 
@@ -499,7 +500,7 @@ def export_ping(self):
 
     return S_OK(dInfo)
 
-  types_echo = [basestring]
+  types_echo = [six.string_types]
 
   @staticmethod
   def export_echo(data):

Core/DISET/private/BaseClient.py

@@ -9,7 +9,7 @@
 
 import six
 import time
-import thread
+from six.moves import _thread as thread
 import DIRAC
 from DIRAC.Core.DISET.private.Protocols import gProtocolDict
 from DIRAC.FrameworkSystem.Client.Logger import gLogger

Core/DISET/private/FileHelper.py

@@ -7,7 +7,7 @@
 import os
 import hashlib
 import threading
-import cStringIO
+from six import StringIO
 import tarfile
 import tempfile
 
@@ -132,7 +132,7 @@ def networkToString(self, maxFileSize=0):
     """ Receive the input from a DISET client and return it as a string
     """
 
-    stringIO = cStringIO.StringIO()
+    stringIO = StringIO()
     result = self.networkToDataSink(stringIO, maxFileSize=maxFileSize)
     if not result['OK']:
       return result
@@ -184,7 +184,7 @@ def stringToNetwork(self, stringVal):
     """ Send a given string to the DISET client over the network
     """
 
-    stringIO = cStringIO.StringIO(stringVal)
+    stringIO = StringIO(stringVal)
 
     iPacketSize = self.packetSize
     ioffset = 0
@@ -234,7 +234,7 @@ def FDToNetwork(self, iFD):
     return S_OK()
 
   def BufferToNetwork(self, stringToSend):
-    sIO = cStringIO.StringIO(stringToSend)
+    sIO = StringIO(stringToSend)
     try:
       return self.DataSourceToNetwork(sIO)
     finally:

Core/DISET/private/GatewayService.py

@@ -20,7 +20,7 @@
 
 
 import sys
-import cStringIO
+from six import BytesIO
 import os
 
 # TODO: Remove ThreadPool later
@@ -312,7 +312,7 @@ def errMsg(self, msg, dynMsg=""):
     gLogger.error("[%s] %s" % (self.__currentMethod, msg), dynMsg)
 
   def getDataFromClient(self, clientFileHelper):
-    sIO = cStringIO.StringIO()
+    sIO = BytesIO()
     self.infoMsg("About to get data from client")
     result = clientFileHelper.networkToDataSink(sIO, self.__transferBytesLimit)
     if not result['OK']:
@@ -363,7 +363,7 @@ def getDataFromService(self, srvMethod, params):
     _, srvTransport = result['Value']
     srvFileHelper = FileHelper(srvTransport)
     srvFileHelper.setDirection("receive")
-    sIO = cStringIO.StringIO()
+    sIO = BytesIO()
     result = srvFileHelper.networkToDataSink(sIO, self.__transferBytesLimit)
     if not result['OK']:
       self.errMsg("Could not receive data from server", result['Message'])

Core/DISET/private/Transports/BaseTransport.py

@@ -25,7 +25,7 @@
 
 import time
 import select
-import cStringIO
+from six import BytesIO
 from hashlib import md5
 
 from DIRAC.Core.Utilities.ReturnValues import S_ERROR, S_OK
@@ -55,7 +55,7 @@ def __init__(self, stServerAddress, bServerMode=False, **kwargs):
     self.remoteAddress = False
     self.appData = ""
     self.startedKeepAlives = set()
-    self.keepAliveId = md5(str(stServerAddress) + str(bServerMode)).hexdigest()
+    self.keepAliveId = md5((str(stServerAddress) + str(bServerMode)).encode()).hexdigest()
     self.receivedMessages = []
     self.sentKeepAlives = 0
     self.waitingForKeepAlivePong = False
@@ -235,7 +235,7 @@ def receiveData(self, maxBufferSize=0, blockAfterKeepAlive=True, idleReceive=Fal
         self.byteStream = pkgData[pkgSize:]
       else:
         # If we still need to read stuff
-        pkgMem = cStringIO.StringIO()
+        pkgMem = BytesIO()
         pkgMem.write(pkgData)
         # Receive while there's still data to be received
         while readSize < pkgSize:

Core/LCG/GGUSTicketsClient.py

@@ -4,7 +4,7 @@
 from __future__ import division
 from __future__ import print_function
 
-import urllib2
+from six.moves.urllib_error import URLError
 
 from suds import WebFault
 from suds.client import Client
@@ -76,7 +76,7 @@ def getTicketsList(self, siteName=None, startDate=None, endDate=None):
       ticketList = self.gclient.service.TicketGetList(query)
     except WebFault as e:
       return S_ERROR(e)
-    except urllib2.URLError as e:
+    except URLError as e:
       return S_ERROR(e)
 
     return self.globalStatistics(ticketList)

Core/Utilities/DEncode.py

@@ -19,7 +19,6 @@
 
 from past.builtins import long
 import six
-import types
 import datetime
 import os
 
@@ -31,6 +30,22 @@
 from pprint import pprint
 
 
+# This is a hack for Python 3 to make it possible to import DEncode
+# There is not point in porting DEncode to Python 3 as it will be removed as
+# part of the HTTPS transition.
+class types(object):
+    IntType = int
+    LongType = long if six.PY2 else int
+    FloatType = float
+    BooleanType = bool
+    StringType = str
+    UnicodeType = type(u"")
+    NoneType = type(None)
+    ListType = list
+    TupleType = tuple
+    DictType = dict
+
+
 # Setting this environment variable to any value will enable the dump of the debugging
 # call stack
 DIRAC_DEBUG_DENCODE_CALLSTACK = bool(os.environ.get('DIRAC_DEBUG_DENCODE_CALLSTACK', False))

Core/Utilities/File.py

@@ -78,13 +78,13 @@ def makeGuid(fileName=None):
   myMd5 = hashlib.md5()
   if fileName:
     try:
-      with open(fileName, 'r') as fd:
+      with open(fileName, 'rb') as fd:
         data = fd.read(10 * 1024 * 1024)
         myMd5.update(data)
     except BaseException:
       return None
   else:
-    myMd5.update(str(random.getrandbits(128)))
+    myMd5.update(str(random.getrandbits(128)).encode())
 
   md5HexString = myMd5.hexdigest().upper()
   return generateGuid(md5HexString, "MD5")
@@ -112,7 +112,7 @@ def generateGuid(checksum, checksumtype):
 
   # Failed to use the check sum, generate a new guid
   myMd5 = hashlib.md5()
-  myMd5.update(str(random.getrandbits(128)))
+  myMd5.update(str(random.getrandbits(128)).encode())
   md5HexString = myMd5.hexdigest()
   guid = "%s-%s-%s-%s-%s" % (md5HexString[0:8],
                              md5HexString[8:12],

Core/Utilities/Grid.py

@@ -147,7 +147,7 @@ def ldapsearchBDII(filt=None, attr=None, host=None, base=None, selectionString="
 def ldapSite(site, attr=None, host=None):
   """ Site information from bdii.
 
-:param  site: Site as it defined in GOCDB or part of it with globing, for example: \UKI-*
+:param  site: Site as it defined in GOCDB or part of it with globing, for example: \\UKI-*
 :return: standard DIRAC answer with Value equals to list of sites.
 
 Each site is dictionary which contains attributes of site.

Core/Utilities/LockRing.py

@@ -5,7 +5,7 @@
 import random
 import time
 import threading
-import thread
+from six.moves import _thread as thread
 from hashlib import md5
 
 from DIRAC.Core.Utilities.ReturnValues import S_ERROR, S_OK
@@ -21,10 +21,11 @@ def __init__(self):
     self.__events = {}
 
   def __genName(self, container):
-    name = md5(str(time.time() + random.random())).hexdigest()
+    # TODO: Shouldn't this be a UUID?
+    name = md5(str(time.time() + random.random()).encode()).hexdigest()
     retries = 10
     while name in container and retries:
-      name = md5(str(time.time() + random.random())).hexdigest()
+      name = md5(str(time.time() + random.random()).encode()).hexdigest()
       retries -= 1
     return name
 

Core/Utilities/MySQL.py

@@ -161,11 +161,15 @@
 from DIRAC.Core.Utilities.Time import fromString
 from DIRAC.Core.Utilities import DErrno
 
-# This is for proper initialization of embedded server, it should only be called once
-try:
-  MySQLdb.server_init(['--defaults-file=/opt/dirac/etc/my.cnf', '--datadir=/opt/mysql/db'], ['mysqld'])
-except MySQLdb.ProgrammingError:
-  pass
+# The mysql_server_init function called here is deprecated and not used by mysqlclient
+# https://dev.mysql.com/doc/c-api/8.0/en/mysql-server-init.html
+# TODO: Check if any changes are needed
+if six.PY2:
+  # This is for proper initialization of embedded server, it should only be called once
+  try:
+    MySQLdb.server_init(['--defaults-file=/opt/dirac/etc/my.cnf', '--datadir=/opt/mysql/db'], ['mysqld'])
+  except MySQLdb.ProgrammingError:
+    pass
 
 gInstancesCount = 0
 

Core/Utilities/Network.py

@@ -8,7 +8,7 @@
 __RCSID__ = "$Id$"
 
 import socket
-import urlparse
+from six.moves.urllib import parse as urlparse
 import os
 import struct
 import array

Core/Utilities/Pfn.py

@@ -20,7 +20,7 @@
 import os
 # # from DIRAC
 from DIRAC import S_OK, S_ERROR, gLogger
-import urlparse
+from six.moves.urllib import parse as urlparse
 
 
 def pfnunparse(pfnDict, srmSpecific=True):