[v7r2] Run client integration tests for Python 3

.github/workflows/basic-python3.yml

@@ -0,0 +1,42 @@
+name: Python 3 tests
+
+on: [push, pull_request]
+
+jobs:
+  check:
+    runs-on: ubuntu-latest
+    if: github.event_name != 'push' || github.repository == 'DIRACGrid/DIRAC'
+    timeout-minutes: 30
+
+    strategy:
+      fail-fast: False
+      matrix:
+        command:
+          # TODO These three tests fail on Python 3:
+          #   * `test_BaseType_Unicode` and `test_nestedStructure` fail due to
+          #     DISET's string and unicode types being poorly defined
+          #   * `testLockedClass` deadlocks frequently due to the use of
+          #     self.__processThread._Thread__stop(). Officially "threads
+          #     cannot be destroyed, stopped, suspended, resumed, or
+          #     interrupted" so the code should be rewritten to follow the
+          #     supported alternative: "If you want your threads to stop
+          #     gracefully, make them non-daemonic and use a suitable
+          #     signalling mechanism such as an Event."
+          - pytest --no-cov -k 'not test_BaseType_Unicode and not test_nestedStructure and not testLockedClass'
+
+    steps:
+    - uses: actions/checkout@v2
+    - name: Fail-fast for outdated pipelines
+      run: .github/workflows/fail-fast.sh
+    - name: Prepare environment
+      run: |
+        conda env create --name dirac-testing --file environment-py3.yml
+    - name: Run tests
+      run: |
+        source "${CONDA}/bin/activate"
+        conda activate dirac-testing
+        set -euxo pipefail
+        export PYTHONPATH=${PWD%/*}
+        ${{ matrix.command }}
+      env:
+        REFERENCE_BRANCH: ${{ github['base_ref'] || github['head_ref'] }}

.github/workflows/integration.yml

@@ -14,42 +14,44 @@ jobs:
       MATRIX_DEFAULT_ES_VER: 6.6.0
       MATRIX_DEFAULT_USE_NEWTHREADPOOL: default
       MATRIX_DEFAULT_DIRACOSVER: default
-      MATRIX_DEFAULT_SERVER_USE_M2CRYPTO: Yes
-      MATRIX_DEFAULT_CLIENT_USE_M2CRYPTO: Yes
       MATRIX_DEFAULT_TEST_HTTPS: No
+      MATRIX_DEFAULT_SERVER_USE_PYTHON3: No
+      MATRIX_DEFAULT_CLIENT_USE_PYTHON3: No
 
     strategy:
       fail-fast: False
       matrix:
+        # TEST_NAME is a dummy variable used to make it easier to read the web interface
         include:
           ###### MySQL versions
-          - MYSQL_VER: 5.6
+          - TEST_NAME: "MySQL 5.6"
+            MYSQL_VER: 5.6
           # IMPLICIT: - MYSQL_VER: 5.7
-          - MYSQL_VER: 8.0
+          - TEST_NAME: "MySQL 8.0"
+            MYSQL_VER: 8.0
           ###### ES versions
-          - ES_VER: 7.9.1
+          - TEST_NAME: "Elasticsearch 7.9.1"
+            ES_VER: 7.9.1
           # IMPLICIT: - ES_VER: 6.6.0
           ###### Host OS
-          - HOST_OS: slc6
+          - TEST_NAME: "SLC 6"
+            HOST_OS: slc6
           # IMPLICIT: - HOST_OS: cc7
           ###### Thread pool
           # IMPLICIT: - USE_NEWTHREADPOOL: default
-          - USE_NEWTHREADPOOL: No
-          ###### M2Crypto
-          # IMPLICIT: - SERVER_USE_M2CRYPTO: Yes
-          #             CLIENT_USE_M2CRYPTO: Yes
-          - SERVER_USE_M2CRYPTO: Yes
-            CLIENT_USE_M2CRYPTO: No
-          - SERVER_USE_M2CRYPTO: No
-            CLIENT_USE_M2CRYPTO: Yes
-          - SERVER_USE_M2CRYPTO: No
-            CLIENT_USE_M2CRYPTO: No
+          - TEST_NAME: "New thread pool"
+            USE_NEWTHREADPOOL: No
           ###### HTTPS tests
           # IMPLICIT: - DIRACOSVER: default
           #             TEST_HTTPS: No
-          - TEST_HTTPS: Yes
+          - TEST_NAME: "HTTPS"
+            TEST_HTTPS: Yes
             DIRACOSVER: https
-
+          ###### Python 3
+          # IMPLICIT: - SERVER_USE_PYTHON3: No
+          #             CLIENT_USE_PYTHON3: No
+          - TEST_NAME: "Python 3 client"
+            CLIENT_USE_PYTHON3: Yes
 
     steps:
     - uses: actions/checkout@v2
@@ -73,8 +75,8 @@ jobs:
         if [[ "${{ matrix.USE_NEWTHREADPOOL || env.MATRIX_DEFAULT_USE_NEWTHREADPOOL }}" != "default" ]]; then echo -n "-e DIRAC_USE_NEWTHREADPOOL=${{ matrix.USE_NEWTHREADPOOL || env.MATRIX_DEFAULT_USE_NEWTHREADPOOL }} " >> run_in_container; fi
         if [[ "${{ matrix.DIRACOSVER || env.MATRIX_DEFAULT_DIRACOSVER }}" != "default" ]]; then echo -n "-e DIRACOSVER=${{ matrix.DIRACOSVER || env.MATRIX_DEFAULT_DIRACOSVER }} " >> run_in_container; fi
         echo -n "-e TEST_HTTPS=${{ matrix.TEST_HTTPS || env.MATRIX_DEFAULT_TEST_HTTPS }} " >> run_in_container
-        echo -n "-e SERVER_USE_M2CRYPTO=${{ matrix.SERVER_USE_M2CRYPTO || env.MATRIX_DEFAULT_SERVER_USE_M2CRYPTO }} " >> run_in_container
-        echo -n "-e CLIENT_USE_M2CRYPTO=${{ matrix.CLIENT_USE_M2CRYPTO || env.MATRIX_DEFAULT_CLIENT_USE_M2CRYPTO }} " >> run_in_container
+        echo -n "-e SERVER_USE_PYTHON3=${{ matrix.SERVER_USE_PYTHON3 || env.MATRIX_DEFAULT_SERVER_USE_PYTHON3 }} " >> run_in_container
+        echo -n "-e CLIENT_USE_PYTHON3=${{ matrix.CLIENT_USE_PYTHON3 || env.MATRIX_DEFAULT_CLIENT_USE_PYTHON3 }} " >> run_in_container
         # Finish wrapper script
         echo -n "dirac-testing-host \"\$@\"" >> run_in_container
         chmod +x run_in_container

AccountingSystem/Agent/NetworkAgent.py

@@ -102,7 +102,7 @@ def updateNameDictionary(self):
       return S_ERROR('Unable to fetch perfSONAR endpoints from CS.')
 
     tmpDict = {}
-    for path, value in result['Value'].iteritems():
+    for path, value in result['Value'].items():
       if value == 'True':
         elements = path.split('/')
         diracName = elements[4]
@@ -211,7 +211,7 @@ def processMessage(self, headers, body):
           OWDMax = 0
           total = 0
           count = 0
-          for value, items in metricData.iteritems():
+          for value, items in metricData.items():
             floatValue = float(value)
             total += floatValue * items
             count += items

AccountingSystem/Service/DataStoreHandler.py

@@ -220,7 +220,7 @@ def export_removeRegisters(self, entriesList):
     for entry in entriesList:
       if len(entry) != 4:
         return S_ERROR("Invalid records")
-      for i in xrange(len(entry)):
+      for i in range(len(entry)):
         if not isinstance(entry[i], expectedTypes[i]):
           return S_ERROR("%s field in the records should be %s" % (i, expectedTypes[i]))
     ok = 0

AccountingSystem/private/MainReporter.py

@@ -46,8 +46,8 @@ def __calculateReportHash(self, reportRequest):
       epoch = requestToHash[key]
       requestToHash[key] = epoch - epoch % granularity
     md5Hash = hashlib.md5()
-    md5Hash.update(repr(requestToHash))
-    md5Hash.update(self.setup)
+    md5Hash.update(repr(requestToHash).encode())
+    md5Hash.update(self.setup.encode())
     return md5Hash.hexdigest()
 
   def generate(self, reportRequest, credDict):

AccountingSystem/private/Plotters/PilotPlotter.py

@@ -168,7 +168,7 @@ def _plotJobsPerPilot(self, reportRequest, plotInfo, filename):
                 'endtime': reportRequest['endTime'],
                 'span': plotInfo['granularity'],
                 'ylabel': "jobs/pilot",
-                'normalization': max(x for y in plotInfo['data'].itervalues() for x in y.itervalues())}
+                'normalization': max(x for y in plotInfo['data'].values() for x in y.values())}
     return self._generateQualityPlot(filename, plotInfo['data'], metadata)
 
   def _reportTotalNumberOfPilots(self, reportRequest):

AccountingSystem/scripts/dirac-accounting-decode-fileid.py

@@ -13,7 +13,7 @@
 
 import pprint
 import sys
-import urlparse
+from six.moves.urllib import parse as urlparse
 import cgi
 from DIRAC import gLogger
 from DIRAC.Core.Base import Script

ConfigurationSystem/Agent/GOCDB2CSAgent.py

@@ -58,7 +58,7 @@ def execute(self):
     """
 
     # __functionMap is at the end of the class definition
-    for option, functionCall in GOCDB2CSAgent.__functionMap.iteritems():
+    for option, functionCall in GOCDB2CSAgent.__functionMap.items():
       optionValue = self.am_getOption(option, True)
       if optionValue:
         result = functionCall(self)
@@ -157,12 +157,12 @@ def __preparePerfSONARConfiguration(self, endpointList):
 
       split = endpoint['DIRACSITENAME'].split('.')
       path = cfgPath(rootPath, split[0], endpoint['DIRACSITENAME'], extPath, endpoint['HOSTNAME'])
-      for name, defaultValue in options.iteritems():
+      for name, defaultValue in options.items():
         newConfiguration[cfgPath(path, name)] = defaultValue
 
     # get current configuration
     currentConfiguration = {}
-    for option in options.iterkeys():
+    for option in options:
       result = gConfig.getConfigurationTree(rootPath, extPath + '/', '/' + option)
       if not result['OK']:
         log.error("getConfigurationTree() failed with message: %s" % result['Message'])
@@ -242,7 +242,7 @@ def __updateConfiguration(self, setElements=None, delElements=None):
     log.debug('Begin function ...')
 
     # assure existence and proper value of a section or an option
-    for path, value in setElements.iteritems():
+    for path, value in setElements.items():
 
       if value is None:
         section = path

ConfigurationSystem/Client/Helpers/CSGlobals.py

@@ -11,12 +11,12 @@
 __RCSID__ = "$Id$"
 
 import imp
+import six
 from DIRAC.Core.Utilities.DIRACSingleton import DIRACSingleton
 
 
+@six.add_metaclass(DIRACSingleton)
 class Extensions(object):
-  __metaclass__ = DIRACSingleton
-
   def __init__(self):
     self.__modules = {}
     self.__orderedExtNames = []

ConfigurationSystem/Client/Helpers/Operations.py

@@ -74,7 +74,7 @@
 from __future__ import print_function
 
 import six
-import thread
+from six.moves import _thread as thread
 import os
 from diraccfg import CFG
 from DIRAC import S_OK, S_ERROR, gConfig

ConfigurationSystem/Client/Helpers/Resources.py

@@ -6,7 +6,7 @@
 
 __RCSID__ = "$Id$"
 
-import urlparse
+from six.moves.urllib import parse as urlparse
 from distutils.version import LooseVersion  # pylint: disable=no-name-in-module,import-error
 
 import six

ConfigurationSystem/Client/PathFinder.py

@@ -6,7 +6,7 @@
 
 __RCSID__ = "$Id$"
 
-import urlparse
+from six.moves.urllib import parse as urlparse
 
 from DIRAC.Core.Utilities import List
 from DIRAC.ConfigurationSystem.Client.ConfigurationData import gConfigurationData

ConfigurationSystem/Client/Utilities.py

@@ -17,7 +17,7 @@
 
 import re
 import socket
-from urlparse import urlparse
+from six.moves.urllib.parse import urlparse
 
 import six
 
@@ -280,9 +280,7 @@ def addToChangeSet(entry, changeSet):
         if submissionMode == "Unknown" and newSubmissionMode:
           addToChangeSet((ceSection, 'SubmissionMode', submissionMode, newSubmissionMode), changeSet)
 
-        queues = ceInfo['Queues'].keys()
-        for queue in queues:
-          queueInfo = ceInfo['Queues'][queue]
+        for queue, queueInfo in ceInfo['Queues'].items():
           queueStatus = queueInfo['GlueCEStateStatus']
           queueSection = cfgPath(ceSection, 'Queues', queue)
           queueDict = {}
@@ -470,8 +468,7 @@ def getGridSRMs(vo, bdiiInfo=None, srmBlackList=None, unUsed=False):
 
   srmSeDict = {}
   for site in siteSRMDict:
-    srms = siteSRMDict[site].keys()
-    for srm in srms:
+    for srm in siteSRMDict[site]:
       if seBdiiDict.get(site, {}).get('SEs', {}).get(srm, {}):
         srmSeDict.setdefault(site, {})
         srmSeDict[site].setdefault(srm, {})

ConfigurationSystem/private/ConfigurationClient.py

@@ -281,7 +281,7 @@ def getConfigurationTree(self, root='', *filters):
       if not options['OK']:
         return S_ERROR("getOptionsDict() failed with message: %s" % options['Message'])
 
-      for key, value in options['Value'].iteritems():
+      for key, value in options['Value'].items():
         path = cfgPath(root, key)
         addOption = True
         for substr in filters:

ConfigurationSystem/private/ConfigurationData.py

@@ -7,7 +7,8 @@
 import os.path
 import zlib
 import zipfile
-import thread
+import six
+from six.moves import _thread as thread
 import time
 import DIRAC
 
@@ -87,7 +88,9 @@ def mergeWithLocal(self, extraCFG):
     return S_OK()
 
   def loadRemoteCFGFromCompressedMem(self, data):
-    sUncompressedData = zlib.decompress(data)
+    if six.PY3 and isinstance(data, str):
+      data = data.encode(errors="surrogateescape")
+    sUncompressedData = zlib.decompress(data).decode()
     self.loadRemoteCFGFromMem(sUncompressedData)
 
   def loadRemoteCFGFromMem(self, data):

ConfigurationSystem/private/Modificator.py

@@ -6,6 +6,7 @@
 
 import zlib
 import difflib
+import six
 
 from diraccfg import CFG
 from DIRAC.Core.Utilities import List, Time
@@ -42,7 +43,10 @@ def loadFromRemote(self):
     retVal = self.rpcClient.getCompressedData()
     if retVal['OK']:
       self.cfgData = CFG()
-      self.cfgData.loadFromBuffer(zlib.decompress(retVal['Value']))
+      data = retVal['Value']
+      if six.PY3 and isinstance(data, str):
+        data = data.encode(errors="surrogateescape")
+      self.cfgData.loadFromBuffer(zlib.decompress(data).decode())
     return retVal
 
   def getCFG(self):
@@ -251,24 +255,38 @@ def getHistory(self, limit=0):
   def showCurrentDiff(self):
     retVal = self.rpcClient.getCompressedData()
     if retVal['OK']:
-      remoteData = zlib.decompress(retVal['Value']).splitlines()
+      data = retVal['Value']
+      if six.PY3 and isinstance(data, str):
+        data = data.encode(errors="surrogateescape")
+      remoteData = zlib.decompress(data).decode().splitlines()
       localData = str(self.cfgData).splitlines()
       return difflib.ndiff(remoteData, localData)
     return []
 
   def getVersionDiff(self, fromDate, toDate):
     retVal = self.rpcClient.getVersionContents([fromDate, toDate])
     if retVal['OK']:
-      fromData = zlib.decompress(retVal['Value'][0])
-      toData = zlib.decompress(retVal['Value'][1])
+      fromData = retVal['Value'][0]
+      if six.PY3 and isinstance(fromData, str):
+        fromData = fromData.encode(errors="surrogateescape")
+      fromData = zlib.decompress(fromData).decode()
+
+      toData = retVal['Value'][1]
+      if six.PY3 and isinstance(toData, str):
+        toData = toData.encode(errors="surrogateescape")
+      toData = zlib.decompress(toData).decode()
+
       return difflib.ndiff(fromData.split("\n"), toData.split("\n"))
     return []
 
   def mergeWithServer(self):
     retVal = self.rpcClient.getCompressedData()
     if retVal['OK']:
       remoteCFG = CFG()
-      remoteCFG.loadFromBuffer(zlib.decompress(retVal['Value']))
+      data = retVal['Value']
+      if six.PY3 and isinstance(data, str):
+        data = data.encode(errors="surrogateescape")
+      remoteCFG.loadFromBuffer(zlib.decompress(data).decode())
       serverVersion = gConfigurationData.getVersion(remoteCFG)
       self.cfgData = remoteCFG.mergeWith(self.cfgData)
       gConfigurationData.setVersion(serverVersion, self.cfgData)

ConfigurationSystem/private/Refresher.py

@@ -31,11 +31,12 @@ def _updateFromRemoteLocation(serviceClient):
   retVal = serviceClient.getCompressedDataIfNewer(localVersion)
   if retVal['OK']:
     dataDict = retVal['Value']
-    if localVersion < dataDict['newestVersion']:
-      gLogger.debug("New version available", "Updating to version %s..." % dataDict['newestVersion'])
+    newestVersion = dataDict['newestVersion']
+    if localVersion < newestVersion:
+      gLogger.debug("New version available", "Updating to version %s..." % newestVersion)
       gConfigurationData.loadRemoteCFGFromCompressedMem(dataDict['data'])
       gLogger.debug("Updated to version %s" % gConfigurationData.getVersion())
-      gEventDispatcher.triggerEvent("CSNewVersion", dataDict['newestVersion'], threaded=True)
+      gEventDispatcher.triggerEvent("CSNewVersion", newestVersion, threaded=True)
     return S_OK()
   return retVal