DIRACGrid · chrisburr · Aug 15, 2020 · Aug 15, 2020 · Aug 15, 2020 · Aug 15, 2020
@@ -0,0 +1,42 @@
+name: Python 3 tests
+
+on: [push, pull_request]
+
+jobs:
+  check:
+    runs-on: ubuntu-latest
+    if: github.event_name != 'push' || github.repository == 'DIRACGrid/DIRAC'
+    timeout-minutes: 30
+
+    strategy:
+      fail-fast: False
+      matrix:
+        command:
+          # TODO These three tests fail on Python 3:
+          #   * `test_BaseType_Unicode` and `test_nestedStructure` fail due to
+          #     DISET's string and unicode types being poorly defined
+          #   * `testLockedClass` deadlocks frequently due to the use of
+          #     self.__processThread._Thread__stop(). Officially "threads
+          #     cannot be destroyed, stopped, suspended, resumed, or
+          #     interrupted" so the code should be rewritten to follow the
+          #     supported alternative: "If you want your threads to stop
+          #     gracefully, make them non-daemonic and use a suitable
+          #     signalling mechanism such as an Event."
+          - pytest --no-cov -k 'not test_BaseType_Unicode and not test_nestedStructure and not testLockedClass'
+
+    steps:
+    - uses: actions/checkout@v2
+    - name: Fail-fast for outdated pipelines
+      run: .github/workflows/fail-fast.sh
+    - name: Prepare environment
+      run: |
+        conda env create --name dirac-testing --file environment-py3.yml
+    - name: Run tests
+      run: |
+        source "${CONDA}/bin/activate"
+        conda activate dirac-testing
+        set -euxo pipefail
+        export PYTHONPATH=${PWD%/*}
+        ${{ matrix.command }}
+      env:
+        REFERENCE_BRANCH: ${{ github['base_ref'] || github['head_ref'] }}
@@ -102,7 +102,7 @@ def updateNameDictionary(self):
       return S_ERROR('Unable to fetch perfSONAR endpoints from CS.')
 
     tmpDict = {}
-    for path, value in result['Value'].iteritems():
+    for path, value in result['Value'].items():
       if value == 'True':
         elements = path.split('/')
         diracName = elements[4]
@@ -211,7 +211,7 @@ def processMessage(self, headers, body):
           OWDMax = 0
           total = 0
           count = 0
-          for value, items in metricData.iteritems():
+          for value, items in metricData.items():
             floatValue = float(value)
             total += floatValue * items
             count += items

@@ -220,7 +220,7 @@ def export_removeRegisters(self, entriesList):
     for entry in entriesList:
       if len(entry) != 4:
         return S_ERROR("Invalid records")
-      for i in xrange(len(entry)):
+      for i in range(len(entry)):
         if not isinstance(entry[i], expectedTypes[i]):
           return S_ERROR("%s field in the records should be %s" % (i, expectedTypes[i]))
     ok = 0

@@ -46,8 +46,8 @@ def __calculateReportHash(self, reportRequest):
       epoch = requestToHash[key]
       requestToHash[key] = epoch - epoch % granularity
     md5Hash = hashlib.md5()
-    md5Hash.update(repr(requestToHash))
-    md5Hash.update(self.setup)
+    md5Hash.update(repr(requestToHash).encode())
+    md5Hash.update(self.setup.encode())
     return md5Hash.hexdigest()
 
   def generate(self, reportRequest, credDict):

@@ -168,7 +168,7 @@ def _plotJobsPerPilot(self, reportRequest, plotInfo, filename):
                 'endtime': reportRequest['endTime'],
                 'span': plotInfo['granularity'],
                 'ylabel': "jobs/pilot",
-                'normalization': max(x for y in plotInfo['data'].itervalues() for x in y.itervalues())}
+                'normalization': max(x for y in plotInfo['data'].values() for x in y.values())}
     return self._generateQualityPlot(filename, plotInfo['data'], metadata)
 
   def _reportTotalNumberOfPilots(self, reportRequest):

@@ -13,7 +13,7 @@
 
 import pprint
 import sys
-import urlparse
+from six.moves.urllib import parse as urlparse
 import cgi
 from DIRAC import gLogger
 from DIRAC.Core.Base import Script

@@ -58,7 +58,7 @@ def execute(self):
     """
 
     # __functionMap is at the end of the class definition
-    for option, functionCall in GOCDB2CSAgent.__functionMap.iteritems():
+    for option, functionCall in GOCDB2CSAgent.__functionMap.items():
       optionValue = self.am_getOption(option, True)
       if optionValue:
         result = functionCall(self)
@@ -157,12 +157,12 @@ def __preparePerfSONARConfiguration(self, endpointList):
 
       split = endpoint['DIRACSITENAME'].split('.')
       path = cfgPath(rootPath, split[0], endpoint['DIRACSITENAME'], extPath, endpoint['HOSTNAME'])
-      for name, defaultValue in options.iteritems():
+      for name, defaultValue in options.items():
         newConfiguration[cfgPath(path, name)] = defaultValue
 
     # get current configuration
     currentConfiguration = {}
-    for option in options.iterkeys():
+    for option in options.keys():
       result = gConfig.getConfigurationTree(rootPath, extPath + '/', '/' + option)
       if not result['OK']:
         log.error("getConfigurationTree() failed with message: %s" % result['Message'])
@@ -242,7 +242,7 @@ def __updateConfiguration(self, setElements=None, delElements=None):
     log.debug('Begin function ...')
 
     # assure existence and proper value of a section or an option
-    for path, value in setElements.iteritems():
+    for path, value in setElements.items():
 
       if value is None:
         section = path

@@ -11,12 +11,12 @@
 __RCSID__ = "$Id$"
 
 import imp
+import six
 from DIRAC.Core.Utilities.DIRACSingleton import DIRACSingleton
 
 
+@six.add_metaclass(DIRACSingleton)
 class Extensions(object):
-  __metaclass__ = DIRACSingleton
-
   def __init__(self):
     self.__modules = {}
     self.__orderedExtNames = []

@@ -74,7 +74,7 @@
 from __future__ import print_function
 
 import six
-import thread
+from six.moves import _thread as thread
 import os
 from diraccfg import CFG
 from DIRAC import S_OK, S_ERROR, gConfig

@@ -6,7 +6,7 @@
 
 __RCSID__ = "$Id$"
 
-import urlparse
+from six.moves.urllib import parse as urlparse
 from distutils.version import LooseVersion  # pylint: disable=no-name-in-module,import-error
 
 import six

@@ -6,7 +6,7 @@
 
 __RCSID__ = "$Id$"
 
-import urlparse
+from six.moves.urllib import parse as urlparse
 
 from DIRAC.Core.Utilities import List
 from DIRAC.ConfigurationSystem.Client.ConfigurationData import gConfigurationData

@@ -17,7 +17,7 @@
 
 import re
 import socket
-from urlparse import urlparse
+from six.moves.urllib.parse import urlparse
 
 import six
 
@@ -274,9 +274,7 @@ def addToChangeSet(entry, changeSet):
         if submissionMode == "Unknown" and newSubmissionMode:
           addToChangeSet((ceSection, 'SubmissionMode', submissionMode, newSubmissionMode), changeSet)
 
-        queues = ceInfo['Queues'].keys()
-        for queue in queues:
-          queueInfo = ceInfo['Queues'][queue]
+        for queue, queueInfo in ceInfo['Queues'].items():
           queueStatus = queueInfo['GlueCEStateStatus']
           queueSection = cfgPath(ceSection, 'Queues', queue)
           queueDict = {}
@@ -455,8 +453,7 @@ def getGridSRMs(vo, bdiiInfo=None, srmBlackList=None, unUsed=False):
 
   srmSeDict = {}
   for site in siteSRMDict:
-    srms = siteSRMDict[site].keys()
-    for srm in srms:
+    for srm in siteSRMDict[site]:
       if seBdiiDict.get(site, {}).get('SEs', {}).get(srm, {}):
         srmSeDict.setdefault(site, {})
         srmSeDict[site].setdefault(srm, {})

@@ -281,7 +281,7 @@ def getConfigurationTree(self, root='', *filters):
       if not options['OK']:
         return S_ERROR("getOptionsDict() failed with message: %s" % options['Message'])
 
-      for key, value in options['Value'].iteritems():
+      for key, value in options['Value'].items():
         path = cfgPath(root, key)
         addOption = True
         for substr in filters:

@@ -7,7 +7,7 @@
 import os.path
 import zlib
 import zipfile
-import thread
+from six.moves import _thread as thread
 import time
 import DIRAC
 

@@ -8,7 +8,7 @@
 __RCSID__ = "$Id$"
 
 import threading
-import thread
+from six.moves import _thread as thread
 import time
 import random
 from DIRAC.ConfigurationSystem.Client.ConfigurationData import gConfigurationData

@@ -16,7 +16,7 @@
 import re
 import os
 import shlex
-from urlparse import urlparse
+from six.moves.urllib.parse import urlparse
 
 from DIRAC.Core.Base import Script
 from DIRAC import gLogger, exit as DIRACExit, S_OK
@@ -97,7 +97,7 @@ def checkUnusedCEs():
       result = getDIRACSiteName(site)
       if result['OK']:
         diracSite = ','.join(result['Value'])
-      ces = siteDict[site].keys()  # pylint: disable=no-member
+      ces = list(siteDict[site])
       if ces:
         gLogger.notice("  %s, DIRAC site %s" % (site, diracSite))
         for ce in ces:
@@ -120,8 +120,7 @@ def checkUnusedCEs():
   for site in siteDict:
     # Get the country code:
     country = ''
-    ces = siteDict[site].keys()  # pylint: disable=no-member
-    for ce in ces:
+    for ce in siteDict[site]:
       country = ce.strip().split('.')[-1].lower()
       if len(country) == 2:
         break

@@ -129,7 +129,7 @@ def showInfo(result, info):
     if info == 'site-se' or info == 'all':
       print("SE: %s \n{" % element.get('GlueSEUniqueID', 'Unknown'))
 
-    for item in element.iteritems():
+    for item in element.items():
       print("  %s: %s" % item)
     print("}")
 

@@ -88,6 +88,8 @@ def __getstate__(self):
     # because of the thread locks
     if isinstance(state['log'], Logging):
       state['log'] = state['log'].getSubName()
+    # In Python 3 PrettyPrinter can't be pickled
+    del state['pPrint']
     return state
 
   def __setstate__(self, state):
@@ -101,6 +103,7 @@ def __setstate__(self, state):
     # due to the thread locks
     if isinstance(state['log'], six.string_types):
       self.log = gLogger.getSubLogger(state['log'])
+    self.pPrint = pprint.PrettyPrinter()
 
   #############################################################################
 

@@ -4,9 +4,9 @@
 from __future__ import division
 from __future__ import print_function
 
-import types
 import pprint
 
+import six
 from DIRAC import gLogger
 from DIRAC.Core.Utilities.ReturnValues import S_OK, S_ERROR, isReturnStructure
 from DIRAC.Core.Utilities.ThreadScheduler import gThreadScheduler
@@ -16,21 +16,21 @@
 
 class ExecutorMindHandler(RequestHandler):
 
-  MSG_DEFINITIONS = {'ProcessTask': {'taskId': (types.IntType, types.LongType),
-                                     'taskStub': types.StringTypes,
-                                     'eType': types.StringTypes},
-                     'TaskDone': {'taskId': (types.IntType, types.LongType),
-                                  'taskStub': types.StringTypes},
-                     'TaskFreeze': {'taskId': (types.IntType, types.LongType),
-                                    'taskStub': types.StringTypes,
-                                    'freezeTime': (types.IntType, types.LongType)},
-                     'TaskError': {'taskId': (types.IntType, types.LongType),
-                                   'errorMsg': types.StringTypes,
-                                   'taskStub': types.StringTypes,
-                                   'eType': types.StringTypes},
-                     'ExecutorError': {'taskId': (types.IntType, types.LongType),
-                                       'errorMsg': types.StringTypes,
-                                       'eType': types.StringTypes}}
+  MSG_DEFINITIONS = {'ProcessTask': {'taskId': six.integer_types,
+                                     'taskStub': six.string_types,
+                                     'eType': six.string_types},
+                     'TaskDone': {'taskId': six.integer_types,
+                                  'taskStub': six.string_types},
+                     'TaskFreeze': {'taskId': six.integer_types,
+                                    'taskStub': six.string_types,
+                                    'freezeTime': six.integer_types},
+                     'TaskError': {'taskId': six.integer_types,
+                                   'errorMsg': six.string_types,
+                                   'taskStub': six.string_types,
+                                   'eType': six.string_types},
+                     'ExecutorError': {'taskId': six.integer_types,
+                                       'errorMsg': six.string_types,
+                                       'eType': six.string_types}}
 
   class MindCallbacks(ExecutorDispatcherCallbacks):
 

@@ -9,6 +9,7 @@
 
 __RCSID__ = "$Id$"
 
+import six
 import datetime
 from sqlalchemy import create_engine, desc, exc
 from sqlalchemy.engine.reflection import Inspector
@@ -189,13 +190,13 @@ def select(self, table, params):
         select = Query(columns, session=session)
 
       # query conditions
-      for columnName, columnValue in params.iteritems():
+      for columnName, columnValue in params.items():
         if not columnValue:
           continue
         column_a = getattr(table_c, columnName.lower())
         if isinstance(columnValue, (list, tuple)):
           select = select.filter(column_a.in_(list(columnValue)))
-        elif isinstance(columnValue, (basestring, datetime.datetime, bool)):
+        elif isinstance(columnValue, six.string_types + (datetime.datetime, bool)):
           select = select.filter(column_a == columnValue)
         else:
           self.log.error("type(columnValue) == %s" % type(columnValue))
@@ -206,7 +207,7 @@ def select(self, table, params):
         column_a = getattr(table_c, newer[0].lower())
         select = select.filter(column_a > newer[1])
       if order:
-        order = [order] if isinstance(order, basestring) else list(order)
+        order = [order] if isinstance(order, six.string_types) else list(order)
         column_a = getattr(table_c, order[0].lower())
         if len(order) == 2 and order[1].lower() == 'desc':
           select = select.order_by(desc(column_a))
@@ -268,13 +269,13 @@ def delete(self, table, params):
 
     try:
       deleteQuery = Query(table_c, session=session)
-      for columnName, columnValue in params.iteritems():
+      for columnName, columnValue in params.items():
         if not columnValue:
           continue
         column_a = getattr(table_c, columnName.lower())
         if isinstance(columnValue, (list, tuple)):
           deleteQuery = deleteQuery.filter(column_a.in_(list(columnValue)))
-        elif isinstance(columnValue, (basestring, datetime.datetime, bool)):
+        elif isinstance(columnValue, six.string_types + (datetime.datetime, bool)):
           deleteQuery = deleteQuery.filter(column_a == columnValue)
         else:
           self.log.error("type(columnValue) == %s" % type(columnValue))
@@ -285,7 +286,7 @@ def delete(self, table, params):
         column_a = getattr(table_c, newer[0].lower())
         deleteQuery = deleteQuery.filter(column_a > newer[1])
       if order:
-        order = [order] if isinstance(order, basestring) else list(order)
+        order = [order] if isinstance(order, six.string_types) else list(order)
         column_a = getattr(table_c, order[0].lower())
         if len(order) == 2 and order[1].lower() == 'desc':
           deleteQuery = deleteQuery.order_by(desc(column_a))

@@ -33,7 +33,7 @@ def _initialize(self):
 
   def __generateUniqueClientName(self):
     hashStr = ":".join((Time.toString(), str(random.random()), Network.getFQDN(), gLogger.getName()))
-    hexHash = md5(hashStr).hexdigest()
+    hexHash = md5(hashStr.encode()).hexdigest()
     return hexHash
 
   def setUniqueName(self, uniqueName):