[v7r2] Use DIRACOS2 for Python 3 client integration tests

.github/workflows/integration.yml

@@ -50,6 +50,7 @@ jobs:
           #             CLIENT_USE_PYTHON3: No
           - TEST_NAME: "Python 3 client"
             CLIENT_USE_PYTHON3: Yes
+            CLIENT_DIRACOSVER: 2.0a8
 
     steps:
     - uses: actions/checkout@v2
@@ -74,7 +75,9 @@ jobs:
         if [[ "${{ matrix.DIRACOSVER || env.MATRIX_DEFAULT_DIRACOSVER }}" != "default" ]]; then echo -n "-e DIRACOSVER=${{ matrix.DIRACOSVER || env.MATRIX_DEFAULT_DIRACOSVER }} " >> run_in_container; fi
         echo -n "-e TEST_HTTPS=${{ matrix.TEST_HTTPS || env.MATRIX_DEFAULT_TEST_HTTPS }} " >> run_in_container
         echo -n "-e SERVER_USE_PYTHON3=${{ matrix.SERVER_USE_PYTHON3 || env.MATRIX_DEFAULT_SERVER_USE_PYTHON3 }} " >> run_in_container
+        if [[ "${{ matrix.SERVER_DIRACOSVER || env.MATRIX_DEFAULT_SERVER_DIRACOSVER }}" != "default" ]]; then echo -n "-e SERVER_DIRACOSVER=${{ matrix.SERVER_DIRACOSVER || env.MATRIX_DEFAULT_SERVER_DIRACOSVER }} " >> run_in_container; fi
         echo -n "-e CLIENT_USE_PYTHON3=${{ matrix.CLIENT_USE_PYTHON3 || env.MATRIX_DEFAULT_CLIENT_USE_PYTHON3 }} " >> run_in_container
+        if [[ "${{ matrix.CLIENT_DIRACOSVER || env.MATRIX_DEFAULT_CLIENT_DIRACOSVER }}" != "default" ]]; then echo -n "-e CLIENT_DIRACOSVER=${{ matrix.CLIENT_DIRACOSVER || env.MATRIX_DEFAULT_CLIENT_DIRACOSVER }} " >> run_in_container; fi
         # Finish wrapper script
         echo -n "dirac-testing-host \"\$@\"" >> run_in_container
         chmod +x run_in_container

docs/source/UserGuide/Tutorials/JobManagementAdvanced/index.rst

@@ -395,7 +395,7 @@ Let's perform this exercise in the python shell.
           $ python Test-API-JDL.py 
 
               Priority = "1";
-              Executable = "$DIRACROOT/scripts/dirac-jobexec";
+              Executable = "dirac-jobexec";
               ExecutionEnvironment = "MYVARIABLE=TEST";
               StdError = "std.err";
               LogLevel = "DEBUG";

setup.cfg

@@ -35,7 +35,6 @@ install_requires =
     botocore
     certifi
     diraccfg
-    fts3-rest
     future
     gfal2-python
     M2Crypto >=0.36

src/DIRAC/Core/scripts/dirac_platform.py

@@ -26,23 +26,22 @@
 
 __RCSID__ = "$Id$"
 
-import sys
-import argparse
-
-parser = argparse.ArgumentParser(
-    description=__doc__,
-    formatter_class=argparse.RawDescriptionHelpFormatter)
-parser.parse_known_args()
 
 try:
   from DIRAC.Core.Utilities.Platform import getPlatformString
 except Exception:
+  import argparse
   import platform
   import os
   import sys
   import re
   import subprocess
 
+  parser = argparse.ArgumentParser(
+      description=__doc__,
+      formatter_class=argparse.RawDescriptionHelpFormatter)
+  parser.parse_known_args()
+
   # We need to patch python platform module. It does a string comparison for the libc versions.
   # it fails when going from 2.9 to 2.10,
   # the fix converts the version to a tuple and attempts a numeric comparison

src/DIRAC/Core/scripts/dirac_version.py

@@ -21,17 +21,18 @@
 
 import argparse
 
-parser = argparse.ArgumentParser(
-    description=__doc__,
-    formatter_class=argparse.RawDescriptionHelpFormatter)
-parser.parse_known_args()
-
 import DIRAC
 from DIRAC.Core.Utilities.DIRACScript import DIRACScript
 
 
 @DIRACScript()
 def main():
+  parser = argparse.ArgumentParser(
+      description=__doc__,
+      formatter_class=argparse.RawDescriptionHelpFormatter,
+  )
+  parser.parse_known_args()
+
   print(DIRAC.version)
 
 

src/DIRAC/Interfaces/API/Job.py

@@ -83,7 +83,7 @@ def __init__(self, script=None, stdout='std.out', stderr='std.err'):
     self.stdout = stdout
     self.stderr = stderr
     self.logLevel = 'info'
-    self.executable = '$DIRACROOT/scripts/dirac-jobexec'  # to be clarified
+    self.executable = 'dirac-jobexec'  # to be clarified
     # $DIRACROOT is set by the JobWrapper at execution time
     self.addToInputSandbox = []
     self.addToOutputSandbox = []

src/DIRAC/Interfaces/API/test/Test_DIRAC.py

@@ -127,8 +127,14 @@ def test_runLocal(dirac, job, mocker, osmock, confMock):
   ret = dirac.runLocal(job)
   LOG.info("dirac log calls: %s", dirac.log.call_args_list)
   LOG.info("CallStack: %s", pformat(ret.get('CallStack', {})))
-  assert sysMock.call_args_list[0][1]['cmdSeq'] == ['/root/dirac/scripts/dirac-jobexec',
-                                                    'jobDescription.xml', '-o', 'LogLevel=DEBUG']
+  try:
+    assert sysMock.call_args_list[0][1]['cmdSeq'] == [
+        'dirac-jobexec', 'jobDescription.xml', '-o', 'LogLevel=DEBUG'
+    ]
+  except AssertionError:
+    assert sysMock.call_args_list[0][1]['cmdSeq'] == [
+        '/root/dirac/scripts/dirac-jobexec', 'jobDescription.xml', '-o', 'LogLevel=DEBUG'
+    ]
   assert ret.get('Message', None) is None
   assert ret['OK']
   assert call('/abspath/absfile.xml', '/pwd/tempFolder/') in shMock.copy.call_args_list

src/DIRAC/Interfaces/API/test/testWF.jdl

@@ -1,6 +1,6 @@
 
     Arguments = "jobDescription.xml -o LogLevel=DEBUG  -p JOB_ID=%(JOB_ID)s  -p InputData=%(InputData)s";
-    Executable = "$DIRACROOT/scripts/dirac-jobexec";
+    Executable = "dirac-jobexec";
     InputData = %(InputData)s;
     InputSandbox = jobDescription.xml;
     JOB_ID = %(JOB_ID)s;

src/DIRAC/Interfaces/API/test/testWFSIO.jdl

@@ -1,6 +1,6 @@
 
     Arguments = "jobDescription.xml -o LogLevel=info";
-    Executable = "$DIRACROOT/scripts/dirac-jobexec";
+    Executable = "dirac-jobexec";
     JobGroup = jobGroup;
     JobName = jobName;
     JobType = jobType;

src/DIRAC/TransformationSystem/test/Test_JobInfo.py

@@ -39,7 +39,7 @@ def setUp(self):
 
     self.jdl2 = {
         'LogTargetPath': "/ilc/prod/clic/500gev/yyveyx_o/ILD/REC/00006326/LOG/00006326_015.tar",
-        'Executable': "$DIRACROOT/scripts/dirac-jobexec",
+        'Executable': "dirac-jobexec",
         'TaskID': 15,
         'SoftwareDistModule': "ILCDIRAC.Core.Utilities.CombinedSoftwareInstallation",
         'JobName': "00006326_00000015",
@@ -109,7 +109,7 @@ def setUp(self):
 
     self.jdlBrokenContent = {
         'LogTargetPath': "/ilc/prod/clic/500gev/yyveyx_o/ILD/REC/00006326/LOG/00006326_015.tar",
-        'Executable': "$DIRACROOT/scripts/dirac-jobexec",
+        'Executable': "dirac-jobexec",
         'TaskID': 'muahahaha',
         'SoftwareDistModule': "ILCDIRAC.Core.Utilities.CombinedSoftwareInstallation",
         'JobName': "00006326_00000015",
@@ -180,7 +180,7 @@ def setUp(self):
     # jdl with single outputdata,
     self.jdl1 = {
         'LogTargetPath': "/ilc/prod/clic/3tev/e1e1_o/SID/SIM/00006301/LOG/00006301_10256.tar",
-        'Executable': "$DIRACROOT/scripts/dirac-jobexec",
+        'Executable': "dirac-jobexec",
         'TaskID': 10256,
         'SoftwareDistModule': "ILCDIRAC.Core.Utilities.CombinedSoftwareInstallation",
         'JobName': "00006301_00010256",
@@ -250,7 +250,7 @@ def setUp(self):
 
     self.jdlNoInput = {
         'LogTargetPath': "/ilc/prod/clic/1.4tev/ea_qqqqnu/gen/00006498/LOG/00006498_1307.tar",
-        'Executable': "$DIRACROOT/scripts/dirac-jobexec",
+        'Executable': "dirac-jobexec",
         'TaskID': 1307,
         'SoftwareDistModule': "ILCDIRAC.Core.Utilities.CombinedSoftwareInstallation",
         'JobName': "00006498_00001307",

src/DIRAC/WorkloadManagementSystem/Agent/test/Test_Agent_JobAgent.py

@@ -35,7 +35,7 @@ def test__getJDLParameters(mocker):
 
   jdl = """
         [
-            Executable = "$DIRACROOT/scripts/dirac-jobexec";
+            Executable = "dirac-jobexec";
             StdError = "std.err";
             LogLevel = "info";
             Site = "ANY";

src/DIRAC/WorkloadManagementSystem/JobWrapper/JobWrapper.py

@@ -28,6 +28,7 @@
 import glob
 import json
 import six
+import distutils.spawn
 
 from six.moves.urllib.parse import unquote as urlunquote
 
@@ -231,6 +232,7 @@ def initialize(self, arguments):
         if os.path.exists('%s/%s' % (self.root, extraOpts)):
           shutil.copyfile('%s/%s' % (self.root, extraOpts), extraOpts)
         self.__loadLocalCFGFiles(self.localSiteRoot)
+
     else:
       self.log.info('JobID is not defined, running in current directory')
 
@@ -333,6 +335,14 @@ def execute(self):
     # the argument should include the jobDescription.xml file
     jobArguments = self.jobArgs.get('Arguments', '')
 
+    # This is a workaround for Python 2 style installations
+    if six.PY3 and executable == "$DIRACROOT/scripts/dirac-jobexec":
+      self.log.warn(
+          'Replaced job executable "$DIRACROOT/scripts/dirac-jobexec" with '
+          '"dirac-jobexec". Please fix your submission script!'
+      )
+      executable = "dirac-jobexec"
+
     executable = os.path.expandvars(executable)
     exeThread = None
     spObject = None
@@ -341,6 +351,11 @@ def execute(self):
       executable = executable.replace('$DIRACROOT', self.localSiteRoot)
       self.log.verbose('Replaced $DIRACROOT for executable as %s' % (self.localSiteRoot))
 
+    # Try to find the executable on PATH
+    if "/" not in executable:
+      # Returns None if the executable is not found so use "or" to leave it unchanged
+      executable = distutils.spawn.find_executable(executable) or executable
+
     # Make the full path since . is not always in the PATH
     executable = os.path.abspath(executable)
     if not os.access(executable, os.X_OK):

tests/CI/run_docker_setup.sh

@@ -111,13 +111,6 @@ prepareEnvironment() {
     fi
 
     echo "declare -a ALTERNATIVE_MODULES"
-    if [[ -n "${ALTERNATIVE_MODULES+x}" ]]; then
-      for module_path in "${ALTERNATIVE_MODULES[@]}"; do
-        echo "ALTERNATIVE_MODULES+=(\"${module_path}\")"
-      done
-    else
-      echo "ALTERNATIVE_MODULES+=(\"${DIRAC_BASE_DIR}/src/DIRAC\")"
-    fi
 
     echo "declare -a INSTALLOPTIONS"
     if [[ -n "${INSTALLOPTIONS+x}" ]]; then
@@ -154,12 +147,37 @@ prepareEnvironment() {
   cp "${SERVERCONFIG}" "${CLIENTCONFIG}"
 
   if [[ "${SERVER_USE_PYTHON3:-}" == "Yes" ]]; then
-    echo "INSTALLOPTIONS+=(\"--pythonVersion=3\")" >> "${SERVERCONFIG}"
+    echo "export SERVER_USE_PYTHON3=${SERVER_USE_PYTHON3}" >> "${SERVERCONFIG}"
+    echo "ALTERNATIVE_MODULES+=(\"${DIRAC_BASE_DIR}/\")" >> "${SERVERCONFIG}"
+  else
+    if [[ -n "${ALTERNATIVE_MODULES+x}" ]]; then
+      for module_path in "${ALTERNATIVE_MODULES[@]}"; do
+        echo "ALTERNATIVE_MODULES+=(\"${module_path}\")" >> "${SERVERCONFIG}"
+      done
+    else
+      echo "ALTERNATIVE_MODULES+=(\"${DIRAC_BASE_DIR}/src/DIRAC\")" >> "${SERVERCONFIG}"
+    fi
+  fi
+
+  if [[ -n "${SERVER_DIRACOSVER+x}" ]]; then
+    echo "export DIRACOSVER=${SERVER_DIRACOSVER}" >> "${SERVERCONFIG}"
   fi
 
-  echo "${CLIENT_USE_PYTHON3}"
   if [[ "${CLIENT_USE_PYTHON3:-}" == "Yes" ]]; then
-    echo "INSTALLOPTIONS+=(\"--pythonVersion=3\")" >> "${CLIENTCONFIG}"
+    echo "export CLIENT_USE_PYTHON3=${CLIENT_USE_PYTHON3}" >> "${CLIENTCONFIG}"
+    echo "ALTERNATIVE_MODULES+=(\"${DIRAC_BASE_DIR}/\")" >> "${CLIENTCONFIG}"
+  else
+    if [[ -n "${ALTERNATIVE_MODULES+x}" ]]; then
+      for module_path in "${ALTERNATIVE_MODULES[@]}"; do
+        echo "ALTERNATIVE_MODULES+=(\"${module_path}\")" >> "${CLIENTCONFIG}"
+      done
+    else
+      echo "ALTERNATIVE_MODULES+=(\"${DIRAC_BASE_DIR}/src/DIRAC\")" >> "${CLIENTCONFIG}"
+    fi
+  fi
+
+  if [[ -n "${CLIENT_DIRACOSVER+x}" ]]; then
+    echo "export DIRACOSVER=${CLIENT_DIRACOSVER}" >> "${CLIENTCONFIG}"
   fi
 
   echo "Generated server config file is:"

tests/CI/run_tests.sh

@@ -45,12 +45,8 @@ elif [[ "$INSTALLTYPE" == "client" ]]; then
     source "$WORKSPACE/ClientInstallDIR/bashrc"
     set -o pipefail
     ERR=0
-    # TODO: The tests should be refactored to remove the need for this
     for repo_path in "${TESTREPO[@]}"; do
-        cp -r "${repo_path}/tests" "$WORKSPACE/ClientInstallDIR/$(basename "${repo_path}")"
-    done
-    for repo_path in "${TESTREPO[@]}"; do
-        source "$WORKSPACE/ClientInstallDIR/$(basename "${repo_path}")/tests/Integration/all_integration_client_tests.sh"
+        source "${repo_path}/tests/Integration/all_integration_client_tests.sh"
     done
 fi
 

tests/Integration/WorkloadManagementSystem/Test_JobDB.py

@@ -20,7 +20,7 @@
 from DIRAC.WorkloadManagementSystem.DB.JobDB import JobDB
 
 jdl = """[
-    Executable = "$DIRACROOT/scripts/dirac-jobexec";
+    Executable = "dirac-jobexec";
     StdError = "std.err";
     LogLevel = "info";
     Site = "ANY";

tests/Integration/WorkloadManagementSystem/Test_JobWrapper.py

@@ -42,7 +42,7 @@ def test_CreateAndSubmit(self):
     jobParams = {'JobID': '1',
                  'JobType': 'Merge',
                  'CPUTime': '1000000',
-                 'Executable': '$DIRACROOT/scripts/dirac-jobexec',
+                 'Executable': 'dirac-jobexec',
                  'Arguments': "helloWorld.xml -o LogLevel=DEBUG --cfg pilot.cfg",
                  'InputSandbox': ['helloWorld.xml', 'exe-script.py']}
     resourceParams = {}

tests/Integration/WorkloadManagementSystem/Test_TimeLeft.sh

@@ -19,7 +19,7 @@ SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
 ###############################################################################
 # Can't find anywhere a batch plugin, not even MJF
 
-$DIRACSCRIPTS/dirac-wms-get-queue-cpu-time --cfg "${SCRIPT_DIR}/pilot.cfg" $DEBUG
+dirac-wms-get-queue-cpu-time --cfg "${SCRIPT_DIR}/pilot.cfg" $DEBUG
 
 if [[ "${?}" -eq 0 ]]; then
   echo -e "\nSuccess\n\n"
@@ -35,7 +35,7 @@ fi
 export MACHINEFEATURES="${SCRIPT_DIR}/sb.cfg"
 export JOBFEATURES="${SCRIPT_DIR}/sb.cfg"
 
-$DIRACSCRIPTS/dirac-wms-get-queue-cpu-time --cfg "${SCRIPT_DIR}/pilot.cfg" $DEBUG
+dirac-wms-get-queue-cpu-time --cfg "${SCRIPT_DIR}/pilot.cfg" $DEBUG
 
 if [[ "${?}" -eq 0 ]]; then
   echo -e "\nSuccess\n\n"
@@ -51,7 +51,7 @@ fi
 export MACHINEFEATURES=${SCRIPT_DIR}/MJF/
 export JOBFEATURES=${SCRIPT_DIR}/MJF/
 
-$DIRACSCRIPTS/dirac-wms-get-queue-cpu-time --cfg "${SCRIPT_DIR}/pilot.cfg" $DEBUG
+dirac-wms-get-queue-cpu-time --cfg "${SCRIPT_DIR}/pilot.cfg" $DEBUG
 
 if [[ "${?}" -eq 0 ]]; then
   echo -e "\nSuccess\n\n"

tests/Integration/WorkloadManagementSystem/Test_dirac-jobexec.sh

@@ -23,7 +23,7 @@ python "${SCRIPT_DIR}/createJobXMLDescriptions.py" $DEBUG
 # Running the real tests
 
 # OK
-$DIRACSCRIPTS/dirac-jobexec jobDescription-OK.xml --cfg "${SCRIPT_DIR}/pilot.cfg" $DEBUG
+dirac-jobexec jobDescription-OK.xml --cfg "${SCRIPT_DIR}/pilot.cfg" $DEBUG
 if [[ "${?}" -eq 0 ]]; then
   echo -e "\nSuccess\n\n"
 else
@@ -32,7 +32,7 @@ else
 fi
 
 # OK2
-$DIRACSCRIPTS/dirac-jobexec jobDescription-OK-multiSteps.xml --cfg "${SCRIPT_DIR}/pilot.cfg" $DEBUG
+dirac-jobexec jobDescription-OK-multiSteps.xml --cfg "${SCRIPT_DIR}/pilot.cfg" $DEBUG
 if [[ "${?}" -eq 0 ]]; then
   echo -e "\nSuccess\n\n"
 else
@@ -42,7 +42,7 @@ fi
 
 
 # FAIL
-$DIRACSCRIPTS/dirac-jobexec jobDescription-FAIL.xml --cfg "${SCRIPT_DIR}/pilot.cfg" $DEBUG
+dirac-jobexec jobDescription-FAIL.xml --cfg "${SCRIPT_DIR}/pilot.cfg" $DEBUG
 if [[ "${?}" -eq 111 ]]; then
   echo -e "\nSuccess\n\n"
 else
@@ -51,7 +51,7 @@ else
 fi
 
 # FAIL2
-$DIRACSCRIPTS/dirac-jobexec jobDescription-FAIL-multiSteps.xml --cfg "${SCRIPT_DIR}/pilot.cfg" $DEBUG
+dirac-jobexec jobDescription-FAIL-multiSteps.xml --cfg "${SCRIPT_DIR}/pilot.cfg" $DEBUG
 if [[ "${?}" -eq 111 ]]; then
   echo -e "\nSuccess\n\n"
 else
@@ -61,7 +61,7 @@ fi
 
 
 # FAIL with exit code > 255
-$DIRACSCRIPTS/dirac-jobexec jobDescription-FAIL1502.xml --cfg "${SCRIPT_DIR}/pilot.cfg" $DEBUG
+dirac-jobexec jobDescription-FAIL1502.xml --cfg "${SCRIPT_DIR}/pilot.cfg" $DEBUG
 if [[ "${?}" -eq 222 ]]; then # This is 1502 & 255 (0xDE)
   echo -e "\nSuccess\n\n"
 else