From 821de56a86090dc9c3c9a16fbbe27adb068b9a9a Mon Sep 17 00:00:00 2001 From: Chris Burr Date: Thu, 18 Feb 2021 10:36:08 +0100 Subject: [PATCH 1/4] Remove fts3-rest from the setuptools configuration for now --- setup.cfg | 1 - 1 file changed, 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index 3d9a3bd8546..e3a2d21d273 100644 --- a/setup.cfg +++ b/setup.cfg @@ -35,7 +35,6 @@ install_requires = botocore certifi diraccfg - fts3-rest future gfal2-python M2Crypto >=0.36 From 0ca53b861a4aaf00f65c9aeb054a3a5db05cd71b Mon Sep 17 00:00:00 2001 From: Chris Burr Date: Fri, 19 Feb 2021 17:36:06 +0100 Subject: [PATCH 2/4] Fix script global state for Python 3 again --- src/DIRAC/Core/scripts/dirac_platform.py | 13 ++++++------- src/DIRAC/Core/scripts/dirac_version.py | 11 ++++++----- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/src/DIRAC/Core/scripts/dirac_platform.py b/src/DIRAC/Core/scripts/dirac_platform.py index 613f45f7234..cc0e8a42526 100755 --- a/src/DIRAC/Core/scripts/dirac_platform.py +++ b/src/DIRAC/Core/scripts/dirac_platform.py @@ -26,23 +26,22 @@ __RCSID__ = "$Id$" -import sys -import argparse - -parser = argparse.ArgumentParser( - description=__doc__, - formatter_class=argparse.RawDescriptionHelpFormatter) -parser.parse_known_args() try: from DIRAC.Core.Utilities.Platform import getPlatformString except Exception: + import argparse import platform import os import sys import re import subprocess + parser = argparse.ArgumentParser( + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + parser.parse_known_args() + # We need to patch python platform module. It does a string comparison for the libc versions. # it fails when going from 2.9 to 2.10, # the fix converts the version to a tuple and attempts a numeric comparison diff --git a/src/DIRAC/Core/scripts/dirac_version.py b/src/DIRAC/Core/scripts/dirac_version.py index 9b7a7797db4..3cb0af19cc3 100755 --- a/src/DIRAC/Core/scripts/dirac_version.py +++ b/src/DIRAC/Core/scripts/dirac_version.py @@ -21,17 +21,18 @@ import argparse -parser = argparse.ArgumentParser( - description=__doc__, - formatter_class=argparse.RawDescriptionHelpFormatter) -parser.parse_known_args() - import DIRAC from DIRAC.Core.Utilities.DIRACScript import DIRACScript @DIRACScript() def main(): + parser = argparse.ArgumentParser( + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter, + ) + parser.parse_known_args() + print(DIRAC.version) From faa81e504da687622f6553b7f8a02e7f5869459b Mon Sep 17 00:00:00 2001 From: Chris Burr Date: Wed, 24 Feb 2021 10:39:40 +0100 Subject: [PATCH 3/4] Get dirac-jobexec from PATH by default --- .../Tutorials/JobManagementAdvanced/index.rst | 2 +- src/DIRAC/Interfaces/API/Job.py | 2 +- src/DIRAC/Interfaces/API/test/Test_DIRAC.py | 10 ++++++++-- src/DIRAC/Interfaces/API/test/testWF.jdl | 2 +- src/DIRAC/Interfaces/API/test/testWFSIO.jdl | 2 +- .../TransformationSystem/test/Test_JobInfo.py | 8 ++++---- .../Agent/test/Test_Agent_JobAgent.py | 2 +- .../JobWrapper/JobWrapper.py | 15 +++++++++++++++ .../WorkloadManagementSystem/Test_JobDB.py | 2 +- .../WorkloadManagementSystem/Test_JobWrapper.py | 2 +- 10 files changed, 34 insertions(+), 13 deletions(-) diff --git a/docs/source/UserGuide/Tutorials/JobManagementAdvanced/index.rst b/docs/source/UserGuide/Tutorials/JobManagementAdvanced/index.rst index 1b8163f07aa..53ce38533d4 100644 --- a/docs/source/UserGuide/Tutorials/JobManagementAdvanced/index.rst +++ b/docs/source/UserGuide/Tutorials/JobManagementAdvanced/index.rst @@ -395,7 +395,7 @@ Let's perform this exercise in the python shell. $ python Test-API-JDL.py Priority = "1"; - Executable = "$DIRACROOT/scripts/dirac-jobexec"; + Executable = "dirac-jobexec"; ExecutionEnvironment = "MYVARIABLE=TEST"; StdError = "std.err"; LogLevel = "DEBUG"; diff --git a/src/DIRAC/Interfaces/API/Job.py b/src/DIRAC/Interfaces/API/Job.py index a3c65a3a0dc..116006d9a63 100755 --- a/src/DIRAC/Interfaces/API/Job.py +++ b/src/DIRAC/Interfaces/API/Job.py @@ -83,7 +83,7 @@ def __init__(self, script=None, stdout='std.out', stderr='std.err'): self.stdout = stdout self.stderr = stderr self.logLevel = 'info' - self.executable = '$DIRACROOT/scripts/dirac-jobexec' # to be clarified + self.executable = 'dirac-jobexec' # to be clarified # $DIRACROOT is set by the JobWrapper at execution time self.addToInputSandbox = [] self.addToOutputSandbox = [] diff --git a/src/DIRAC/Interfaces/API/test/Test_DIRAC.py b/src/DIRAC/Interfaces/API/test/Test_DIRAC.py index 38dd7118f27..47fafce187e 100644 --- a/src/DIRAC/Interfaces/API/test/Test_DIRAC.py +++ b/src/DIRAC/Interfaces/API/test/Test_DIRAC.py @@ -127,8 +127,14 @@ def test_runLocal(dirac, job, mocker, osmock, confMock): ret = dirac.runLocal(job) LOG.info("dirac log calls: %s", dirac.log.call_args_list) LOG.info("CallStack: %s", pformat(ret.get('CallStack', {}))) - assert sysMock.call_args_list[0][1]['cmdSeq'] == ['/root/dirac/scripts/dirac-jobexec', - 'jobDescription.xml', '-o', 'LogLevel=DEBUG'] + try: + assert sysMock.call_args_list[0][1]['cmdSeq'] == [ + 'dirac-jobexec', 'jobDescription.xml', '-o', 'LogLevel=DEBUG' + ] + except AssertionError: + assert sysMock.call_args_list[0][1]['cmdSeq'] == [ + '/root/dirac/scripts/dirac-jobexec', 'jobDescription.xml', '-o', 'LogLevel=DEBUG' + ] assert ret.get('Message', None) is None assert ret['OK'] assert call('/abspath/absfile.xml', '/pwd/tempFolder/') in shMock.copy.call_args_list diff --git a/src/DIRAC/Interfaces/API/test/testWF.jdl b/src/DIRAC/Interfaces/API/test/testWF.jdl index 332c6aafebf..2d6a4b6446e 100644 --- a/src/DIRAC/Interfaces/API/test/testWF.jdl +++ b/src/DIRAC/Interfaces/API/test/testWF.jdl @@ -1,6 +1,6 @@ Arguments = "jobDescription.xml -o LogLevel=DEBUG -p JOB_ID=%(JOB_ID)s -p InputData=%(InputData)s"; - Executable = "$DIRACROOT/scripts/dirac-jobexec"; + Executable = "dirac-jobexec"; InputData = %(InputData)s; InputSandbox = jobDescription.xml; JOB_ID = %(JOB_ID)s; diff --git a/src/DIRAC/Interfaces/API/test/testWFSIO.jdl b/src/DIRAC/Interfaces/API/test/testWFSIO.jdl index 68386046516..9c3c75d083f 100644 --- a/src/DIRAC/Interfaces/API/test/testWFSIO.jdl +++ b/src/DIRAC/Interfaces/API/test/testWFSIO.jdl @@ -1,6 +1,6 @@ Arguments = "jobDescription.xml -o LogLevel=info"; - Executable = "$DIRACROOT/scripts/dirac-jobexec"; + Executable = "dirac-jobexec"; JobGroup = jobGroup; JobName = jobName; JobType = jobType; diff --git a/src/DIRAC/TransformationSystem/test/Test_JobInfo.py b/src/DIRAC/TransformationSystem/test/Test_JobInfo.py index ba86e6700e7..fd48fe4ebfd 100644 --- a/src/DIRAC/TransformationSystem/test/Test_JobInfo.py +++ b/src/DIRAC/TransformationSystem/test/Test_JobInfo.py @@ -39,7 +39,7 @@ def setUp(self): self.jdl2 = { 'LogTargetPath': "/ilc/prod/clic/500gev/yyveyx_o/ILD/REC/00006326/LOG/00006326_015.tar", - 'Executable': "$DIRACROOT/scripts/dirac-jobexec", + 'Executable': "dirac-jobexec", 'TaskID': 15, 'SoftwareDistModule': "ILCDIRAC.Core.Utilities.CombinedSoftwareInstallation", 'JobName': "00006326_00000015", @@ -109,7 +109,7 @@ def setUp(self): self.jdlBrokenContent = { 'LogTargetPath': "/ilc/prod/clic/500gev/yyveyx_o/ILD/REC/00006326/LOG/00006326_015.tar", - 'Executable': "$DIRACROOT/scripts/dirac-jobexec", + 'Executable': "dirac-jobexec", 'TaskID': 'muahahaha', 'SoftwareDistModule': "ILCDIRAC.Core.Utilities.CombinedSoftwareInstallation", 'JobName': "00006326_00000015", @@ -180,7 +180,7 @@ def setUp(self): # jdl with single outputdata, self.jdl1 = { 'LogTargetPath': "/ilc/prod/clic/3tev/e1e1_o/SID/SIM/00006301/LOG/00006301_10256.tar", - 'Executable': "$DIRACROOT/scripts/dirac-jobexec", + 'Executable': "dirac-jobexec", 'TaskID': 10256, 'SoftwareDistModule': "ILCDIRAC.Core.Utilities.CombinedSoftwareInstallation", 'JobName': "00006301_00010256", @@ -250,7 +250,7 @@ def setUp(self): self.jdlNoInput = { 'LogTargetPath': "/ilc/prod/clic/1.4tev/ea_qqqqnu/gen/00006498/LOG/00006498_1307.tar", - 'Executable': "$DIRACROOT/scripts/dirac-jobexec", + 'Executable': "dirac-jobexec", 'TaskID': 1307, 'SoftwareDistModule': "ILCDIRAC.Core.Utilities.CombinedSoftwareInstallation", 'JobName': "00006498_00001307", diff --git a/src/DIRAC/WorkloadManagementSystem/Agent/test/Test_Agent_JobAgent.py b/src/DIRAC/WorkloadManagementSystem/Agent/test/Test_Agent_JobAgent.py index 787a3a692a4..89342fcad0f 100644 --- a/src/DIRAC/WorkloadManagementSystem/Agent/test/Test_Agent_JobAgent.py +++ b/src/DIRAC/WorkloadManagementSystem/Agent/test/Test_Agent_JobAgent.py @@ -35,7 +35,7 @@ def test__getJDLParameters(mocker): jdl = """ [ - Executable = "$DIRACROOT/scripts/dirac-jobexec"; + Executable = "dirac-jobexec"; StdError = "std.err"; LogLevel = "info"; Site = "ANY"; diff --git a/src/DIRAC/WorkloadManagementSystem/JobWrapper/JobWrapper.py b/src/DIRAC/WorkloadManagementSystem/JobWrapper/JobWrapper.py index a95675bbd63..e31848c8d39 100755 --- a/src/DIRAC/WorkloadManagementSystem/JobWrapper/JobWrapper.py +++ b/src/DIRAC/WorkloadManagementSystem/JobWrapper/JobWrapper.py @@ -28,6 +28,7 @@ import glob import json import six +import distutils.spawn from six.moves.urllib.parse import unquote as urlunquote @@ -231,6 +232,7 @@ def initialize(self, arguments): if os.path.exists('%s/%s' % (self.root, extraOpts)): shutil.copyfile('%s/%s' % (self.root, extraOpts), extraOpts) self.__loadLocalCFGFiles(self.localSiteRoot) + else: self.log.info('JobID is not defined, running in current directory') @@ -333,6 +335,14 @@ def execute(self): # the argument should include the jobDescription.xml file jobArguments = self.jobArgs.get('Arguments', '') + # This is a workaround for Python 2 style installations + if six.PY3 and executable == "$DIRACROOT/scripts/dirac-jobexec": + self.log.warn( + 'Replaced job executable "$DIRACROOT/scripts/dirac-jobexec" with ' + '"dirac-jobexec". Please fix your submission script!' + ) + executable = "dirac-jobexec" + executable = os.path.expandvars(executable) exeThread = None spObject = None @@ -341,6 +351,11 @@ def execute(self): executable = executable.replace('$DIRACROOT', self.localSiteRoot) self.log.verbose('Replaced $DIRACROOT for executable as %s' % (self.localSiteRoot)) + # Try to find the executable on PATH + if "/" not in executable: + # Returns None if the executable is not found so use "or" to leave it unchanged + executable = distutils.spawn.find_executable(executable) or executable + # Make the full path since . is not always in the PATH executable = os.path.abspath(executable) if not os.access(executable, os.X_OK): diff --git a/tests/Integration/WorkloadManagementSystem/Test_JobDB.py b/tests/Integration/WorkloadManagementSystem/Test_JobDB.py index cc844fa9473..4b007ac854a 100644 --- a/tests/Integration/WorkloadManagementSystem/Test_JobDB.py +++ b/tests/Integration/WorkloadManagementSystem/Test_JobDB.py @@ -20,7 +20,7 @@ from DIRAC.WorkloadManagementSystem.DB.JobDB import JobDB jdl = """[ - Executable = "$DIRACROOT/scripts/dirac-jobexec"; + Executable = "dirac-jobexec"; StdError = "std.err"; LogLevel = "info"; Site = "ANY"; diff --git a/tests/Integration/WorkloadManagementSystem/Test_JobWrapper.py b/tests/Integration/WorkloadManagementSystem/Test_JobWrapper.py index 88cbd548eec..04707d3a117 100644 --- a/tests/Integration/WorkloadManagementSystem/Test_JobWrapper.py +++ b/tests/Integration/WorkloadManagementSystem/Test_JobWrapper.py @@ -42,7 +42,7 @@ def test_CreateAndSubmit(self): jobParams = {'JobID': '1', 'JobType': 'Merge', 'CPUTime': '1000000', - 'Executable': '$DIRACROOT/scripts/dirac-jobexec', + 'Executable': 'dirac-jobexec', 'Arguments': "helloWorld.xml -o LogLevel=DEBUG --cfg pilot.cfg", 'InputSandbox': ['helloWorld.xml', 'exe-script.py']} resourceParams = {} From 193f0f4e0d9dbe39d7ebd7933a0c89eb1919be44 Mon Sep 17 00:00:00 2001 From: Chris Burr Date: Wed, 24 Feb 2021 13:24:58 +0100 Subject: [PATCH 4/4] Use DIRACOS2 for Python 3 client integration tests --- .github/workflows/integration.yml | 3 + tests/CI/run_docker_setup.sh | 38 +++++--- tests/CI/run_tests.sh | 6 +- .../WorkloadManagementSystem/Test_TimeLeft.sh | 6 +- .../Test_dirac-jobexec.sh | 10 +-- .../all_integration_client_tests.sh | 52 +++++------ tests/Jenkins/utilities.sh | 87 +++++++++++-------- 7 files changed, 118 insertions(+), 84 deletions(-) diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index 733a559ef8c..efc189abd0d 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -50,6 +50,7 @@ jobs: # CLIENT_USE_PYTHON3: No - TEST_NAME: "Python 3 client" CLIENT_USE_PYTHON3: Yes + CLIENT_DIRACOSVER: 2.0a8 steps: - uses: actions/checkout@v2 @@ -74,7 +75,9 @@ jobs: if [[ "${{ matrix.DIRACOSVER || env.MATRIX_DEFAULT_DIRACOSVER }}" != "default" ]]; then echo -n "-e DIRACOSVER=${{ matrix.DIRACOSVER || env.MATRIX_DEFAULT_DIRACOSVER }} " >> run_in_container; fi echo -n "-e TEST_HTTPS=${{ matrix.TEST_HTTPS || env.MATRIX_DEFAULT_TEST_HTTPS }} " >> run_in_container echo -n "-e SERVER_USE_PYTHON3=${{ matrix.SERVER_USE_PYTHON3 || env.MATRIX_DEFAULT_SERVER_USE_PYTHON3 }} " >> run_in_container + if [[ "${{ matrix.SERVER_DIRACOSVER || env.MATRIX_DEFAULT_SERVER_DIRACOSVER }}" != "default" ]]; then echo -n "-e SERVER_DIRACOSVER=${{ matrix.SERVER_DIRACOSVER || env.MATRIX_DEFAULT_SERVER_DIRACOSVER }} " >> run_in_container; fi echo -n "-e CLIENT_USE_PYTHON3=${{ matrix.CLIENT_USE_PYTHON3 || env.MATRIX_DEFAULT_CLIENT_USE_PYTHON3 }} " >> run_in_container + if [[ "${{ matrix.CLIENT_DIRACOSVER || env.MATRIX_DEFAULT_CLIENT_DIRACOSVER }}" != "default" ]]; then echo -n "-e CLIENT_DIRACOSVER=${{ matrix.CLIENT_DIRACOSVER || env.MATRIX_DEFAULT_CLIENT_DIRACOSVER }} " >> run_in_container; fi # Finish wrapper script echo -n "dirac-testing-host \"\$@\"" >> run_in_container chmod +x run_in_container diff --git a/tests/CI/run_docker_setup.sh b/tests/CI/run_docker_setup.sh index b533b02fb28..0a284425e63 100755 --- a/tests/CI/run_docker_setup.sh +++ b/tests/CI/run_docker_setup.sh @@ -111,13 +111,6 @@ prepareEnvironment() { fi echo "declare -a ALTERNATIVE_MODULES" - if [[ -n "${ALTERNATIVE_MODULES+x}" ]]; then - for module_path in "${ALTERNATIVE_MODULES[@]}"; do - echo "ALTERNATIVE_MODULES+=(\"${module_path}\")" - done - else - echo "ALTERNATIVE_MODULES+=(\"${DIRAC_BASE_DIR}/src/DIRAC\")" - fi echo "declare -a INSTALLOPTIONS" if [[ -n "${INSTALLOPTIONS+x}" ]]; then @@ -154,12 +147,37 @@ prepareEnvironment() { cp "${SERVERCONFIG}" "${CLIENTCONFIG}" if [[ "${SERVER_USE_PYTHON3:-}" == "Yes" ]]; then - echo "INSTALLOPTIONS+=(\"--pythonVersion=3\")" >> "${SERVERCONFIG}" + echo "export SERVER_USE_PYTHON3=${SERVER_USE_PYTHON3}" >> "${SERVERCONFIG}" + echo "ALTERNATIVE_MODULES+=(\"${DIRAC_BASE_DIR}/\")" >> "${SERVERCONFIG}" + else + if [[ -n "${ALTERNATIVE_MODULES+x}" ]]; then + for module_path in "${ALTERNATIVE_MODULES[@]}"; do + echo "ALTERNATIVE_MODULES+=(\"${module_path}\")" >> "${SERVERCONFIG}" + done + else + echo "ALTERNATIVE_MODULES+=(\"${DIRAC_BASE_DIR}/src/DIRAC\")" >> "${SERVERCONFIG}" + fi + fi + + if [[ -n "${SERVER_DIRACOSVER+x}" ]]; then + echo "export DIRACOSVER=${SERVER_DIRACOSVER}" >> "${SERVERCONFIG}" fi - echo "${CLIENT_USE_PYTHON3}" if [[ "${CLIENT_USE_PYTHON3:-}" == "Yes" ]]; then - echo "INSTALLOPTIONS+=(\"--pythonVersion=3\")" >> "${CLIENTCONFIG}" + echo "export CLIENT_USE_PYTHON3=${CLIENT_USE_PYTHON3}" >> "${CLIENTCONFIG}" + echo "ALTERNATIVE_MODULES+=(\"${DIRAC_BASE_DIR}/\")" >> "${CLIENTCONFIG}" + else + if [[ -n "${ALTERNATIVE_MODULES+x}" ]]; then + for module_path in "${ALTERNATIVE_MODULES[@]}"; do + echo "ALTERNATIVE_MODULES+=(\"${module_path}\")" >> "${CLIENTCONFIG}" + done + else + echo "ALTERNATIVE_MODULES+=(\"${DIRAC_BASE_DIR}/src/DIRAC\")" >> "${CLIENTCONFIG}" + fi + fi + + if [[ -n "${CLIENT_DIRACOSVER+x}" ]]; then + echo "export DIRACOSVER=${CLIENT_DIRACOSVER}" >> "${CLIENTCONFIG}" fi echo "Generated server config file is:" diff --git a/tests/CI/run_tests.sh b/tests/CI/run_tests.sh index d8c9d82e68c..c3a626e1f7c 100755 --- a/tests/CI/run_tests.sh +++ b/tests/CI/run_tests.sh @@ -45,12 +45,8 @@ elif [[ "$INSTALLTYPE" == "client" ]]; then source "$WORKSPACE/ClientInstallDIR/bashrc" set -o pipefail ERR=0 - # TODO: The tests should be refactored to remove the need for this for repo_path in "${TESTREPO[@]}"; do - cp -r "${repo_path}/tests" "$WORKSPACE/ClientInstallDIR/$(basename "${repo_path}")" - done - for repo_path in "${TESTREPO[@]}"; do - source "$WORKSPACE/ClientInstallDIR/$(basename "${repo_path}")/tests/Integration/all_integration_client_tests.sh" + source "${repo_path}/tests/Integration/all_integration_client_tests.sh" done fi diff --git a/tests/Integration/WorkloadManagementSystem/Test_TimeLeft.sh b/tests/Integration/WorkloadManagementSystem/Test_TimeLeft.sh index 8c4e8070827..af9473c3592 100755 --- a/tests/Integration/WorkloadManagementSystem/Test_TimeLeft.sh +++ b/tests/Integration/WorkloadManagementSystem/Test_TimeLeft.sh @@ -19,7 +19,7 @@ SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" ############################################################################### # Can't find anywhere a batch plugin, not even MJF -$DIRACSCRIPTS/dirac-wms-get-queue-cpu-time --cfg "${SCRIPT_DIR}/pilot.cfg" $DEBUG +dirac-wms-get-queue-cpu-time --cfg "${SCRIPT_DIR}/pilot.cfg" $DEBUG if [[ "${?}" -eq 0 ]]; then echo -e "\nSuccess\n\n" @@ -35,7 +35,7 @@ fi export MACHINEFEATURES="${SCRIPT_DIR}/sb.cfg" export JOBFEATURES="${SCRIPT_DIR}/sb.cfg" -$DIRACSCRIPTS/dirac-wms-get-queue-cpu-time --cfg "${SCRIPT_DIR}/pilot.cfg" $DEBUG +dirac-wms-get-queue-cpu-time --cfg "${SCRIPT_DIR}/pilot.cfg" $DEBUG if [[ "${?}" -eq 0 ]]; then echo -e "\nSuccess\n\n" @@ -51,7 +51,7 @@ fi export MACHINEFEATURES=${SCRIPT_DIR}/MJF/ export JOBFEATURES=${SCRIPT_DIR}/MJF/ -$DIRACSCRIPTS/dirac-wms-get-queue-cpu-time --cfg "${SCRIPT_DIR}/pilot.cfg" $DEBUG +dirac-wms-get-queue-cpu-time --cfg "${SCRIPT_DIR}/pilot.cfg" $DEBUG if [[ "${?}" -eq 0 ]]; then echo -e "\nSuccess\n\n" diff --git a/tests/Integration/WorkloadManagementSystem/Test_dirac-jobexec.sh b/tests/Integration/WorkloadManagementSystem/Test_dirac-jobexec.sh index 259a18e8883..b6bf4d7fd19 100755 --- a/tests/Integration/WorkloadManagementSystem/Test_dirac-jobexec.sh +++ b/tests/Integration/WorkloadManagementSystem/Test_dirac-jobexec.sh @@ -23,7 +23,7 @@ python "${SCRIPT_DIR}/createJobXMLDescriptions.py" $DEBUG # Running the real tests # OK -$DIRACSCRIPTS/dirac-jobexec jobDescription-OK.xml --cfg "${SCRIPT_DIR}/pilot.cfg" $DEBUG +dirac-jobexec jobDescription-OK.xml --cfg "${SCRIPT_DIR}/pilot.cfg" $DEBUG if [[ "${?}" -eq 0 ]]; then echo -e "\nSuccess\n\n" else @@ -32,7 +32,7 @@ else fi # OK2 -$DIRACSCRIPTS/dirac-jobexec jobDescription-OK-multiSteps.xml --cfg "${SCRIPT_DIR}/pilot.cfg" $DEBUG +dirac-jobexec jobDescription-OK-multiSteps.xml --cfg "${SCRIPT_DIR}/pilot.cfg" $DEBUG if [[ "${?}" -eq 0 ]]; then echo -e "\nSuccess\n\n" else @@ -42,7 +42,7 @@ fi # FAIL -$DIRACSCRIPTS/dirac-jobexec jobDescription-FAIL.xml --cfg "${SCRIPT_DIR}/pilot.cfg" $DEBUG +dirac-jobexec jobDescription-FAIL.xml --cfg "${SCRIPT_DIR}/pilot.cfg" $DEBUG if [[ "${?}" -eq 111 ]]; then echo -e "\nSuccess\n\n" else @@ -51,7 +51,7 @@ else fi # FAIL2 -$DIRACSCRIPTS/dirac-jobexec jobDescription-FAIL-multiSteps.xml --cfg "${SCRIPT_DIR}/pilot.cfg" $DEBUG +dirac-jobexec jobDescription-FAIL-multiSteps.xml --cfg "${SCRIPT_DIR}/pilot.cfg" $DEBUG if [[ "${?}" -eq 111 ]]; then echo -e "\nSuccess\n\n" else @@ -61,7 +61,7 @@ fi # FAIL with exit code > 255 -$DIRACSCRIPTS/dirac-jobexec jobDescription-FAIL1502.xml --cfg "${SCRIPT_DIR}/pilot.cfg" $DEBUG +dirac-jobexec jobDescription-FAIL1502.xml --cfg "${SCRIPT_DIR}/pilot.cfg" $DEBUG if [[ "${?}" -eq 222 ]]; then # This is 1502 & 255 (0xDE) echo -e "\nSuccess\n\n" else diff --git a/tests/Integration/all_integration_client_tests.sh b/tests/Integration/all_integration_client_tests.sh index ad30ad67b82..368e26f4e3a 100644 --- a/tests/Integration/all_integration_client_tests.sh +++ b/tests/Integration/all_integration_client_tests.sh @@ -9,13 +9,16 @@ echo -e '****************************************' echo -e '********' "client -> server tests" '********\n' +THIS_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" +echo -e "THIS_DIR=${THIS_DIR}" |& tee -a clientTestOutputs.txt + echo -e "*** $(date -u) Getting a non privileged user\n" |& tee -a clientTestOutputs.txt dirac-proxy-init -C "${SERVERINSTALLDIR}/user/client.pem" -K "${SERVERINSTALLDIR}/user/client.key" "${DEBUG}" |& tee -a clientTestOutputs.txt; (( ERR |= "${?}" )) #-------------------------------------------------------------------------------# echo -e "*** $(date -u) **** Accounting TESTS ****\n" -pytest "${CLIENTINSTALLDIR}/DIRAC/tests/Integration/AccountingSystem/Test_DataStoreClient.py" |& tee -a clientTestOutputs.txt; (( ERR |= "${?}" )) -pytest "${CLIENTINSTALLDIR}/DIRAC/tests/Integration/AccountingSystem/Test_ReportsClient.py" |& tee -a clientTestOutputs.txt; (( ERR |= "${?}" )) +pytest "${THIS_DIR}/AccountingSystem/Test_DataStoreClient.py" |& tee -a clientTestOutputs.txt; (( ERR |= "${?}" )) +pytest "${THIS_DIR}/AccountingSystem/Test_ReportsClient.py" |& tee -a clientTestOutputs.txt; (( ERR |= "${?}" )) #-------------------------------------------------------------------------------# @@ -25,50 +28,49 @@ echo -e "*** $(date -u) Getting a non privileged user\n" |& tee -a clientTestOu dirac-proxy-init -C "${SERVERINSTALLDIR}/user/client.pem" -K "${SERVERINSTALLDIR}/user/client.key" "${DEBUG}" |& tee -a clientTestOutputs.txt echo -e "*** $(date -u) Starting RMS Client test as a non privileged user\n" |& tee -a clientTestOutputs.txt -pytest "${CLIENTINSTALLDIR}/DIRAC/tests/Integration/RequestManagementSystem/Test_Client_Req.py" |& tee -a clientTestOutputs.txt; (( ERR |= "${?}" )) +pytest "${THIS_DIR}/RequestManagementSystem/Test_Client_Req.py" |& tee -a clientTestOutputs.txt; (( ERR |= "${?}" )) echo -e "*** $(date -u) getting the prod role again\n" |& tee -a clientTestOutputs.txt dirac-proxy-init -g prod -C "${SERVERINSTALLDIR}/user/client.pem" -K "${SERVERINSTALLDIR}/user/client.key" "${DEBUG}" |& tee -a clientTestOutputs.txt echo -e "*** $(date -u) Starting RMS Client test as an admin user\n" |& tee -a clientTestOutputs.txt -pytest "${CLIENTINSTALLDIR}/DIRAC/tests/Integration/RequestManagementSystem/Test_Client_Req.py" |& tee -a clientTestOutputs.txt; (( ERR |= "${?}" )) +pytest "${THIS_DIR}/RequestManagementSystem/Test_Client_Req.py" |& tee -a clientTestOutputs.txt; (( ERR |= "${?}" )) #-------------------------------------------------------------------------------# echo -e "*** $(date -u) **** RSS TESTS ****\n" -pytest "${CLIENTINSTALLDIR}/DIRAC/tests/Integration/ResourceStatusSystem/Test_ResourceManagement.py" |& tee -a clientTestOutputs.txt; (( ERR |= "${?}" )) -pytest "${CLIENTINSTALLDIR}/DIRAC/tests/Integration/ResourceStatusSystem/Test_ResourceStatus.py" |& tee -a clientTestOutputs.txt; (( ERR |= "${?}" )) -pytest "${CLIENTINSTALLDIR}/DIRAC/tests/Integration/ResourceStatusSystem/Test_SiteStatus.py" |& tee -a clientTestOutputs.txt; (( ERR |= "${?}" )) -pytest "${CLIENTINSTALLDIR}/DIRAC/tests/Integration/ResourceStatusSystem/Test_Publisher.py" |& tee -a clientTestOutputs.txt; (( ERR |= "${?}" )) -pytest "${CLIENTINSTALLDIR}/DIRAC/tests/Integration/ResourceStatusSystem/Test_EmailActionAgent.py" |& tee -a clientTestOutputs.txt; (( ERR |= "${?}" )) +pytest "${THIS_DIR}/ResourceStatusSystem/Test_ResourceManagement.py" |& tee -a clientTestOutputs.txt; (( ERR |= "${?}" )) +pytest "${THIS_DIR}/ResourceStatusSystem/Test_ResourceStatus.py" |& tee -a clientTestOutputs.txt; (( ERR |= "${?}" )) +pytest "${THIS_DIR}/ResourceStatusSystem/Test_SiteStatus.py" |& tee -a clientTestOutputs.txt; (( ERR |= "${?}" )) +pytest "${THIS_DIR}/ResourceStatusSystem/Test_Publisher.py" |& tee -a clientTestOutputs.txt; (( ERR |= "${?}" )) +pytest "${THIS_DIR}/ResourceStatusSystem/Test_EmailActionAgent.py" |& tee -a clientTestOutputs.txt; (( ERR |= "${?}" )) #-------------------------------------------------------------------------------# echo -e "*** $(date -u) **** WMS TESTS ****\n" # pytest "${CLIENTINSTALLDIR}"/DIRAC/tests/Integration/WorkloadManagementSystem/Test_PilotsLoggingClient.py |& tee -a clientTestOutputs.txt -python "${CLIENTINSTALLDIR}/DIRAC/tests/Integration/WorkloadManagementSystem/Test_SandboxStoreClient.py" --cfg "$WORKSPACE/TestCode/DIRAC/tests/Integration/WorkloadManagementSystem/sb.cfg" |& tee -a clientTestOutputs.txt; (( ERR |= "${?}" )) -pytest "${CLIENTINSTALLDIR}/DIRAC/tests/Integration/WorkloadManagementSystem/Test_JobWrapper.py" |& tee -a clientTestOutputs.txt; (( ERR |= "${?}" )) -pytest "${CLIENTINSTALLDIR}/DIRAC/tests/Integration/WorkloadManagementSystem/Test_PilotsClient.py" |& tee -a clientTestOutputs.txt; (( ERR |= "${?}" )) +python "${THIS_DIR}/WorkloadManagementSystem/Test_SandboxStoreClient.py" --cfg "$WORKSPACE/TestCode/DIRAC/tests/Integration/WorkloadManagementSystem/sb.cfg" |& tee -a clientTestOutputs.txt; (( ERR |= "${?}" )) +pytest "${THIS_DIR}/WorkloadManagementSystem/Test_JobWrapper.py" |& tee -a clientTestOutputs.txt; (( ERR |= "${?}" )) +pytest "${THIS_DIR}/WorkloadManagementSystem/Test_PilotsClient.py" |& tee -a clientTestOutputs.txt; (( ERR |= "${?}" )) ## no real tests -python "${CLIENTINSTALLDIR}/DIRAC/tests/Integration/WorkloadManagementSystem/createJobXMLDescriptions.py" |& tee -a clientTestOutputs.txt; (( ERR |= "${?}" )) -"${CLIENTINSTALLDIR}/DIRAC/tests/Integration/WorkloadManagementSystem/Test_dirac-jobexec.sh" |& tee -a clientTestOutputs.txt; (( ERR |= "${?}" )) -"${CLIENTINSTALLDIR}/DIRAC/tests/Integration/WorkloadManagementSystem/Test_TimeLeft.sh" |& tee -a clientTestOutputs.txt; (( ERR |= "${?}" )) +python "${THIS_DIR}/WorkloadManagementSystem/createJobXMLDescriptions.py" |& tee -a clientTestOutputs.txt; (( ERR |= "${?}" )) +"${THIS_DIR}/WorkloadManagementSystem/Test_dirac-jobexec.sh" |& tee -a clientTestOutputs.txt; (( ERR |= "${?}" )) +"${THIS_DIR}/WorkloadManagementSystem/Test_TimeLeft.sh" |& tee -a clientTestOutputs.txt; (( ERR |= "${?}" )) #-------------------------------------------------------------------------------# echo -e "*** $(date -u) **** MONITORING TESTS ****\n" -pytest "${CLIENTINSTALLDIR}/DIRAC/tests/Integration/Monitoring/Test_MonitoringSystem.py" |& tee -a clientTestOutputs.txt; (( ERR |= "${?}" )) +pytest "${THIS_DIR}/Monitoring/Test_MonitoringSystem.py" |& tee -a clientTestOutputs.txt; (( ERR |= "${?}" )) #-------------------------------------------------------------------------------# echo -e "*** $(date -u) **** TS TESTS ****\n" -pytest "${CLIENTINSTALLDIR}/DIRAC/tests/Integration/TransformationSystem/Test_Client_Transformation.py" |& tee -a clientTestOutputs.txt; (( ERR |= "${?}" )) -# pytest "${CLIENTINSTALLDIR}/DIRAC/tests/Integration/TransformationSystem/Test_TS_DFC_Catalog.py" |& tee -a clientTestOutputs.txt; (( ERR |= "${?}" )) - +pytest "${THIS_DIR}/TransformationSystem/Test_Client_Transformation.py" |& tee -a clientTestOutputs.txt; (( ERR |= "${?}" )) +# pytest "${THIS_DIR}/TransformationSystem/Test_TS_DFC_Catalog.py" |& tee -a clientTestOutputs.txt; (( ERR |= "${?}" )) #-------------------------------------------------------------------------------# echo -e "*** $(date -u) **** PS TESTS ****\n" -pytest "${CLIENTINSTALLDIR}/DIRAC/tests/Integration/ProductionSystem/Test_Client_Production.py" |& tee -a clientTestOutputs.txt; (( ERR |= "${?}" )) -pytest "${CLIENTINSTALLDIR}/DIRAC/tests/Integration/ProductionSystem/Test_Client_TS_Prod.py" |& tee -a clientTestOutputs.txt; (( ERR |= "${?}" )) +pytest "${THIS_DIR}/ProductionSystem/Test_Client_Production.py" |& tee -a clientTestOutputs.txt; (( ERR |= "${?}" )) +pytest "${THIS_DIR}/ProductionSystem/Test_Client_TS_Prod.py" |& tee -a clientTestOutputs.txt; (( ERR |= "${?}" )) #-------------------------------------------------------------------------------# echo -e "*** $(date -u) **** DataManager TESTS ****\n" @@ -77,7 +79,7 @@ echo -e "*** $(date -u) **** DataManager TESTS ****\n" echo -e "*** $(date -u) Getting a non privileged user to find its VO dynamically\n" |& tee -a clientTestOutputs.txt dirac-proxy-init -g jenkins_user -C $SERVERINSTALLDIR/user/client.pem -K $SERVERINSTALLDIR/user/client.key $DEBUG |& tee -a clientTestOutputs.txt -userVO=$(python -c "from DIRAC.Core.Base.Script import parseCommandLine; parseCommandLine(); from DIRAC.Core.Security.ProxyInfo import getVOfromProxyGroup; print getVOfromProxyGroup().get('Value','')") +userVO=$(python -c "from DIRAC.Core.Base.Script import parseCommandLine; parseCommandLine(); from DIRAC.Core.Security.ProxyInfo import getVOfromProxyGroup; print(getVOfromProxyGroup().get('Value',''))") userVO="${userVO:-Jenkins}" echo -e "*** $(date -u) VO is "${userVO}"\n" |& tee -a clientTestOutputs.txt @@ -100,7 +102,7 @@ dirac-dms-filecatalog-cli -f FileCatalog < dataManager_create_folders echo -e "*** $(date -u) Getting a non privileged user\n" |& tee -a clientTestOutputs.txt dirac-proxy-init -g jenkins_user -C "${SERVERINSTALLDIR}/user/client.pem" -K "${SERVERINSTALLDIR}/user/client.key" "${DEBUG}" |& tee -a clientTestOutputs.txt -pytest "${CLIENTINSTALLDIR}/DIRAC/tests/Integration/DataManagementSystem/Test_DataManager.py" |& tee -a clientTestOutputs.txt; (( ERR |= "${?}" )) +pytest "${THIS_DIR}/DataManagementSystem/Test_DataManager.py" |& tee -a clientTestOutputs.txt; (( ERR |= "${?}" )) #-------------------------------------------------------------------------------# # MultiVO File Catalog tests are configured to use MultiVOFileCatalog module with a separate DB. @@ -110,7 +112,7 @@ pytest "${CLIENTINSTALLDIR}/DIRAC/tests/Integration/DataManagementSystem/Test_Da # normal user proxy dirac-proxy-init -g jenkins_user -C "${SERVERINSTALLDIR}/user/client.pem" -K "${SERVERINSTALLDIR}/user/client.key" "${DEBUG}" |& tee -a clientTestOutputs.txt echo -e "*** $(date -u) **** MultiVO User Metadata TESTS ****\n" -python -m pytest "${CLIENTINSTALLDIR}/DIRAC/tests/Integration/DataManagementSystem/Test_UserMetadata.py" |& tee -a clientTestOutputs.txt; (( ERR |= "${?}" )) +python -m pytest "${THIS_DIR}/DataManagementSystem/Test_UserMetadata.py" |& tee -a clientTestOutputs.txt; (( ERR |= "${?}" )) echo -e "*** $(date -u) **** S3 TESTS ****\n" -pytest "${CLIENTINSTALLDIR}/DIRAC/tests/Integration/Resources/Storage/Test_Resources_S3.py" |& tee -a clientTestOutputs.txt; (( ERR |= "${?}" )) +pytest "${THIS_DIR}/Resources/Storage/Test_Resources_S3.py" |& tee -a clientTestOutputs.txt; (( ERR |= "${?}" )) diff --git a/tests/Jenkins/utilities.sh b/tests/Jenkins/utilities.sh index 0bb9d03d9bc..003bd6fa697 100644 --- a/tests/Jenkins/utilities.sh +++ b/tests/Jenkins/utilities.sh @@ -308,51 +308,69 @@ getCFGFile() { # --UseServerCertificate -o /DIRAC/Security/CertFile=some/location.pem -o /DIRAC/Security/KeyFile=some/location.pem installDIRAC() { - echo -n > "${CLIENTINSTALLDIR}/dirac-ci-install.cfg" - echo '==> Installing DIRAC client' if ! cd "${CLIENTINSTALLDIR}"; then echo "ERROR: cannot change to ${CLIENTINSTALLDIR}" >&2 exit 1 fi - cp "${TESTCODE}/DIRAC/src/DIRAC/Core/scripts/dirac-install.py" "${CLIENTINSTALLDIR}/dirac-install" - chmod +x "${CLIENTINSTALLDIR}/dirac-install" - - if [[ -n "${DEBUG+x}" ]]; then - INSTALLOPTIONS+=("${DEBUG}") - fi - - if [[ -n "${ALTERNATIVE_MODULES+x}" ]]; then - echo "Installing from non-release code" - local option="--module=" + if [[ "${CLIENT_USE_PYTHON3:-}" == "Yes" ]]; then + if [[ -n "${DIRACOSVER+x}" ]]; then + DIRACOS2_URL="https://github.com/DIRACGrid/DIRACOS2/releases/download/${DIRACOSVER}/DIRACOS-Linux-x86_64.sh" + else + DIRACOS2_URL="https://github.com/DIRACGrid/DIRACOS2/releases/latest/download/DIRACOS-Linux-x86_64.sh" + fi + curl -L "${DIRACOS2_URL}" > "installer.sh" + bash "installer.sh" + rm "installer.sh" + # TODO: Remove + echo "source \"$PWD/diracos/diracosrc\"" > "$PWD/bashrc" + source diracos/diracosrc for module_path in "${ALTERNATIVE_MODULES[@]}"; do - if [[ -d "${module_path}" ]]; then - option+="${module_path}:::$(basename "${module_path}"):::local," - else - option+="${module_path}," - fi + pip install "${module_path}" done - INSTALLOPTIONS+=("${option: :$((${#option} - 1))}") - fi + else + echo -n > "${CLIENTINSTALLDIR}/dirac-ci-install.cfg" - if [[ "${DIRACOSVER}" ]]; then - INSTALLOPTIONS+=("--dirac-os") - INSTALLOPTIONS+=("--dirac-os-version=${DIRACOSVER}") - fi + curl -L "https://raw.githubusercontent.com/DIRACGrid/DIRAC/integration/src/DIRAC/Core/scripts/dirac-install.py" \ + > "${CLIENTINSTALLDIR}/dirac-install" + chmod +x "${CLIENTINSTALLDIR}/dirac-install" - if [[ "$DIRACOS_TARBALL_PATH" ]]; then - { - echo "DIRACOS = $DIRACOS_TARBALL_PATH" - } >> "${CLIENTINSTALLDIR}/dirac-ci-install.cfg" - fi + if [[ -n "${DEBUG+x}" ]]; then + INSTALLOPTIONS+=("${DEBUG}") + fi - if ! ./dirac-install -r "${DIRAC_RELEASE}" -t client "${INSTALLOPTIONS[@]}" "${CLIENTINSTALLDIR}/dirac-ci-install.cfg"; then - echo 'ERROR: DIRAC client installation failed' >&2 - exit 1 - fi + if [[ -n "${ALTERNATIVE_MODULES+x}" ]]; then + echo "Installing from non-release code" + local option="--module=" + for module_path in "${ALTERNATIVE_MODULES[@]}"; do + if [[ -d "${module_path}" ]]; then + option+="${module_path}:::$(basename "${module_path}"):::local," + else + option+="${module_path}," + fi + done + INSTALLOPTIONS+=("${option: :$((${#option} - 1))}") + fi + + if [[ "${DIRACOSVER}" ]]; then + INSTALLOPTIONS+=("--dirac-os") + INSTALLOPTIONS+=("--dirac-os-version=${DIRACOSVER}") + fi + + if [[ "$DIRACOS_TARBALL_PATH" ]]; then + { + echo "DIRACOS = $DIRACOS_TARBALL_PATH" + } >> "${CLIENTINSTALLDIR}/dirac-ci-install.cfg" + fi + + if ! ./dirac-install -r "${DIRAC_RELEASE}" -t client "${INSTALLOPTIONS[@]}" "${CLIENTINSTALLDIR}/dirac-ci-install.cfg"; then + echo 'ERROR: DIRAC client installation failed' >&2 + exit 1 + fi - source bashrc + source bashrc + fi echo "$DIRAC" echo "$PATH" @@ -363,9 +381,6 @@ installDIRAC() { exit 1 fi - echo 'Content of etc/dirac.cfg:' - cat "${CLIENTINSTALLDIR}/etc/dirac.cfg" - echo '==> Done installDIRAC' }