[v7r2] Move to a src/ layout

.github/workflows/basic-python3.yml

@@ -23,6 +23,7 @@ jobs:
           #     gracefully, make them non-daemonic and use a suitable
           #     signalling mechanism such as an Event."
           - pytest --no-cov -k 'not test_BaseType_Unicode and not test_nestedStructure and not testLockedClass'
+          # - pytest --no-cov src/DIRAC/Core/Security/test
 
     steps:
     - uses: actions/checkout@v2
@@ -36,7 +37,7 @@ jobs:
         source "${CONDA}/bin/activate"
         conda activate dirac-testing
         set -euxo pipefail
-        export PYTHONPATH=${PWD%/*}
+        export PYTHONPATH=${PWD}/src
         ${{ matrix.command }}
       env:
         REFERENCE_BRANCH: ${{ github['base_ref'] || github['head_ref'] }}

.github/workflows/basic.yml

@@ -13,7 +13,7 @@ jobs:
       matrix:
         command:
           - pytest --no-cov
-          - pytest --no-cov Core/Security/test
+          - pytest --no-cov src/DIRAC/Core/Security/test
           - tests/checkDocs.sh
           # TODO This should cover more than just tests/CI
           # Excluded codes related to sourcing files
@@ -43,7 +43,7 @@ jobs:
         source "${CONDA}/bin/activate"
         conda activate dirac-testing
         set -euxo pipefail
-        export PYTHONPATH=${PWD%/*}
+        export PYTHONPATH=${PWD}/src
         ${{ matrix.command }}
       env:
         REFERENCE_BRANCH: ${{ github['base_ref'] || github['head_ref'] }}

.github/workflows/diracinstall.yml

@@ -27,4 +27,4 @@ jobs:
     - name: run dirac-install
       run: |
         eval "$(conda shell.bash hook)" && conda activate python_${{ matrix.python }}
-        python Core/scripts/dirac-install.py -l DIRAC -r integration -t server --dirac-os -ddd
+        python src/DIRAC/Core/scripts/dirac-install.py -l DIRAC -r integration -t server --dirac-os -ddd

.github/workflows/pilotWrapper.yml

@@ -28,4 +28,4 @@ jobs:
       run: |
         cp tests/Integration/WorkloadManagementSystem/Test_GenerateAndExecutePilotWrapper.py .
         eval "$(conda shell.bash hook)" && conda activate python_${{ matrix.python }}
-        python Test_GenerateAndExecutePilotWrapper.py file://${{ github.workspace }}/WorkloadManagementSystem/Utilities/PilotWrapper.py file://${{ github.workspace }}/Core/scripts/dirac-install.py
+        python Test_GenerateAndExecutePilotWrapper.py file://${{ github.workspace }}/src/DIRAC/WorkloadManagementSystem/Utilities/PilotWrapper.py file://${{ github.workspace }}/src/DIRAC/Core/scripts/dirac-install.py

container/Dockerfile

@@ -24,7 +24,7 @@ RUN mkdir -p /opt/dirac/DIRAC && \
 # Installing DIRAC in /opt/dirac
 RUN \
     cd /opt/dirac && \
-    curl -L -o dirac-install https://raw.githubusercontent.com/DIRACGrid/DIRAC/integration/Core/scripts/dirac-install.py && \
+    curl -L -o dirac-install https://raw.githubusercontent.com/DIRACGrid/DIRAC/integration/src/DIRAC/Core/scripts/dirac-install.py && \
     chmod +x dirac-install && \
     ./dirac-install -r v6r20 -t server -i 27 -g v14r1 && \
     rm -rf /opt/dirac/.installCache && \

docs/diracdoctools/Utilities.py

@@ -78,21 +78,21 @@ def setUpReadTheDocsEnvironment(moduleName='DIRAC'):
   """
   LOG.info('Running for READTHEDOCS')
   sys.path.append(os.path.abspath('.'))
-  diracPath = os.path.abspath(os.path.join(os.getcwd(), '../..'))
+  diracPath = os.path.abspath(os.path.join(os.getcwd(), '../src'))
   LOG.info('Path To Module(?): %r', diracPath)
 
   buildfolder = '_build'
   mkdir(os.path.abspath('../' + buildfolder))
 
   # We need to have the moduleName somewhere, or we cannot import it, as
   # readtheDocs clones the repo into something based on the branchname
-  if not os.path.exists(os.path.join('../../', moduleName)):
+  if not os.path.exists(os.path.join('../src', moduleName)):
     diracLink = os.path.abspath(os.path.join(os.getcwd(), '../', buildfolder, moduleName))
     LOG.info('Link: %r', diracLink)
     if not os.path.exists(diracLink):
       LOG.info('Creating symbolic link %r -> %r', diracPath, diracLink)
       os.symlink(diracPath, diracLink)
-    diracPath = os.path.abspath(os.path.join(diracLink, '..'))
+    diracPath = os.path.abspath(os.path.join(diracLink, 'src'))
 
   sys.path.insert(0, diracPath)
 

docs/docs.conf

@@ -4,7 +4,7 @@
 module_name = DIRAC
 
 # path to the source folder relative to docs.conf
-source_folder = ../
+source_folder = ../src/DIRAC
 
 [Code]
 # building code reference

docs/source/AdministratorGuide/Tutorials/basicTutoSetup.rst

@@ -214,7 +214,7 @@ the user's home folder):
 
 The output should look something like this::
 
-  --2019-04-11 08:51:21--  https://github.com/DIRACGrid/DIRAC/raw/integration/Core/scripts/dirac-install.py
+  --2019-04-11 08:51:21--  https://github.com/DIRACGrid/DIRAC/raw/integration/src/DIRAC/Core/scripts/dirac-install.py
   Resolving github.com... 140.82.118.4, 140.82.118.3
   Connecting to github.com|140.82.118.4|:443... connected.
   HTTP request sent, awaiting response... 302 Found

docs/source/AdministratorGuide/Tutorials/basicTutoSetup.sh

@@ -225,7 +225,7 @@ chown -R diracuser:diracuser ~diracuser/.globus/
 cat > InstallDiracClient <<EOF
 # START installClient1
 mkdir -p ~/DiracInstallation && cd ~/DiracInstallation
-curl -O -L https://github.com/DIRACGrid/DIRAC/raw/integration/Core/scripts/dirac-install.py
+curl -O -L https://github.com/DIRACGrid/DIRAC/raw/integration/src/DIRAC/Core/scripts/dirac-install.py
 chmod +x dirac-install.py
 ./dirac-install.py -r v7r0p36 --dirac-os
 # END installClient1

docs/source/DeveloperGuide/WebAppDIRAC/InstallWebAppDIRAC/index.rst

@@ -19,7 +19,7 @@ continue to install DIRAC `<https://github.com/DIRACGrid/DIRAC/wiki/GitSetup>`_.
 #. Create a directory where you will install DIRAC and WebAppDIRAC: mkdir portal; cd portal
 #. git clone git://github.com/zmathe/DIRAC.git. (NOTE: This works when you forked the DIRAC repository) or execute: git clone https://github.com/DIRACGrid/DIRAC.git
 #. git clone git://github.com/zmathe/WebAppDIRAC.git (NOTE: This works when you forked the WebAppDIRAC repository on github)  or git clone `<https://github.com/DIRACGrid/WebAppDIRAC.git>`_.
-#. ./DIRAC/Core/scripts/dirac-install.py -r v6r21-pre1 -X -t server --dirac-os --dirac-os-version=0.0.6** (You can use the current production version of DIRAC which can found http://diracgrid.org. NOTE: The current version of dirac-os is 0.0.6 but this might change. The available versions can be found at http://lhcbproject.web.cern.ch/lhcbproject/dist/Dirac_project/installSource/.)
+#. ./DIRAC/src/DIRAC/Core/scripts/dirac-install.py -r v6r21-pre1 -X -t server --dirac-os --dirac-os-version=0.0.6** (You can use the current production version of DIRAC which can found http://diracgrid.org. NOTE: The current version of dirac-os is 0.0.6 but this might change. The available versions can be found at http://lhcbproject.web.cern.ch/lhcbproject/dist/Dirac_project/installSource/.)
 #. python DIRAC/Core/scripts/dirac-deploy-scripts.py
 #. ./WebAppDIRAC/dirac-postInstall.py
 #. source bashrc (we have to use the correct python in order to install tornado)

docs/source/UserGuide/GettingStarted/InstallingClient/index.rst

@@ -20,7 +20,7 @@ Install script
 
 Download the *dirac-install* script from::
 
-  wget -np -O dirac-install https://github.com/DIRACGrid/DIRAC/raw/integration/Core/scripts/dirac-install.py --no-check-certificate
+  wget -np -O dirac-install https://github.com/DIRACGrid/DIRAC/raw/integration/src/DIRAC/Core/scripts/dirac-install.py --no-check-certificate
   chmod +x dirac-install
 
 Choose the directory where you want to install the DIRAC software and run the dirac-install script from

pytest.ini

@@ -5,4 +5,4 @@ python_files=Test_*.py assert*.py
 # The reason here is that we do nasty things with the pythonpath
 # in order to make sure that M2Crypto and pyGSI do not step
 # on each other's feet
-addopts = -rx -v --color=yes --showlocals --tb=long --ignore=tests --ignore=Core/Security/test
+addopts = -rx -v --color=yes --showlocals --tb=long --ignore=tests --ignore=src/DIRAC/Core/Security/test

Core/Security/m2crypto/X509Chain.py → src/DIRAC/Core/Security/m2crypto/X509Chain.py

@@ -895,7 +895,7 @@ def dumpChainToString(self):
 
      :returns: S_OK(pem chain)
     """
-    data = ''
+    data = b""
     for cert in self._certList:
       data += cert.asPem()
     return S_OK(data)
@@ -1060,9 +1060,9 @@ def hash(self):
       return S_OK(self.__hash)
     sha1 = hashlib.sha1()
     for cert in self._certList:
-      sha1.update(str(cert.getSubjectNameObject()["Value"]))
-    sha1.update(str(self.getRemainingSecs()['Value'] / 3600))
-    sha1.update(self.getDIRACGroup()['Value'])
+      sha1.update(str(cert.getSubjectNameObject()["Value"]).encode())
+    sha1.update(str(self.getRemainingSecs()['Value'] / 3600).encode())
+    sha1.update(self.getDIRACGroup()['Value'].encode())
     if self.isVOMS():
       sha1.update(b"VOMS")
       from DIRAC.Core.Security.VOMS import VOMS

Core/Security/test/Test_X509Chain.py → src/DIRAC/Core/Security/test/Test_X509Chain.py

@@ -60,7 +60,7 @@ def get_proxy(request):
   # When you do the delegation, you call both Request and Proxy generation fixtures.
   # So if you do the cleaning twice, you end up in a terrible mess.
   # So, do not do the cleaning if you are in the test_delegation method
-  if request.function.func_name != 'test_delegation':
+  if request.function.__name__ != 'test_delegation':
     deimportDIRAC()
   x509Class = request.param
 
@@ -335,7 +335,7 @@ def test_dumpChainToString_on_cert(cert_file, get_X509Chain_class):
 
   assert res['OK']
 
-  assert res['Value'] == "%s" % getCertOption(cert_file, 'content')
+  assert res['Value'] == getCertOption(cert_file, 'content')
 
 
 @parametrize('cert_file', CERTS)

Core/Security/test/x509TestUtilities.py → src/DIRAC/Core/Security/test/x509TestUtilities.py

@@ -23,7 +23,7 @@
 CERTKEYS = (HOSTKEY, USERKEY)
 
 CERTCONTENTS = {
-    'HOSTCERTCONTENT': """-----BEGIN CERTIFICATE-----
+    'HOSTCERTCONTENT': b"""-----BEGIN CERTIFICATE-----
 MIIGQTCCBCmgAwIBAgICEAIwDQYJKoZIhvcNAQELBQAwVDEYMBYGA1UECgwPRElS
 QUMgQ29tcHV0aW5nMTgwNgYDVQQDDC9ESVJBQyBDb21wdXRpbmcgU2lnbmluZyBD
 ZXJ0aWZpY2F0aW9uIEF1dGhvcml0eTAeFw0xODA4MjIwOTE4MTdaFw0zNzEwMjEw
@@ -60,7 +60,7 @@
 Hia1xkpBKOPRY0BrSGCdEUT5+ict
 -----END CERTIFICATE-----
 """,
-    'USERCERTCONTENT': """-----BEGIN CERTIFICATE-----
+    'USERCERTCONTENT': b"""-----BEGIN CERTIFICATE-----
 MIIFszCCA5ugAwIBAgICEAEwDQYJKoZIhvcNAQELBQAwVDEYMBYGA1UECgwPRElS
 QUMgQ29tcHV0aW5nMTgwNgYDVQQDDC9ESVJBQyBDb21wdXRpbmcgU2lnbmluZyBD
 ZXJ0aWZpY2F0aW9uIEF1dGhvcml0eTAeFw0xODA4MjIwOTE1MTRaFw0zNzEwMjEw
@@ -105,7 +105,7 @@
 # https://tls.mbed.org/kb/cryptography/asn1-key-structures-in-der-and-pem
 
 KEYCONTENTS_PKCS8 = {
-    HOSTKEY: """-----BEGIN PRIVATE KEY-----
+    HOSTKEY: b"""-----BEGIN PRIVATE KEY-----
 MIIJQwIBADANBgkqhkiG9w0BAQEFAASCCS0wggkpAgEAAoICAQDjV5Y6AQI61nZH
 y6hjr1MziFFeh/z1DdAgkPfiUnHQLxWtvXGcc4sX/tBcD6tvNKTzJCwyFVAML0WN
 TD/w480TUmGILlRtg+17qfSWfeCvDygSbGNINX+la0auEqY7u5oXtwhFAEnqBe+6
@@ -158,7 +158,7 @@
 CrerHYr0zdgIXTt+xus9RGGmZi1mfjI=
 -----END PRIVATE KEY-----
 """,
-    USERKEY: """-----BEGIN PRIVATE KEY-----
+    USERKEY: b"""-----BEGIN PRIVATE KEY-----
 MIIJQgIBADANBgkqhkiG9w0BAQEFAASCCSwwggkoAgEAAoICAQCp9md/3Ar5r6rH
 yR9GUhp6HMPaLH7pMpkn7D1oGWHXVvlw9+Ygw6ZMRd7ysPkbpYqh84o0GJDi9fa0
 S/umxil60zBD1HmSzrtE39gYWTMQQ7uPNgW1axLaNux0dV//bB6MgI+EAFg/WwZp

Core/Utilities/test/Test_Encode.py → src/DIRAC/Core/Utilities/test/Test_Encode.py

@@ -97,11 +97,9 @@ def test_everyBaseTypeIsTested():
       We rely on the fact that the test function will be called
       "test_BaseType"
   """
-  current_module = sys.modules[__name__]
-
   for encodeFunc in g_dEncodeFunctions.values():
     testFuncName = ('test_BaseType_%s' % encodeFunc.__name__).replace('encode', '')
-    getattr(current_module, testFuncName)
+    globals()[testFuncName]
 
 
 def agnosticTestFunction(enc_dec_tuple, data):