[v7r1] Various minor fixes for FrameworkSystem

ConfigurationSystem/Client/Helpers/Registry.py

@@ -23,6 +23,7 @@ def getUsernameForDN(dn, usersList=None):
 
       :return: S_OK(str)/S_ERROR()
   """
+  dn = dn.strip()
   if not usersList:
     result = gConfig.getSections("%s/Users" % gBaseRegistrySection)
     if not result['OK']:
@@ -63,6 +64,7 @@ def getGroupsForDN(dn):
 
       :return: S_OK(list)/S_ERROR() -- contain list of groups
   """
+  dn = dn.strip()
   result = getUsernameForDN(dn)
   if not result['OK']:
     return result
@@ -128,6 +130,7 @@ def getHostnameForDN(dn):
 
       :return: S_OK()/S_ERROR()
   """
+  dn = dn.strip()
   result = gConfig.getSections("%s/Hosts" % gBaseRegistrySection)
   if not result['OK']:
     return result
@@ -153,6 +156,7 @@ def findDefaultGroupForDN(dn):
 
       :return: S_OK()/S_ERROR()
   """
+  dn = dn.strip()
   result = getUsernameForDN(dn)
   if not result['OK']:
     return result

DataManagementSystem/Client/FTS3Client.py

@@ -11,7 +11,7 @@ class FTS3Client(Client):
   def __init__(self, url=None, **kwargs):
     """ Constructor function.
     """
-    Client.__init__(self, **kwargs)
+    super(FTS3Client, self).__init__(**kwargs)
     self.setServer('DataManagement/FTS3Manager')
     if url:
       self.setServer(url)

DataManagementSystem/Client/S3GatewayClient.py

@@ -11,7 +11,7 @@ class S3GatewayClient(Client):
   def __init__(self, url=None, **kwargs):
     """ Constructor function.
     """
-    Client.__init__(self, **kwargs)
+    super(S3GatewayClient, self).__init__(**kwargs)
     self.setServer('DataManagement/S3Gateway')
     if url:
       self.setServer(url)

FrameworkSystem/Client/BundleDeliveryClient.py

@@ -2,30 +2,30 @@
 """
 
 import os
+import io
 import tarfile
 import cStringIO
 
 from DIRAC import S_OK, gLogger
-from DIRAC.Core.DISET.RPCClient import RPCClient
+from DIRAC.Core.Base.Client import Client, createClient
 from DIRAC.Core.DISET.TransferClient import TransferClient
 from DIRAC.Core.Security import Locations, Utilities
+from DIRAC.Core.Utilities.File import mkDir
 from DIRAC.ConfigurationSystem.Client.Helpers.CSGlobals import skipCACheck
+
+
 __RCSID__ = "$Id$"
 
 
-class BundleDeliveryClient(object):
+@createClient('Framework/BundleDelivery')
+class BundleDeliveryClient(Client):
 
-  def __init__(self, rpcClient=False, transferClient=False):
-    self.rpcClient = rpcClient
+  def __init__(self, transferClient=False, **kwargs):
+    super(BundleDeliveryClient, self).__init__(**kwargs)
+    self.setServer('Framework/BundleDelivery')
     self.transferClient = transferClient
     self.log = gLogger.getSubLogger("BundleDelivery")
 
-  def __getRPCClient(self):
-    if self.rpcClient:
-      return self.rpcClient
-    return RPCClient("Framework/BundleDelivery",
-                     skipCACheck=skipCACheck())
-
   def __getTransferClient(self):
     if self.transferClient:
       return self.transferClient
@@ -34,7 +34,7 @@ def __getTransferClient(self):
 
   def __getHash(self, bundleID, dirToSyncTo):
     try:
-      with open(os.path.join(dirToSyncTo, ".dab.%s" % bundleID), "rb") as fd:
+      with io.open(os.path.join(dirToSyncTo, ".dab.%s" % bundleID), "rb") as fd:
         bdHash = fd.read().strip()
         return bdHash
     except BaseException:
@@ -43,7 +43,7 @@ def __getHash(self, bundleID, dirToSyncTo):
   def __setHash(self, bundleID, dirToSyncTo, bdHash):
     try:
       fileName = os.path.join(dirToSyncTo, ".dab.%s" % bundleID)
-      with open(fileName, "wb") as fd:
+      with io.open(fileName, "wb") as fd:
         fd.write(bdHash)
     except Exception as e:
       self.log.error("Could not save hash after synchronization", "%s: %s" % (fileName, str(e)))
@@ -52,7 +52,7 @@ def syncDir(self, bundleID, dirToSyncTo):
     dirCreated = False
     if not os.path.isdir(dirToSyncTo):
       self.log.info("Creating dir %s" % dirToSyncTo)
-      os.makedirs(dirToSyncTo)
+      mkDir(dirToSyncTo)
       dirCreated = True
     currentHash = self.__getHash(bundleID, dirToSyncTo)
     self.log.info("Current hash for bundle %s in dir %s is '%s'" % (bundleID, dirToSyncTo, currentHash))
@@ -113,7 +113,7 @@ def getCAs(self):
       # if we can not found the file, we return the directory, where the file should be
       transferClient = self.__getTransferClient()
       casFile = os.path.join(os.path.dirname(retVal['Message']), "cas.pem")
-      with open(casFile, "w") as fd:
+      with io.open(casFile, "w") as fd:
         result = transferClient.receiveFile(fd, 'CAs')
         if not result['OK']:
           return result
@@ -131,7 +131,7 @@ def getCLRs(self):
       # if we can not found the file, we return the directory, where the file should be
       transferClient = self.__getTransferClient()
       casFile = os.path.join(os.path.dirname(retVal['Message']), "crls.pem")
-      with open(casFile, "w") as fd:
+      with io.open(casFile, "w") as fd:
         result = transferClient.receiveFile(fd, 'CRLs')
         if not result['OK']:
           return result

FrameworkSystem/Client/ComponentInstaller.py

@@ -58,6 +58,7 @@
 from __future__ import print_function, absolute_import
 
 import os
+import io
 import re
 import glob
 import stat
@@ -1014,7 +1015,7 @@ def getSoftwareComponents(self, extensions):
           for agent in agentList:
             if os.path.splitext(agent)[1] == ".py":
               agentFile = os.path.join(agentDir, agent)
-              with open(agentFile, 'r') as afile:
+              with io.open(agentFile, 'rt') as afile:
                 body = afile.read()
               if body.find('AgentModule') != -1 or body.find('OptimizerModule') != -1:
                 if system not in agents:
@@ -1040,7 +1041,7 @@ def getSoftwareComponents(self, extensions):
           for executor in executorList:
             if os.path.splitext(executor)[1] == ".py":
               executorFile = os.path.join(executorDir, executor)
-              with open(executorFile, 'r') as afile:
+              with io.open(executorFile, 'rt') as afile:
                 body = afile.read()
               if body.find('OptimizerExecutor') != -1:
                 if system not in executors:
@@ -1091,9 +1092,8 @@ def getInstalledComponents(self):
       for component in components:
         try:
           runFile = os.path.join(systemDir, component, 'run')
-          rfile = open(runFile, 'r')
-          body = rfile.read()
-          rfile.close()
+          with io.open(runFile, 'rt') as rFile:
+            body = rFile.read()
 
           for cType in self.componentTypes:
             if body.find('dirac-%s' % (cType)) != -1:
@@ -1126,9 +1126,8 @@ def getSetupComponents(self):
     for component in componentList:
       try:
         runFile = os.path.join(self.startDir, component, 'run')
-        rfile = open(runFile, 'r')
-        body = rfile.read()
-        rfile.close()
+        with io.open(runFile, 'rt') as rfile:
+          body = rfile.read()
 
         for cType in self.componentTypes:
           if body.find('dirac-%s' % (cType)) != -1:
@@ -1463,9 +1462,8 @@ def getLogTail(self, system, component, length=100):
       if not os.path.exists(logFileName):
         retDict[compName] = 'No log file found'
       else:
-        logFile = open(logFileName, 'r')
-        lines = [line.strip() for line in logFile.readlines()]
-        logFile.close()
+        with io.open(logFileName, 'rt') as logFile:
+          lines = [line.strip() for line in logFile.readlines()]
 
         if len(lines) < length:
           retDict[compName] = '\n'.join(lines)
@@ -1601,7 +1599,7 @@ def setupSite(self, scriptCfg, cfg=None):
 
       if not cmdFound:
         gLogger.notice('Starting runsvdir ...')
-        with open(os.devnull, 'w') as devnull:
+        with io.open(os.devnull, 'w') as devnull:
           subprocess.Popen(['nohup', 'runsvdir', self.startDir, 'log:  DIRAC runsv'],
                            stdout=devnull, stderr=devnull, universal_newlines=True)
 
@@ -1815,22 +1813,21 @@ def _createRunitLog(self, runitCompDir):
     mkDir(logDir)
 
     logConfigFile = os.path.join(logDir, 'config')
-    with open(logConfigFile, 'w') as fd:
+    with io.open(logConfigFile, 'w') as fd:
       fd.write(
-          """s10000000
+          u"""s10000000
   n20
   """)
 
     logRunFile = os.path.join(logDir, 'run')
-    with open(logRunFile, 'w') as fd:
+    with io.open(logRunFile, 'w') as fd:
       fd.write(
-          """#!/bin/bash
-  #
-  rcfile=%(bashrc)s
-  [ -e $rcfile ] && source $rcfile
-  #
-  exec svlogd .
+          u"""#!/bin/bash
 
+rcfile=%(bashrc)s
+[[ -e $rcfile ]] && source ${rcfile}
+#
+exec svlogd .
   """ % {'bashrc': os.path.join(self.instancePath, 'bashrc')})
 
     os.chmod(logRunFile, self.gDefaultPerms)
@@ -1896,31 +1893,30 @@ def installComponent(self, componentType, system, component, extensions, compone
     try:
       componentCfg = os.path.join(self.linkedRootPath, 'etc', '%s_%s.cfg' % (system, component))
       if not os.path.exists(componentCfg):
-        fd = open(componentCfg, 'w')
-        fd.close()
+        io.open(componentCfg, 'w').close()
 
       self._createRunitLog(runitCompDir)
 
       runFile = os.path.join(runitCompDir, 'run')
-      fd = open(runFile, 'w')
-      fd.write(
-          """#!/bin/bash
-  rcfile=%(bashrc)s
-  [ -e $rcfile ] && source $rcfile
-  #
-  exec 2>&1
-  #
-  [ "%(componentType)s" = "agent" ] && renice 20 -p $$
-  #%(bashVariables)s
-  #
-  exec python $DIRAC/DIRAC/Core/scripts/dirac-%(componentType)s.py %(system)s/%(component)s %(componentCfg)s < /dev/null
-  """ % {'bashrc': os.path.join(self.instancePath, 'bashrc'),
-              'bashVariables': bashVars,
-              'componentType': componentType,
-              'system': system,
-              'component': component,
-              'componentCfg': componentCfg})
-      fd.close()
+      with io.open(runFile, 'w') as fd:
+        fd.write(
+            u"""#!/bin/bash
+
+rcfile=%(bashrc)s
+[[ -e $rcfile ]] && source ${rcfile}
+#
+exec 2>&1
+#
+[[ "%(componentType)s" = "agent" ]] && renice 20 -p $$
+#%(bashVariables)s
+#
+exec python $DIRAC/DIRAC/Core/scripts/dirac-%(componentType)s.py %(system)s/%(component)s %(componentCfg)s < /dev/null
+    """ % {'bashrc': os.path.join(self.instancePath, 'bashrc'),
+                'bashVariables': bashVars,
+                'componentType': componentType,
+                'system': system,
+                'component': component,
+                'componentCfg': componentCfg})
 
       os.chmod(runFile, self.gDefaultPerms)
 
@@ -1930,8 +1926,9 @@ def installComponent(self, componentType, system, component, extensions, compone
         stopFile = os.path.join(runitCompDir, 'control', 't')
         # This is, e.g., /opt/dirac/control/WorkfloadManagementSystem/Matcher/
         controlDir = self.runitDir.replace('runit', 'control')
-        with open(stopFile, 'w') as fd:
-          fd.write("""#!/bin/bash
+        with io.open(stopFile, 'w') as fd:
+          fd.write(u"""#!/bin/bash
+
 echo %(controlDir)s/%(system)s/%(component)s/stop_%(type)s
 touch %(controlDir)s/%(system)s/%(component)s/stop_%(type)s
 """ % {'controlDir': controlDir,
@@ -2093,15 +2090,16 @@ def installPortal(self):
       try:
         self._createRunitLog(runitWebAppDir)
         runFile = os.path.join(runitWebAppDir, 'run')
-        with open(runFile, 'w') as fd:
+        with io.open(runFile, 'w') as fd:
           fd.write(
-              """#!/bin/bash
-  rcfile=%(bashrc)s
-  [ -e $rcfile ] && source $rcfile
-  #
-  exec 2>&1
-  #
-  exec python %(DIRAC)s/WebAppDIRAC/scripts/dirac-webapp-run.py -p < /dev/null
+              u"""#!/bin/bash
+
+rcfile=%(bashrc)s
+[[ -e $rcfile ]] && source $rcfile
+#
+exec 2>&1
+#
+exec python %(DIRAC)s/WebAppDIRAC/scripts/dirac-webapp-run.py -p < /dev/null
   """ % {'bashrc': os.path.join(self.instancePath, 'bashrc'),
                   'DIRAC': self.linkedRootPath})
 
@@ -2405,19 +2403,17 @@ def _createMySQLCMDLines(self, dbFile):
 
     cmdLines = []
 
-    fd = open(dbFile)
-    dbLines = fd.readlines()
-    fd.close()
+    with io.open(dbFile, 'rt') as fd:
+      dbLines = fd.readlines()
 
     for line in dbLines:
       # Should we first source an SQL file (is this sql file an extension)?
       if line.lower().startswith('source'):
         sourcedDBbFileName = line.split(' ')[1].replace('\n', '')
         gLogger.info("Found file to source: %s" % sourcedDBbFileName)
         sourcedDBbFile = os.path.join(rootPath, sourcedDBbFileName)
-        fdSourced = open(sourcedDBbFile)
-        dbLinesSourced = fdSourced.readlines()
-        fdSourced.close()
+        with io.open(sourcedDBbFile, 'rt') as fdSourced:
+          dbLinesSourced = fdSourced.readlines()
         for lineSourced in dbLinesSourced:
           if lineSourced.strip():
             cmdLines.append(lineSourced.strip())