diff --git a/Core/scripts/dirac-install.py b/Core/scripts/dirac-install.py index 6fe5e5d1dbe..1215d086596 100755 --- a/Core/scripts/dirac-install.py +++ b/Core/scripts/dirac-install.py @@ -1,2724 +1,52 @@ #!/usr/bin/env python -""" -The main DIRAC installer script. It can be used to install the main DIRAC software, its -modules, web, rest etc. and DIRAC extensions. - -In order to deploy DIRAC you have to provide: globalDefaultsURL, which is by default: -"http://diracproject.web.cern.ch/diracproject/configs/globalDefaults.cfg", but it can be -in the local file system in a separate directory. The content of this file is the following:: - - Installations - { - DIRAC - { - DefaultsLocation = http://diracproject.web.cern.ch/diracproject/dirac.cfg - LocalInstallation - { - PythonVersion = 27 - } - # in case you have a DIRAC extension - LHCb - { - DefaultsLocation = http://lhcb-rpm.web.cern.ch/lhcb-rpm/lhcbdirac/lhcb.cfg - } - } - } - Projects - { - DIRAC - { - DefaultsLocation = http://diracproject.web.cern.ch/diracproject/dirac.cfg - } - # in case you have a DIRAC extension - LHCb - { - DefaultsLocation = http://lhcb-rpm.web.cern.ch/lhcb-rpm/lhcbdirac/lhcb.cfg - } - } - -the DefaultsLocation for example:: - - DefaultsLocation = http://diracproject.web.cern.ch/diracproject/dirac.cfg - -must contain a minimal configuration. The following options must be in this -file:: - - Releases=,UploadCommand=,BaseURL= - -In case you want to overwrite the global configuration file, you have to use --defaultsURL - -After providing the default configuration files, DIRAC or your extension can be installed from: - -1. in a directory you have to be present globalDefaults.cfg, dirac.cfg and all binaries. - For example:: - - zmathe@dzmathe zmathe]$ ls tars/ - dirac.cfg diracos-0.1.md5 diracos-0.1.tar.gz DIRAC-v6r20-pre16.md5 DIRAC-v6r20-pre16.tar.gz - globalDefaults.cfg release-DIRAC-v6r20-pre16.cfg release-DIRAC-v6r20-pre16.md5 - zmathe@dzmathe zmathe]$ - - For example:: - - dirac-install -r v6r20-pre16 --dirac-os --dirac-os-version=0.0.1 -u /home/zmathe/tars - - this command will use /home/zmathe/tars directory for the source code. - It will install DIRAC v6r20-pre16, DIRAC OS 0.1 version - -2. You can use your dedicated web server or the official DIRAC web server - - for example:: - - dirac-install -r v6r20-pre16 --dirac-os --dirac-os-version=0.0.1 - - It will install DIRAC v6r20-pre16 - - You can install an extension of diracos. - - for example:: - - dirac-install -r v9r4-pre2 -l LHCb --dirac-os --dirac-os-version=LHCb:master - -3. You have possibility to install a not-yet-released DIRAC, module or extension using -m or --tag options. - The non release version can be specified. - - for example:: - - dirac-install -l DIRAC -r v6r20-pre16 -g v14r0 -t client -m DIRAC --tag=integration - - It will install DIRAC v6r20-pre16, where the DIRAC package based on integration, other other packages will be - the same what is specified in release.cfg file in v6r20-pre16 tarball. - - dirac-install -l DIRAC -r v6r20-pre16 -g v14r0 -t client -m DIRAC --tag=v6r20-pre22 - - It installs a specific tag - - Note: If the source is not provided, DIRAC repository is used, which is defined in the global - configuration file. - - We can provide the repository url:code repository:::Project:::branch. for example:: - - dirac-install -l DIRAC -r v6r20-pre16 -g v14r0 -t client \\ - -m https://github.com/zmathe/DIRAC.git:::DIRAC:::dev_main_branch, \\ - https://github.com/zmathe/WebAppDIRAC.git:::WebAppDIRAC:::extjs6 -e WebAppDIRAC - - it will install DIRAC based on dev_main_branch and WebAppDIRAC based on extjs6:: - - dirac-install -l DIRAC -r v6r20-pre16 -g v14r0 -t client \\ - -m WebAppDIRAC --tag=integration -e WebAppDIRAC - - it will install DIRAC v6r20-pre16 and WebAppDIRAC integration branch - -You can use install.cfg configuration file:: - - DIRACOS = http://lhcb-rpm.web.cern.ch/lhcb-rpm/dirac/DIRACOS/ - WebAppDIRAC = https://github.com/zmathe/WebAppDIRAC.git - DIRAC=https://github.com/DIRACGrid/DIRAC.git - LocalInstallation - { - # Project = LHCbDIRAC - # The project LHCbDIRAC is not defined in the globalsDefaults.cfg - Project = LHCb - Release = v9r2-pre8 - Extensions = LHCb - ConfigurationServer = dips://lhcb-conf-dirac.cern.ch:9135/Configuration/Server - Setup = LHCb-Production - SkipCAChecks = True - SkipCADownload = True - WebAppDIRAC=extjs6 - DIRAC=rel-v6r20 - } - - dirac-install -l LHCb -r v9r2-pre8 -t server --dirac-os --dirac-os-version=0.0.6 install.cfg - -""" - -from __future__ import unicode_literals, absolute_import, division, print_function - -import sys import os -import getopt -import imp -import signal -import time import stat -import shutil -import subprocess -import ssl -import hashlib - -from distutils.version import LooseVersion # pylint: disable=no-name-in-module,import-error - +import sys +import tempfile try: # For Python 3.0 and later from urllib.request import urlopen, HTTPError, URLError except ImportError: # Fall back to Python 2's urllib2 from urllib2 import urlopen, HTTPError, URLError -try: - str_type = basestring -except NameError: - str_type = str - -__RCSID__ = "$Id$" - -executablePerms = stat.S_IWUSR | stat.S_IRUSR | stat.S_IXUSR | stat.S_IRGRP | stat.S_IXGRP | stat.S_IROTH | stat.S_IXOTH - - -def S_OK(value=""): - return {'OK': True, 'Value': value} - - -def S_ERROR(msg=""): - return {'OK': False, 'Message': msg} - -############ -# Start of CFG -############ - - -class Params(object): - - def __init__(self): - self.extensions = [] - self.project = 'DIRAC' - self.installation = 'DIRAC' - self.release = "" - self.externalsType = 'client' - self.pythonVersion = '27' - self.platform = "" - self.basePath = os.getcwd() - self.targetPath = os.getcwd() - self.buildExternals = False - self.noAutoBuild = False - self.debug = False - self.externalsOnly = False - self.lcgVer = '' - self.noLcg = False - self.useVersionsDir = False - self.installSource = "" - self.globalDefaults = False - self.timeout = 300 - self.diracOSVersion = '' - self.diracOS = False - self.tag = "" - self.modules = {} - self.externalVersion = "" - self.createLink = False - self.scriptSymlink = False - self.userEnvVariables = {} - - -cliParams = Params() - -### -# Release config manager -### - - -class ReleaseConfig(object): - - class CFG(object): - - def __init__(self, cfgData=""): - """ c'tor - :param self: self reference - :param str cfgData: the content of the configuration file - """ - self.data = {} - self.children = {} - if cfgData: - self.parse(cfgData) - - def parse(self, cfgData): - """ - It parses the configuration file and propagate the data and children - with the content of the cfg file - :param str cfgData: configuration data, which is the content of the configuration file - """ - try: - self.__parse(cfgData) - except BaseException: - import traceback - traceback.print_exc() - raise - return self - - def getChild(self, path): - """ - It return the child of a given section - :param str, list, tuple path: for example: Installations/DIRAC, Projects/DIRAC - :return object It returns a CFG instance - """ - - child = self - if isinstance(path, (list, tuple)): - pathList = path - else: - pathList = [sec.strip() for sec in path.split("/") if sec.strip()] - for childName in pathList: - if childName not in child.children: - return False - child = child.children[childName] - return child - - def __parse(self, cfgData, cIndex=0): - """ - It parse a given DIRAC cfg file and store the result in self.data variable. - - :param str cfgData: the content of the configuration file - :param int cIndex: it is the new line counter - """ - - childName = "" - numLine = 0 - while cIndex < len(cfgData): - eol = cfgData.find("\n", cIndex) - if eol < cIndex: - # End? - return cIndex - numLine += 1 - if eol == cIndex: - cIndex += 1 - continue - line = cfgData[cIndex: eol].strip() - # Jump EOL - cIndex = eol + 1 - if not line or line[0] == "#": - continue - if line.find("+=") > -1: - fields = line.split("+=") - opName = fields[0].strip() - if opName in self.data: - self.data[opName] += ', %s' % '+='.join(fields[1:]).strip() - else: - self.data[opName] = '+='.join(fields[1:]).strip() - continue - - if line.find("=") > -1: - fields = line.split("=") - self.data[fields[0].strip()] = "=".join(fields[1:]).strip() - continue - - opFound = line.find("{") - if opFound > -1: - childName += line[:opFound].strip() - if not childName: - raise Exception("No section name defined for opening in line %s" % numLine) - childName = childName.strip() - self.children[childName] = ReleaseConfig.CFG() - eoc = self.children[childName].__parse(cfgData, cIndex) - cIndex = eoc - childName = "" - continue - - if line == "}": - return cIndex - # Must be name for section - childName += line.strip() - return cIndex - - def createSection(self, name, cfg=None): - """ - It creates a subsection for an existing CS section. - :param str name: the name of the section - :param object cfg: the ReleaseConfig.CFG object loaded into memory - """ - - if isinstance(name, (list, tuple)): - pathList = name - else: - pathList = [sec.strip() for sec in name.split("/") if sec.strip()] - parent = self - for lev in pathList[:-1]: - if lev not in parent.children: - parent.children[lev] = ReleaseConfig.CFG() - parent = parent.children[lev] - secName = pathList[-1] - if secName not in parent.children: - if not cfg: - cfg = ReleaseConfig.CFG() - parent.children[secName] = cfg - return parent.children[secName] - - def isSection(self, obList): - """ - Checks if a given path is a section - :param str objList: is a path: for example: Releases/v6r20-pre16 - """ - return self.__exists([ob.strip() for ob in obList.split("/") if ob.strip()]) == 2 - - def sections(self): - """ - Returns all sections - """ - return [k for k in self.children] - - def isOption(self, obList): - return self.__exists([ob.strip() for ob in obList.split("/") if ob.strip()]) == 1 - - def options(self): - """ - Returns the options - """ - return [k for k in self.data] - - def __exists(self, obList): - """ - Check the existence of a certain element - - :param list obList: the list of cfg element names. - for example: [Releases,v6r20-pre16] - """ - if len(obList) == 1: - if obList[0] in self.children: - return 2 - elif obList[0] in self.data: - return 1 - else: - return 0 - if obList[0] in self.children: - return self.children[obList[0]].__exists(obList[1:]) - return 0 - - def get(self, opName, defaultValue=None): - """ - It return the value of a certain option - - :param str opName: the name of the option - :param str defaultValue: the default value of a given option - """ - try: - value = self.__get([op.strip() for op in opName.split("/") if op.strip()]) - except KeyError: - if defaultValue is not None: - return defaultValue - raise - if defaultValue is None: - return value - defType = type(defaultValue) - if isinstance(defType, bool): - return value.lower() in ("1", "true", "yes") - try: - return defType(value) - except ValueError: - return defaultValue - - def __get(self, obList): - """ - It return a given section - - :param list obList: the list of cfg element names. - """ - if len(obList) == 1: - if obList[0] in self.data: - return self.data[obList[0]] - raise KeyError("Missing option %s" % obList[0]) - if obList[0] in self.children: - return self.children[obList[0]].__get(obList[1:]) - raise KeyError("Missing section %s" % obList[0]) - - def toString(self, tabs=0): - """ - It return the configuration file as a string - :param int tabs: the number of tabs used to format the CS string - """ - - lines = ["%s%s = %s" % (" " * tabs, opName, self.data[opName]) for opName in self.data] - for secName in self.children: - lines.append("%s%s" % (" " * tabs, secName)) - lines.append("%s{" % (" " * tabs)) - lines.append(self.children[secName].toString(tabs + 1)) - lines.append("%s}" % (" " * tabs)) - return "\n".join(lines) - - def getOptions(self, path=""): - """ - Rturns the options for a given path - - :param str path: the path to the CS element - """ - parentPath = [sec.strip() for sec in path.split("/") if sec.strip()][:-1] - if parentPath: - parent = self.getChild(parentPath) - else: - parent = self - if not parent: - return [] - return tuple(parent.data) - - def delPath(self, path): - """ - It deletes a given CS element - - :param str path: the path to the CS element - """ - path = [sec.strip() for sec in path.split("/") if sec.strip()] - if not path: - return - keyName = path[-1] - parentPath = path[:-1] - if parentPath: - parent = self.getChild(parentPath) - else: - parent = self - if parent: - parent.data.pop(keyName) - - def update(self, path, cfg): - """ - Used to update the CS - - :param str path: path to the CS element - :param object cfg: the CS object - """ - parent = self.getChild(path) - if not parent: - self.createSection(path, cfg) - return - parent.__apply(cfg) - - def __apply(self, cfg): - """ - It adds a certain cfg subsection to a given section - - :param object cfg: the CS object - """ - for k in cfg.sections(): - if k in self.children: - self.children[k].__apply(cfg.getChild(k)) - else: - self.children[k] = cfg.getChild(k) - for k in cfg.options(): - self.data[k] = cfg.get(k) -############################################################################ -# END OF CFG CLASS -############################################################################ - - def __init__(self, instName='DIRAC', projectName='DIRAC', globalDefaultsURL=None): - """ c'tor - :param str instName: the name of the installation - :param str projectName: the name of the project - :param str globalDefaultsURL: the default url - """ - if globalDefaultsURL: - self.globalDefaultsURL = globalDefaultsURL - else: - self.globalDefaultsURL = "http://diracproject.web.cern.ch/diracproject/configs/globalDefaults.cfg" - self.globalDefaults = ReleaseConfig.CFG() - self.loadedCfgs = [] - self.prjDepends = {} - self.diracBaseModules = {} - self.prjRelCFG = {} - self.projectsLoadedBy = {} - self.cfgCache = {} - - self.debugCB = False - self.instName = instName - self.projectName = projectName - - def setDebugCB(self, debFunc): - """ - It is used by the dirac-distribution. It sets the debug function - """ - self.debugCB = debFunc - - def __dbgMsg(self, msg): - """ - :param str msg: the debug message - """ - if self.debugCB: - self.debugCB(msg) - - def __loadCFGFromURL(self, urlcfg, checkHash=False): - """ - It is used to load the configuration file - - :param str urlcfg: the location of the source repository and - where the default configuration file is exists. - :param bool checkHash: check if the file is corrupted. - """ - # This can be a local file - if os.path.exists(urlcfg): - with open(urlcfg, 'r') as relFile: - cfgData = relFile.read() - else: - if urlcfg in self.cfgCache: - return S_OK(self.cfgCache[urlcfg]) - try: - cfgData = urlretrieveTimeout(urlcfg, timeout=cliParams.timeout) - if not cfgData: - return S_ERROR("Could not get data from %s" % urlcfg) - except BaseException: - return S_ERROR("Could not open %s" % urlcfg) - try: - # cfgData = cfgFile.read() - cfg = ReleaseConfig.CFG(cfgData) - except Exception as excp: - return S_ERROR("Could not parse %s: %s" % (urlcfg, excp)) - # cfgFile.close() - if not checkHash: - self.cfgCache[urlcfg] = cfg - return S_OK(cfg) - try: - md5path = urlcfg[:-4] + ".md5" - if os.path.exists(md5path): - md5File = open(md5path, 'r') - md5Data = md5File.read() - md5File.close() - else: - md5Data = urlretrieveTimeout(md5path, timeout=60) - md5Hex = md5Data.strip() - # md5File.close() - if md5Hex != hashlib.md5(cfgData.encode('utf-8')).hexdigest(): - return S_ERROR("Hash check failed on %s" % urlcfg) - except Exception as excp: - return S_ERROR("Hash check failed on %s: %s" % (urlcfg, excp)) - self.cfgCache[urlcfg] = cfg - return S_OK(cfg) - - def loadInstallationDefaults(self): - """ - Load the default configurations - """ - result = self.__loadGlobalDefaults() - if not result['OK']: - return result - return self.__loadObjectDefaults("Installations", self.instName) - - def loadProjectDefaults(self): - """ - Load default configurations - """ - result = self.__loadGlobalDefaults() - if not result['OK']: - return result - return self.__loadObjectDefaults("Projects", self.projectName) - - def __loadGlobalDefaults(self): - """ - It loads the default configuration files - """ - - globalDefaultsCVMFSPath = "/cvmfs/dirac.egi.eu/admin/globalDefaults.cfg" - self.__dbgMsg("Loading global defaults from: %s" % globalDefaultsCVMFSPath) - result = self.__loadCFGFromURL(globalDefaultsCVMFSPath) - if not result['OK']: - self.__dbgMsg("Loading global defaults from: %s" % self.globalDefaultsURL) - result = self.__loadCFGFromURL(self.globalDefaultsURL) - if not result['OK']: - return result - self.globalDefaults = result['Value'] - for k in ("Installations", "Projects"): - if not self.globalDefaults.isSection(k): - self.globalDefaults.createSection(k) - self.__dbgMsg("Loaded global defaults") - return S_OK() - - def __loadObjectDefaults(self, rootPath, objectName): - """ - It loads the CFG, if it is not loaded. - :param str rootPath: the main section. for example: Installations - :param str objectName: The name of the section. for example: DIRAC - """ - - basePath = "%s/%s" % (rootPath, objectName) - if basePath in self.loadedCfgs: - return S_OK() - - # Check if it's a direct alias - try: - aliasTo = self.globalDefaults.get(basePath) - except KeyError: - aliasTo = False - - if aliasTo: - self.__dbgMsg("%s is an alias to %s" % (objectName, aliasTo)) - result = self.__loadObjectDefaults(rootPath, aliasTo) - if not result['OK']: - return result - cfg = result['Value'] - self.globalDefaults.update(basePath, cfg) - return S_OK() - - # Load the defaults - if self.globalDefaults.get("%s/SkipDefaults" % basePath, False): - defaultsLocation = "" - else: - defaultsLocation = self.globalDefaults.get("%s/DefaultsLocation" % basePath, "") - - if not defaultsLocation: - self.__dbgMsg("No defaults file defined for %s %s" % (rootPath.lower()[:-1], objectName)) - else: - self.__dbgMsg("Defaults for %s are in %s" % (basePath, defaultsLocation)) - result = self.__loadCFGFromURL(defaultsLocation) - if not result['OK']: - return result - cfg = result['Value'] - self.globalDefaults.update(basePath, cfg) - - # Check if the defaults have a sub alias - try: - aliasTo = self.globalDefaults.get("%s/Alias" % basePath) - except KeyError: - aliasTo = False - - if aliasTo: - self.__dbgMsg("%s is an alias to %s" % (objectName, aliasTo)) - result = self.__loadObjectDefaults(rootPath, aliasTo) - if not result['OK']: - return result - cfg = result['Value'] - self.globalDefaults.update(basePath, cfg) - - self.loadedCfgs.append(basePath) - return S_OK(self.globalDefaults.getChild(basePath)) - - def loadInstallationLocalDefaults(self, args): - """ - Load the configuration file from a file - - :param str args: the arguments in which to look for configuration file names - """ - - # at the end we load the local configuration and merge it with the global cfg - argList = list(args) - if os.path.exists('etc/dirac.cfg') and 'etc/dirac.cfg' not in args: - argList = ['etc/dirac.cfg'] + argList - - for arg in argList: - if arg.endswith(".cfg") and ':::' not in arg: - fileName = arg - else: - continue - - logNOTICE("Defaults for LocalInstallation are in %s" % fileName) - try: - fd = open(fileName, "r") - cfg = ReleaseConfig.CFG().parse(fd.read()) - fd.close() - except Exception as excp: - logERROR("Could not load %s: %s" % (fileName, excp)) - continue - - self.globalDefaults.update("Installations/%s" % self.instName, cfg) - self.globalDefaults.update("Projects/%s" % self.instName, cfg) - if self.projectName: - # we have an extension and have a local cfg file - self.globalDefaults.update("Projects/%s" % self.projectName, cfg) - - logNOTICE("Loaded %s" % arg) - - def getModuleVersionFromLocalCfg(self, moduleName): - """ - It returns the version of a certain module defined in the LocalInstallation section - :param str moduleName: - :return str: the version of a certain module - """ - return self.globalDefaults.get("Installations/%s/LocalInstallation/%s" % (self.instName, moduleName), "") - - def getInstallationCFG(self, instName=None): - """ - Returns the installation name - - :param str instName: the installation name - """ - if not instName: - instName = self.instName - return self.globalDefaults.getChild("Installations/%s" % instName) - - def getInstallationConfig(self, opName, instName=None): - """ - It returns the configurations from the Installations section. - This is usually provided in the local configuration file - - :param str opName: the option name for example: LocalInstallation/Release - :param str instName: - """ - if not instName: - instName = self.instName - return self.globalDefaults.get("Installations/%s/%s" % (instName, opName)) - - def isProjectLoaded(self, project): - """ - Checks if the project is loaded. - - :param str project: the name of the project - """ - return project in self.prjRelCFG - - def getTarsLocation(self, project, module=None): - """ - Returns the location of the binaries for a given project for example: LHCb or DIRAC, etc... - - :param str project: the name of the project - """ - sourceUrl = self.globalDefaults.get("Projects/%s/BaseURL" % project, "") - if module: - # in case we define a different URL in the CS - differntSourceUrl = self.globalDefaults.get("Projects/%s/%s" % (project, module), "") - if differntSourceUrl: - sourceUrl = differntSourceUrl - if sourceUrl: - return S_OK(sourceUrl) - return S_ERROR("Don't know how to find the installation tarballs for project %s" % project) - - def getDiracOsLocation(self, useVanillaDiracOS=False): - """ - Returns the location of the DIRAC os binary for a given project for example: LHCb or DIRAC, etc... - :param bool useVanillaDiracOS: flag to take diracos distribution from the default location - :return: the location of the tar balls - """ - keysToConsider = [] - if not useVanillaDiracOS: - keysToConsider += [ - "Installations/%s/DIRACOS" % self.projectName, - "Projects/%s/DIRACOS" % self.projectName, - ] - keysToConsider += [ - "Installations/DIRAC/DIRACOS", - "Projects/DIRAC/DIRACOS", - ] - - for key in keysToConsider: - location = self.globalDefaults.get(key, "") - if location: - logDEBUG("Using DIRACOS tarball URL from configuration key %s" % key) - return location - - def getUploadCommand(self, project=None): - """ - It returns the command used to upload the binary - - :param str project: the name of the project - """ - if not project: - project = self.projectName - defLoc = self.globalDefaults.get("Projects/%s/UploadCommand" % project, "") - if defLoc: - return S_OK(defLoc) - return S_ERROR("No UploadCommand for %s" % project) - - def __loadReleaseConfig(self, project, release, releaseMode, sourceURL=None, relLocation=None): - """ - It loads the release configuration file - - :param str project: the name of the project - :param str release: the release version - :param str releaseMode: the type of the release server/client - :param str sourceURL: the source of the binary - :param str relLocation: the release configuration file - """ - if project not in self.prjRelCFG: - self.prjRelCFG[project] = {} - if release in self.prjRelCFG[project]: - self.__dbgMsg("Release config for %s:%s has already been loaded" % (project, release)) - return S_OK() - - if relLocation: - relcfgLoc = relLocation - else: - if releaseMode: - try: - relcfgLoc = self.globalDefaults.get("Projects/%s/Releases" % project) - except KeyError: - return S_ERROR("Missing Releases file for project %s" % project) - else: - if not sourceURL: - result = self.getTarsLocation(project) - if not result['OK']: - return result - siu = result['Value'] - else: - siu = sourceURL - relcfgLoc = "%s/release-%s-%s.cfg" % (siu, project, release) - self.__dbgMsg("Releases file is %s" % relcfgLoc) - result = self.__loadCFGFromURL(relcfgLoc, checkHash=not releaseMode) - if not result['OK']: - return result - self.prjRelCFG[project][release] = result['Value'] - self.__dbgMsg("Loaded releases file %s" % relcfgLoc) - - return S_OK(self.prjRelCFG[project][release]) - - def getReleaseCFG(self, project, release): - """ - Returns the release configuration object - - :param str project: the name of the project - :param str release: the release version - """ - return self.prjRelCFG[project][release] - - def dumpReleasesToPath(self): - """ - It dumps the content of the loaded configuration (memory content) to - a given file - """ - for project in self.prjRelCFG: - prjRels = self.prjRelCFG[project] - for release in prjRels: - self.__dbgMsg("Dumping releases file for %s:%s" % (project, release)) - fd = open( - os.path.join( - cliParams.targetPath, "releases-%s-%s.cfg" % - (project, release)), "w") - fd.write(prjRels[release].toString()) - fd.close() - - def __checkCircularDependencies(self, key, routePath=None): - """ - Check the dependencies - - :param str key: the name of the project and the release version - :param list routePath: it stores the software packages, used to check the - dependency - """ - - if not routePath: - routePath = [] - if key not in self.projectsLoadedBy: - return S_OK() - routePath.insert(0, key) - for lKey in self.projectsLoadedBy[key]: - if lKey in routePath: - routePath.insert(0, lKey) - route = "->".join(["%s:%s" % sKey for sKey in routePath]) - return S_ERROR("Circular dependency found for %s: %s" % ("%s:%s" % lKey, route)) - result = self.__checkCircularDependencies(lKey, routePath) - if not result['OK']: - return result - routePath.pop(0) - return S_OK() - - def loadProjectRelease(self, releases, - project=None, - sourceURL=None, - releaseMode=None, - relLocation=None): - """ - This method loads all project configurations (*.cfg). If a project is an extension of DIRAC, - it will load the extension and after will load the base DIRAC module. - - :param list releases: list of releases, which will be loaded: for example: v6r19 - :param str project: the name of the project, if it is given. For example: DIRAC - :param str sourceURL: the code repository - :param str releaseMode: - :param str relLocation: local configuration file, - which contains the releases. for example: file:///`pwd`/releases.cfg - """ - - if not project: - project = self.projectName - - if not isinstance(releases, (list, tuple)): - releases = [releases] - - # Load defaults - result = self.__loadObjectDefaults("Projects", project) - if not result['OK']: - self.__dbgMsg("Could not load defaults for project %s" % project) - return result - - if project not in self.prjDepends: - self.prjDepends[project] = {} - - for release in releases: - self.__dbgMsg("Processing dependencies for %s:%s" % (project, release)) - result = self.__loadReleaseConfig(project, release, releaseMode, sourceURL, relLocation) - if not result['OK']: - return result - relCFG = result['Value'] - # Calculate dependencies and avoid circular deps - self.prjDepends[project][release] = [(project, release)] - relDeps = self.prjDepends[project][release] - - if not relCFG.getChild("Releases/%s" % (release)): # pylint: disable=no-member - return S_ERROR( - "Release %s is not defined for project %s in the release file" % - (release, project)) - - initialDeps = self.getReleaseDependencies(project, release) - if initialDeps: - self.__dbgMsg("%s %s depends on %s" % - (project, release, ", ".join(["%s:%s" % - (k, initialDeps[k]) for k in initialDeps]))) - relDeps.extend([(p, initialDeps[p]) for p in initialDeps]) - for depProject in initialDeps: - depVersion = initialDeps[depProject] - # Check if already processed - dKey = (depProject, depVersion) - if dKey not in self.projectsLoadedBy: - self.projectsLoadedBy[dKey] = [] - self.projectsLoadedBy[dKey].append((project, release)) - result = self.__checkCircularDependencies(dKey) - if not result['OK']: - return result - # if it has already been processed just return OK - if len(self.projectsLoadedBy[dKey]) > 1: - return S_OK() - - # Load dependencies and calculate incompatibilities - result = self.loadProjectRelease(depVersion, project=depProject) - if not result['OK']: - return result - subDep = self.prjDepends[depProject][depVersion] - # Merge dependencies - for sKey in subDep: - if sKey not in relDeps: - relDeps.append(sKey) - continue - prj, vrs = sKey - for pKey in relDeps: - if pKey[0] == prj and pKey[1] != vrs: - errMsg = "%s is required with two different versions ( %s and %s ) \ - starting with %s:%s" % (prj, - pKey[1], vrs, - project, release) - return S_ERROR(errMsg) - - # Same version already required - if project in relDeps and relDeps[project] != release: - errMsg = "%s:%s requires itself with a different version through dependencies ( %s )" % ( - project, release, relDeps[project]) - return S_ERROR(errMsg) - - # we have now all dependencies, let's retrieve the resources (code repository) - for project, version in relDeps: - if project in self.diracBaseModules: - continue - modules = self.getModulesForRelease(version, project) - if modules['OK']: - for dependency in modules['Value']: - self.diracBaseModules.setdefault(dependency, {}) - self.diracBaseModules[dependency]['Version'] = modules['Value'][dependency] - res = self.getModSource(version, dependency, project) - if not res['OK']: - self.__dbgMsg( - "Unable to found the source URL for %s : %s" % - (dependency, res['Message'])) - else: - self.diracBaseModules[dependency]['sourceUrl'] = res['Value'][1] - - return S_OK() - - def getReleaseOption(self, project, release, option): - """ - Returns a given option - - :param str project: the name of the project - :param str release: the release version - :param str option: the option name - """ - try: - return self.prjRelCFG[project][release].get(option) - except KeyError: - self.__dbgMsg("Missing option %s for %s:%s" % (option, project, release)) - # try to found the option in a different release - for project in self.prjRelCFG: - for release in self.prjRelCFG[project]: - if self.prjRelCFG[project][release].isOption(option): - return self.prjRelCFG[project][release].get(option) - return False - - def getReleaseDependencies(self, project, release): - """ - It return the dependencies for a certain project - - :param str project: the name of the project - :param str release: the release version - """ - try: - data = self.prjRelCFG[project][release].get("Releases/%s/Depends" % release) - except KeyError: - return {} - data = [field for field in data.split(",") if field.strip()] - deps = {} - for field in data: - field = field.strip() - if not field: - continue - pv = field.split(":") - if len(pv) == 1: - deps[pv[0].strip()] = release - else: - deps[pv[0].strip()] = ":".join(pv[1:]).strip() - return deps - - def getModulesForRelease(self, release, project=None): - """ - Returns the modules for a given release for example: WebAppDIRAC, - RESTDIRAC, LHCbWebAppDIRAC, etc - - :param str release: the release version - :param str project: the project name - """ - if not project: - project = self.projectName - if project not in self.prjRelCFG: - return S_ERROR("Project %s has not been loaded. I'm a MEGA BUG! Please report me!" % project) - if release not in self.prjRelCFG[project]: - return S_ERROR("Version %s has not been loaded for project %s" % (release, project)) - config = self.prjRelCFG[project][release] - if not config.isSection("Releases/%s" % release): - return S_ERROR("Release %s is not defined for project %s" % (release, project)) - # Defined Modules explicitly in the release - modules = self.getReleaseOption(project, release, "Releases/%s/Modules" % release) - if modules: - dMods = {} - for entry in [entry.split(":") for entry in modules.split( - ",") if entry.strip()]: # pylint: disable=no-member - if len(entry) == 1: - dMods[entry[0].strip()] = release - else: - dMods[entry[0].strip()] = entry[1].strip() - modules = dMods - else: - # Default modules with the same version as the release version - modules = self.getReleaseOption(project, release, "DefaultModules") - if modules: - modules = dict((modName.strip(), release) for modName in modules.split(",") - if modName.strip()) # pylint: disable=no-member - else: - # Mod = project and same version - modules = {project: release} - # Check project is in the modNames if not DIRAC - if project != "DIRAC": - for modName in modules: - if modName.find(project) != 0: - return S_ERROR("Module %s does not start with the name %s" % (modName, project)) - return S_OK(modules) - - def getModSource(self, release, modName, project=None): - """ - It reads the Sources section from the .cfg file for example: - Sources - { - Web = git://github.com/DIRACGrid/DIRACWeb.git - VMDIRAC = git://github.com/DIRACGrid/VMDIRAC.git - DIRAC = git://github.com/DIRACGrid/DIRAC.git - BoincDIRAC = git://github.com/DIRACGrid/BoincDIRAC.git - RESTDIRAC = git://github.com/DIRACGrid/RESTDIRAC.git - COMDIRAC = git://github.com/DIRACGrid/COMDIRAC.git - WebAppDIRAC = git://github.com/DIRACGrid/WebAppDIRAC.git - } - - :param str release: the release which is already loaded for example: v6r19 - :param str modName: the name of the DIRAC module for example: WebAppDIRAC - :param str project: the name of the project for example: DIRAC - """ - - if self.projectName not in self.prjRelCFG: - return S_ERROR( - "Project %s has not been loaded. I'm a MEGA BUG! Please report me!" % - self.projectName) - - if not project: - project = self.projectName - modLocation = self.getReleaseOption(project, release, "Sources/%s" % modName) - if not modLocation: - return S_ERROR("Source origin for module %s is not defined" % modName) - modTpl = [field.strip() for field in modLocation.split( - "|") if field.strip()] # pylint: disable=no-member - if len(modTpl) == 1: - return S_OK((False, modTpl[0])) - return S_OK((modTpl[0], modTpl[1])) - - def getExternalsVersion(self, release=None): - """ - It returns the version of DIRAC Externals. If it is not provided, - uses the default cfg - - :param str release: the release version - """ - - if 'DIRAC' not in self.prjRelCFG: - return False - if not release: - release = list(self.prjRelCFG['DIRAC']) - release = sorted(release, key=LooseVersion)[-1] - try: - return self.prjRelCFG['DIRAC'][release].get('Releases/%s/Externals' % release) - except KeyError: - return False - - def getDiracOSExtensionAndVersion(self, diracOSVersion): - """ - This method return the diracos and version taking into - account the extension. The file format will be .tar.gz - - :param str diracOSVersion: column separated string for example: LHCb:v1 - :return: if the extension is not provided, it will return DIRACOS defined in DIRAC otherwise - the DIRACOS specified in the extension - """ - if ":" in diracOSVersion: - package, packageVersion = [i.strip() for i in diracOSVersion.split(':')] - return [package + 'diracos', packageVersion] - else: - return ['diracos', diracOSVersion] - - def getDiracOSVersion(self, diracOSVersion=None): - """ - It returns the DIRACOS version - :param str diracOSVersion: the OS version - """ - - if diracOSVersion: - return self.getDiracOSExtensionAndVersion(diracOSVersion) - try: - diracOSVersion = self.prjRelCFG[self.projectName][cliParams.release].get( - "Releases/%s/DIRACOS" % cliParams.release, diracOSVersion) - if not diracOSVersion: - # the DIRAC extension does not specify DIRACOS version - for release in self.prjRelCFG['DIRAC']: - logWARN("Getting DIRACOS version from DIRAC %s!" % release) - diracOSVersion = self.prjRelCFG['DIRAC'][release].get( - "Releases/%s/DIRACOS" % release, diracOSVersion) - except KeyError: - pass - return self.getDiracOSExtensionAndVersion(diracOSVersion) - - def getLCGVersion(self, lcgVersion=None): - """ - It returns the LCG version - :param str lcgVersion: LCG version - """ - if lcgVersion: - return lcgVersion - try: - return self.prjRelCFG[self.projectName][cliParams.release].get( - "Releases/%s/LcgVer" % cliParams.release, lcgVersion) - except KeyError: - pass - return lcgVersion - - def getModulesToInstall(self, release, extensions=None): - """ - It returns the modules to be installed. - :param str release: the release version to be deployed - :param str extensions: DIRAC extension - :return: the order of the nodules and modules to be installed. - """ - if not extensions: - extensions = [] - extraFound = [] - modsToInstall = {} - modsOrder = [] - if self.projectName not in self.prjDepends: - return S_ERROR("Project %s has not been loaded" % self.projectName) - if release not in self.prjDepends[self.projectName]: - return S_ERROR( - "Version %s has not been loaded for project %s" % - (release, self.projectName)) - # Get a list of projects with their releases - projects = list(self.prjDepends[self.projectName][release]) - for project, relVersion in projects: - try: - requiredModules = self.prjRelCFG[project][relVersion].get("RequiredExtraModules") - requiredModules = [modName.strip() - for modName in requiredModules.split("/") if modName.strip()] - except KeyError: - requiredModules = [] - for modName in requiredModules: - if modName not in extensions: - extensions.append(modName) - self.__dbgMsg("Discovering modules to install for %s (%s)" % (project, relVersion)) - result = self.getModulesForRelease(relVersion, project) - if not result['OK']: - return result - modVersions = result['Value'] - try: - defaultMods = self.prjRelCFG[project][relVersion].get("DefaultModules") - modNames = [mod.strip() for mod in defaultMods.split(",") if mod.strip()] - except KeyError: - modNames = [] - for extension in extensions: - # Check if the version of the extension module is specified in the command line - extraVersion = None - if ":" in extension: - extension, extraVersion = extension.split(":") - modVersions[extension] = extraVersion - if extension in modVersions: - modNames.append(extension) - extraFound.append(extension) - if 'DIRAC' not in extension: - dextension = "%sDIRAC" % extension - if dextension in modVersions: - modNames.append(dextension) - extraFound.append(extension) - modNameVer = ["%s:%s" % (modName, modVersions[modName]) for modName in modNames] - self.__dbgMsg("Modules to be installed for %s are: %s" % (project, ", ".join(modNameVer))) - for modName in modNames: - result = self.getTarsLocation(project, modName) - if not result['OK']: - return result - tarsURL = result['Value'] - modVersion = modVersions[modName] - defLoc = self.getModuleVersionFromLocalCfg(modName) - if defLoc: - modVersion = defLoc # this overwrite the version which are defined in the release.cfg - modsToInstall[modName] = (tarsURL, modVersion) - modsOrder.insert(0, modName) - - for modName in extensions: - if modName.split(":")[0] not in extraFound: - return S_ERROR("No module %s defined. You sure it's defined for this release?" % modName) - - return S_OK((modsOrder, modsToInstall)) - - -################################################################################# -# End of ReleaseConfig -################################################################################# - - -# platformAlias = { 'Darwin_i386_10.6' : 'Darwin_i386_10.5' } -platformAlias = {} - -#### -# Start of helper functions -#### - - -def logDEBUG(msg): - """ - :param str msg: debug message - """ - if cliParams.debug: - for line in msg.split("\n"): - print("%s UTC dirac-install [DEBUG] %s" % (time.strftime('%Y-%m-%d %H:%M:%S', - time.gmtime()), - line)) - sys.stdout.flush() - - -def logERROR(msg): - """ - :param str msg: error message - """ - for line in msg.split("\n"): - print("%s UTC dirac-install [ERROR] %s" % (time.strftime('%Y-%m-%d %H:%M:%S', - time.gmtime()), - line)) - sys.stdout.flush() - - -def logWARN(msg): - """ - :param str msg: warning message - """ - for line in msg.split("\n"): - print("%s UTC dirac-install [WARN] %s" % (time.strftime('%Y-%m-%d %H:%M:%S', - time.gmtime()), - line)) - sys.stdout.flush() - - -def logNOTICE(msg): - """ - :param str msg: notice message - """ - for line in msg.split("\n"): - print("%s UTC dirac-install [NOTICE] %s" % (time.strftime('%Y-%m-%d %H:%M:%S', - time.gmtime()), - line)) - sys.stdout.flush() - - -def alarmTimeoutHandler(*args): - """ - When a connection time out then raise and exception - """ - raise Exception('Timeout') - - -def urlretrieveTimeout(url, fileName='', timeout=0, retries=3): - """ - Retrieve remote url to local file, with timeout wrapper - - :param str fileName: file name - :param int timeout: time out in second used for downloading the files. - """ - if fileName: - # This can be a local file - if os.path.exists(url): # we do not download from web, use locally - logDEBUG('Local file used: "%s"' % url) - shutil.copy(url, fileName) - return True - localFD = open(fileName, "wb") - - # NOTE: Not thread-safe, since all threads will catch same alarm. - # This is OK for dirac-install, since there are no threads. - logDEBUG('Retrieving remote file "%s"' % url) - - urlData = '' - if timeout: - signal.signal(signal.SIGALRM, alarmTimeoutHandler) - # set timeout alarm - signal.alarm(timeout + 5) - try: - # if "http_proxy" in os.environ and os.environ['http_proxy']: - # proxyIP = os.environ['http_proxy'] - # proxy = urllib2.ProxyHandler( {'http': proxyIP} ) - # opener = urllib2.build_opener( proxy ) - # #opener = urllib2.build_opener() - # urllib2.install_opener( opener ) - - # Try to use insecure context explicitly, needed for python >= 2.7.9 - try: - context = ssl._create_unverified_context() - remoteFD = urlopen(url, context=context) # pylint: disable=unexpected-keyword-arg - # the keyword 'context' is present from 2.7.9+ - except AttributeError: - remoteFD = urlopen(url) - - expectedBytes = 0 - # Sometimes repositories do not return Content-Length parameter - try: - expectedBytes = int(remoteFD.info()['Content-Length']) - except Exception: - logWARN('Content-Length parameter not returned, skipping expectedBytes check') - - receivedBytes = 0 - data = remoteFD.read(16384) - count = 1 - progressBar = False - while data: - receivedBytes += len(data) - if fileName: - localFD.write(data) - else: - urlData += data.decode('utf8', 'ignore') - data = remoteFD.read(16384) - if count % 20 == 0 and sys.stdout.isatty(): - print(u'\033[1D' + ".", end=" ") - sys.stdout.flush() - progressBar = True - count += 1 - if progressBar and sys.stdout.isatty(): - # return cursor to the beginning of the line - print('\033[1K', end=" ") - print('\033[1A') - if fileName: - localFD.close() - remoteFD.close() - if receivedBytes != expectedBytes and expectedBytes > 0: - logERROR("File should be %s bytes but received %s" % (expectedBytes, receivedBytes)) - return False - except HTTPError as x: - if x.code == 404: - logERROR("%s does not exist" % url) - if timeout: - signal.alarm(0) - return False - else: - logWARN('Status code accessing URL %s was %s' % (url, x.getcode())) - if retries: - logWARN('Will retry after 30 seconds, %s retries remaining' % retries) - time.sleep(30) - return urlretrieveTimeout(url, fileName=fileName, timeout=timeout, retries=retries - 1) - else: - logERROR("No retries remaining for %s" % url) - sys.exit(1) - except URLError: - logERROR('Timeout after %s seconds on transfer request for "%s"' % (str(timeout), url)) - except Exception as x: - if x == 'Timeout': - logERROR('Timeout after %s seconds on transfer request for "%s"' % (str(timeout), url)) - if timeout: - signal.alarm(0) - raise x - if timeout: - signal.alarm(0) - - if fileName: - return True - else: - return urlData - - -def downloadAndExtractTarball(tarsURL, pkgName, pkgVer, checkHash=True, cache=False): - """ - It downloads and extracts a given tarball from a given destination: file system, - web server or code repository. - - :param str tarsURL: the location of the source repository - :param str pkgName: the name of the package to be installed - :param str pkgVer: the version of the package - :param bool checkHash: check the sanity of the file - :param bool cache: use local cache for the tarballs - - """ - tarName = "%s-%s.tar.gz" % (pkgName, pkgVer) - tarPath = os.path.join(cliParams.targetPath, tarName) - tarFileURL = "%s/%s" % (tarsURL, tarName) - tarFileCVMFS = "/cvmfs/dirac.egi.eu/installSource/%s" % tarName - cacheDir = os.path.join(cliParams.basePath, ".installCache") - tarCachePath = os.path.join(cacheDir, tarName) - isSource = False - if cache and os.path.isfile(tarCachePath): - logNOTICE("Using cached copy of %s" % tarName) - shutil.copy(tarCachePath, tarPath) - elif os.path.exists(tarFileCVMFS): - logNOTICE("Using CVMFS copy of %s" % tarName) - tarPath = tarFileCVMFS - checkHash = False - cache = False - else: - logNOTICE("Retrieving %s" % tarFileURL) - try: - if not urlretrieveTimeout(tarFileURL, tarPath, cliParams.timeout): - if os.path.exists(tarPath): - os.unlink(tarPath) - retVal = checkoutFromGit(pkgName, tarsURL, pkgVer) - if not retVal['OK']: - logERROR("Cannot download %s" % tarName) - logERROR("Cannot download %s" % retVal['Message']) - return False - else: - isSource = True - except Exception as e: - logERROR("Cannot download %s: %s" % (tarName, str(e))) - sys.exit(1) - if not isSource and checkHash: - md5Name = "%s-%s.md5" % (pkgName, pkgVer) - md5Path = os.path.join(cliParams.targetPath, md5Name) - md5FileURL = "%s/%s" % (tarsURL, md5Name) - md5CachePath = os.path.join(cacheDir, md5Name) - if cache and os.path.isfile(md5CachePath): - logNOTICE("Using cached copy of %s" % md5Name) - shutil.copy(md5CachePath, md5Path) - else: - logNOTICE("Retrieving %s" % md5FileURL) - try: - if not urlretrieveTimeout(md5FileURL, md5Path, 60): - logERROR("Cannot download %s" % tarName) - return False - except Exception as e: - logERROR("Cannot download %s: %s" % (md5Name, str(e))) - return False - # Read md5 - fd = open(os.path.join(cliParams.targetPath, md5Name), "r") - md5Expected = fd.read().strip() - fd.close() - # Calculate md5 - md5Calculated = hashlib.md5() - with open(os.path.join(cliParams.targetPath, tarName), "rb") as fd: - buf = fd.read(4096) - while buf: - md5Calculated.update(buf) - buf = fd.read(4096) - - # Check - if md5Expected != md5Calculated.hexdigest(): - logERROR("Oops... md5 for package %s failed!" % pkgVer) - sys.exit(1) - # Delete md5 file - if cache: - if not os.path.isdir(cacheDir): - os.makedirs(cacheDir) - os.rename(md5Path, md5CachePath) - else: - os.unlink(md5Path) - # Extract - # cwd = os.getcwd() - # os.chdir(cliParams.targetPath) - # tf = tarfile.open( tarPath, "r" ) - # for member in tf.getmembers(): - # tf.extract( member ) - # os.chdir(cwd) - if not isSource: - logNOTICE("Extract using system tar: %s" % tarPath) - tarCmd = "tar xzf '%s' -C '%s'" % (tarPath, cliParams.targetPath) - if os.system(tarCmd): - logERROR("Extraction of tarball %s failed" % tarPath) - raise RuntimeError("Failed to extract tarball") - # Delete tar - if cache: - if not os.path.isdir(cacheDir): - os.makedirs(cacheDir) - os.rename(tarPath, tarCachePath) - else: - if tarPath != tarFileCVMFS: - os.unlink(tarPath) - - postInstallScript = os.path.join(cliParams.targetPath, pkgName, 'dirac-postInstall.py') - if os.path.isfile(postInstallScript): - os.chmod(postInstallScript, executablePerms) - logNOTICE("Executing %s..." % postInstallScript) - if os.system("python '%s' > '%s.out' 2> '%s.err'" % (postInstallScript, - postInstallScript, - postInstallScript)): - logERROR("Post installation script %s failed. Check %s.err" % (postInstallScript, - postInstallScript)) - return True - - -def fixBuildPaths(): - """ - At compilation time many scripts get the building directory inserted, - this needs to be changed to point to the current installation path: - cliParams.targetPath -""" - - # Locate build path (from header of pydoc) - binaryPath = os.path.join(cliParams.targetPath, cliParams.platform) - pydocPath = os.path.join(binaryPath, 'bin', 'pydoc') - try: - fd = open(pydocPath) - line = fd.readline() - fd.close() - buildPath = line[2:line.find(cliParams.platform) - 1] - replaceCmd = "grep -rIl '%s' %s | xargs sed -i'.org' 's:%s:%s:g'" % (buildPath, - binaryPath, - buildPath, - cliParams.targetPath) - os.system(replaceCmd) - - except BaseException: - pass - - -def fixPythonShebang(): - """ - Some scripts (like the gfal2 scripts) come with a shebang pointing to the system python. - We replace it with the environment one - """ - - binaryPath = os.path.join(cliParams.targetPath, cliParams.platform) - try: - replaceCmd = "grep -rIl '#!/usr/bin/python' %s/bin |\ - xargs sed -i'.org' 's:#!/usr/bin/python:#!/usr/bin/env python:g'" % binaryPath - os.system(replaceCmd) - except BaseException: - pass - - -def runExternalsPostInstall(): - """ - If there are any postInstall in externals, run them - """ - if cliParams.diracOS or cliParams.diracOSVersion: - postInstallPath = os.path.join(cliParams.targetPath, "postInstall") - else: - postInstallPath = os.path.join(cliParams.targetPath, cliParams.platform, "postInstall") - if not os.path.isdir(postInstallPath): - logDEBUG("There's no %s directory. Skipping postInstall step" % postInstallPath) - return - postInstallSuffix = "-postInstall" - for scriptName in os.listdir(postInstallPath): - if not scriptName.endswith(postInstallSuffix): - logDEBUG("%s does not have the %s suffix. Skipping.." % (scriptName, postInstallSuffix)) - continue - scriptPath = os.path.join(postInstallPath, scriptName) - os.chmod(scriptPath, executablePerms) - logNOTICE("Executing %s..." % scriptPath) - if os.system("'%s' > '%s.out' 2> '%s.err'" % (scriptPath, scriptPath, scriptPath)): - logERROR("Post installation script %s failed. Check %s.err" % (scriptPath, scriptPath)) - sys.exit(1) - - -def fixMySQLScript(): - """ - Update the mysql.server script (if installed) to point to the proper datadir - """ - scriptPath = os.path.join(cliParams.targetPath, 'scripts', 'dirac-fix-mysql-script') - bashrcFile = os.path.join(cliParams.targetPath, 'bashrc') - if cliParams.useVersionsDir: - bashrcFile = os.path.join(cliParams.basePath, 'bashrc') - command = 'source %s; %s > /dev/null' % (bashrcFile, scriptPath) - if os.path.exists(scriptPath): - logNOTICE("Executing %s..." % command) - os.system('bash -c "%s"' % command) - - -def checkPlatformAliasLink(): - """ - Make a link if there's an alias - """ - if cliParams.platform in platformAlias: - os.symlink(os.path.join(cliParams.targetPath, platformAlias[cliParams.platform]), - os.path.join(cliParams.targetPath, cliParams.platform)) - - -def installExternalRequirements(extType): - """ Install the extension requirements if any - """ - reqScript = os.path.join(cliParams.targetPath, "scripts", 'dirac-externals-requirements') - bashrcFile = os.path.join(cliParams.targetPath, 'bashrc') - if cliParams.useVersionsDir: - bashrcFile = os.path.join(cliParams.basePath, 'bashrc') - if os.path.isfile(reqScript): - os.chmod(reqScript, executablePerms) - logNOTICE("Executing %s..." % reqScript) - command = "%s -t '%s' > '%s.out' 2> '%s.err'" % (reqScript, extType, reqScript, reqScript) - if os.system('bash -c "source %s; %s"' % (bashrcFile, command)): - logERROR("Requirements installation script %s failed. Check %s.err" % (reqScript, - reqScript)) - return True - - -def discoverModules(modules): - """ - Created the dictionary which contains all modules, which can be installed - for example: {"DIRAC:{"sourceUrl":"https://github.com/zmathe/DIRAC.git","Vesrion:v6r20p11"}} - - :param: str modules: it contains meta information for the module, - which will be installed: https://github.com/zmathe/DIRAC.git:::DIRAC:::dev_main_branch - """ - - projects = {} - - for module in modules.split(","): - s = m = v = None - try: - s, m, v = module.split(":::") - except ValueError: - m = module.split(":::")[0] # the source and version is not provided - - projects[m] = {} - if s and v: - projects[m] = {"sourceUrl": s, "Version": v} - else: - logWARN('Unable to parse module: %s' % module) - return projects - -#### -# End of helper functions -#### - - -cmdOpts = (('r:', 'release=', 'Release version to install'), - ('l:', 'project=', 'Project to install'), - ('e:', 'extensions=', 'Extensions to install (comma separated)'), - ('t:', 'installType=', 'Installation type (client/server)'), - ('i:', 'pythonVersion=', 'Python version to compile (27/26)'), - ('p:', 'platform=', 'Platform to install'), - ('P:', 'installationPath=', 'Path where to install (default current working dir)'), - ('b', 'build', 'Force local compilation'), - ('g:', 'grid=', 'lcg tools package version'), - (' ', 'no-lcg-bundle', 'lcg tools not to be installed'), - ('B', 'noAutoBuild', 'Do not build if not available'), - ('v', 'useVersionsDir', 'Use versions directory'), - ('u:', 'baseURL=', "Use URL as the source for installation tarballs"), - ('d', 'debug', 'Show debug messages'), - ('V:', 'installation=', 'Installation from which to extract parameter values'), - ('X', 'externalsOnly', 'Only install external binaries'), - ('M:', 'defaultsURL=', 'Where to retrieve the global defaults from'), - ('h', 'help', 'Show this help'), - ('T:', 'Timeout=', 'Timeout for downloads (default = %s)'), - (' ', 'dirac-os-version=', 'the version of the DIRAC OS'), - (' ', 'dirac-os', 'Enable installation of DIRAC OS'), - (' ', 'tag=', 'release version to install from git, http or local'), - ('m:', 'module=', - 'Module to be installed. for example: -m DIRAC or -m git://github.com/DIRACGrid/DIRAC.git:DIRAC'), - ('x:', 'external=', 'external version'), - (' ', 'createLink', 'create version symbolic link from the versions directory. This is equivalent to the \ - following command: ln -s /opt/dirac/versions/vArBpC vArBpC'), - (' ', 'scriptSymlink', 'Symlink the scripts instead of creating wrapper'), - (' ', 'userEnvVariables=', - 'User-requested environment variables (comma-separated, name and value separated by ":::")') - ) - - -def usage(): - print("\nUsage:\n\n %s " % os.path.basename(sys.argv[0])) - print("\nOptions:") - for cmdOpt in cmdOpts: - print(" %s %s : %s" % (cmdOpt[0].ljust(3), cmdOpt[1].ljust(20), cmdOpt[2])) - print() - print("Known options and default values from /defaults section of releases file:") - for options in [('Release', cliParams.release), - ('Project', cliParams.project), - ('ModulesToInstall', []), - ('ExternalsType', cliParams.externalsType), - ('PythonVersion', cliParams.pythonVersion), - ('LcgVer', cliParams.lcgVer), - ('UseVersionsDir', cliParams.useVersionsDir), - ('BuildExternals', cliParams.buildExternals), - ('NoAutoBuild', cliParams.noAutoBuild), - ('Debug', cliParams.debug), - ('Timeout', cliParams.timeout)]: - print(" %s = %s" % options) - - sys.exit(0) - - -def loadConfiguration(): - """ - It loads the configuration file - """ - optList, args = getopt.getopt(sys.argv[1:], - "".join([opt[0] for opt in cmdOpts]), - [opt[1] for opt in cmdOpts]) - - # First check if the name is defined - for o, v in optList: - if o in ('-h', '--help'): - usage() - elif o in ('-V', '--installation'): - cliParams.installation = v - elif o in ("-d", "--debug"): - cliParams.debug = True - elif o in ("-M", "--defaultsURL"): - cliParams.globalDefaults = v - - releaseConfig = ReleaseConfig( - instName=cliParams.installation, - globalDefaultsURL=cliParams.globalDefaults) - if cliParams.debug: - releaseConfig.debugCB = logDEBUG - - result = releaseConfig.loadInstallationDefaults() - if not result['OK']: - logERROR("Could not load defaults: %s" % result['Message']) - - releaseConfig.loadInstallationLocalDefaults(args) - - for opName in ('release', 'externalsType', 'installType', 'pythonVersion', - 'buildExternals', 'noAutoBuild', 'debug', 'globalDefaults', - 'lcgVer', 'useVersionsDir', 'targetPath', - 'project', 'release', 'extensions', 'timeout'): - try: - opVal = releaseConfig.getInstallationConfig( - "LocalInstallation/%s" % (opName[0].upper() + opName[1:])) - except KeyError: - continue - - if opName == 'installType': - opName = 'externalsType' - - if isinstance(getattr(cliParams, opName), str_type): - setattr(cliParams, opName, opVal) - elif isinstance(getattr(cliParams, opName), bool): - setattr(cliParams, opName, opVal.lower() in ("y", "yes", "true", "1")) - elif isinstance(getattr(cliParams, opName), list): - setattr(cliParams, opName, [opV.strip() for opV in opVal.split(",") if opV]) - - # Now parse the ops - for o, v in optList: - if o in ('-r', '--release'): - cliParams.release = v - elif o in ('-l', '--project'): - cliParams.project = v - elif o in ('-e', '--extensions'): - for pkg in [p.strip() for p in v.split(",") if p.strip()]: - if pkg not in cliParams.extensions: - cliParams.extensions.append(pkg) - elif o in ('-t', '--installType'): - cliParams.externalsType = v - elif o in ('-i', '--pythonVersion'): - cliParams.pythonVersion = v - elif o in ('-p', '--platform'): - cliParams.platform = v - elif o in ('-d', '--debug'): - cliParams.debug = True - elif o in ('-g', '--grid'): - cliParams.lcgVer = v - elif o in ('--no-lcg-bundle'): - cliParams.noLcg = True - elif o in ('-u', '--baseURL'): - cliParams.installSource = v - elif o in ('-P', '--installationPath'): - cliParams.targetPath = v - try: - os.makedirs(v) - except BaseException: - pass - elif o in ('-v', '--useVersionsDir'): - cliParams.useVersionsDir = True - elif o in ('-b', '--build'): - cliParams.buildExternals = True - elif o in ("-B", '--noAutoBuild'): - cliParams.noAutoBuild = True - elif o in ('-X', '--externalsOnly'): - cliParams.externalsOnly = True - elif o in ('-T', '--Timeout'): - try: - cliParams.timeout = max(cliParams.timeout, int(v)) - cliParams.timeout = min(cliParams.timeout, 3600) - except ValueError: - pass - elif o == '--dirac-os-version': - cliParams.diracOSVersion = v - elif o == '--dirac-os': - cliParams.diracOS = True - elif o == '--tag': - cliParams.tag = v - elif o in ('-m', '--module'): - cliParams.modules = discoverModules(v) - elif o in ('-x', '--external'): - cliParams.externalVersion = v - elif o == '--createLink': - cliParams.createLink = True - elif o == '--scriptSymlink': - cliParams.scriptSymlink = True - elif o == '--userEnvVariables': - cliParams.userEnvVariables = dict(zip([name.split(':::')[0] for name in v.replace(' ', '').split(',')], - [value.split(':::')[1] for value in v.replace(' ', '').split(',')])) - - if not cliParams.release and not cliParams.modules: - logERROR("Missing release to install") - usage() - - cliParams.basePath = cliParams.targetPath - if cliParams.useVersionsDir: - # install under /versions/_ - cliParams.targetPath = os.path.join( - cliParams.targetPath, 'versions', '%s_%s' % (cliParams.release, int(time.time()))) - try: - os.makedirs(cliParams.targetPath) - except BaseException: - pass - - # If we are running an update, DIRACOS will be set in the environment - if not cliParams.diracOS and 'DIRACOS' in os.environ: - logWARN("Forcing to install DIRACOS, because it is already installed!") - cliParams.diracOS = True - - logNOTICE("Destination path for installation is %s" % cliParams.targetPath) - releaseConfig.projectName = cliParams.project - - result = releaseConfig.loadProjectRelease(cliParams.release, - project=cliParams.project, - sourceURL=cliParams.installSource) - if not result['OK']: - return result - - if not releaseConfig.isProjectLoaded("DIRAC"): - return S_ERROR("DIRAC is not depended by this installation. Aborting") - - # Reload the local configuration to ensure it takes prescience - releaseConfig.loadInstallationLocalDefaults(args) - - return S_OK(releaseConfig) - - -def compileExternals(extVersion): - """ - It is used to compile the external for a given platform - - :param str extVersion: the external version - """ - logNOTICE("Compiling externals %s" % extVersion) - buildCmd = os.path.join( - cliParams.targetPath, - "DIRAC", - "Core", - "scripts", - "dirac-compile-externals.py") - buildCmd = "%s -t '%s' -D '%s' -v '%s' -i '%s'" % (buildCmd, - cliParams.externalsType, - os.path.join( - cliParams.targetPath, - cliParams.platform), - extVersion, - cliParams.pythonVersion) - if os.system(buildCmd): - logERROR("Could not compile binaries") - return False - return True - - -def getPlatform(): - """ - It returns the platform, where this script is running using Platform.py - """ - platformPath = os.path.join(cliParams.targetPath, "DIRAC", "Core", "Utilities", "Platform.py") - try: - platFD = open(platformPath, "r") - except IOError: - logERROR("Cannot open Platform.py. Is DIRAC installed?") - return '' - - Platform = imp.load_module("Platform", platFD, platformPath, ("", "r", imp.PY_SOURCE)) - platFD.close() - return Platform.getPlatformString() - - -def installExternals(releaseConfig): - """ - It install the DIRAC external. The version of the external is provided by - the cmd or in the configuration file. - - :param object releaseConfig: - """ - if not releaseConfig: - externalsVersion = cliParams.externalVersion - else: - externalsVersion = releaseConfig.getExternalsVersion() - if not externalsVersion: - logERROR("No externals defined") - return False - - if not cliParams.platform: - cliParams.platform = getPlatform() - if not cliParams.platform: - return False - - if cliParams.installSource: - tarsURL = cliParams.installSource - else: - tarsURL = releaseConfig.getTarsLocation('DIRAC')['Value'] - - if cliParams.buildExternals: - compileExternals(externalsVersion) - else: - logDEBUG("Using platform: %s" % cliParams.platform) - extVer = "%s-%s-%s-python%s" % (cliParams.externalsType, externalsVersion, - cliParams.platform, cliParams.pythonVersion) - logDEBUG("Externals %s are to be installed" % extVer) - if not downloadAndExtractTarball(tarsURL, "Externals", extVer, cache=True): - return (not cliParams.noAutoBuild) and compileExternals(externalsVersion) - logNOTICE("Fixing externals paths...") - fixBuildPaths() - logNOTICE("Running externals post install...") - checkPlatformAliasLink() - return True - - -def installLCGutils(releaseConfig): - """ - DIRAC uses various tools from LCG area. This method install a given - lcg version. - :param object releaseConfig: the configuration file object (class ReleaseConfig) - """ - if not cliParams.platform: - cliParams.platform = getPlatform() - if not cliParams.platform: - return False - - if cliParams.installSource: - tarsURL = cliParams.installSource - else: - tarsURL = releaseConfig.getTarsLocation('DIRAC')['Value'] - - # lcg utils? - # LCG utils if required - if not releaseConfig: - lcgVer = cliParams.lcgVer - else: - lcgVer = releaseConfig.getLCGVersion(cliParams.lcgVer) - if lcgVer: - verString = "%s-%s-python%s" % (lcgVer, cliParams.platform, cliParams.pythonVersion) - # HACK: try to find a more elegant solution for the lcg bundles location - if not downloadAndExtractTarball(tarsURL + "/../lcgBundles", "DIRAC-lcg", verString, False, cache=True): - logERROR( - "\nThe requested LCG software version %s for the local operating system could not be downloaded." % - verString) - logERROR("Please, check the availability of the LCG software bindings for you \ - platform 'DIRAC-lcg-%s' \n in the repository %s/lcgBundles/." % - (verString, os.path.dirname(tarsURL))) - logERROR( - "\nIf you would like to skip the installation of the LCG software, redo the installation with \ - adding the option --no-lcg-bundle to the command line.") - return False - - logNOTICE("Fixing Python Shebang...") - fixPythonShebang() - return True - - -def createPermanentDirLinks(): - """ Create links to permanent directories from within the version directory - """ - if cliParams.useVersionsDir: - try: - # Directories - for directory in ['startup', 'runit', 'data', 'work', 'control', 'sbin', 'etc', 'webRoot']: - fake = os.path.join(cliParams.targetPath, directory) - real = os.path.join(cliParams.basePath, directory) - if not os.path.exists(real): - os.makedirs(real) - if os.path.exists(fake): - # Try to reproduce the directory structure to avoid lacking directories - fakeDirs = os.listdir(fake) - for fd in fakeDirs: - if os.path.isdir(os.path.join(fake, fd)): - if not os.path.exists(os.path.join(real, fd)): - os.makedirs(os.path.join(real, fd)) - os.rename(fake, fake + '.bak') - os.symlink(real, fake) - - # Files - for filename in ['bashrc']: - fake = os.path.join(cliParams.targetPath, filename) - real = os.path.join(cliParams.basePath, filename) - os.symlink(real, fake) - except Exception as x: - logERROR(str(x)) - return False - - return True - - -def createOldProLinks(): - """ Create links to permanent directories from within the version directory - """ - proPath = cliParams.targetPath - if cliParams.useVersionsDir: - oldPath = os.path.join(cliParams.basePath, 'old') - proPath = os.path.join(cliParams.basePath, 'pro') - try: - if os.path.exists(proPath) or os.path.islink(proPath): - if os.path.exists(oldPath) or os.path.islink(oldPath): - os.unlink(oldPath) - os.rename(proPath, oldPath) - os.symlink(cliParams.targetPath, proPath) - except Exception as x: - logERROR(str(x)) - return False - - return True - - -def createBashrc(): - """ Create DIRAC environment setting script for the bash shell - """ - - proPath = cliParams.targetPath - # Now create bashrc at basePath - try: - bashrcFile = os.path.join(cliParams.targetPath, 'bashrc') - if cliParams.useVersionsDir: - bashrcFile = os.path.join(cliParams.basePath, 'bashrc') - proPath = os.path.join(cliParams.basePath, 'pro') - logNOTICE('Creating %s' % bashrcFile) - if not os.path.exists(bashrcFile): - lines = ['# DIRAC bashrc file, used by service and agent run scripts to set environment', - 'export PYTHONUNBUFFERED=yes', - 'export PYTHONOPTIMIZE=x'] - if 'HOME' in os.environ: - lines.append('[ -z "$HOME" ] && export HOME=%s' % os.environ['HOME']) - - # Determining where the CAs are... - if 'X509_CERT_DIR' in os.environ: - certDir = os.environ['X509_CERT_DIR'] - else: - if os.path.isdir('/etc/grid-security/certificates') and \ - os.listdir('/etc/grid-security/certificates'): - # Assuming that, if present, it is not empty, and has correct CAs - certDir = '/etc/grid-security/certificates' - else: - # But this will have to be created at some point (dirac-configure) - certDir = '%s/etc/grid-security/certificates' % proPath - lines.extend(['# CAs path for SSL verification', - 'export X509_CERT_DIR=${X509_CERT_DIR:-%s}' % certDir, - 'export SSL_CERT_DIR=${SSL_CERT_DIR:-%s}' % certDir]) - - lines.append( - 'export X509_VOMS_DIR=${X509_VOMS_DIR:-%s}' % - os.path.join( - proPath, - 'etc', - 'grid-security', - 'vomsdir')) - lines.append( - 'export X509_VOMSES=${X509_VOMSES:-%s}' % - os.path.join( - proPath, - 'etc', - 'grid-security', - 'vomses')) - lines.extend( - [ - '# Some DIRAC locations', - '[ -z "$DIRAC" ] && export DIRAC=%s' % - proPath, - 'export DIRACBIN=%s' % - os.path.join( - "$DIRAC", - cliParams.platform, - 'bin'), - 'export DIRACSCRIPTS=%s' % - os.path.join( - "$DIRAC", - 'scripts'), - 'export DIRACLIB=%s' % - os.path.join( - "$DIRAC", - cliParams.platform, - 'lib'), - 'export TERMINFO=%s' % - __getTerminfoLocations( - os.path.join( - "$DIRAC", - cliParams.platform, - 'share', - 'terminfo')), - 'export RRD_DEFAULT_FONT=%s' % - os.path.join( - "$DIRAC", - cliParams.platform, - 'share', - 'rrdtool', - 'fonts', - 'DejaVuSansMono-Roman.ttf')]) - - lines.extend(['# Prepend the PYTHONPATH, the LD_LIBRARY_PATH, and the DYLD_LIBRARY_PATH']) - lines.extend(['( echo $PATH | grep -q $DIRACBIN ) || export PATH=$DIRACBIN:$PATH', - '( echo $PATH | grep -q $DIRACSCRIPTS ) || export PATH=$DIRACSCRIPTS:$PATH', - '( echo $LD_LIBRARY_PATH | grep -q $DIRACLIB ) || \ - export LD_LIBRARY_PATH=$DIRACLIB:$LD_LIBRARY_PATH', - '( echo $LD_LIBRARY_PATH | grep -q $DIRACLIB/mysql ) || \ - export LD_LIBRARY_PATH=$DIRACLIB/mysql:$LD_LIBRARY_PATH', - '( echo $DYLD_LIBRARY_PATH | grep -q $DIRACLIB ) || \ - export DYLD_LIBRARY_PATH=$DIRACLIB:$DYLD_LIBRARY_PATH', - '( echo $DYLD_LIBRARY_PATH | grep -q $DIRACLIB/mysql ) || \ - export DYLD_LIBRARY_PATH=$DIRACLIB/mysql:$DYLD_LIBRARY_PATH']) - - lines.extend(['export PYTHONPATH=$DIRAC']) - - lines.extend(['# new OpenSSL version require OPENSSL_CONF to point to some accessible location', - 'export OPENSSL_CONF=/tmp']) - - # gfal2 requires some environment variables to be set - lines.extend(['# Gfal2 configuration and plugins', 'export GFAL_CONFIG_DIR=%s' % - os.path.join("$DIRAC", cliParams.platform, 'etc/gfal2.d'), 'export GFAL_PLUGIN_DIR=%s' % - os.path.join("$DIRACLIB", 'gfal2-plugins')]) - # add DIRACPLAT environment variable for client installations - if cliParams.externalsType == 'client': - lines.extend(['# DIRAC platform', - '[ -z "$DIRACPLAT" ] && export DIRACPLAT=`$DIRAC/scripts/dirac-platform`']) - # Add the lines required for globus-* tools to use IPv6 - lines.extend(['# IPv6 support', - 'export GLOBUS_IO_IPV6=TRUE', - 'export GLOBUS_FTP_CLIENT_IPV6=TRUE']) - # Add the lines required for ARC CE support - lines.extend(['# ARC Computing Element', - 'export ARC_PLUGIN_PATH=$DIRACLIB/arc']) - - # Add the lines required for fork support for xrootd - lines.extend(['# Fork support for xrootd', - 'export XRD_RUNFORKHANDLER=1']) - - # Add the lines required for further env variables requested - if cliParams.userEnvVariables: - lines.extend(['# User-requested variables']) - for envName, envValue in cliParams.userEnvVariables.items(): - lines.extend(['( echo $%s | grep -q $%s ) || export %s=$%s:$%s' % ( - envName, envValue, - envName, envName, envValue)]) - - # Add possible DIRAC environment variables - lines.append('') - lines.append('# before enabling any of these variables, please see the documentation ') - lines.append('# https://dirac.readthedocs.io/en/latest/AdministratorGuide/' + - 'ServerInstallations/environment_variable_configuration.html') - lines.append('# export DIRAC_DEBUG_DENCODE_CALLSTACK=1') - lines.append('# export DIRAC_DEBUG_STOMP=1') - lines.append('# export DIRAC_DEPRECATED_FAIL=1') - lines.append('# export DIRAC_GFAL_GRIDFTP_SESSION_REUSE=true') - lines.append('# export DIRAC_USE_JSON_DECODE=no') - lines.append('# export DIRAC_USE_JSON_ENCODE=no') - lines.append('# export DIRAC_USE_M2CRYPTO=true') - lines.append('# export DIRAC_USE_NEWTHREADPOOL=yes') - lines.append('# export DIRAC_NO_CFG=true') - lines.append('') - f = open(bashrcFile, 'w') - f.write('\n'.join(lines)) - f.close() - except Exception as x: - logERROR(str(x)) - return False - - return True - - -def createCshrc(): - """ Create DIRAC environment setting script for the (t)csh shell - """ - proPath = cliParams.targetPath - # Now create cshrc at basePath - try: - cshrcFile = os.path.join(cliParams.targetPath, 'cshrc') - if cliParams.useVersionsDir: - cshrcFile = os.path.join(cliParams.basePath, 'cshrc') - proPath = os.path.join(cliParams.basePath, 'pro') - logNOTICE('Creating %s' % cshrcFile) - if not os.path.exists(cshrcFile): - lines = ['# DIRAC cshrc file, used by clients to set up the environment', - 'setenv PYTHONUNBUFFERED yes', - 'setenv PYTHONOPTIMIZE x'] - - # Determining where the CAs are... - if 'X509_CERT_DIR' in os.environ: - certDir = os.environ['X509_CERT_DIR'] - else: - if os.path.isdir('/etc/grid-security/certificates') and \ - os.listdir('/etc/grid-security/certificates'): - # Assuming that, if present, it is not empty, and has correct CAs - certDir = '/etc/grid-security/certificates' - else: - # But this will have to be created at some point (dirac-configure) - certDir = '%s/etc/grid-security/certificates' % proPath - lines.extend(['# CAs path for SSL verification', - 'setenv X509_CERT_DIR %s' % certDir, - 'setenv SSL_CERT_DIR %s' % certDir]) - - lines.append( - 'setenv X509_VOMS_DIR %s' % - os.path.join( - proPath, - 'etc', - 'grid-security', - 'vomsdir')) - lines.append( - 'setenv X509_VOMSES %s' % - os.path.join( - proPath, - 'etc', - 'grid-security', - 'vomses')) - lines.extend(['# Some DIRAC locations', - '( test $?DIRAC -eq 1 ) || setenv DIRAC %s' % proPath, - 'setenv DIRACBIN %s' % os.path.join("$DIRAC", cliParams.platform, 'bin'), - 'setenv DIRACSCRIPTS %s' % os.path.join("$DIRAC", 'scripts'), - 'setenv DIRACLIB %s' % os.path.join("$DIRAC", cliParams.platform, 'lib'), - 'setenv TERMINFO %s' % __getTerminfoLocations(os.path.join("$DIRAC", - cliParams.platform, - 'share', - 'terminfo'))]) - - lines.extend(['# Prepend the PYTHONPATH, the LD_LIBRARY_PATH, and the DYLD_LIBRARY_PATH']) - - lines.extend(['( test $?PATH -eq 1 ) || setenv PATH ""', - '( test $?LD_LIBRARY_PATH -eq 1 ) || setenv LD_LIBRARY_PATH ""', - '( test $?DY_LD_LIBRARY_PATH -eq 1 ) || setenv DYLD_LIBRARY_PATH ""', - '( test $?PYTHONPATH -eq 1 ) || setenv PYTHONPATH ""', - '( echo $PATH | grep -q $DIRACBIN ) || setenv PATH ${DIRACBIN}:$PATH', - '( echo $PATH | grep -q $DIRACSCRIPTS ) || setenv PATH ${DIRACSCRIPTS}:$PATH', - '( echo $LD_LIBRARY_PATH | grep -q $DIRACLIB ) || \ - setenv LD_LIBRARY_PATH ${DIRACLIB}:$LD_LIBRARY_PATH', - '( echo $LD_LIBRARY_PATH | grep -q $DIRACLIB/mysql ) || \ - setenv LD_LIBRARY_PATH ${DIRACLIB}/mysql:$LD_LIBRARY_PATH', - '( echo $DYLD_LIBRARY_PATH | grep -q $DIRACLIB ) || \ - setenv DYLD_LIBRARY_PATH ${DIRACLIB}:$DYLD_LIBRARY_PATH', - '( echo $DYLD_LIBRARY_PATH | grep -q $DIRACLIB/mysql ) || \ - setenv DYLD_LIBRARY_PATH ${DIRACLIB}/mysql:$DYLD_LIBRARY_PATH']) - - lines.extend(['setenv PYTHONPATH ${DIRAC}']) - - lines.extend(['# new OpenSSL version require OPENSSL_CONF to point to some accessible location', - 'setenv OPENSSL_CONF /tmp']) - lines.extend(['# IPv6 support', - 'setenv GLOBUS_IO_IPV6 TRUE', - 'setenv GLOBUS_FTP_CLIENT_IPV6 TRUE']) - # gfal2 requires some environment variables to be set - lines.extend(['# Gfal2 configuration and plugins', 'setenv GFAL_CONFIG_DIR %s' % - os.path.join("$DIRAC", cliParams.platform, 'etc/gfal2.d'), 'setenv GFAL_PLUGIN_DIR %s' % - os.path.join("$DIRACLIB", 'gfal2-plugins')]) - # add DIRACPLAT environment variable for client installations - if cliParams.externalsType == 'client': - lines.extend(['# DIRAC platform', - 'test $?DIRACPLAT -eq 1 || setenv DIRACPLAT `$DIRAC/scripts/dirac-platform`']) - # Add the lines required for ARC CE support - lines.extend(['# ARC Computing Element', - 'setenv ARC_PLUGIN_PATH $DIRACLIB/arc']) - - # Add the lines required for fork support for xrootd - lines.extend(['# Fork support for xrootd', - 'setenv XRD_RUNFORKHANDLER 1']) - - # Add the lines required for further env variables requested - if cliParams.userEnvVariables: - lines.extend(['# User-requested variables']) - for envName, envValue in cliParams.userEnvVariables.items(): - lines.extend(['setenv %s %s' % (envName, envValue)]) - - lines.append('') - f = open(cshrcFile, 'w') - f.write('\n'.join(lines)) - f.close() - except Exception as x: - logERROR(str(x)) - return False - - return True - - -def writeDefaultConfiguration(): - """ - After DIRAC is installed a default configuration file is created, - which contains a minimal setup. - """ - if not releaseConfig: - return - instCFG = releaseConfig.getInstallationCFG() - if not instCFG: - return - for opName in instCFG.getOptions(): - instCFG.delPath(opName) - - # filePath = os.path.join( cliParams.targetPath, "defaults-%s.cfg" % cliParams.installation ) - # Keep the default configuration file in the working directory - filePath = "defaults-%s.cfg" % cliParams.installation - try: - fd = open(filePath, "w") - fd.write(instCFG.toString()) - fd.close() - except Exception as excp: - logERROR("Could not write %s: %s" % (filePath, excp)) - logNOTICE("Defaults written to %s" % filePath) - - -def __getTerminfoLocations(defaultLocation=None): - """returns the terminfo locations as a colon separated string""" - - terminfoLocations = [] - if defaultLocation: - terminfoLocations = [defaultLocation] - - for termpath in ['/usr/share/terminfo', '/etc/terminfo']: - if os.path.exists(termpath): - terminfoLocations.append(termpath) - - return ":".join(terminfoLocations) - - -def installDiracOS(releaseConfig): - """ - Install DIRAC OS. - - :param ReleaseConfig releaseConfig: The ReleaseConfig object for configuring the installation - """ - diracos, diracOSVersion = releaseConfig.getDiracOSVersion(cliParams.diracOSVersion) - if not diracOSVersion: - logERROR("No diracos defined") - return False - tarsURL = cliParams.installSource - if not tarsURL: - tarsURL = releaseConfig.getDiracOsLocation(useVanillaDiracOS=(diracos.lower() == 'diracos')) - if not tarsURL: - tarsURL = releaseConfig.getTarsLocation('DIRAC')['Value'] - logWARN("DIRACOS location is not specified using %s" % tarsURL) - if not downloadAndExtractTarball(tarsURL, diracos, diracOSVersion, cache=True): - return False - logNOTICE("Fixing externals paths...") - fixBuildPaths() - logNOTICE("Running externals post install...") - checkPlatformAliasLink() - return True - - -def installDiracOSPython3(releaseConfig): - """ - Install DIRAC OS for Python 3. - - :param ReleaseConfig releaseConfig: The ReleaseConfig object for configuring the installation - """ - url = "https://github.com/chrisburr/DIRACOS2/releases/latest/download/DIRACOS-Linux-x86_64.sh" - installerFn = os.path.join(cliParams.basePath, "DIRACOS-Linux-x86_64.sh") - if not urlretrieveTimeout(url, installerFn, cliParams.timeout): - raise Exception("Failed to download DIRACOS from " + url) - subprocess.check_call(["bash", installerFn, "-b", "-p", os.path.join(cliParams.basePath, "diracos")]) - return True - - -def createBashrcForDiracOS(): - """ Create DIRAC environment setting script for the bash shell - """ - - proPath = cliParams.targetPath - # Now create bashrc at basePath - try: - bashrcFile = os.path.join(cliParams.targetPath, 'bashrc') - if cliParams.useVersionsDir: - bashrcFile = os.path.join(cliParams.basePath, 'bashrc') - proPath = os.path.join(cliParams.basePath, 'pro') - logNOTICE('Creating %s' % bashrcFile) - if not os.path.exists(bashrcFile): - lines = ['# DIRAC bashrc file, used by service and agent run scripts to set environment', - 'export PYTHONUNBUFFERED=yes', - 'export PYTHONOPTIMIZE=x', - '[ -z "$DIRAC" ] && export DIRAC=%s' % proPath, - '[ -z "$DIRACOS" ] && export DIRACOS=$DIRAC/diracos', - '. $DIRACOS/diracosrc'] - if 'HOME' in os.environ: - lines.append('[ -z "$HOME" ] && export HOME=%s' % os.environ['HOME']) - - # Determining where the CAs are... - if 'X509_CERT_DIR' in os.environ: - certDir = os.environ['X509_CERT_DIR'] - else: - if os.path.isdir('/etc/grid-security/certificates') and \ - os.listdir('/etc/grid-security/certificates'): - # Assuming that, if present, it is not empty, and has correct CAs - certDir = '/etc/grid-security/certificates' - else: - # But this will have to be created at some point (dirac-configure) - certDir = '%s/etc/grid-security/certificates' % proPath - lines.extend(['# CAs path for SSL verification', - 'export X509_CERT_DIR=${X509_CERT_DIR:-%s}' % certDir, - 'export SSL_CERT_DIR=${SSL_CERT_DIR:-%s}' % certDir]) - - lines.append( - 'export X509_VOMS_DIR=${X509_VOMS_DIR:-%s}' % - os.path.join( - proPath, - 'etc', - 'grid-security', - 'vomsdir')) - lines.append( - 'export X509_VOMSES=${X509_VOMSES:-%s}' % - os.path.join( - proPath, - 'etc', - 'grid-security', - 'vomses')) - lines.extend( - [ - '# Some DIRAC locations', - 'export DIRACSCRIPTS=%s' % - os.path.join( - "$DIRAC", - 'scripts')]) - - lines.extend(['# Prepend the PATH and set the PYTHONPATH']) - - lines.extend(['( echo $PATH | grep -q $DIRACSCRIPTS ) || export PATH=$DIRACSCRIPTS:$PATH']) - - lines.extend(['export PYTHONPATH=$DIRAC']) - - lines.extend(['# new OpenSSL version require OPENSSL_CONF to point to some accessible location', - 'export OPENSSL_CONF=/tmp']) - - # add DIRACPLAT environment variable for client installations - if cliParams.externalsType == 'client': - lines.extend(['# DIRAC platform', - '[ -z "$DIRACPLAT" ] && export DIRACPLAT=`$DIRAC/scripts/dirac-platform`']) - # Add the lines required for globus-* tools to use IPv6 - lines.extend(['# IPv6 support', - 'export GLOBUS_IO_IPV6=TRUE', - 'export GLOBUS_FTP_CLIENT_IPV6=TRUE']) - - # Add the lines required for fork support for xrootd - lines.extend(['# Fork support for xrootd', - 'export XRD_RUNFORKHANDLER=1']) - - # Add possible DIRAC environment variables - lines.append('') - lines.append('# before enabling any of these variables, please see the documentation ') - lines.append('# https://dirac.readthedocs.io/en/latest/AdministratorGuide/' + - 'ServerInstallations/environment_variable_configuration.html') - lines.append('# export DIRAC_DEBUG_DENCODE_CALLSTACK=1') - lines.append('# export DIRAC_DEBUG_STOMP=1') - lines.append('# export DIRAC_DEPRECATED_FAIL=1') - lines.append('# export DIRAC_GFAL_GRIDFTP_SESSION_REUSE=true') - lines.append('# export DIRAC_USE_JSON_DECODE=no') - lines.append('# export DIRAC_USE_JSON_ENCODE=no') - lines.append('# export DIRAC_USE_M2CRYPTO=true') - lines.append('# export DIRAC_USE_NEWTHREADPOOL=yes') - - # Add the lines required for further env variables requested - if cliParams.userEnvVariables: - lines.extend(['# User-requested variables']) - for envName, envValue in cliParams.userEnvVariables.items(): - lines.extend(['export %s=%s' % (envName, envValue)]) - - lines.append('') - with open(bashrcFile, 'w') as f: - f.write('\n'.join(lines)) - except Exception as x: - logERROR(str(x)) - return False - - return True - - -def checkoutFromGit(moduleName, sourceURL, tagVersion, destinationDir=None): - """ - This method checkout a given tag from a git repository. - Note: we can checkout any project form a git repository. - - :param str moduleName: The name of the Module: for example: LHCbWebDIRAC - :param str sourceURL: The code repository: https://github.com/DIRACGrid/WebAppDIRAC.git - :param str tagVersion: the tag for example: v3r1p10 - - """ - - codeRepo = moduleName + 'Repo' - - fDirName = os.path.join(cliParams.targetPath, codeRepo) - if os.path.isdir(sourceURL): - logNOTICE("Using local copy for source: %s" % sourceURL) - shutil.copytree(sourceURL, fDirName) - else: - cmd = "git clone '%s' '%s'" % (sourceURL, fDirName) - - logNOTICE("Executing: %s" % cmd) - if os.system(cmd): - return S_ERROR("Error while retrieving sources from git") - - branchName = "%s-%s" % (tagVersion, os.getpid()) - - isTagCmd = "( cd '%s'; git tag -l | grep '%s' )" % (fDirName, tagVersion) - if os.system(isTagCmd): - # No tag found, assume branch - branchSource = 'origin/%s' % tagVersion - else: - branchSource = tagVersion - - cmd = "( cd '%s'; git checkout -b '%s' '%s' )" % (fDirName, branchName, branchSource) - - logNOTICE("Executing: %s" % cmd) - exportRes = os.system(cmd) - - if exportRes: - return S_ERROR("Error while exporting from git") - - # replacing the code - if os.path.exists(fDirName + '/' + moduleName): - cmd = "ln -s %s/%s %s" % (codeRepo, moduleName, os.path.join(cliParams.targetPath, moduleName)) - else: - cmd = "mv %s %s" % (fDirName, os.path.join(cliParams.targetPath, moduleName)) - logNOTICE("Executing: %s" % cmd) - retVal = os.system(cmd) - - if retVal: - return S_ERROR("Error while creating module: %s" % (moduleName)) - - return S_OK() - - -def createSymbolicLink(): - """ - It creates a symbolic link to the actual directory from versions - directory. - """ - - cmd = "ln -s %s %s" % (cliParams.targetPath, cliParams.release) - logNOTICE("Executing: %s" % cmd) - retVal = os.system(cmd) - if retVal: - return S_ERROR("Error while creating symbolic link!") - - return S_OK() - - -if __name__ == "__main__": - logNOTICE("Processing installation requirements") - result = loadConfiguration() - releaseConfig = None - modsToInstall = {} - modsOrder = [] - if not result['OK']: - # the configuration files does not exists, which means the module is not released. - if cliParams.modules: - logNOTICE(str(cliParams.modules)) - for i in cliParams.modules: - modsOrder.append(i) - modsToInstall[i] = (cliParams.modules[i]['sourceUrl'], cliParams.modules[i]['Version']) - else: - # there is no module provided which can be deployed - logERROR(result['Message']) - sys.exit(1) - else: - releaseConfig = result['Value'] - if not createPermanentDirLinks(): - sys.exit(1) - - if not cliParams.externalsOnly: - logNOTICE("Discovering modules to install") - if releaseConfig: - result = releaseConfig.getModulesToInstall(cliParams.release, cliParams.extensions) - if not result['OK']: - logERROR(result['Message']) - sys.exit(1) - modsOrder, modsToInstall = result['Value'] - if cliParams.debug and releaseConfig: - logNOTICE("Writing down the releases files") - releaseConfig.dumpReleasesToPath() - logNOTICE("Installing modules...") - for modName in set(modsOrder): - tarsURL, modVersion = modsToInstall[modName] - if cliParams.installSource and not cliParams.modules: - # we install not release version of DIRAC - tarsURL = cliParams.installSource - if modName in cliParams.modules: - sourceURL = cliParams.modules[modName].get('sourceUrl') - if 'Version' in cliParams.modules[modName]: - modVersion = cliParams.modules[modName]['Version'] - if not sourceURL: - retVal = releaseConfig.getModSource(cliParams.release, modName) - if retVal['OK']: - tarsURL = retVal['Value'][1] # this is the git repository url - modVersion = cliParams.tag - else: - tarsURL = sourceURL - retVal = checkoutFromGit(modName, tarsURL, modVersion) - if not retVal['OK']: - logERROR("Cannot checkout %s" % retVal['Message']) - sys.exit(1) - else: - logNOTICE("Installing %s:%s" % (modName, modVersion)) - if not downloadAndExtractTarball(tarsURL, modName, modVersion): - sys.exit(1) - logNOTICE("Deploying scripts...") - ddeLocation = os.path.join(cliParams.targetPath, "DIRAC", "Core", - "scripts", "dirac-deploy-scripts.py") - if not os.path.isfile(ddeLocation): - ddeLocation = os.path.join(cliParams.targetPath, "DIRAC", "Core", - "scripts", "dirac_deploy_scripts.py") - if not os.path.isfile(ddeLocation): - ddeLocation = os.path.join(cliParams.targetPath, "DIRAC", "src", "DIRAC", "Core", - "scripts", "dirac_deploy_scripts.py") - if os.path.isfile(ddeLocation): - cmd = ddeLocation - - # if specified, create symlink instead of wrapper. - if cliParams.scriptSymlink: - cmd += ' --symlink' - - # In MacOS /usr/bin/env does not find python in the $PATH, passing binary path - # as an argument to the dirac-deploy-scripts - if not cliParams.platform: - cliParams.platform = getPlatform() - if "Darwin" in cliParams.platform: - binaryPath = os.path.join(cliParams.targetPath, cliParams.platform) - logNOTICE("For MacOS (Darwin) use explicit binary path %s" % binaryPath) - cmd += ' %s' % binaryPath - - os.system(cmd) - else: - logERROR("No dirac-deploy-scripts found. This doesn't look good") - else: - logNOTICE("Skipping installing DIRAC") - # we install with DIRACOS from v7rX DIRAC release - if cliParams.diracOS \ - or list(releaseConfig.prjRelCFG['DIRAC'])[0][1] not in '0123456789' \ - or int(list(releaseConfig.prjRelCFG['DIRAC'])[0][1]) > 6: - logNOTICE("Installing DIRAC OS %s..." % cliParams.diracOSVersion) - if cliParams.pythonVersion.startswith("3"): - if not installDiracOSPython3(releaseConfig): - sys.exit(1) - else: - if not installDiracOS(releaseConfig): - sys.exit(1) - if not createBashrcForDiracOS(): - sys.exit(1) - else: - logNOTICE("Installing %s externals..." % cliParams.externalsType) - if not installExternals(releaseConfig): - sys.exit(1) - if cliParams.noLcg: - logNOTICE("Skipping installation of LCG software...") - else: - logNOTICE("Installing LCG software...") - if not installLCGutils(releaseConfig): - sys.exit(1) - if not createBashrc(): - sys.exit(1) - if not createCshrc(): - sys.exit(1) - if not createOldProLinks(): - sys.exit(1) - runExternalsPostInstall() - writeDefaultConfiguration() - if cliParams.externalsType == "server": - fixMySQLScript() - installExternalRequirements(cliParams.externalsType) - if cliParams.createLink: - createSymbolicLink() - logNOTICE("%s properly installed" % cliParams.installation) - sys.exit(0) +sys.stderr.write("#" * 100 + "\n") +sys.stderr.write("#" * 100 + "\n") +sys.stderr.write("#" * 100 + "\n") +sys.stderr.write("#" * 100 + "\n") +sys.stderr.write("#" * 100 + "\n") +sys.stderr.write("\n") +sys.stderr.write("Getting dirac-install from this location is no longer supported!\n") +sys.stderr.write("\n") +sys.stderr.write("Please update your scripts to use:\n") +sys.stderr.write(" https://raw.githubusercontent.com/DIRACGrid/management/master/dirac-install.py\n") +sys.stderr.write("\n") +sys.stderr.write("#" * 100 + "\n") +sys.stderr.write("#" * 100 + "\n") +sys.stderr.write("#" * 100 + "\n") +sys.stderr.write("#" * 100 + "\n") +sys.stderr.write("#" * 100 + "\n") + +if os.environ.get("DIRAC_DEPRECATED_FAIL", "No").lower() in ("y", "yes", "t", "true", "on", "1"): + raise RuntimeError("Failing as DIRAC_DEPRECATED_FAIL is set") + +# Download dirac-install.py +response = urlopen( + "https://raw.githubusercontent.com/DIRACGrid/management/master/dirac-install.py" +) +code = response.getcode() +if code > 200 or code >= 300: + raise RuntimeError("Failed to download dirac-install.py with code %s" % code) + +# Write dirac-install.py to a temporay file +tmpHandle, tmp = tempfile.mkstemp() +fp = os.fdopen(tmpHandle, "wb") +fp.write(response.read()) +fp.close() + +# Make the dirac-install.py temporay file executable +st = os.stat(tmp) +os.chmod(tmp, st.st_mode | stat.S_IEXEC) + +# Replace the current process with the actual dirac-install.py script +os.execv(tmp, sys.argv) diff --git a/src/DIRAC/Core/scripts/dirac-install.py b/src/DIRAC/Core/scripts/dirac-install.py index 6fe5e5d1dbe..7b281725ba3 100755 --- a/src/DIRAC/Core/scripts/dirac-install.py +++ b/src/DIRAC/Core/scripts/dirac-install.py @@ -3,2722 +3,36 @@ The main DIRAC installer script. It can be used to install the main DIRAC software, its modules, web, rest etc. and DIRAC extensions. -In order to deploy DIRAC you have to provide: globalDefaultsURL, which is by default: -"http://diracproject.web.cern.ch/diracproject/configs/globalDefaults.cfg", but it can be -in the local file system in a separate directory. The content of this file is the following:: - - Installations - { - DIRAC - { - DefaultsLocation = http://diracproject.web.cern.ch/diracproject/dirac.cfg - LocalInstallation - { - PythonVersion = 27 - } - # in case you have a DIRAC extension - LHCb - { - DefaultsLocation = http://lhcb-rpm.web.cern.ch/lhcb-rpm/lhcbdirac/lhcb.cfg - } - } - } - Projects - { - DIRAC - { - DefaultsLocation = http://diracproject.web.cern.ch/diracproject/dirac.cfg - } - # in case you have a DIRAC extension - LHCb - { - DefaultsLocation = http://lhcb-rpm.web.cern.ch/lhcb-rpm/lhcbdirac/lhcb.cfg - } - } - -the DefaultsLocation for example:: - - DefaultsLocation = http://diracproject.web.cern.ch/diracproject/dirac.cfg - -must contain a minimal configuration. The following options must be in this -file:: - - Releases=,UploadCommand=,BaseURL= - -In case you want to overwrite the global configuration file, you have to use --defaultsURL - -After providing the default configuration files, DIRAC or your extension can be installed from: - -1. in a directory you have to be present globalDefaults.cfg, dirac.cfg and all binaries. - For example:: - - zmathe@dzmathe zmathe]$ ls tars/ - dirac.cfg diracos-0.1.md5 diracos-0.1.tar.gz DIRAC-v6r20-pre16.md5 DIRAC-v6r20-pre16.tar.gz - globalDefaults.cfg release-DIRAC-v6r20-pre16.cfg release-DIRAC-v6r20-pre16.md5 - zmathe@dzmathe zmathe]$ - - For example:: - - dirac-install -r v6r20-pre16 --dirac-os --dirac-os-version=0.0.1 -u /home/zmathe/tars - - this command will use /home/zmathe/tars directory for the source code. - It will install DIRAC v6r20-pre16, DIRAC OS 0.1 version - -2. You can use your dedicated web server or the official DIRAC web server - - for example:: - - dirac-install -r v6r20-pre16 --dirac-os --dirac-os-version=0.0.1 - - It will install DIRAC v6r20-pre16 - - You can install an extension of diracos. - - for example:: - - dirac-install -r v9r4-pre2 -l LHCb --dirac-os --dirac-os-version=LHCb:master - -3. You have possibility to install a not-yet-released DIRAC, module or extension using -m or --tag options. - The non release version can be specified. - - for example:: - - dirac-install -l DIRAC -r v6r20-pre16 -g v14r0 -t client -m DIRAC --tag=integration - - It will install DIRAC v6r20-pre16, where the DIRAC package based on integration, other other packages will be - the same what is specified in release.cfg file in v6r20-pre16 tarball. - - dirac-install -l DIRAC -r v6r20-pre16 -g v14r0 -t client -m DIRAC --tag=v6r20-pre22 - - It installs a specific tag - - Note: If the source is not provided, DIRAC repository is used, which is defined in the global - configuration file. - - We can provide the repository url:code repository:::Project:::branch. for example:: - - dirac-install -l DIRAC -r v6r20-pre16 -g v14r0 -t client \\ - -m https://github.com/zmathe/DIRAC.git:::DIRAC:::dev_main_branch, \\ - https://github.com/zmathe/WebAppDIRAC.git:::WebAppDIRAC:::extjs6 -e WebAppDIRAC - - it will install DIRAC based on dev_main_branch and WebAppDIRAC based on extjs6:: - - dirac-install -l DIRAC -r v6r20-pre16 -g v14r0 -t client \\ - -m WebAppDIRAC --tag=integration -e WebAppDIRAC - - it will install DIRAC v6r20-pre16 and WebAppDIRAC integration branch - -You can use install.cfg configuration file:: - - DIRACOS = http://lhcb-rpm.web.cern.ch/lhcb-rpm/dirac/DIRACOS/ - WebAppDIRAC = https://github.com/zmathe/WebAppDIRAC.git - DIRAC=https://github.com/DIRACGrid/DIRAC.git - LocalInstallation - { - # Project = LHCbDIRAC - # The project LHCbDIRAC is not defined in the globalsDefaults.cfg - Project = LHCb - Release = v9r2-pre8 - Extensions = LHCb - ConfigurationServer = dips://lhcb-conf-dirac.cern.ch:9135/Configuration/Server - Setup = LHCb-Production - SkipCAChecks = True - SkipCADownload = True - WebAppDIRAC=extjs6 - DIRAC=rel-v6r20 - } - - dirac-install -l LHCb -r v9r2-pre8 -t server --dirac-os --dirac-os-version=0.0.6 install.cfg - +See https://github.com/DIRACGrid/management/blob/master/dirac-install.py """ - -from __future__ import unicode_literals, absolute_import, division, print_function - -import sys import os -import getopt -import imp -import signal -import time import stat -import shutil -import subprocess -import ssl -import hashlib - -from distutils.version import LooseVersion # pylint: disable=no-name-in-module,import-error - +import sys +import tempfile try: # For Python 3.0 and later from urllib.request import urlopen, HTTPError, URLError except ImportError: # Fall back to Python 2's urllib2 from urllib2 import urlopen, HTTPError, URLError -try: - str_type = basestring -except NameError: - str_type = str - -__RCSID__ = "$Id$" - -executablePerms = stat.S_IWUSR | stat.S_IRUSR | stat.S_IXUSR | stat.S_IRGRP | stat.S_IXGRP | stat.S_IROTH | stat.S_IXOTH - - -def S_OK(value=""): - return {'OK': True, 'Value': value} - - -def S_ERROR(msg=""): - return {'OK': False, 'Message': msg} - -############ -# Start of CFG -############ - - -class Params(object): - - def __init__(self): - self.extensions = [] - self.project = 'DIRAC' - self.installation = 'DIRAC' - self.release = "" - self.externalsType = 'client' - self.pythonVersion = '27' - self.platform = "" - self.basePath = os.getcwd() - self.targetPath = os.getcwd() - self.buildExternals = False - self.noAutoBuild = False - self.debug = False - self.externalsOnly = False - self.lcgVer = '' - self.noLcg = False - self.useVersionsDir = False - self.installSource = "" - self.globalDefaults = False - self.timeout = 300 - self.diracOSVersion = '' - self.diracOS = False - self.tag = "" - self.modules = {} - self.externalVersion = "" - self.createLink = False - self.scriptSymlink = False - self.userEnvVariables = {} - - -cliParams = Params() - -### -# Release config manager -### - - -class ReleaseConfig(object): - - class CFG(object): - - def __init__(self, cfgData=""): - """ c'tor - :param self: self reference - :param str cfgData: the content of the configuration file - """ - self.data = {} - self.children = {} - if cfgData: - self.parse(cfgData) - - def parse(self, cfgData): - """ - It parses the configuration file and propagate the data and children - with the content of the cfg file - :param str cfgData: configuration data, which is the content of the configuration file - """ - try: - self.__parse(cfgData) - except BaseException: - import traceback - traceback.print_exc() - raise - return self - - def getChild(self, path): - """ - It return the child of a given section - :param str, list, tuple path: for example: Installations/DIRAC, Projects/DIRAC - :return object It returns a CFG instance - """ - - child = self - if isinstance(path, (list, tuple)): - pathList = path - else: - pathList = [sec.strip() for sec in path.split("/") if sec.strip()] - for childName in pathList: - if childName not in child.children: - return False - child = child.children[childName] - return child - - def __parse(self, cfgData, cIndex=0): - """ - It parse a given DIRAC cfg file and store the result in self.data variable. - - :param str cfgData: the content of the configuration file - :param int cIndex: it is the new line counter - """ - - childName = "" - numLine = 0 - while cIndex < len(cfgData): - eol = cfgData.find("\n", cIndex) - if eol < cIndex: - # End? - return cIndex - numLine += 1 - if eol == cIndex: - cIndex += 1 - continue - line = cfgData[cIndex: eol].strip() - # Jump EOL - cIndex = eol + 1 - if not line or line[0] == "#": - continue - if line.find("+=") > -1: - fields = line.split("+=") - opName = fields[0].strip() - if opName in self.data: - self.data[opName] += ', %s' % '+='.join(fields[1:]).strip() - else: - self.data[opName] = '+='.join(fields[1:]).strip() - continue - - if line.find("=") > -1: - fields = line.split("=") - self.data[fields[0].strip()] = "=".join(fields[1:]).strip() - continue - - opFound = line.find("{") - if opFound > -1: - childName += line[:opFound].strip() - if not childName: - raise Exception("No section name defined for opening in line %s" % numLine) - childName = childName.strip() - self.children[childName] = ReleaseConfig.CFG() - eoc = self.children[childName].__parse(cfgData, cIndex) - cIndex = eoc - childName = "" - continue - - if line == "}": - return cIndex - # Must be name for section - childName += line.strip() - return cIndex - - def createSection(self, name, cfg=None): - """ - It creates a subsection for an existing CS section. - :param str name: the name of the section - :param object cfg: the ReleaseConfig.CFG object loaded into memory - """ - - if isinstance(name, (list, tuple)): - pathList = name - else: - pathList = [sec.strip() for sec in name.split("/") if sec.strip()] - parent = self - for lev in pathList[:-1]: - if lev not in parent.children: - parent.children[lev] = ReleaseConfig.CFG() - parent = parent.children[lev] - secName = pathList[-1] - if secName not in parent.children: - if not cfg: - cfg = ReleaseConfig.CFG() - parent.children[secName] = cfg - return parent.children[secName] - - def isSection(self, obList): - """ - Checks if a given path is a section - :param str objList: is a path: for example: Releases/v6r20-pre16 - """ - return self.__exists([ob.strip() for ob in obList.split("/") if ob.strip()]) == 2 - - def sections(self): - """ - Returns all sections - """ - return [k for k in self.children] - - def isOption(self, obList): - return self.__exists([ob.strip() for ob in obList.split("/") if ob.strip()]) == 1 - - def options(self): - """ - Returns the options - """ - return [k for k in self.data] - - def __exists(self, obList): - """ - Check the existence of a certain element - - :param list obList: the list of cfg element names. - for example: [Releases,v6r20-pre16] - """ - if len(obList) == 1: - if obList[0] in self.children: - return 2 - elif obList[0] in self.data: - return 1 - else: - return 0 - if obList[0] in self.children: - return self.children[obList[0]].__exists(obList[1:]) - return 0 - - def get(self, opName, defaultValue=None): - """ - It return the value of a certain option - - :param str opName: the name of the option - :param str defaultValue: the default value of a given option - """ - try: - value = self.__get([op.strip() for op in opName.split("/") if op.strip()]) - except KeyError: - if defaultValue is not None: - return defaultValue - raise - if defaultValue is None: - return value - defType = type(defaultValue) - if isinstance(defType, bool): - return value.lower() in ("1", "true", "yes") - try: - return defType(value) - except ValueError: - return defaultValue - - def __get(self, obList): - """ - It return a given section - - :param list obList: the list of cfg element names. - """ - if len(obList) == 1: - if obList[0] in self.data: - return self.data[obList[0]] - raise KeyError("Missing option %s" % obList[0]) - if obList[0] in self.children: - return self.children[obList[0]].__get(obList[1:]) - raise KeyError("Missing section %s" % obList[0]) - - def toString(self, tabs=0): - """ - It return the configuration file as a string - :param int tabs: the number of tabs used to format the CS string - """ - - lines = ["%s%s = %s" % (" " * tabs, opName, self.data[opName]) for opName in self.data] - for secName in self.children: - lines.append("%s%s" % (" " * tabs, secName)) - lines.append("%s{" % (" " * tabs)) - lines.append(self.children[secName].toString(tabs + 1)) - lines.append("%s}" % (" " * tabs)) - return "\n".join(lines) - - def getOptions(self, path=""): - """ - Rturns the options for a given path - - :param str path: the path to the CS element - """ - parentPath = [sec.strip() for sec in path.split("/") if sec.strip()][:-1] - if parentPath: - parent = self.getChild(parentPath) - else: - parent = self - if not parent: - return [] - return tuple(parent.data) - - def delPath(self, path): - """ - It deletes a given CS element - - :param str path: the path to the CS element - """ - path = [sec.strip() for sec in path.split("/") if sec.strip()] - if not path: - return - keyName = path[-1] - parentPath = path[:-1] - if parentPath: - parent = self.getChild(parentPath) - else: - parent = self - if parent: - parent.data.pop(keyName) - - def update(self, path, cfg): - """ - Used to update the CS - - :param str path: path to the CS element - :param object cfg: the CS object - """ - parent = self.getChild(path) - if not parent: - self.createSection(path, cfg) - return - parent.__apply(cfg) - - def __apply(self, cfg): - """ - It adds a certain cfg subsection to a given section - - :param object cfg: the CS object - """ - for k in cfg.sections(): - if k in self.children: - self.children[k].__apply(cfg.getChild(k)) - else: - self.children[k] = cfg.getChild(k) - for k in cfg.options(): - self.data[k] = cfg.get(k) -############################################################################ -# END OF CFG CLASS -############################################################################ - - def __init__(self, instName='DIRAC', projectName='DIRAC', globalDefaultsURL=None): - """ c'tor - :param str instName: the name of the installation - :param str projectName: the name of the project - :param str globalDefaultsURL: the default url - """ - if globalDefaultsURL: - self.globalDefaultsURL = globalDefaultsURL - else: - self.globalDefaultsURL = "http://diracproject.web.cern.ch/diracproject/configs/globalDefaults.cfg" - self.globalDefaults = ReleaseConfig.CFG() - self.loadedCfgs = [] - self.prjDepends = {} - self.diracBaseModules = {} - self.prjRelCFG = {} - self.projectsLoadedBy = {} - self.cfgCache = {} - - self.debugCB = False - self.instName = instName - self.projectName = projectName - - def setDebugCB(self, debFunc): - """ - It is used by the dirac-distribution. It sets the debug function - """ - self.debugCB = debFunc - - def __dbgMsg(self, msg): - """ - :param str msg: the debug message - """ - if self.debugCB: - self.debugCB(msg) - - def __loadCFGFromURL(self, urlcfg, checkHash=False): - """ - It is used to load the configuration file - - :param str urlcfg: the location of the source repository and - where the default configuration file is exists. - :param bool checkHash: check if the file is corrupted. - """ - # This can be a local file - if os.path.exists(urlcfg): - with open(urlcfg, 'r') as relFile: - cfgData = relFile.read() - else: - if urlcfg in self.cfgCache: - return S_OK(self.cfgCache[urlcfg]) - try: - cfgData = urlretrieveTimeout(urlcfg, timeout=cliParams.timeout) - if not cfgData: - return S_ERROR("Could not get data from %s" % urlcfg) - except BaseException: - return S_ERROR("Could not open %s" % urlcfg) - try: - # cfgData = cfgFile.read() - cfg = ReleaseConfig.CFG(cfgData) - except Exception as excp: - return S_ERROR("Could not parse %s: %s" % (urlcfg, excp)) - # cfgFile.close() - if not checkHash: - self.cfgCache[urlcfg] = cfg - return S_OK(cfg) - try: - md5path = urlcfg[:-4] + ".md5" - if os.path.exists(md5path): - md5File = open(md5path, 'r') - md5Data = md5File.read() - md5File.close() - else: - md5Data = urlretrieveTimeout(md5path, timeout=60) - md5Hex = md5Data.strip() - # md5File.close() - if md5Hex != hashlib.md5(cfgData.encode('utf-8')).hexdigest(): - return S_ERROR("Hash check failed on %s" % urlcfg) - except Exception as excp: - return S_ERROR("Hash check failed on %s: %s" % (urlcfg, excp)) - self.cfgCache[urlcfg] = cfg - return S_OK(cfg) - - def loadInstallationDefaults(self): - """ - Load the default configurations - """ - result = self.__loadGlobalDefaults() - if not result['OK']: - return result - return self.__loadObjectDefaults("Installations", self.instName) - - def loadProjectDefaults(self): - """ - Load default configurations - """ - result = self.__loadGlobalDefaults() - if not result['OK']: - return result - return self.__loadObjectDefaults("Projects", self.projectName) - - def __loadGlobalDefaults(self): - """ - It loads the default configuration files - """ - - globalDefaultsCVMFSPath = "/cvmfs/dirac.egi.eu/admin/globalDefaults.cfg" - self.__dbgMsg("Loading global defaults from: %s" % globalDefaultsCVMFSPath) - result = self.__loadCFGFromURL(globalDefaultsCVMFSPath) - if not result['OK']: - self.__dbgMsg("Loading global defaults from: %s" % self.globalDefaultsURL) - result = self.__loadCFGFromURL(self.globalDefaultsURL) - if not result['OK']: - return result - self.globalDefaults = result['Value'] - for k in ("Installations", "Projects"): - if not self.globalDefaults.isSection(k): - self.globalDefaults.createSection(k) - self.__dbgMsg("Loaded global defaults") - return S_OK() - - def __loadObjectDefaults(self, rootPath, objectName): - """ - It loads the CFG, if it is not loaded. - :param str rootPath: the main section. for example: Installations - :param str objectName: The name of the section. for example: DIRAC - """ - - basePath = "%s/%s" % (rootPath, objectName) - if basePath in self.loadedCfgs: - return S_OK() - - # Check if it's a direct alias - try: - aliasTo = self.globalDefaults.get(basePath) - except KeyError: - aliasTo = False - - if aliasTo: - self.__dbgMsg("%s is an alias to %s" % (objectName, aliasTo)) - result = self.__loadObjectDefaults(rootPath, aliasTo) - if not result['OK']: - return result - cfg = result['Value'] - self.globalDefaults.update(basePath, cfg) - return S_OK() - - # Load the defaults - if self.globalDefaults.get("%s/SkipDefaults" % basePath, False): - defaultsLocation = "" - else: - defaultsLocation = self.globalDefaults.get("%s/DefaultsLocation" % basePath, "") - - if not defaultsLocation: - self.__dbgMsg("No defaults file defined for %s %s" % (rootPath.lower()[:-1], objectName)) - else: - self.__dbgMsg("Defaults for %s are in %s" % (basePath, defaultsLocation)) - result = self.__loadCFGFromURL(defaultsLocation) - if not result['OK']: - return result - cfg = result['Value'] - self.globalDefaults.update(basePath, cfg) - - # Check if the defaults have a sub alias - try: - aliasTo = self.globalDefaults.get("%s/Alias" % basePath) - except KeyError: - aliasTo = False - - if aliasTo: - self.__dbgMsg("%s is an alias to %s" % (objectName, aliasTo)) - result = self.__loadObjectDefaults(rootPath, aliasTo) - if not result['OK']: - return result - cfg = result['Value'] - self.globalDefaults.update(basePath, cfg) - - self.loadedCfgs.append(basePath) - return S_OK(self.globalDefaults.getChild(basePath)) - - def loadInstallationLocalDefaults(self, args): - """ - Load the configuration file from a file - - :param str args: the arguments in which to look for configuration file names - """ - - # at the end we load the local configuration and merge it with the global cfg - argList = list(args) - if os.path.exists('etc/dirac.cfg') and 'etc/dirac.cfg' not in args: - argList = ['etc/dirac.cfg'] + argList - - for arg in argList: - if arg.endswith(".cfg") and ':::' not in arg: - fileName = arg - else: - continue - - logNOTICE("Defaults for LocalInstallation are in %s" % fileName) - try: - fd = open(fileName, "r") - cfg = ReleaseConfig.CFG().parse(fd.read()) - fd.close() - except Exception as excp: - logERROR("Could not load %s: %s" % (fileName, excp)) - continue - - self.globalDefaults.update("Installations/%s" % self.instName, cfg) - self.globalDefaults.update("Projects/%s" % self.instName, cfg) - if self.projectName: - # we have an extension and have a local cfg file - self.globalDefaults.update("Projects/%s" % self.projectName, cfg) - - logNOTICE("Loaded %s" % arg) - - def getModuleVersionFromLocalCfg(self, moduleName): - """ - It returns the version of a certain module defined in the LocalInstallation section - :param str moduleName: - :return str: the version of a certain module - """ - return self.globalDefaults.get("Installations/%s/LocalInstallation/%s" % (self.instName, moduleName), "") - - def getInstallationCFG(self, instName=None): - """ - Returns the installation name - - :param str instName: the installation name - """ - if not instName: - instName = self.instName - return self.globalDefaults.getChild("Installations/%s" % instName) - - def getInstallationConfig(self, opName, instName=None): - """ - It returns the configurations from the Installations section. - This is usually provided in the local configuration file - - :param str opName: the option name for example: LocalInstallation/Release - :param str instName: - """ - if not instName: - instName = self.instName - return self.globalDefaults.get("Installations/%s/%s" % (instName, opName)) - - def isProjectLoaded(self, project): - """ - Checks if the project is loaded. - - :param str project: the name of the project - """ - return project in self.prjRelCFG - - def getTarsLocation(self, project, module=None): - """ - Returns the location of the binaries for a given project for example: LHCb or DIRAC, etc... - - :param str project: the name of the project - """ - sourceUrl = self.globalDefaults.get("Projects/%s/BaseURL" % project, "") - if module: - # in case we define a different URL in the CS - differntSourceUrl = self.globalDefaults.get("Projects/%s/%s" % (project, module), "") - if differntSourceUrl: - sourceUrl = differntSourceUrl - if sourceUrl: - return S_OK(sourceUrl) - return S_ERROR("Don't know how to find the installation tarballs for project %s" % project) - - def getDiracOsLocation(self, useVanillaDiracOS=False): - """ - Returns the location of the DIRAC os binary for a given project for example: LHCb or DIRAC, etc... - :param bool useVanillaDiracOS: flag to take diracos distribution from the default location - :return: the location of the tar balls - """ - keysToConsider = [] - if not useVanillaDiracOS: - keysToConsider += [ - "Installations/%s/DIRACOS" % self.projectName, - "Projects/%s/DIRACOS" % self.projectName, - ] - keysToConsider += [ - "Installations/DIRAC/DIRACOS", - "Projects/DIRAC/DIRACOS", - ] - - for key in keysToConsider: - location = self.globalDefaults.get(key, "") - if location: - logDEBUG("Using DIRACOS tarball URL from configuration key %s" % key) - return location - - def getUploadCommand(self, project=None): - """ - It returns the command used to upload the binary - - :param str project: the name of the project - """ - if not project: - project = self.projectName - defLoc = self.globalDefaults.get("Projects/%s/UploadCommand" % project, "") - if defLoc: - return S_OK(defLoc) - return S_ERROR("No UploadCommand for %s" % project) - - def __loadReleaseConfig(self, project, release, releaseMode, sourceURL=None, relLocation=None): - """ - It loads the release configuration file - - :param str project: the name of the project - :param str release: the release version - :param str releaseMode: the type of the release server/client - :param str sourceURL: the source of the binary - :param str relLocation: the release configuration file - """ - if project not in self.prjRelCFG: - self.prjRelCFG[project] = {} - if release in self.prjRelCFG[project]: - self.__dbgMsg("Release config for %s:%s has already been loaded" % (project, release)) - return S_OK() - - if relLocation: - relcfgLoc = relLocation - else: - if releaseMode: - try: - relcfgLoc = self.globalDefaults.get("Projects/%s/Releases" % project) - except KeyError: - return S_ERROR("Missing Releases file for project %s" % project) - else: - if not sourceURL: - result = self.getTarsLocation(project) - if not result['OK']: - return result - siu = result['Value'] - else: - siu = sourceURL - relcfgLoc = "%s/release-%s-%s.cfg" % (siu, project, release) - self.__dbgMsg("Releases file is %s" % relcfgLoc) - result = self.__loadCFGFromURL(relcfgLoc, checkHash=not releaseMode) - if not result['OK']: - return result - self.prjRelCFG[project][release] = result['Value'] - self.__dbgMsg("Loaded releases file %s" % relcfgLoc) - - return S_OK(self.prjRelCFG[project][release]) - - def getReleaseCFG(self, project, release): - """ - Returns the release configuration object - - :param str project: the name of the project - :param str release: the release version - """ - return self.prjRelCFG[project][release] - - def dumpReleasesToPath(self): - """ - It dumps the content of the loaded configuration (memory content) to - a given file - """ - for project in self.prjRelCFG: - prjRels = self.prjRelCFG[project] - for release in prjRels: - self.__dbgMsg("Dumping releases file for %s:%s" % (project, release)) - fd = open( - os.path.join( - cliParams.targetPath, "releases-%s-%s.cfg" % - (project, release)), "w") - fd.write(prjRels[release].toString()) - fd.close() - - def __checkCircularDependencies(self, key, routePath=None): - """ - Check the dependencies - - :param str key: the name of the project and the release version - :param list routePath: it stores the software packages, used to check the - dependency - """ - - if not routePath: - routePath = [] - if key not in self.projectsLoadedBy: - return S_OK() - routePath.insert(0, key) - for lKey in self.projectsLoadedBy[key]: - if lKey in routePath: - routePath.insert(0, lKey) - route = "->".join(["%s:%s" % sKey for sKey in routePath]) - return S_ERROR("Circular dependency found for %s: %s" % ("%s:%s" % lKey, route)) - result = self.__checkCircularDependencies(lKey, routePath) - if not result['OK']: - return result - routePath.pop(0) - return S_OK() - - def loadProjectRelease(self, releases, - project=None, - sourceURL=None, - releaseMode=None, - relLocation=None): - """ - This method loads all project configurations (*.cfg). If a project is an extension of DIRAC, - it will load the extension and after will load the base DIRAC module. - - :param list releases: list of releases, which will be loaded: for example: v6r19 - :param str project: the name of the project, if it is given. For example: DIRAC - :param str sourceURL: the code repository - :param str releaseMode: - :param str relLocation: local configuration file, - which contains the releases. for example: file:///`pwd`/releases.cfg - """ - - if not project: - project = self.projectName - - if not isinstance(releases, (list, tuple)): - releases = [releases] - - # Load defaults - result = self.__loadObjectDefaults("Projects", project) - if not result['OK']: - self.__dbgMsg("Could not load defaults for project %s" % project) - return result - - if project not in self.prjDepends: - self.prjDepends[project] = {} - - for release in releases: - self.__dbgMsg("Processing dependencies for %s:%s" % (project, release)) - result = self.__loadReleaseConfig(project, release, releaseMode, sourceURL, relLocation) - if not result['OK']: - return result - relCFG = result['Value'] - # Calculate dependencies and avoid circular deps - self.prjDepends[project][release] = [(project, release)] - relDeps = self.prjDepends[project][release] - - if not relCFG.getChild("Releases/%s" % (release)): # pylint: disable=no-member - return S_ERROR( - "Release %s is not defined for project %s in the release file" % - (release, project)) - - initialDeps = self.getReleaseDependencies(project, release) - if initialDeps: - self.__dbgMsg("%s %s depends on %s" % - (project, release, ", ".join(["%s:%s" % - (k, initialDeps[k]) for k in initialDeps]))) - relDeps.extend([(p, initialDeps[p]) for p in initialDeps]) - for depProject in initialDeps: - depVersion = initialDeps[depProject] - # Check if already processed - dKey = (depProject, depVersion) - if dKey not in self.projectsLoadedBy: - self.projectsLoadedBy[dKey] = [] - self.projectsLoadedBy[dKey].append((project, release)) - result = self.__checkCircularDependencies(dKey) - if not result['OK']: - return result - # if it has already been processed just return OK - if len(self.projectsLoadedBy[dKey]) > 1: - return S_OK() - - # Load dependencies and calculate incompatibilities - result = self.loadProjectRelease(depVersion, project=depProject) - if not result['OK']: - return result - subDep = self.prjDepends[depProject][depVersion] - # Merge dependencies - for sKey in subDep: - if sKey not in relDeps: - relDeps.append(sKey) - continue - prj, vrs = sKey - for pKey in relDeps: - if pKey[0] == prj and pKey[1] != vrs: - errMsg = "%s is required with two different versions ( %s and %s ) \ - starting with %s:%s" % (prj, - pKey[1], vrs, - project, release) - return S_ERROR(errMsg) - - # Same version already required - if project in relDeps and relDeps[project] != release: - errMsg = "%s:%s requires itself with a different version through dependencies ( %s )" % ( - project, release, relDeps[project]) - return S_ERROR(errMsg) - - # we have now all dependencies, let's retrieve the resources (code repository) - for project, version in relDeps: - if project in self.diracBaseModules: - continue - modules = self.getModulesForRelease(version, project) - if modules['OK']: - for dependency in modules['Value']: - self.diracBaseModules.setdefault(dependency, {}) - self.diracBaseModules[dependency]['Version'] = modules['Value'][dependency] - res = self.getModSource(version, dependency, project) - if not res['OK']: - self.__dbgMsg( - "Unable to found the source URL for %s : %s" % - (dependency, res['Message'])) - else: - self.diracBaseModules[dependency]['sourceUrl'] = res['Value'][1] - - return S_OK() - - def getReleaseOption(self, project, release, option): - """ - Returns a given option - - :param str project: the name of the project - :param str release: the release version - :param str option: the option name - """ - try: - return self.prjRelCFG[project][release].get(option) - except KeyError: - self.__dbgMsg("Missing option %s for %s:%s" % (option, project, release)) - # try to found the option in a different release - for project in self.prjRelCFG: - for release in self.prjRelCFG[project]: - if self.prjRelCFG[project][release].isOption(option): - return self.prjRelCFG[project][release].get(option) - return False - - def getReleaseDependencies(self, project, release): - """ - It return the dependencies for a certain project - - :param str project: the name of the project - :param str release: the release version - """ - try: - data = self.prjRelCFG[project][release].get("Releases/%s/Depends" % release) - except KeyError: - return {} - data = [field for field in data.split(",") if field.strip()] - deps = {} - for field in data: - field = field.strip() - if not field: - continue - pv = field.split(":") - if len(pv) == 1: - deps[pv[0].strip()] = release - else: - deps[pv[0].strip()] = ":".join(pv[1:]).strip() - return deps - - def getModulesForRelease(self, release, project=None): - """ - Returns the modules for a given release for example: WebAppDIRAC, - RESTDIRAC, LHCbWebAppDIRAC, etc - - :param str release: the release version - :param str project: the project name - """ - if not project: - project = self.projectName - if project not in self.prjRelCFG: - return S_ERROR("Project %s has not been loaded. I'm a MEGA BUG! Please report me!" % project) - if release not in self.prjRelCFG[project]: - return S_ERROR("Version %s has not been loaded for project %s" % (release, project)) - config = self.prjRelCFG[project][release] - if not config.isSection("Releases/%s" % release): - return S_ERROR("Release %s is not defined for project %s" % (release, project)) - # Defined Modules explicitly in the release - modules = self.getReleaseOption(project, release, "Releases/%s/Modules" % release) - if modules: - dMods = {} - for entry in [entry.split(":") for entry in modules.split( - ",") if entry.strip()]: # pylint: disable=no-member - if len(entry) == 1: - dMods[entry[0].strip()] = release - else: - dMods[entry[0].strip()] = entry[1].strip() - modules = dMods - else: - # Default modules with the same version as the release version - modules = self.getReleaseOption(project, release, "DefaultModules") - if modules: - modules = dict((modName.strip(), release) for modName in modules.split(",") - if modName.strip()) # pylint: disable=no-member - else: - # Mod = project and same version - modules = {project: release} - # Check project is in the modNames if not DIRAC - if project != "DIRAC": - for modName in modules: - if modName.find(project) != 0: - return S_ERROR("Module %s does not start with the name %s" % (modName, project)) - return S_OK(modules) - - def getModSource(self, release, modName, project=None): - """ - It reads the Sources section from the .cfg file for example: - Sources - { - Web = git://github.com/DIRACGrid/DIRACWeb.git - VMDIRAC = git://github.com/DIRACGrid/VMDIRAC.git - DIRAC = git://github.com/DIRACGrid/DIRAC.git - BoincDIRAC = git://github.com/DIRACGrid/BoincDIRAC.git - RESTDIRAC = git://github.com/DIRACGrid/RESTDIRAC.git - COMDIRAC = git://github.com/DIRACGrid/COMDIRAC.git - WebAppDIRAC = git://github.com/DIRACGrid/WebAppDIRAC.git - } - - :param str release: the release which is already loaded for example: v6r19 - :param str modName: the name of the DIRAC module for example: WebAppDIRAC - :param str project: the name of the project for example: DIRAC - """ - - if self.projectName not in self.prjRelCFG: - return S_ERROR( - "Project %s has not been loaded. I'm a MEGA BUG! Please report me!" % - self.projectName) - - if not project: - project = self.projectName - modLocation = self.getReleaseOption(project, release, "Sources/%s" % modName) - if not modLocation: - return S_ERROR("Source origin for module %s is not defined" % modName) - modTpl = [field.strip() for field in modLocation.split( - "|") if field.strip()] # pylint: disable=no-member - if len(modTpl) == 1: - return S_OK((False, modTpl[0])) - return S_OK((modTpl[0], modTpl[1])) - - def getExternalsVersion(self, release=None): - """ - It returns the version of DIRAC Externals. If it is not provided, - uses the default cfg - - :param str release: the release version - """ - - if 'DIRAC' not in self.prjRelCFG: - return False - if not release: - release = list(self.prjRelCFG['DIRAC']) - release = sorted(release, key=LooseVersion)[-1] - try: - return self.prjRelCFG['DIRAC'][release].get('Releases/%s/Externals' % release) - except KeyError: - return False - - def getDiracOSExtensionAndVersion(self, diracOSVersion): - """ - This method return the diracos and version taking into - account the extension. The file format will be .tar.gz - - :param str diracOSVersion: column separated string for example: LHCb:v1 - :return: if the extension is not provided, it will return DIRACOS defined in DIRAC otherwise - the DIRACOS specified in the extension - """ - if ":" in diracOSVersion: - package, packageVersion = [i.strip() for i in diracOSVersion.split(':')] - return [package + 'diracos', packageVersion] - else: - return ['diracos', diracOSVersion] - - def getDiracOSVersion(self, diracOSVersion=None): - """ - It returns the DIRACOS version - :param str diracOSVersion: the OS version - """ - - if diracOSVersion: - return self.getDiracOSExtensionAndVersion(diracOSVersion) - try: - diracOSVersion = self.prjRelCFG[self.projectName][cliParams.release].get( - "Releases/%s/DIRACOS" % cliParams.release, diracOSVersion) - if not diracOSVersion: - # the DIRAC extension does not specify DIRACOS version - for release in self.prjRelCFG['DIRAC']: - logWARN("Getting DIRACOS version from DIRAC %s!" % release) - diracOSVersion = self.prjRelCFG['DIRAC'][release].get( - "Releases/%s/DIRACOS" % release, diracOSVersion) - except KeyError: - pass - return self.getDiracOSExtensionAndVersion(diracOSVersion) - - def getLCGVersion(self, lcgVersion=None): - """ - It returns the LCG version - :param str lcgVersion: LCG version - """ - if lcgVersion: - return lcgVersion - try: - return self.prjRelCFG[self.projectName][cliParams.release].get( - "Releases/%s/LcgVer" % cliParams.release, lcgVersion) - except KeyError: - pass - return lcgVersion - - def getModulesToInstall(self, release, extensions=None): - """ - It returns the modules to be installed. - :param str release: the release version to be deployed - :param str extensions: DIRAC extension - :return: the order of the nodules and modules to be installed. - """ - if not extensions: - extensions = [] - extraFound = [] - modsToInstall = {} - modsOrder = [] - if self.projectName not in self.prjDepends: - return S_ERROR("Project %s has not been loaded" % self.projectName) - if release not in self.prjDepends[self.projectName]: - return S_ERROR( - "Version %s has not been loaded for project %s" % - (release, self.projectName)) - # Get a list of projects with their releases - projects = list(self.prjDepends[self.projectName][release]) - for project, relVersion in projects: - try: - requiredModules = self.prjRelCFG[project][relVersion].get("RequiredExtraModules") - requiredModules = [modName.strip() - for modName in requiredModules.split("/") if modName.strip()] - except KeyError: - requiredModules = [] - for modName in requiredModules: - if modName not in extensions: - extensions.append(modName) - self.__dbgMsg("Discovering modules to install for %s (%s)" % (project, relVersion)) - result = self.getModulesForRelease(relVersion, project) - if not result['OK']: - return result - modVersions = result['Value'] - try: - defaultMods = self.prjRelCFG[project][relVersion].get("DefaultModules") - modNames = [mod.strip() for mod in defaultMods.split(",") if mod.strip()] - except KeyError: - modNames = [] - for extension in extensions: - # Check if the version of the extension module is specified in the command line - extraVersion = None - if ":" in extension: - extension, extraVersion = extension.split(":") - modVersions[extension] = extraVersion - if extension in modVersions: - modNames.append(extension) - extraFound.append(extension) - if 'DIRAC' not in extension: - dextension = "%sDIRAC" % extension - if dextension in modVersions: - modNames.append(dextension) - extraFound.append(extension) - modNameVer = ["%s:%s" % (modName, modVersions[modName]) for modName in modNames] - self.__dbgMsg("Modules to be installed for %s are: %s" % (project, ", ".join(modNameVer))) - for modName in modNames: - result = self.getTarsLocation(project, modName) - if not result['OK']: - return result - tarsURL = result['Value'] - modVersion = modVersions[modName] - defLoc = self.getModuleVersionFromLocalCfg(modName) - if defLoc: - modVersion = defLoc # this overwrite the version which are defined in the release.cfg - modsToInstall[modName] = (tarsURL, modVersion) - modsOrder.insert(0, modName) - - for modName in extensions: - if modName.split(":")[0] not in extraFound: - return S_ERROR("No module %s defined. You sure it's defined for this release?" % modName) - - return S_OK((modsOrder, modsToInstall)) - - -################################################################################# -# End of ReleaseConfig -################################################################################# - - -# platformAlias = { 'Darwin_i386_10.6' : 'Darwin_i386_10.5' } -platformAlias = {} - -#### -# Start of helper functions -#### - - -def logDEBUG(msg): - """ - :param str msg: debug message - """ - if cliParams.debug: - for line in msg.split("\n"): - print("%s UTC dirac-install [DEBUG] %s" % (time.strftime('%Y-%m-%d %H:%M:%S', - time.gmtime()), - line)) - sys.stdout.flush() - - -def logERROR(msg): - """ - :param str msg: error message - """ - for line in msg.split("\n"): - print("%s UTC dirac-install [ERROR] %s" % (time.strftime('%Y-%m-%d %H:%M:%S', - time.gmtime()), - line)) - sys.stdout.flush() - - -def logWARN(msg): - """ - :param str msg: warning message - """ - for line in msg.split("\n"): - print("%s UTC dirac-install [WARN] %s" % (time.strftime('%Y-%m-%d %H:%M:%S', - time.gmtime()), - line)) - sys.stdout.flush() - - -def logNOTICE(msg): - """ - :param str msg: notice message - """ - for line in msg.split("\n"): - print("%s UTC dirac-install [NOTICE] %s" % (time.strftime('%Y-%m-%d %H:%M:%S', - time.gmtime()), - line)) - sys.stdout.flush() - - -def alarmTimeoutHandler(*args): - """ - When a connection time out then raise and exception - """ - raise Exception('Timeout') - - -def urlretrieveTimeout(url, fileName='', timeout=0, retries=3): - """ - Retrieve remote url to local file, with timeout wrapper - - :param str fileName: file name - :param int timeout: time out in second used for downloading the files. - """ - if fileName: - # This can be a local file - if os.path.exists(url): # we do not download from web, use locally - logDEBUG('Local file used: "%s"' % url) - shutil.copy(url, fileName) - return True - localFD = open(fileName, "wb") - - # NOTE: Not thread-safe, since all threads will catch same alarm. - # This is OK for dirac-install, since there are no threads. - logDEBUG('Retrieving remote file "%s"' % url) - - urlData = '' - if timeout: - signal.signal(signal.SIGALRM, alarmTimeoutHandler) - # set timeout alarm - signal.alarm(timeout + 5) - try: - # if "http_proxy" in os.environ and os.environ['http_proxy']: - # proxyIP = os.environ['http_proxy'] - # proxy = urllib2.ProxyHandler( {'http': proxyIP} ) - # opener = urllib2.build_opener( proxy ) - # #opener = urllib2.build_opener() - # urllib2.install_opener( opener ) - - # Try to use insecure context explicitly, needed for python >= 2.7.9 - try: - context = ssl._create_unverified_context() - remoteFD = urlopen(url, context=context) # pylint: disable=unexpected-keyword-arg - # the keyword 'context' is present from 2.7.9+ - except AttributeError: - remoteFD = urlopen(url) - - expectedBytes = 0 - # Sometimes repositories do not return Content-Length parameter - try: - expectedBytes = int(remoteFD.info()['Content-Length']) - except Exception: - logWARN('Content-Length parameter not returned, skipping expectedBytes check') - - receivedBytes = 0 - data = remoteFD.read(16384) - count = 1 - progressBar = False - while data: - receivedBytes += len(data) - if fileName: - localFD.write(data) - else: - urlData += data.decode('utf8', 'ignore') - data = remoteFD.read(16384) - if count % 20 == 0 and sys.stdout.isatty(): - print(u'\033[1D' + ".", end=" ") - sys.stdout.flush() - progressBar = True - count += 1 - if progressBar and sys.stdout.isatty(): - # return cursor to the beginning of the line - print('\033[1K', end=" ") - print('\033[1A') - if fileName: - localFD.close() - remoteFD.close() - if receivedBytes != expectedBytes and expectedBytes > 0: - logERROR("File should be %s bytes but received %s" % (expectedBytes, receivedBytes)) - return False - except HTTPError as x: - if x.code == 404: - logERROR("%s does not exist" % url) - if timeout: - signal.alarm(0) - return False - else: - logWARN('Status code accessing URL %s was %s' % (url, x.getcode())) - if retries: - logWARN('Will retry after 30 seconds, %s retries remaining' % retries) - time.sleep(30) - return urlretrieveTimeout(url, fileName=fileName, timeout=timeout, retries=retries - 1) - else: - logERROR("No retries remaining for %s" % url) - sys.exit(1) - except URLError: - logERROR('Timeout after %s seconds on transfer request for "%s"' % (str(timeout), url)) - except Exception as x: - if x == 'Timeout': - logERROR('Timeout after %s seconds on transfer request for "%s"' % (str(timeout), url)) - if timeout: - signal.alarm(0) - raise x - if timeout: - signal.alarm(0) - - if fileName: - return True - else: - return urlData - - -def downloadAndExtractTarball(tarsURL, pkgName, pkgVer, checkHash=True, cache=False): - """ - It downloads and extracts a given tarball from a given destination: file system, - web server or code repository. - - :param str tarsURL: the location of the source repository - :param str pkgName: the name of the package to be installed - :param str pkgVer: the version of the package - :param bool checkHash: check the sanity of the file - :param bool cache: use local cache for the tarballs - - """ - tarName = "%s-%s.tar.gz" % (pkgName, pkgVer) - tarPath = os.path.join(cliParams.targetPath, tarName) - tarFileURL = "%s/%s" % (tarsURL, tarName) - tarFileCVMFS = "/cvmfs/dirac.egi.eu/installSource/%s" % tarName - cacheDir = os.path.join(cliParams.basePath, ".installCache") - tarCachePath = os.path.join(cacheDir, tarName) - isSource = False - if cache and os.path.isfile(tarCachePath): - logNOTICE("Using cached copy of %s" % tarName) - shutil.copy(tarCachePath, tarPath) - elif os.path.exists(tarFileCVMFS): - logNOTICE("Using CVMFS copy of %s" % tarName) - tarPath = tarFileCVMFS - checkHash = False - cache = False - else: - logNOTICE("Retrieving %s" % tarFileURL) - try: - if not urlretrieveTimeout(tarFileURL, tarPath, cliParams.timeout): - if os.path.exists(tarPath): - os.unlink(tarPath) - retVal = checkoutFromGit(pkgName, tarsURL, pkgVer) - if not retVal['OK']: - logERROR("Cannot download %s" % tarName) - logERROR("Cannot download %s" % retVal['Message']) - return False - else: - isSource = True - except Exception as e: - logERROR("Cannot download %s: %s" % (tarName, str(e))) - sys.exit(1) - if not isSource and checkHash: - md5Name = "%s-%s.md5" % (pkgName, pkgVer) - md5Path = os.path.join(cliParams.targetPath, md5Name) - md5FileURL = "%s/%s" % (tarsURL, md5Name) - md5CachePath = os.path.join(cacheDir, md5Name) - if cache and os.path.isfile(md5CachePath): - logNOTICE("Using cached copy of %s" % md5Name) - shutil.copy(md5CachePath, md5Path) - else: - logNOTICE("Retrieving %s" % md5FileURL) - try: - if not urlretrieveTimeout(md5FileURL, md5Path, 60): - logERROR("Cannot download %s" % tarName) - return False - except Exception as e: - logERROR("Cannot download %s: %s" % (md5Name, str(e))) - return False - # Read md5 - fd = open(os.path.join(cliParams.targetPath, md5Name), "r") - md5Expected = fd.read().strip() - fd.close() - # Calculate md5 - md5Calculated = hashlib.md5() - with open(os.path.join(cliParams.targetPath, tarName), "rb") as fd: - buf = fd.read(4096) - while buf: - md5Calculated.update(buf) - buf = fd.read(4096) - - # Check - if md5Expected != md5Calculated.hexdigest(): - logERROR("Oops... md5 for package %s failed!" % pkgVer) - sys.exit(1) - # Delete md5 file - if cache: - if not os.path.isdir(cacheDir): - os.makedirs(cacheDir) - os.rename(md5Path, md5CachePath) - else: - os.unlink(md5Path) - # Extract - # cwd = os.getcwd() - # os.chdir(cliParams.targetPath) - # tf = tarfile.open( tarPath, "r" ) - # for member in tf.getmembers(): - # tf.extract( member ) - # os.chdir(cwd) - if not isSource: - logNOTICE("Extract using system tar: %s" % tarPath) - tarCmd = "tar xzf '%s' -C '%s'" % (tarPath, cliParams.targetPath) - if os.system(tarCmd): - logERROR("Extraction of tarball %s failed" % tarPath) - raise RuntimeError("Failed to extract tarball") - # Delete tar - if cache: - if not os.path.isdir(cacheDir): - os.makedirs(cacheDir) - os.rename(tarPath, tarCachePath) - else: - if tarPath != tarFileCVMFS: - os.unlink(tarPath) - - postInstallScript = os.path.join(cliParams.targetPath, pkgName, 'dirac-postInstall.py') - if os.path.isfile(postInstallScript): - os.chmod(postInstallScript, executablePerms) - logNOTICE("Executing %s..." % postInstallScript) - if os.system("python '%s' > '%s.out' 2> '%s.err'" % (postInstallScript, - postInstallScript, - postInstallScript)): - logERROR("Post installation script %s failed. Check %s.err" % (postInstallScript, - postInstallScript)) - return True - - -def fixBuildPaths(): - """ - At compilation time many scripts get the building directory inserted, - this needs to be changed to point to the current installation path: - cliParams.targetPath -""" - - # Locate build path (from header of pydoc) - binaryPath = os.path.join(cliParams.targetPath, cliParams.platform) - pydocPath = os.path.join(binaryPath, 'bin', 'pydoc') - try: - fd = open(pydocPath) - line = fd.readline() - fd.close() - buildPath = line[2:line.find(cliParams.platform) - 1] - replaceCmd = "grep -rIl '%s' %s | xargs sed -i'.org' 's:%s:%s:g'" % (buildPath, - binaryPath, - buildPath, - cliParams.targetPath) - os.system(replaceCmd) - - except BaseException: - pass - - -def fixPythonShebang(): - """ - Some scripts (like the gfal2 scripts) come with a shebang pointing to the system python. - We replace it with the environment one - """ - - binaryPath = os.path.join(cliParams.targetPath, cliParams.platform) - try: - replaceCmd = "grep -rIl '#!/usr/bin/python' %s/bin |\ - xargs sed -i'.org' 's:#!/usr/bin/python:#!/usr/bin/env python:g'" % binaryPath - os.system(replaceCmd) - except BaseException: - pass - - -def runExternalsPostInstall(): - """ - If there are any postInstall in externals, run them - """ - if cliParams.diracOS or cliParams.diracOSVersion: - postInstallPath = os.path.join(cliParams.targetPath, "postInstall") - else: - postInstallPath = os.path.join(cliParams.targetPath, cliParams.platform, "postInstall") - if not os.path.isdir(postInstallPath): - logDEBUG("There's no %s directory. Skipping postInstall step" % postInstallPath) - return - postInstallSuffix = "-postInstall" - for scriptName in os.listdir(postInstallPath): - if not scriptName.endswith(postInstallSuffix): - logDEBUG("%s does not have the %s suffix. Skipping.." % (scriptName, postInstallSuffix)) - continue - scriptPath = os.path.join(postInstallPath, scriptName) - os.chmod(scriptPath, executablePerms) - logNOTICE("Executing %s..." % scriptPath) - if os.system("'%s' > '%s.out' 2> '%s.err'" % (scriptPath, scriptPath, scriptPath)): - logERROR("Post installation script %s failed. Check %s.err" % (scriptPath, scriptPath)) - sys.exit(1) - - -def fixMySQLScript(): - """ - Update the mysql.server script (if installed) to point to the proper datadir - """ - scriptPath = os.path.join(cliParams.targetPath, 'scripts', 'dirac-fix-mysql-script') - bashrcFile = os.path.join(cliParams.targetPath, 'bashrc') - if cliParams.useVersionsDir: - bashrcFile = os.path.join(cliParams.basePath, 'bashrc') - command = 'source %s; %s > /dev/null' % (bashrcFile, scriptPath) - if os.path.exists(scriptPath): - logNOTICE("Executing %s..." % command) - os.system('bash -c "%s"' % command) - - -def checkPlatformAliasLink(): - """ - Make a link if there's an alias - """ - if cliParams.platform in platformAlias: - os.symlink(os.path.join(cliParams.targetPath, platformAlias[cliParams.platform]), - os.path.join(cliParams.targetPath, cliParams.platform)) - - -def installExternalRequirements(extType): - """ Install the extension requirements if any - """ - reqScript = os.path.join(cliParams.targetPath, "scripts", 'dirac-externals-requirements') - bashrcFile = os.path.join(cliParams.targetPath, 'bashrc') - if cliParams.useVersionsDir: - bashrcFile = os.path.join(cliParams.basePath, 'bashrc') - if os.path.isfile(reqScript): - os.chmod(reqScript, executablePerms) - logNOTICE("Executing %s..." % reqScript) - command = "%s -t '%s' > '%s.out' 2> '%s.err'" % (reqScript, extType, reqScript, reqScript) - if os.system('bash -c "source %s; %s"' % (bashrcFile, command)): - logERROR("Requirements installation script %s failed. Check %s.err" % (reqScript, - reqScript)) - return True - - -def discoverModules(modules): - """ - Created the dictionary which contains all modules, which can be installed - for example: {"DIRAC:{"sourceUrl":"https://github.com/zmathe/DIRAC.git","Vesrion:v6r20p11"}} - - :param: str modules: it contains meta information for the module, - which will be installed: https://github.com/zmathe/DIRAC.git:::DIRAC:::dev_main_branch - """ - - projects = {} - - for module in modules.split(","): - s = m = v = None - try: - s, m, v = module.split(":::") - except ValueError: - m = module.split(":::")[0] # the source and version is not provided - - projects[m] = {} - if s and v: - projects[m] = {"sourceUrl": s, "Version": v} - else: - logWARN('Unable to parse module: %s' % module) - return projects - -#### -# End of helper functions -#### - - -cmdOpts = (('r:', 'release=', 'Release version to install'), - ('l:', 'project=', 'Project to install'), - ('e:', 'extensions=', 'Extensions to install (comma separated)'), - ('t:', 'installType=', 'Installation type (client/server)'), - ('i:', 'pythonVersion=', 'Python version to compile (27/26)'), - ('p:', 'platform=', 'Platform to install'), - ('P:', 'installationPath=', 'Path where to install (default current working dir)'), - ('b', 'build', 'Force local compilation'), - ('g:', 'grid=', 'lcg tools package version'), - (' ', 'no-lcg-bundle', 'lcg tools not to be installed'), - ('B', 'noAutoBuild', 'Do not build if not available'), - ('v', 'useVersionsDir', 'Use versions directory'), - ('u:', 'baseURL=', "Use URL as the source for installation tarballs"), - ('d', 'debug', 'Show debug messages'), - ('V:', 'installation=', 'Installation from which to extract parameter values'), - ('X', 'externalsOnly', 'Only install external binaries'), - ('M:', 'defaultsURL=', 'Where to retrieve the global defaults from'), - ('h', 'help', 'Show this help'), - ('T:', 'Timeout=', 'Timeout for downloads (default = %s)'), - (' ', 'dirac-os-version=', 'the version of the DIRAC OS'), - (' ', 'dirac-os', 'Enable installation of DIRAC OS'), - (' ', 'tag=', 'release version to install from git, http or local'), - ('m:', 'module=', - 'Module to be installed. for example: -m DIRAC or -m git://github.com/DIRACGrid/DIRAC.git:DIRAC'), - ('x:', 'external=', 'external version'), - (' ', 'createLink', 'create version symbolic link from the versions directory. This is equivalent to the \ - following command: ln -s /opt/dirac/versions/vArBpC vArBpC'), - (' ', 'scriptSymlink', 'Symlink the scripts instead of creating wrapper'), - (' ', 'userEnvVariables=', - 'User-requested environment variables (comma-separated, name and value separated by ":::")') - ) - - -def usage(): - print("\nUsage:\n\n %s " % os.path.basename(sys.argv[0])) - print("\nOptions:") - for cmdOpt in cmdOpts: - print(" %s %s : %s" % (cmdOpt[0].ljust(3), cmdOpt[1].ljust(20), cmdOpt[2])) - print() - print("Known options and default values from /defaults section of releases file:") - for options in [('Release', cliParams.release), - ('Project', cliParams.project), - ('ModulesToInstall', []), - ('ExternalsType', cliParams.externalsType), - ('PythonVersion', cliParams.pythonVersion), - ('LcgVer', cliParams.lcgVer), - ('UseVersionsDir', cliParams.useVersionsDir), - ('BuildExternals', cliParams.buildExternals), - ('NoAutoBuild', cliParams.noAutoBuild), - ('Debug', cliParams.debug), - ('Timeout', cliParams.timeout)]: - print(" %s = %s" % options) - - sys.exit(0) - - -def loadConfiguration(): - """ - It loads the configuration file - """ - optList, args = getopt.getopt(sys.argv[1:], - "".join([opt[0] for opt in cmdOpts]), - [opt[1] for opt in cmdOpts]) - - # First check if the name is defined - for o, v in optList: - if o in ('-h', '--help'): - usage() - elif o in ('-V', '--installation'): - cliParams.installation = v - elif o in ("-d", "--debug"): - cliParams.debug = True - elif o in ("-M", "--defaultsURL"): - cliParams.globalDefaults = v - - releaseConfig = ReleaseConfig( - instName=cliParams.installation, - globalDefaultsURL=cliParams.globalDefaults) - if cliParams.debug: - releaseConfig.debugCB = logDEBUG - - result = releaseConfig.loadInstallationDefaults() - if not result['OK']: - logERROR("Could not load defaults: %s" % result['Message']) - - releaseConfig.loadInstallationLocalDefaults(args) - - for opName in ('release', 'externalsType', 'installType', 'pythonVersion', - 'buildExternals', 'noAutoBuild', 'debug', 'globalDefaults', - 'lcgVer', 'useVersionsDir', 'targetPath', - 'project', 'release', 'extensions', 'timeout'): - try: - opVal = releaseConfig.getInstallationConfig( - "LocalInstallation/%s" % (opName[0].upper() + opName[1:])) - except KeyError: - continue - - if opName == 'installType': - opName = 'externalsType' - - if isinstance(getattr(cliParams, opName), str_type): - setattr(cliParams, opName, opVal) - elif isinstance(getattr(cliParams, opName), bool): - setattr(cliParams, opName, opVal.lower() in ("y", "yes", "true", "1")) - elif isinstance(getattr(cliParams, opName), list): - setattr(cliParams, opName, [opV.strip() for opV in opVal.split(",") if opV]) - - # Now parse the ops - for o, v in optList: - if o in ('-r', '--release'): - cliParams.release = v - elif o in ('-l', '--project'): - cliParams.project = v - elif o in ('-e', '--extensions'): - for pkg in [p.strip() for p in v.split(",") if p.strip()]: - if pkg not in cliParams.extensions: - cliParams.extensions.append(pkg) - elif o in ('-t', '--installType'): - cliParams.externalsType = v - elif o in ('-i', '--pythonVersion'): - cliParams.pythonVersion = v - elif o in ('-p', '--platform'): - cliParams.platform = v - elif o in ('-d', '--debug'): - cliParams.debug = True - elif o in ('-g', '--grid'): - cliParams.lcgVer = v - elif o in ('--no-lcg-bundle'): - cliParams.noLcg = True - elif o in ('-u', '--baseURL'): - cliParams.installSource = v - elif o in ('-P', '--installationPath'): - cliParams.targetPath = v - try: - os.makedirs(v) - except BaseException: - pass - elif o in ('-v', '--useVersionsDir'): - cliParams.useVersionsDir = True - elif o in ('-b', '--build'): - cliParams.buildExternals = True - elif o in ("-B", '--noAutoBuild'): - cliParams.noAutoBuild = True - elif o in ('-X', '--externalsOnly'): - cliParams.externalsOnly = True - elif o in ('-T', '--Timeout'): - try: - cliParams.timeout = max(cliParams.timeout, int(v)) - cliParams.timeout = min(cliParams.timeout, 3600) - except ValueError: - pass - elif o == '--dirac-os-version': - cliParams.diracOSVersion = v - elif o == '--dirac-os': - cliParams.diracOS = True - elif o == '--tag': - cliParams.tag = v - elif o in ('-m', '--module'): - cliParams.modules = discoverModules(v) - elif o in ('-x', '--external'): - cliParams.externalVersion = v - elif o == '--createLink': - cliParams.createLink = True - elif o == '--scriptSymlink': - cliParams.scriptSymlink = True - elif o == '--userEnvVariables': - cliParams.userEnvVariables = dict(zip([name.split(':::')[0] for name in v.replace(' ', '').split(',')], - [value.split(':::')[1] for value in v.replace(' ', '').split(',')])) - - if not cliParams.release and not cliParams.modules: - logERROR("Missing release to install") - usage() - - cliParams.basePath = cliParams.targetPath - if cliParams.useVersionsDir: - # install under /versions/_ - cliParams.targetPath = os.path.join( - cliParams.targetPath, 'versions', '%s_%s' % (cliParams.release, int(time.time()))) - try: - os.makedirs(cliParams.targetPath) - except BaseException: - pass - - # If we are running an update, DIRACOS will be set in the environment - if not cliParams.diracOS and 'DIRACOS' in os.environ: - logWARN("Forcing to install DIRACOS, because it is already installed!") - cliParams.diracOS = True - - logNOTICE("Destination path for installation is %s" % cliParams.targetPath) - releaseConfig.projectName = cliParams.project - - result = releaseConfig.loadProjectRelease(cliParams.release, - project=cliParams.project, - sourceURL=cliParams.installSource) - if not result['OK']: - return result - - if not releaseConfig.isProjectLoaded("DIRAC"): - return S_ERROR("DIRAC is not depended by this installation. Aborting") - - # Reload the local configuration to ensure it takes prescience - releaseConfig.loadInstallationLocalDefaults(args) - - return S_OK(releaseConfig) - - -def compileExternals(extVersion): - """ - It is used to compile the external for a given platform - - :param str extVersion: the external version - """ - logNOTICE("Compiling externals %s" % extVersion) - buildCmd = os.path.join( - cliParams.targetPath, - "DIRAC", - "Core", - "scripts", - "dirac-compile-externals.py") - buildCmd = "%s -t '%s' -D '%s' -v '%s' -i '%s'" % (buildCmd, - cliParams.externalsType, - os.path.join( - cliParams.targetPath, - cliParams.platform), - extVersion, - cliParams.pythonVersion) - if os.system(buildCmd): - logERROR("Could not compile binaries") - return False - return True - - -def getPlatform(): - """ - It returns the platform, where this script is running using Platform.py - """ - platformPath = os.path.join(cliParams.targetPath, "DIRAC", "Core", "Utilities", "Platform.py") - try: - platFD = open(platformPath, "r") - except IOError: - logERROR("Cannot open Platform.py. Is DIRAC installed?") - return '' - - Platform = imp.load_module("Platform", platFD, platformPath, ("", "r", imp.PY_SOURCE)) - platFD.close() - return Platform.getPlatformString() - - -def installExternals(releaseConfig): - """ - It install the DIRAC external. The version of the external is provided by - the cmd or in the configuration file. - - :param object releaseConfig: - """ - if not releaseConfig: - externalsVersion = cliParams.externalVersion - else: - externalsVersion = releaseConfig.getExternalsVersion() - if not externalsVersion: - logERROR("No externals defined") - return False - - if not cliParams.platform: - cliParams.platform = getPlatform() - if not cliParams.platform: - return False - - if cliParams.installSource: - tarsURL = cliParams.installSource - else: - tarsURL = releaseConfig.getTarsLocation('DIRAC')['Value'] - - if cliParams.buildExternals: - compileExternals(externalsVersion) - else: - logDEBUG("Using platform: %s" % cliParams.platform) - extVer = "%s-%s-%s-python%s" % (cliParams.externalsType, externalsVersion, - cliParams.platform, cliParams.pythonVersion) - logDEBUG("Externals %s are to be installed" % extVer) - if not downloadAndExtractTarball(tarsURL, "Externals", extVer, cache=True): - return (not cliParams.noAutoBuild) and compileExternals(externalsVersion) - logNOTICE("Fixing externals paths...") - fixBuildPaths() - logNOTICE("Running externals post install...") - checkPlatformAliasLink() - return True - - -def installLCGutils(releaseConfig): - """ - DIRAC uses various tools from LCG area. This method install a given - lcg version. - :param object releaseConfig: the configuration file object (class ReleaseConfig) - """ - if not cliParams.platform: - cliParams.platform = getPlatform() - if not cliParams.platform: - return False - - if cliParams.installSource: - tarsURL = cliParams.installSource - else: - tarsURL = releaseConfig.getTarsLocation('DIRAC')['Value'] - - # lcg utils? - # LCG utils if required - if not releaseConfig: - lcgVer = cliParams.lcgVer - else: - lcgVer = releaseConfig.getLCGVersion(cliParams.lcgVer) - if lcgVer: - verString = "%s-%s-python%s" % (lcgVer, cliParams.platform, cliParams.pythonVersion) - # HACK: try to find a more elegant solution for the lcg bundles location - if not downloadAndExtractTarball(tarsURL + "/../lcgBundles", "DIRAC-lcg", verString, False, cache=True): - logERROR( - "\nThe requested LCG software version %s for the local operating system could not be downloaded." % - verString) - logERROR("Please, check the availability of the LCG software bindings for you \ - platform 'DIRAC-lcg-%s' \n in the repository %s/lcgBundles/." % - (verString, os.path.dirname(tarsURL))) - logERROR( - "\nIf you would like to skip the installation of the LCG software, redo the installation with \ - adding the option --no-lcg-bundle to the command line.") - return False - - logNOTICE("Fixing Python Shebang...") - fixPythonShebang() - return True - - -def createPermanentDirLinks(): - """ Create links to permanent directories from within the version directory - """ - if cliParams.useVersionsDir: - try: - # Directories - for directory in ['startup', 'runit', 'data', 'work', 'control', 'sbin', 'etc', 'webRoot']: - fake = os.path.join(cliParams.targetPath, directory) - real = os.path.join(cliParams.basePath, directory) - if not os.path.exists(real): - os.makedirs(real) - if os.path.exists(fake): - # Try to reproduce the directory structure to avoid lacking directories - fakeDirs = os.listdir(fake) - for fd in fakeDirs: - if os.path.isdir(os.path.join(fake, fd)): - if not os.path.exists(os.path.join(real, fd)): - os.makedirs(os.path.join(real, fd)) - os.rename(fake, fake + '.bak') - os.symlink(real, fake) - - # Files - for filename in ['bashrc']: - fake = os.path.join(cliParams.targetPath, filename) - real = os.path.join(cliParams.basePath, filename) - os.symlink(real, fake) - except Exception as x: - logERROR(str(x)) - return False - - return True - - -def createOldProLinks(): - """ Create links to permanent directories from within the version directory - """ - proPath = cliParams.targetPath - if cliParams.useVersionsDir: - oldPath = os.path.join(cliParams.basePath, 'old') - proPath = os.path.join(cliParams.basePath, 'pro') - try: - if os.path.exists(proPath) or os.path.islink(proPath): - if os.path.exists(oldPath) or os.path.islink(oldPath): - os.unlink(oldPath) - os.rename(proPath, oldPath) - os.symlink(cliParams.targetPath, proPath) - except Exception as x: - logERROR(str(x)) - return False - - return True - - -def createBashrc(): - """ Create DIRAC environment setting script for the bash shell - """ - - proPath = cliParams.targetPath - # Now create bashrc at basePath - try: - bashrcFile = os.path.join(cliParams.targetPath, 'bashrc') - if cliParams.useVersionsDir: - bashrcFile = os.path.join(cliParams.basePath, 'bashrc') - proPath = os.path.join(cliParams.basePath, 'pro') - logNOTICE('Creating %s' % bashrcFile) - if not os.path.exists(bashrcFile): - lines = ['# DIRAC bashrc file, used by service and agent run scripts to set environment', - 'export PYTHONUNBUFFERED=yes', - 'export PYTHONOPTIMIZE=x'] - if 'HOME' in os.environ: - lines.append('[ -z "$HOME" ] && export HOME=%s' % os.environ['HOME']) - - # Determining where the CAs are... - if 'X509_CERT_DIR' in os.environ: - certDir = os.environ['X509_CERT_DIR'] - else: - if os.path.isdir('/etc/grid-security/certificates') and \ - os.listdir('/etc/grid-security/certificates'): - # Assuming that, if present, it is not empty, and has correct CAs - certDir = '/etc/grid-security/certificates' - else: - # But this will have to be created at some point (dirac-configure) - certDir = '%s/etc/grid-security/certificates' % proPath - lines.extend(['# CAs path for SSL verification', - 'export X509_CERT_DIR=${X509_CERT_DIR:-%s}' % certDir, - 'export SSL_CERT_DIR=${SSL_CERT_DIR:-%s}' % certDir]) - - lines.append( - 'export X509_VOMS_DIR=${X509_VOMS_DIR:-%s}' % - os.path.join( - proPath, - 'etc', - 'grid-security', - 'vomsdir')) - lines.append( - 'export X509_VOMSES=${X509_VOMSES:-%s}' % - os.path.join( - proPath, - 'etc', - 'grid-security', - 'vomses')) - lines.extend( - [ - '# Some DIRAC locations', - '[ -z "$DIRAC" ] && export DIRAC=%s' % - proPath, - 'export DIRACBIN=%s' % - os.path.join( - "$DIRAC", - cliParams.platform, - 'bin'), - 'export DIRACSCRIPTS=%s' % - os.path.join( - "$DIRAC", - 'scripts'), - 'export DIRACLIB=%s' % - os.path.join( - "$DIRAC", - cliParams.platform, - 'lib'), - 'export TERMINFO=%s' % - __getTerminfoLocations( - os.path.join( - "$DIRAC", - cliParams.platform, - 'share', - 'terminfo')), - 'export RRD_DEFAULT_FONT=%s' % - os.path.join( - "$DIRAC", - cliParams.platform, - 'share', - 'rrdtool', - 'fonts', - 'DejaVuSansMono-Roman.ttf')]) - - lines.extend(['# Prepend the PYTHONPATH, the LD_LIBRARY_PATH, and the DYLD_LIBRARY_PATH']) - lines.extend(['( echo $PATH | grep -q $DIRACBIN ) || export PATH=$DIRACBIN:$PATH', - '( echo $PATH | grep -q $DIRACSCRIPTS ) || export PATH=$DIRACSCRIPTS:$PATH', - '( echo $LD_LIBRARY_PATH | grep -q $DIRACLIB ) || \ - export LD_LIBRARY_PATH=$DIRACLIB:$LD_LIBRARY_PATH', - '( echo $LD_LIBRARY_PATH | grep -q $DIRACLIB/mysql ) || \ - export LD_LIBRARY_PATH=$DIRACLIB/mysql:$LD_LIBRARY_PATH', - '( echo $DYLD_LIBRARY_PATH | grep -q $DIRACLIB ) || \ - export DYLD_LIBRARY_PATH=$DIRACLIB:$DYLD_LIBRARY_PATH', - '( echo $DYLD_LIBRARY_PATH | grep -q $DIRACLIB/mysql ) || \ - export DYLD_LIBRARY_PATH=$DIRACLIB/mysql:$DYLD_LIBRARY_PATH']) - - lines.extend(['export PYTHONPATH=$DIRAC']) - - lines.extend(['# new OpenSSL version require OPENSSL_CONF to point to some accessible location', - 'export OPENSSL_CONF=/tmp']) - - # gfal2 requires some environment variables to be set - lines.extend(['# Gfal2 configuration and plugins', 'export GFAL_CONFIG_DIR=%s' % - os.path.join("$DIRAC", cliParams.platform, 'etc/gfal2.d'), 'export GFAL_PLUGIN_DIR=%s' % - os.path.join("$DIRACLIB", 'gfal2-plugins')]) - # add DIRACPLAT environment variable for client installations - if cliParams.externalsType == 'client': - lines.extend(['# DIRAC platform', - '[ -z "$DIRACPLAT" ] && export DIRACPLAT=`$DIRAC/scripts/dirac-platform`']) - # Add the lines required for globus-* tools to use IPv6 - lines.extend(['# IPv6 support', - 'export GLOBUS_IO_IPV6=TRUE', - 'export GLOBUS_FTP_CLIENT_IPV6=TRUE']) - # Add the lines required for ARC CE support - lines.extend(['# ARC Computing Element', - 'export ARC_PLUGIN_PATH=$DIRACLIB/arc']) - - # Add the lines required for fork support for xrootd - lines.extend(['# Fork support for xrootd', - 'export XRD_RUNFORKHANDLER=1']) - - # Add the lines required for further env variables requested - if cliParams.userEnvVariables: - lines.extend(['# User-requested variables']) - for envName, envValue in cliParams.userEnvVariables.items(): - lines.extend(['( echo $%s | grep -q $%s ) || export %s=$%s:$%s' % ( - envName, envValue, - envName, envName, envValue)]) - - # Add possible DIRAC environment variables - lines.append('') - lines.append('# before enabling any of these variables, please see the documentation ') - lines.append('# https://dirac.readthedocs.io/en/latest/AdministratorGuide/' + - 'ServerInstallations/environment_variable_configuration.html') - lines.append('# export DIRAC_DEBUG_DENCODE_CALLSTACK=1') - lines.append('# export DIRAC_DEBUG_STOMP=1') - lines.append('# export DIRAC_DEPRECATED_FAIL=1') - lines.append('# export DIRAC_GFAL_GRIDFTP_SESSION_REUSE=true') - lines.append('# export DIRAC_USE_JSON_DECODE=no') - lines.append('# export DIRAC_USE_JSON_ENCODE=no') - lines.append('# export DIRAC_USE_M2CRYPTO=true') - lines.append('# export DIRAC_USE_NEWTHREADPOOL=yes') - lines.append('# export DIRAC_NO_CFG=true') - lines.append('') - f = open(bashrcFile, 'w') - f.write('\n'.join(lines)) - f.close() - except Exception as x: - logERROR(str(x)) - return False - - return True - - -def createCshrc(): - """ Create DIRAC environment setting script for the (t)csh shell - """ - proPath = cliParams.targetPath - # Now create cshrc at basePath - try: - cshrcFile = os.path.join(cliParams.targetPath, 'cshrc') - if cliParams.useVersionsDir: - cshrcFile = os.path.join(cliParams.basePath, 'cshrc') - proPath = os.path.join(cliParams.basePath, 'pro') - logNOTICE('Creating %s' % cshrcFile) - if not os.path.exists(cshrcFile): - lines = ['# DIRAC cshrc file, used by clients to set up the environment', - 'setenv PYTHONUNBUFFERED yes', - 'setenv PYTHONOPTIMIZE x'] - - # Determining where the CAs are... - if 'X509_CERT_DIR' in os.environ: - certDir = os.environ['X509_CERT_DIR'] - else: - if os.path.isdir('/etc/grid-security/certificates') and \ - os.listdir('/etc/grid-security/certificates'): - # Assuming that, if present, it is not empty, and has correct CAs - certDir = '/etc/grid-security/certificates' - else: - # But this will have to be created at some point (dirac-configure) - certDir = '%s/etc/grid-security/certificates' % proPath - lines.extend(['# CAs path for SSL verification', - 'setenv X509_CERT_DIR %s' % certDir, - 'setenv SSL_CERT_DIR %s' % certDir]) - - lines.append( - 'setenv X509_VOMS_DIR %s' % - os.path.join( - proPath, - 'etc', - 'grid-security', - 'vomsdir')) - lines.append( - 'setenv X509_VOMSES %s' % - os.path.join( - proPath, - 'etc', - 'grid-security', - 'vomses')) - lines.extend(['# Some DIRAC locations', - '( test $?DIRAC -eq 1 ) || setenv DIRAC %s' % proPath, - 'setenv DIRACBIN %s' % os.path.join("$DIRAC", cliParams.platform, 'bin'), - 'setenv DIRACSCRIPTS %s' % os.path.join("$DIRAC", 'scripts'), - 'setenv DIRACLIB %s' % os.path.join("$DIRAC", cliParams.platform, 'lib'), - 'setenv TERMINFO %s' % __getTerminfoLocations(os.path.join("$DIRAC", - cliParams.platform, - 'share', - 'terminfo'))]) - - lines.extend(['# Prepend the PYTHONPATH, the LD_LIBRARY_PATH, and the DYLD_LIBRARY_PATH']) - - lines.extend(['( test $?PATH -eq 1 ) || setenv PATH ""', - '( test $?LD_LIBRARY_PATH -eq 1 ) || setenv LD_LIBRARY_PATH ""', - '( test $?DY_LD_LIBRARY_PATH -eq 1 ) || setenv DYLD_LIBRARY_PATH ""', - '( test $?PYTHONPATH -eq 1 ) || setenv PYTHONPATH ""', - '( echo $PATH | grep -q $DIRACBIN ) || setenv PATH ${DIRACBIN}:$PATH', - '( echo $PATH | grep -q $DIRACSCRIPTS ) || setenv PATH ${DIRACSCRIPTS}:$PATH', - '( echo $LD_LIBRARY_PATH | grep -q $DIRACLIB ) || \ - setenv LD_LIBRARY_PATH ${DIRACLIB}:$LD_LIBRARY_PATH', - '( echo $LD_LIBRARY_PATH | grep -q $DIRACLIB/mysql ) || \ - setenv LD_LIBRARY_PATH ${DIRACLIB}/mysql:$LD_LIBRARY_PATH', - '( echo $DYLD_LIBRARY_PATH | grep -q $DIRACLIB ) || \ - setenv DYLD_LIBRARY_PATH ${DIRACLIB}:$DYLD_LIBRARY_PATH', - '( echo $DYLD_LIBRARY_PATH | grep -q $DIRACLIB/mysql ) || \ - setenv DYLD_LIBRARY_PATH ${DIRACLIB}/mysql:$DYLD_LIBRARY_PATH']) - - lines.extend(['setenv PYTHONPATH ${DIRAC}']) - - lines.extend(['# new OpenSSL version require OPENSSL_CONF to point to some accessible location', - 'setenv OPENSSL_CONF /tmp']) - lines.extend(['# IPv6 support', - 'setenv GLOBUS_IO_IPV6 TRUE', - 'setenv GLOBUS_FTP_CLIENT_IPV6 TRUE']) - # gfal2 requires some environment variables to be set - lines.extend(['# Gfal2 configuration and plugins', 'setenv GFAL_CONFIG_DIR %s' % - os.path.join("$DIRAC", cliParams.platform, 'etc/gfal2.d'), 'setenv GFAL_PLUGIN_DIR %s' % - os.path.join("$DIRACLIB", 'gfal2-plugins')]) - # add DIRACPLAT environment variable for client installations - if cliParams.externalsType == 'client': - lines.extend(['# DIRAC platform', - 'test $?DIRACPLAT -eq 1 || setenv DIRACPLAT `$DIRAC/scripts/dirac-platform`']) - # Add the lines required for ARC CE support - lines.extend(['# ARC Computing Element', - 'setenv ARC_PLUGIN_PATH $DIRACLIB/arc']) - - # Add the lines required for fork support for xrootd - lines.extend(['# Fork support for xrootd', - 'setenv XRD_RUNFORKHANDLER 1']) - - # Add the lines required for further env variables requested - if cliParams.userEnvVariables: - lines.extend(['# User-requested variables']) - for envName, envValue in cliParams.userEnvVariables.items(): - lines.extend(['setenv %s %s' % (envName, envValue)]) - - lines.append('') - f = open(cshrcFile, 'w') - f.write('\n'.join(lines)) - f.close() - except Exception as x: - logERROR(str(x)) - return False - - return True - - -def writeDefaultConfiguration(): - """ - After DIRAC is installed a default configuration file is created, - which contains a minimal setup. - """ - if not releaseConfig: - return - instCFG = releaseConfig.getInstallationCFG() - if not instCFG: - return - for opName in instCFG.getOptions(): - instCFG.delPath(opName) - - # filePath = os.path.join( cliParams.targetPath, "defaults-%s.cfg" % cliParams.installation ) - # Keep the default configuration file in the working directory - filePath = "defaults-%s.cfg" % cliParams.installation - try: - fd = open(filePath, "w") - fd.write(instCFG.toString()) - fd.close() - except Exception as excp: - logERROR("Could not write %s: %s" % (filePath, excp)) - logNOTICE("Defaults written to %s" % filePath) - - -def __getTerminfoLocations(defaultLocation=None): - """returns the terminfo locations as a colon separated string""" - - terminfoLocations = [] - if defaultLocation: - terminfoLocations = [defaultLocation] - - for termpath in ['/usr/share/terminfo', '/etc/terminfo']: - if os.path.exists(termpath): - terminfoLocations.append(termpath) - - return ":".join(terminfoLocations) - - -def installDiracOS(releaseConfig): - """ - Install DIRAC OS. - - :param ReleaseConfig releaseConfig: The ReleaseConfig object for configuring the installation - """ - diracos, diracOSVersion = releaseConfig.getDiracOSVersion(cliParams.diracOSVersion) - if not diracOSVersion: - logERROR("No diracos defined") - return False - tarsURL = cliParams.installSource - if not tarsURL: - tarsURL = releaseConfig.getDiracOsLocation(useVanillaDiracOS=(diracos.lower() == 'diracos')) - if not tarsURL: - tarsURL = releaseConfig.getTarsLocation('DIRAC')['Value'] - logWARN("DIRACOS location is not specified using %s" % tarsURL) - if not downloadAndExtractTarball(tarsURL, diracos, diracOSVersion, cache=True): - return False - logNOTICE("Fixing externals paths...") - fixBuildPaths() - logNOTICE("Running externals post install...") - checkPlatformAliasLink() - return True - - -def installDiracOSPython3(releaseConfig): - """ - Install DIRAC OS for Python 3. - - :param ReleaseConfig releaseConfig: The ReleaseConfig object for configuring the installation - """ - url = "https://github.com/chrisburr/DIRACOS2/releases/latest/download/DIRACOS-Linux-x86_64.sh" - installerFn = os.path.join(cliParams.basePath, "DIRACOS-Linux-x86_64.sh") - if not urlretrieveTimeout(url, installerFn, cliParams.timeout): - raise Exception("Failed to download DIRACOS from " + url) - subprocess.check_call(["bash", installerFn, "-b", "-p", os.path.join(cliParams.basePath, "diracos")]) - return True - - -def createBashrcForDiracOS(): - """ Create DIRAC environment setting script for the bash shell - """ - - proPath = cliParams.targetPath - # Now create bashrc at basePath - try: - bashrcFile = os.path.join(cliParams.targetPath, 'bashrc') - if cliParams.useVersionsDir: - bashrcFile = os.path.join(cliParams.basePath, 'bashrc') - proPath = os.path.join(cliParams.basePath, 'pro') - logNOTICE('Creating %s' % bashrcFile) - if not os.path.exists(bashrcFile): - lines = ['# DIRAC bashrc file, used by service and agent run scripts to set environment', - 'export PYTHONUNBUFFERED=yes', - 'export PYTHONOPTIMIZE=x', - '[ -z "$DIRAC" ] && export DIRAC=%s' % proPath, - '[ -z "$DIRACOS" ] && export DIRACOS=$DIRAC/diracos', - '. $DIRACOS/diracosrc'] - if 'HOME' in os.environ: - lines.append('[ -z "$HOME" ] && export HOME=%s' % os.environ['HOME']) - - # Determining where the CAs are... - if 'X509_CERT_DIR' in os.environ: - certDir = os.environ['X509_CERT_DIR'] - else: - if os.path.isdir('/etc/grid-security/certificates') and \ - os.listdir('/etc/grid-security/certificates'): - # Assuming that, if present, it is not empty, and has correct CAs - certDir = '/etc/grid-security/certificates' - else: - # But this will have to be created at some point (dirac-configure) - certDir = '%s/etc/grid-security/certificates' % proPath - lines.extend(['# CAs path for SSL verification', - 'export X509_CERT_DIR=${X509_CERT_DIR:-%s}' % certDir, - 'export SSL_CERT_DIR=${SSL_CERT_DIR:-%s}' % certDir]) - - lines.append( - 'export X509_VOMS_DIR=${X509_VOMS_DIR:-%s}' % - os.path.join( - proPath, - 'etc', - 'grid-security', - 'vomsdir')) - lines.append( - 'export X509_VOMSES=${X509_VOMSES:-%s}' % - os.path.join( - proPath, - 'etc', - 'grid-security', - 'vomses')) - lines.extend( - [ - '# Some DIRAC locations', - 'export DIRACSCRIPTS=%s' % - os.path.join( - "$DIRAC", - 'scripts')]) - - lines.extend(['# Prepend the PATH and set the PYTHONPATH']) - - lines.extend(['( echo $PATH | grep -q $DIRACSCRIPTS ) || export PATH=$DIRACSCRIPTS:$PATH']) - - lines.extend(['export PYTHONPATH=$DIRAC']) - - lines.extend(['# new OpenSSL version require OPENSSL_CONF to point to some accessible location', - 'export OPENSSL_CONF=/tmp']) - - # add DIRACPLAT environment variable for client installations - if cliParams.externalsType == 'client': - lines.extend(['# DIRAC platform', - '[ -z "$DIRACPLAT" ] && export DIRACPLAT=`$DIRAC/scripts/dirac-platform`']) - # Add the lines required for globus-* tools to use IPv6 - lines.extend(['# IPv6 support', - 'export GLOBUS_IO_IPV6=TRUE', - 'export GLOBUS_FTP_CLIENT_IPV6=TRUE']) - - # Add the lines required for fork support for xrootd - lines.extend(['# Fork support for xrootd', - 'export XRD_RUNFORKHANDLER=1']) - - # Add possible DIRAC environment variables - lines.append('') - lines.append('# before enabling any of these variables, please see the documentation ') - lines.append('# https://dirac.readthedocs.io/en/latest/AdministratorGuide/' + - 'ServerInstallations/environment_variable_configuration.html') - lines.append('# export DIRAC_DEBUG_DENCODE_CALLSTACK=1') - lines.append('# export DIRAC_DEBUG_STOMP=1') - lines.append('# export DIRAC_DEPRECATED_FAIL=1') - lines.append('# export DIRAC_GFAL_GRIDFTP_SESSION_REUSE=true') - lines.append('# export DIRAC_USE_JSON_DECODE=no') - lines.append('# export DIRAC_USE_JSON_ENCODE=no') - lines.append('# export DIRAC_USE_M2CRYPTO=true') - lines.append('# export DIRAC_USE_NEWTHREADPOOL=yes') - - # Add the lines required for further env variables requested - if cliParams.userEnvVariables: - lines.extend(['# User-requested variables']) - for envName, envValue in cliParams.userEnvVariables.items(): - lines.extend(['export %s=%s' % (envName, envValue)]) - - lines.append('') - with open(bashrcFile, 'w') as f: - f.write('\n'.join(lines)) - except Exception as x: - logERROR(str(x)) - return False - - return True - - -def checkoutFromGit(moduleName, sourceURL, tagVersion, destinationDir=None): - """ - This method checkout a given tag from a git repository. - Note: we can checkout any project form a git repository. - - :param str moduleName: The name of the Module: for example: LHCbWebDIRAC - :param str sourceURL: The code repository: https://github.com/DIRACGrid/WebAppDIRAC.git - :param str tagVersion: the tag for example: v3r1p10 - - """ - - codeRepo = moduleName + 'Repo' - - fDirName = os.path.join(cliParams.targetPath, codeRepo) - if os.path.isdir(sourceURL): - logNOTICE("Using local copy for source: %s" % sourceURL) - shutil.copytree(sourceURL, fDirName) - else: - cmd = "git clone '%s' '%s'" % (sourceURL, fDirName) - - logNOTICE("Executing: %s" % cmd) - if os.system(cmd): - return S_ERROR("Error while retrieving sources from git") - - branchName = "%s-%s" % (tagVersion, os.getpid()) - - isTagCmd = "( cd '%s'; git tag -l | grep '%s' )" % (fDirName, tagVersion) - if os.system(isTagCmd): - # No tag found, assume branch - branchSource = 'origin/%s' % tagVersion - else: - branchSource = tagVersion - - cmd = "( cd '%s'; git checkout -b '%s' '%s' )" % (fDirName, branchName, branchSource) - - logNOTICE("Executing: %s" % cmd) - exportRes = os.system(cmd) - - if exportRes: - return S_ERROR("Error while exporting from git") - - # replacing the code - if os.path.exists(fDirName + '/' + moduleName): - cmd = "ln -s %s/%s %s" % (codeRepo, moduleName, os.path.join(cliParams.targetPath, moduleName)) - else: - cmd = "mv %s %s" % (fDirName, os.path.join(cliParams.targetPath, moduleName)) - logNOTICE("Executing: %s" % cmd) - retVal = os.system(cmd) - - if retVal: - return S_ERROR("Error while creating module: %s" % (moduleName)) - - return S_OK() - - -def createSymbolicLink(): - """ - It creates a symbolic link to the actual directory from versions - directory. - """ - - cmd = "ln -s %s %s" % (cliParams.targetPath, cliParams.release) - logNOTICE("Executing: %s" % cmd) - retVal = os.system(cmd) - if retVal: - return S_ERROR("Error while creating symbolic link!") - - return S_OK() - - -if __name__ == "__main__": - logNOTICE("Processing installation requirements") - result = loadConfiguration() - releaseConfig = None - modsToInstall = {} - modsOrder = [] - if not result['OK']: - # the configuration files does not exists, which means the module is not released. - if cliParams.modules: - logNOTICE(str(cliParams.modules)) - for i in cliParams.modules: - modsOrder.append(i) - modsToInstall[i] = (cliParams.modules[i]['sourceUrl'], cliParams.modules[i]['Version']) - else: - # there is no module provided which can be deployed - logERROR(result['Message']) - sys.exit(1) - else: - releaseConfig = result['Value'] - if not createPermanentDirLinks(): - sys.exit(1) - - if not cliParams.externalsOnly: - logNOTICE("Discovering modules to install") - if releaseConfig: - result = releaseConfig.getModulesToInstall(cliParams.release, cliParams.extensions) - if not result['OK']: - logERROR(result['Message']) - sys.exit(1) - modsOrder, modsToInstall = result['Value'] - if cliParams.debug and releaseConfig: - logNOTICE("Writing down the releases files") - releaseConfig.dumpReleasesToPath() - logNOTICE("Installing modules...") - for modName in set(modsOrder): - tarsURL, modVersion = modsToInstall[modName] - if cliParams.installSource and not cliParams.modules: - # we install not release version of DIRAC - tarsURL = cliParams.installSource - if modName in cliParams.modules: - sourceURL = cliParams.modules[modName].get('sourceUrl') - if 'Version' in cliParams.modules[modName]: - modVersion = cliParams.modules[modName]['Version'] - if not sourceURL: - retVal = releaseConfig.getModSource(cliParams.release, modName) - if retVal['OK']: - tarsURL = retVal['Value'][1] # this is the git repository url - modVersion = cliParams.tag - else: - tarsURL = sourceURL - retVal = checkoutFromGit(modName, tarsURL, modVersion) - if not retVal['OK']: - logERROR("Cannot checkout %s" % retVal['Message']) - sys.exit(1) - else: - logNOTICE("Installing %s:%s" % (modName, modVersion)) - if not downloadAndExtractTarball(tarsURL, modName, modVersion): - sys.exit(1) - logNOTICE("Deploying scripts...") - ddeLocation = os.path.join(cliParams.targetPath, "DIRAC", "Core", - "scripts", "dirac-deploy-scripts.py") - if not os.path.isfile(ddeLocation): - ddeLocation = os.path.join(cliParams.targetPath, "DIRAC", "Core", - "scripts", "dirac_deploy_scripts.py") - if not os.path.isfile(ddeLocation): - ddeLocation = os.path.join(cliParams.targetPath, "DIRAC", "src", "DIRAC", "Core", - "scripts", "dirac_deploy_scripts.py") - if os.path.isfile(ddeLocation): - cmd = ddeLocation - - # if specified, create symlink instead of wrapper. - if cliParams.scriptSymlink: - cmd += ' --symlink' - - # In MacOS /usr/bin/env does not find python in the $PATH, passing binary path - # as an argument to the dirac-deploy-scripts - if not cliParams.platform: - cliParams.platform = getPlatform() - if "Darwin" in cliParams.platform: - binaryPath = os.path.join(cliParams.targetPath, cliParams.platform) - logNOTICE("For MacOS (Darwin) use explicit binary path %s" % binaryPath) - cmd += ' %s' % binaryPath - - os.system(cmd) - else: - logERROR("No dirac-deploy-scripts found. This doesn't look good") - else: - logNOTICE("Skipping installing DIRAC") - # we install with DIRACOS from v7rX DIRAC release - if cliParams.diracOS \ - or list(releaseConfig.prjRelCFG['DIRAC'])[0][1] not in '0123456789' \ - or int(list(releaseConfig.prjRelCFG['DIRAC'])[0][1]) > 6: - logNOTICE("Installing DIRAC OS %s..." % cliParams.diracOSVersion) - if cliParams.pythonVersion.startswith("3"): - if not installDiracOSPython3(releaseConfig): - sys.exit(1) - else: - if not installDiracOS(releaseConfig): - sys.exit(1) - if not createBashrcForDiracOS(): - sys.exit(1) - else: - logNOTICE("Installing %s externals..." % cliParams.externalsType) - if not installExternals(releaseConfig): - sys.exit(1) - if cliParams.noLcg: - logNOTICE("Skipping installation of LCG software...") - else: - logNOTICE("Installing LCG software...") - if not installLCGutils(releaseConfig): - sys.exit(1) - if not createBashrc(): - sys.exit(1) - if not createCshrc(): - sys.exit(1) - if not createOldProLinks(): - sys.exit(1) - runExternalsPostInstall() - writeDefaultConfiguration() - if cliParams.externalsType == "server": - fixMySQLScript() - installExternalRequirements(cliParams.externalsType) - if cliParams.createLink: - createSymbolicLink() - logNOTICE("%s properly installed" % cliParams.installation) - sys.exit(0) +# Download dirac-install.py +response = urlopen( + "https://raw.githubusercontent.com/DIRACGrid/management/master/dirac-install.py" +) +code = response.getcode() +if code > 200 or code >= 300: + raise RuntimeError("Failed to download dirac-install.py with code %s" % code) + +# Write dirac-install.py to a temporay file +tmpHandle, tmp = tempfile.mkstemp() +fp = os.fdopen(tmpHandle, "wb") +fp.write(response.read()) +fp.close() + +# Make the dirac-install.py temporay file executable +st = os.stat(tmp) +os.chmod(tmp, st.st_mode | stat.S_IEXEC) + +# Replace the current process with the actual dirac-install.py script +os.execv(tmp, sys.argv)