From 67ae911f127f121b22d9e99e72111bf934d570b3 Mon Sep 17 00:00:00 2001 From: Chris Burr Date: Thu, 25 Feb 2021 08:10:39 +0100 Subject: [PATCH 1/2] Add dirac-install.py --- dirac-install.py | 2724 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 2724 insertions(+) create mode 100755 dirac-install.py diff --git a/dirac-install.py b/dirac-install.py new file mode 100755 index 0000000..6fe5e5d --- /dev/null +++ b/dirac-install.py @@ -0,0 +1,2724 @@ +#!/usr/bin/env python +""" +The main DIRAC installer script. It can be used to install the main DIRAC software, its +modules, web, rest etc. and DIRAC extensions. + +In order to deploy DIRAC you have to provide: globalDefaultsURL, which is by default: +"http://diracproject.web.cern.ch/diracproject/configs/globalDefaults.cfg", but it can be +in the local file system in a separate directory. The content of this file is the following:: + + Installations + { + DIRAC + { + DefaultsLocation = http://diracproject.web.cern.ch/diracproject/dirac.cfg + LocalInstallation + { + PythonVersion = 27 + } + # in case you have a DIRAC extension + LHCb + { + DefaultsLocation = http://lhcb-rpm.web.cern.ch/lhcb-rpm/lhcbdirac/lhcb.cfg + } + } + } + Projects + { + DIRAC + { + DefaultsLocation = http://diracproject.web.cern.ch/diracproject/dirac.cfg + } + # in case you have a DIRAC extension + LHCb + { + DefaultsLocation = http://lhcb-rpm.web.cern.ch/lhcb-rpm/lhcbdirac/lhcb.cfg + } + } + +the DefaultsLocation for example:: + + DefaultsLocation = http://diracproject.web.cern.ch/diracproject/dirac.cfg + +must contain a minimal configuration. The following options must be in this +file:: + + Releases=,UploadCommand=,BaseURL= + +In case you want to overwrite the global configuration file, you have to use --defaultsURL + +After providing the default configuration files, DIRAC or your extension can be installed from: + +1. in a directory you have to be present globalDefaults.cfg, dirac.cfg and all binaries. + For example:: + + zmathe@dzmathe zmathe]$ ls tars/ + dirac.cfg diracos-0.1.md5 diracos-0.1.tar.gz DIRAC-v6r20-pre16.md5 DIRAC-v6r20-pre16.tar.gz + globalDefaults.cfg release-DIRAC-v6r20-pre16.cfg release-DIRAC-v6r20-pre16.md5 + zmathe@dzmathe zmathe]$ + + For example:: + + dirac-install -r v6r20-pre16 --dirac-os --dirac-os-version=0.0.1 -u /home/zmathe/tars + + this command will use /home/zmathe/tars directory for the source code. + It will install DIRAC v6r20-pre16, DIRAC OS 0.1 version + +2. You can use your dedicated web server or the official DIRAC web server + + for example:: + + dirac-install -r v6r20-pre16 --dirac-os --dirac-os-version=0.0.1 + + It will install DIRAC v6r20-pre16 + + You can install an extension of diracos. + + for example:: + + dirac-install -r v9r4-pre2 -l LHCb --dirac-os --dirac-os-version=LHCb:master + +3. You have possibility to install a not-yet-released DIRAC, module or extension using -m or --tag options. + The non release version can be specified. + + for example:: + + dirac-install -l DIRAC -r v6r20-pre16 -g v14r0 -t client -m DIRAC --tag=integration + + It will install DIRAC v6r20-pre16, where the DIRAC package based on integration, other other packages will be + the same what is specified in release.cfg file in v6r20-pre16 tarball. + + dirac-install -l DIRAC -r v6r20-pre16 -g v14r0 -t client -m DIRAC --tag=v6r20-pre22 + + It installs a specific tag + + Note: If the source is not provided, DIRAC repository is used, which is defined in the global + configuration file. + + We can provide the repository url:code repository:::Project:::branch. for example:: + + dirac-install -l DIRAC -r v6r20-pre16 -g v14r0 -t client \\ + -m https://github.com/zmathe/DIRAC.git:::DIRAC:::dev_main_branch, \\ + https://github.com/zmathe/WebAppDIRAC.git:::WebAppDIRAC:::extjs6 -e WebAppDIRAC + + it will install DIRAC based on dev_main_branch and WebAppDIRAC based on extjs6:: + + dirac-install -l DIRAC -r v6r20-pre16 -g v14r0 -t client \\ + -m WebAppDIRAC --tag=integration -e WebAppDIRAC + + it will install DIRAC v6r20-pre16 and WebAppDIRAC integration branch + +You can use install.cfg configuration file:: + + DIRACOS = http://lhcb-rpm.web.cern.ch/lhcb-rpm/dirac/DIRACOS/ + WebAppDIRAC = https://github.com/zmathe/WebAppDIRAC.git + DIRAC=https://github.com/DIRACGrid/DIRAC.git + LocalInstallation + { + # Project = LHCbDIRAC + # The project LHCbDIRAC is not defined in the globalsDefaults.cfg + Project = LHCb + Release = v9r2-pre8 + Extensions = LHCb + ConfigurationServer = dips://lhcb-conf-dirac.cern.ch:9135/Configuration/Server + Setup = LHCb-Production + SkipCAChecks = True + SkipCADownload = True + WebAppDIRAC=extjs6 + DIRAC=rel-v6r20 + } + + dirac-install -l LHCb -r v9r2-pre8 -t server --dirac-os --dirac-os-version=0.0.6 install.cfg + +""" + +from __future__ import unicode_literals, absolute_import, division, print_function + +import sys +import os +import getopt +import imp +import signal +import time +import stat +import shutil +import subprocess +import ssl +import hashlib + +from distutils.version import LooseVersion # pylint: disable=no-name-in-module,import-error + +try: + # For Python 3.0 and later + from urllib.request import urlopen, HTTPError, URLError +except ImportError: + # Fall back to Python 2's urllib2 + from urllib2 import urlopen, HTTPError, URLError +try: + str_type = basestring +except NameError: + str_type = str + +__RCSID__ = "$Id$" + +executablePerms = stat.S_IWUSR | stat.S_IRUSR | stat.S_IXUSR | stat.S_IRGRP | stat.S_IXGRP | stat.S_IROTH | stat.S_IXOTH + + +def S_OK(value=""): + return {'OK': True, 'Value': value} + + +def S_ERROR(msg=""): + return {'OK': False, 'Message': msg} + +############ +# Start of CFG +############ + + +class Params(object): + + def __init__(self): + self.extensions = [] + self.project = 'DIRAC' + self.installation = 'DIRAC' + self.release = "" + self.externalsType = 'client' + self.pythonVersion = '27' + self.platform = "" + self.basePath = os.getcwd() + self.targetPath = os.getcwd() + self.buildExternals = False + self.noAutoBuild = False + self.debug = False + self.externalsOnly = False + self.lcgVer = '' + self.noLcg = False + self.useVersionsDir = False + self.installSource = "" + self.globalDefaults = False + self.timeout = 300 + self.diracOSVersion = '' + self.diracOS = False + self.tag = "" + self.modules = {} + self.externalVersion = "" + self.createLink = False + self.scriptSymlink = False + self.userEnvVariables = {} + + +cliParams = Params() + +### +# Release config manager +### + + +class ReleaseConfig(object): + + class CFG(object): + + def __init__(self, cfgData=""): + """ c'tor + :param self: self reference + :param str cfgData: the content of the configuration file + """ + self.data = {} + self.children = {} + if cfgData: + self.parse(cfgData) + + def parse(self, cfgData): + """ + It parses the configuration file and propagate the data and children + with the content of the cfg file + :param str cfgData: configuration data, which is the content of the configuration file + """ + try: + self.__parse(cfgData) + except BaseException: + import traceback + traceback.print_exc() + raise + return self + + def getChild(self, path): + """ + It return the child of a given section + :param str, list, tuple path: for example: Installations/DIRAC, Projects/DIRAC + :return object It returns a CFG instance + """ + + child = self + if isinstance(path, (list, tuple)): + pathList = path + else: + pathList = [sec.strip() for sec in path.split("/") if sec.strip()] + for childName in pathList: + if childName not in child.children: + return False + child = child.children[childName] + return child + + def __parse(self, cfgData, cIndex=0): + """ + It parse a given DIRAC cfg file and store the result in self.data variable. + + :param str cfgData: the content of the configuration file + :param int cIndex: it is the new line counter + """ + + childName = "" + numLine = 0 + while cIndex < len(cfgData): + eol = cfgData.find("\n", cIndex) + if eol < cIndex: + # End? + return cIndex + numLine += 1 + if eol == cIndex: + cIndex += 1 + continue + line = cfgData[cIndex: eol].strip() + # Jump EOL + cIndex = eol + 1 + if not line or line[0] == "#": + continue + if line.find("+=") > -1: + fields = line.split("+=") + opName = fields[0].strip() + if opName in self.data: + self.data[opName] += ', %s' % '+='.join(fields[1:]).strip() + else: + self.data[opName] = '+='.join(fields[1:]).strip() + continue + + if line.find("=") > -1: + fields = line.split("=") + self.data[fields[0].strip()] = "=".join(fields[1:]).strip() + continue + + opFound = line.find("{") + if opFound > -1: + childName += line[:opFound].strip() + if not childName: + raise Exception("No section name defined for opening in line %s" % numLine) + childName = childName.strip() + self.children[childName] = ReleaseConfig.CFG() + eoc = self.children[childName].__parse(cfgData, cIndex) + cIndex = eoc + childName = "" + continue + + if line == "}": + return cIndex + # Must be name for section + childName += line.strip() + return cIndex + + def createSection(self, name, cfg=None): + """ + It creates a subsection for an existing CS section. + :param str name: the name of the section + :param object cfg: the ReleaseConfig.CFG object loaded into memory + """ + + if isinstance(name, (list, tuple)): + pathList = name + else: + pathList = [sec.strip() for sec in name.split("/") if sec.strip()] + parent = self + for lev in pathList[:-1]: + if lev not in parent.children: + parent.children[lev] = ReleaseConfig.CFG() + parent = parent.children[lev] + secName = pathList[-1] + if secName not in parent.children: + if not cfg: + cfg = ReleaseConfig.CFG() + parent.children[secName] = cfg + return parent.children[secName] + + def isSection(self, obList): + """ + Checks if a given path is a section + :param str objList: is a path: for example: Releases/v6r20-pre16 + """ + return self.__exists([ob.strip() for ob in obList.split("/") if ob.strip()]) == 2 + + def sections(self): + """ + Returns all sections + """ + return [k for k in self.children] + + def isOption(self, obList): + return self.__exists([ob.strip() for ob in obList.split("/") if ob.strip()]) == 1 + + def options(self): + """ + Returns the options + """ + return [k for k in self.data] + + def __exists(self, obList): + """ + Check the existence of a certain element + + :param list obList: the list of cfg element names. + for example: [Releases,v6r20-pre16] + """ + if len(obList) == 1: + if obList[0] in self.children: + return 2 + elif obList[0] in self.data: + return 1 + else: + return 0 + if obList[0] in self.children: + return self.children[obList[0]].__exists(obList[1:]) + return 0 + + def get(self, opName, defaultValue=None): + """ + It return the value of a certain option + + :param str opName: the name of the option + :param str defaultValue: the default value of a given option + """ + try: + value = self.__get([op.strip() for op in opName.split("/") if op.strip()]) + except KeyError: + if defaultValue is not None: + return defaultValue + raise + if defaultValue is None: + return value + defType = type(defaultValue) + if isinstance(defType, bool): + return value.lower() in ("1", "true", "yes") + try: + return defType(value) + except ValueError: + return defaultValue + + def __get(self, obList): + """ + It return a given section + + :param list obList: the list of cfg element names. + """ + if len(obList) == 1: + if obList[0] in self.data: + return self.data[obList[0]] + raise KeyError("Missing option %s" % obList[0]) + if obList[0] in self.children: + return self.children[obList[0]].__get(obList[1:]) + raise KeyError("Missing section %s" % obList[0]) + + def toString(self, tabs=0): + """ + It return the configuration file as a string + :param int tabs: the number of tabs used to format the CS string + """ + + lines = ["%s%s = %s" % (" " * tabs, opName, self.data[opName]) for opName in self.data] + for secName in self.children: + lines.append("%s%s" % (" " * tabs, secName)) + lines.append("%s{" % (" " * tabs)) + lines.append(self.children[secName].toString(tabs + 1)) + lines.append("%s}" % (" " * tabs)) + return "\n".join(lines) + + def getOptions(self, path=""): + """ + Rturns the options for a given path + + :param str path: the path to the CS element + """ + parentPath = [sec.strip() for sec in path.split("/") if sec.strip()][:-1] + if parentPath: + parent = self.getChild(parentPath) + else: + parent = self + if not parent: + return [] + return tuple(parent.data) + + def delPath(self, path): + """ + It deletes a given CS element + + :param str path: the path to the CS element + """ + path = [sec.strip() for sec in path.split("/") if sec.strip()] + if not path: + return + keyName = path[-1] + parentPath = path[:-1] + if parentPath: + parent = self.getChild(parentPath) + else: + parent = self + if parent: + parent.data.pop(keyName) + + def update(self, path, cfg): + """ + Used to update the CS + + :param str path: path to the CS element + :param object cfg: the CS object + """ + parent = self.getChild(path) + if not parent: + self.createSection(path, cfg) + return + parent.__apply(cfg) + + def __apply(self, cfg): + """ + It adds a certain cfg subsection to a given section + + :param object cfg: the CS object + """ + for k in cfg.sections(): + if k in self.children: + self.children[k].__apply(cfg.getChild(k)) + else: + self.children[k] = cfg.getChild(k) + for k in cfg.options(): + self.data[k] = cfg.get(k) +############################################################################ +# END OF CFG CLASS +############################################################################ + + def __init__(self, instName='DIRAC', projectName='DIRAC', globalDefaultsURL=None): + """ c'tor + :param str instName: the name of the installation + :param str projectName: the name of the project + :param str globalDefaultsURL: the default url + """ + if globalDefaultsURL: + self.globalDefaultsURL = globalDefaultsURL + else: + self.globalDefaultsURL = "http://diracproject.web.cern.ch/diracproject/configs/globalDefaults.cfg" + self.globalDefaults = ReleaseConfig.CFG() + self.loadedCfgs = [] + self.prjDepends = {} + self.diracBaseModules = {} + self.prjRelCFG = {} + self.projectsLoadedBy = {} + self.cfgCache = {} + + self.debugCB = False + self.instName = instName + self.projectName = projectName + + def setDebugCB(self, debFunc): + """ + It is used by the dirac-distribution. It sets the debug function + """ + self.debugCB = debFunc + + def __dbgMsg(self, msg): + """ + :param str msg: the debug message + """ + if self.debugCB: + self.debugCB(msg) + + def __loadCFGFromURL(self, urlcfg, checkHash=False): + """ + It is used to load the configuration file + + :param str urlcfg: the location of the source repository and + where the default configuration file is exists. + :param bool checkHash: check if the file is corrupted. + """ + # This can be a local file + if os.path.exists(urlcfg): + with open(urlcfg, 'r') as relFile: + cfgData = relFile.read() + else: + if urlcfg in self.cfgCache: + return S_OK(self.cfgCache[urlcfg]) + try: + cfgData = urlretrieveTimeout(urlcfg, timeout=cliParams.timeout) + if not cfgData: + return S_ERROR("Could not get data from %s" % urlcfg) + except BaseException: + return S_ERROR("Could not open %s" % urlcfg) + try: + # cfgData = cfgFile.read() + cfg = ReleaseConfig.CFG(cfgData) + except Exception as excp: + return S_ERROR("Could not parse %s: %s" % (urlcfg, excp)) + # cfgFile.close() + if not checkHash: + self.cfgCache[urlcfg] = cfg + return S_OK(cfg) + try: + md5path = urlcfg[:-4] + ".md5" + if os.path.exists(md5path): + md5File = open(md5path, 'r') + md5Data = md5File.read() + md5File.close() + else: + md5Data = urlretrieveTimeout(md5path, timeout=60) + md5Hex = md5Data.strip() + # md5File.close() + if md5Hex != hashlib.md5(cfgData.encode('utf-8')).hexdigest(): + return S_ERROR("Hash check failed on %s" % urlcfg) + except Exception as excp: + return S_ERROR("Hash check failed on %s: %s" % (urlcfg, excp)) + self.cfgCache[urlcfg] = cfg + return S_OK(cfg) + + def loadInstallationDefaults(self): + """ + Load the default configurations + """ + result = self.__loadGlobalDefaults() + if not result['OK']: + return result + return self.__loadObjectDefaults("Installations", self.instName) + + def loadProjectDefaults(self): + """ + Load default configurations + """ + result = self.__loadGlobalDefaults() + if not result['OK']: + return result + return self.__loadObjectDefaults("Projects", self.projectName) + + def __loadGlobalDefaults(self): + """ + It loads the default configuration files + """ + + globalDefaultsCVMFSPath = "/cvmfs/dirac.egi.eu/admin/globalDefaults.cfg" + self.__dbgMsg("Loading global defaults from: %s" % globalDefaultsCVMFSPath) + result = self.__loadCFGFromURL(globalDefaultsCVMFSPath) + if not result['OK']: + self.__dbgMsg("Loading global defaults from: %s" % self.globalDefaultsURL) + result = self.__loadCFGFromURL(self.globalDefaultsURL) + if not result['OK']: + return result + self.globalDefaults = result['Value'] + for k in ("Installations", "Projects"): + if not self.globalDefaults.isSection(k): + self.globalDefaults.createSection(k) + self.__dbgMsg("Loaded global defaults") + return S_OK() + + def __loadObjectDefaults(self, rootPath, objectName): + """ + It loads the CFG, if it is not loaded. + :param str rootPath: the main section. for example: Installations + :param str objectName: The name of the section. for example: DIRAC + """ + + basePath = "%s/%s" % (rootPath, objectName) + if basePath in self.loadedCfgs: + return S_OK() + + # Check if it's a direct alias + try: + aliasTo = self.globalDefaults.get(basePath) + except KeyError: + aliasTo = False + + if aliasTo: + self.__dbgMsg("%s is an alias to %s" % (objectName, aliasTo)) + result = self.__loadObjectDefaults(rootPath, aliasTo) + if not result['OK']: + return result + cfg = result['Value'] + self.globalDefaults.update(basePath, cfg) + return S_OK() + + # Load the defaults + if self.globalDefaults.get("%s/SkipDefaults" % basePath, False): + defaultsLocation = "" + else: + defaultsLocation = self.globalDefaults.get("%s/DefaultsLocation" % basePath, "") + + if not defaultsLocation: + self.__dbgMsg("No defaults file defined for %s %s" % (rootPath.lower()[:-1], objectName)) + else: + self.__dbgMsg("Defaults for %s are in %s" % (basePath, defaultsLocation)) + result = self.__loadCFGFromURL(defaultsLocation) + if not result['OK']: + return result + cfg = result['Value'] + self.globalDefaults.update(basePath, cfg) + + # Check if the defaults have a sub alias + try: + aliasTo = self.globalDefaults.get("%s/Alias" % basePath) + except KeyError: + aliasTo = False + + if aliasTo: + self.__dbgMsg("%s is an alias to %s" % (objectName, aliasTo)) + result = self.__loadObjectDefaults(rootPath, aliasTo) + if not result['OK']: + return result + cfg = result['Value'] + self.globalDefaults.update(basePath, cfg) + + self.loadedCfgs.append(basePath) + return S_OK(self.globalDefaults.getChild(basePath)) + + def loadInstallationLocalDefaults(self, args): + """ + Load the configuration file from a file + + :param str args: the arguments in which to look for configuration file names + """ + + # at the end we load the local configuration and merge it with the global cfg + argList = list(args) + if os.path.exists('etc/dirac.cfg') and 'etc/dirac.cfg' not in args: + argList = ['etc/dirac.cfg'] + argList + + for arg in argList: + if arg.endswith(".cfg") and ':::' not in arg: + fileName = arg + else: + continue + + logNOTICE("Defaults for LocalInstallation are in %s" % fileName) + try: + fd = open(fileName, "r") + cfg = ReleaseConfig.CFG().parse(fd.read()) + fd.close() + except Exception as excp: + logERROR("Could not load %s: %s" % (fileName, excp)) + continue + + self.globalDefaults.update("Installations/%s" % self.instName, cfg) + self.globalDefaults.update("Projects/%s" % self.instName, cfg) + if self.projectName: + # we have an extension and have a local cfg file + self.globalDefaults.update("Projects/%s" % self.projectName, cfg) + + logNOTICE("Loaded %s" % arg) + + def getModuleVersionFromLocalCfg(self, moduleName): + """ + It returns the version of a certain module defined in the LocalInstallation section + :param str moduleName: + :return str: the version of a certain module + """ + return self.globalDefaults.get("Installations/%s/LocalInstallation/%s" % (self.instName, moduleName), "") + + def getInstallationCFG(self, instName=None): + """ + Returns the installation name + + :param str instName: the installation name + """ + if not instName: + instName = self.instName + return self.globalDefaults.getChild("Installations/%s" % instName) + + def getInstallationConfig(self, opName, instName=None): + """ + It returns the configurations from the Installations section. + This is usually provided in the local configuration file + + :param str opName: the option name for example: LocalInstallation/Release + :param str instName: + """ + if not instName: + instName = self.instName + return self.globalDefaults.get("Installations/%s/%s" % (instName, opName)) + + def isProjectLoaded(self, project): + """ + Checks if the project is loaded. + + :param str project: the name of the project + """ + return project in self.prjRelCFG + + def getTarsLocation(self, project, module=None): + """ + Returns the location of the binaries for a given project for example: LHCb or DIRAC, etc... + + :param str project: the name of the project + """ + sourceUrl = self.globalDefaults.get("Projects/%s/BaseURL" % project, "") + if module: + # in case we define a different URL in the CS + differntSourceUrl = self.globalDefaults.get("Projects/%s/%s" % (project, module), "") + if differntSourceUrl: + sourceUrl = differntSourceUrl + if sourceUrl: + return S_OK(sourceUrl) + return S_ERROR("Don't know how to find the installation tarballs for project %s" % project) + + def getDiracOsLocation(self, useVanillaDiracOS=False): + """ + Returns the location of the DIRAC os binary for a given project for example: LHCb or DIRAC, etc... + :param bool useVanillaDiracOS: flag to take diracos distribution from the default location + :return: the location of the tar balls + """ + keysToConsider = [] + if not useVanillaDiracOS: + keysToConsider += [ + "Installations/%s/DIRACOS" % self.projectName, + "Projects/%s/DIRACOS" % self.projectName, + ] + keysToConsider += [ + "Installations/DIRAC/DIRACOS", + "Projects/DIRAC/DIRACOS", + ] + + for key in keysToConsider: + location = self.globalDefaults.get(key, "") + if location: + logDEBUG("Using DIRACOS tarball URL from configuration key %s" % key) + return location + + def getUploadCommand(self, project=None): + """ + It returns the command used to upload the binary + + :param str project: the name of the project + """ + if not project: + project = self.projectName + defLoc = self.globalDefaults.get("Projects/%s/UploadCommand" % project, "") + if defLoc: + return S_OK(defLoc) + return S_ERROR("No UploadCommand for %s" % project) + + def __loadReleaseConfig(self, project, release, releaseMode, sourceURL=None, relLocation=None): + """ + It loads the release configuration file + + :param str project: the name of the project + :param str release: the release version + :param str releaseMode: the type of the release server/client + :param str sourceURL: the source of the binary + :param str relLocation: the release configuration file + """ + if project not in self.prjRelCFG: + self.prjRelCFG[project] = {} + if release in self.prjRelCFG[project]: + self.__dbgMsg("Release config for %s:%s has already been loaded" % (project, release)) + return S_OK() + + if relLocation: + relcfgLoc = relLocation + else: + if releaseMode: + try: + relcfgLoc = self.globalDefaults.get("Projects/%s/Releases" % project) + except KeyError: + return S_ERROR("Missing Releases file for project %s" % project) + else: + if not sourceURL: + result = self.getTarsLocation(project) + if not result['OK']: + return result + siu = result['Value'] + else: + siu = sourceURL + relcfgLoc = "%s/release-%s-%s.cfg" % (siu, project, release) + self.__dbgMsg("Releases file is %s" % relcfgLoc) + result = self.__loadCFGFromURL(relcfgLoc, checkHash=not releaseMode) + if not result['OK']: + return result + self.prjRelCFG[project][release] = result['Value'] + self.__dbgMsg("Loaded releases file %s" % relcfgLoc) + + return S_OK(self.prjRelCFG[project][release]) + + def getReleaseCFG(self, project, release): + """ + Returns the release configuration object + + :param str project: the name of the project + :param str release: the release version + """ + return self.prjRelCFG[project][release] + + def dumpReleasesToPath(self): + """ + It dumps the content of the loaded configuration (memory content) to + a given file + """ + for project in self.prjRelCFG: + prjRels = self.prjRelCFG[project] + for release in prjRels: + self.__dbgMsg("Dumping releases file for %s:%s" % (project, release)) + fd = open( + os.path.join( + cliParams.targetPath, "releases-%s-%s.cfg" % + (project, release)), "w") + fd.write(prjRels[release].toString()) + fd.close() + + def __checkCircularDependencies(self, key, routePath=None): + """ + Check the dependencies + + :param str key: the name of the project and the release version + :param list routePath: it stores the software packages, used to check the + dependency + """ + + if not routePath: + routePath = [] + if key not in self.projectsLoadedBy: + return S_OK() + routePath.insert(0, key) + for lKey in self.projectsLoadedBy[key]: + if lKey in routePath: + routePath.insert(0, lKey) + route = "->".join(["%s:%s" % sKey for sKey in routePath]) + return S_ERROR("Circular dependency found for %s: %s" % ("%s:%s" % lKey, route)) + result = self.__checkCircularDependencies(lKey, routePath) + if not result['OK']: + return result + routePath.pop(0) + return S_OK() + + def loadProjectRelease(self, releases, + project=None, + sourceURL=None, + releaseMode=None, + relLocation=None): + """ + This method loads all project configurations (*.cfg). If a project is an extension of DIRAC, + it will load the extension and after will load the base DIRAC module. + + :param list releases: list of releases, which will be loaded: for example: v6r19 + :param str project: the name of the project, if it is given. For example: DIRAC + :param str sourceURL: the code repository + :param str releaseMode: + :param str relLocation: local configuration file, + which contains the releases. for example: file:///`pwd`/releases.cfg + """ + + if not project: + project = self.projectName + + if not isinstance(releases, (list, tuple)): + releases = [releases] + + # Load defaults + result = self.__loadObjectDefaults("Projects", project) + if not result['OK']: + self.__dbgMsg("Could not load defaults for project %s" % project) + return result + + if project not in self.prjDepends: + self.prjDepends[project] = {} + + for release in releases: + self.__dbgMsg("Processing dependencies for %s:%s" % (project, release)) + result = self.__loadReleaseConfig(project, release, releaseMode, sourceURL, relLocation) + if not result['OK']: + return result + relCFG = result['Value'] + # Calculate dependencies and avoid circular deps + self.prjDepends[project][release] = [(project, release)] + relDeps = self.prjDepends[project][release] + + if not relCFG.getChild("Releases/%s" % (release)): # pylint: disable=no-member + return S_ERROR( + "Release %s is not defined for project %s in the release file" % + (release, project)) + + initialDeps = self.getReleaseDependencies(project, release) + if initialDeps: + self.__dbgMsg("%s %s depends on %s" % + (project, release, ", ".join(["%s:%s" % + (k, initialDeps[k]) for k in initialDeps]))) + relDeps.extend([(p, initialDeps[p]) for p in initialDeps]) + for depProject in initialDeps: + depVersion = initialDeps[depProject] + # Check if already processed + dKey = (depProject, depVersion) + if dKey not in self.projectsLoadedBy: + self.projectsLoadedBy[dKey] = [] + self.projectsLoadedBy[dKey].append((project, release)) + result = self.__checkCircularDependencies(dKey) + if not result['OK']: + return result + # if it has already been processed just return OK + if len(self.projectsLoadedBy[dKey]) > 1: + return S_OK() + + # Load dependencies and calculate incompatibilities + result = self.loadProjectRelease(depVersion, project=depProject) + if not result['OK']: + return result + subDep = self.prjDepends[depProject][depVersion] + # Merge dependencies + for sKey in subDep: + if sKey not in relDeps: + relDeps.append(sKey) + continue + prj, vrs = sKey + for pKey in relDeps: + if pKey[0] == prj and pKey[1] != vrs: + errMsg = "%s is required with two different versions ( %s and %s ) \ + starting with %s:%s" % (prj, + pKey[1], vrs, + project, release) + return S_ERROR(errMsg) + + # Same version already required + if project in relDeps and relDeps[project] != release: + errMsg = "%s:%s requires itself with a different version through dependencies ( %s )" % ( + project, release, relDeps[project]) + return S_ERROR(errMsg) + + # we have now all dependencies, let's retrieve the resources (code repository) + for project, version in relDeps: + if project in self.diracBaseModules: + continue + modules = self.getModulesForRelease(version, project) + if modules['OK']: + for dependency in modules['Value']: + self.diracBaseModules.setdefault(dependency, {}) + self.diracBaseModules[dependency]['Version'] = modules['Value'][dependency] + res = self.getModSource(version, dependency, project) + if not res['OK']: + self.__dbgMsg( + "Unable to found the source URL for %s : %s" % + (dependency, res['Message'])) + else: + self.diracBaseModules[dependency]['sourceUrl'] = res['Value'][1] + + return S_OK() + + def getReleaseOption(self, project, release, option): + """ + Returns a given option + + :param str project: the name of the project + :param str release: the release version + :param str option: the option name + """ + try: + return self.prjRelCFG[project][release].get(option) + except KeyError: + self.__dbgMsg("Missing option %s for %s:%s" % (option, project, release)) + # try to found the option in a different release + for project in self.prjRelCFG: + for release in self.prjRelCFG[project]: + if self.prjRelCFG[project][release].isOption(option): + return self.prjRelCFG[project][release].get(option) + return False + + def getReleaseDependencies(self, project, release): + """ + It return the dependencies for a certain project + + :param str project: the name of the project + :param str release: the release version + """ + try: + data = self.prjRelCFG[project][release].get("Releases/%s/Depends" % release) + except KeyError: + return {} + data = [field for field in data.split(",") if field.strip()] + deps = {} + for field in data: + field = field.strip() + if not field: + continue + pv = field.split(":") + if len(pv) == 1: + deps[pv[0].strip()] = release + else: + deps[pv[0].strip()] = ":".join(pv[1:]).strip() + return deps + + def getModulesForRelease(self, release, project=None): + """ + Returns the modules for a given release for example: WebAppDIRAC, + RESTDIRAC, LHCbWebAppDIRAC, etc + + :param str release: the release version + :param str project: the project name + """ + if not project: + project = self.projectName + if project not in self.prjRelCFG: + return S_ERROR("Project %s has not been loaded. I'm a MEGA BUG! Please report me!" % project) + if release not in self.prjRelCFG[project]: + return S_ERROR("Version %s has not been loaded for project %s" % (release, project)) + config = self.prjRelCFG[project][release] + if not config.isSection("Releases/%s" % release): + return S_ERROR("Release %s is not defined for project %s" % (release, project)) + # Defined Modules explicitly in the release + modules = self.getReleaseOption(project, release, "Releases/%s/Modules" % release) + if modules: + dMods = {} + for entry in [entry.split(":") for entry in modules.split( + ",") if entry.strip()]: # pylint: disable=no-member + if len(entry) == 1: + dMods[entry[0].strip()] = release + else: + dMods[entry[0].strip()] = entry[1].strip() + modules = dMods + else: + # Default modules with the same version as the release version + modules = self.getReleaseOption(project, release, "DefaultModules") + if modules: + modules = dict((modName.strip(), release) for modName in modules.split(",") + if modName.strip()) # pylint: disable=no-member + else: + # Mod = project and same version + modules = {project: release} + # Check project is in the modNames if not DIRAC + if project != "DIRAC": + for modName in modules: + if modName.find(project) != 0: + return S_ERROR("Module %s does not start with the name %s" % (modName, project)) + return S_OK(modules) + + def getModSource(self, release, modName, project=None): + """ + It reads the Sources section from the .cfg file for example: + Sources + { + Web = git://github.com/DIRACGrid/DIRACWeb.git + VMDIRAC = git://github.com/DIRACGrid/VMDIRAC.git + DIRAC = git://github.com/DIRACGrid/DIRAC.git + BoincDIRAC = git://github.com/DIRACGrid/BoincDIRAC.git + RESTDIRAC = git://github.com/DIRACGrid/RESTDIRAC.git + COMDIRAC = git://github.com/DIRACGrid/COMDIRAC.git + WebAppDIRAC = git://github.com/DIRACGrid/WebAppDIRAC.git + } + + :param str release: the release which is already loaded for example: v6r19 + :param str modName: the name of the DIRAC module for example: WebAppDIRAC + :param str project: the name of the project for example: DIRAC + """ + + if self.projectName not in self.prjRelCFG: + return S_ERROR( + "Project %s has not been loaded. I'm a MEGA BUG! Please report me!" % + self.projectName) + + if not project: + project = self.projectName + modLocation = self.getReleaseOption(project, release, "Sources/%s" % modName) + if not modLocation: + return S_ERROR("Source origin for module %s is not defined" % modName) + modTpl = [field.strip() for field in modLocation.split( + "|") if field.strip()] # pylint: disable=no-member + if len(modTpl) == 1: + return S_OK((False, modTpl[0])) + return S_OK((modTpl[0], modTpl[1])) + + def getExternalsVersion(self, release=None): + """ + It returns the version of DIRAC Externals. If it is not provided, + uses the default cfg + + :param str release: the release version + """ + + if 'DIRAC' not in self.prjRelCFG: + return False + if not release: + release = list(self.prjRelCFG['DIRAC']) + release = sorted(release, key=LooseVersion)[-1] + try: + return self.prjRelCFG['DIRAC'][release].get('Releases/%s/Externals' % release) + except KeyError: + return False + + def getDiracOSExtensionAndVersion(self, diracOSVersion): + """ + This method return the diracos and version taking into + account the extension. The file format will be .tar.gz + + :param str diracOSVersion: column separated string for example: LHCb:v1 + :return: if the extension is not provided, it will return DIRACOS defined in DIRAC otherwise + the DIRACOS specified in the extension + """ + if ":" in diracOSVersion: + package, packageVersion = [i.strip() for i in diracOSVersion.split(':')] + return [package + 'diracos', packageVersion] + else: + return ['diracos', diracOSVersion] + + def getDiracOSVersion(self, diracOSVersion=None): + """ + It returns the DIRACOS version + :param str diracOSVersion: the OS version + """ + + if diracOSVersion: + return self.getDiracOSExtensionAndVersion(diracOSVersion) + try: + diracOSVersion = self.prjRelCFG[self.projectName][cliParams.release].get( + "Releases/%s/DIRACOS" % cliParams.release, diracOSVersion) + if not diracOSVersion: + # the DIRAC extension does not specify DIRACOS version + for release in self.prjRelCFG['DIRAC']: + logWARN("Getting DIRACOS version from DIRAC %s!" % release) + diracOSVersion = self.prjRelCFG['DIRAC'][release].get( + "Releases/%s/DIRACOS" % release, diracOSVersion) + except KeyError: + pass + return self.getDiracOSExtensionAndVersion(diracOSVersion) + + def getLCGVersion(self, lcgVersion=None): + """ + It returns the LCG version + :param str lcgVersion: LCG version + """ + if lcgVersion: + return lcgVersion + try: + return self.prjRelCFG[self.projectName][cliParams.release].get( + "Releases/%s/LcgVer" % cliParams.release, lcgVersion) + except KeyError: + pass + return lcgVersion + + def getModulesToInstall(self, release, extensions=None): + """ + It returns the modules to be installed. + :param str release: the release version to be deployed + :param str extensions: DIRAC extension + :return: the order of the nodules and modules to be installed. + """ + if not extensions: + extensions = [] + extraFound = [] + modsToInstall = {} + modsOrder = [] + if self.projectName not in self.prjDepends: + return S_ERROR("Project %s has not been loaded" % self.projectName) + if release not in self.prjDepends[self.projectName]: + return S_ERROR( + "Version %s has not been loaded for project %s" % + (release, self.projectName)) + # Get a list of projects with their releases + projects = list(self.prjDepends[self.projectName][release]) + for project, relVersion in projects: + try: + requiredModules = self.prjRelCFG[project][relVersion].get("RequiredExtraModules") + requiredModules = [modName.strip() + for modName in requiredModules.split("/") if modName.strip()] + except KeyError: + requiredModules = [] + for modName in requiredModules: + if modName not in extensions: + extensions.append(modName) + self.__dbgMsg("Discovering modules to install for %s (%s)" % (project, relVersion)) + result = self.getModulesForRelease(relVersion, project) + if not result['OK']: + return result + modVersions = result['Value'] + try: + defaultMods = self.prjRelCFG[project][relVersion].get("DefaultModules") + modNames = [mod.strip() for mod in defaultMods.split(",") if mod.strip()] + except KeyError: + modNames = [] + for extension in extensions: + # Check if the version of the extension module is specified in the command line + extraVersion = None + if ":" in extension: + extension, extraVersion = extension.split(":") + modVersions[extension] = extraVersion + if extension in modVersions: + modNames.append(extension) + extraFound.append(extension) + if 'DIRAC' not in extension: + dextension = "%sDIRAC" % extension + if dextension in modVersions: + modNames.append(dextension) + extraFound.append(extension) + modNameVer = ["%s:%s" % (modName, modVersions[modName]) for modName in modNames] + self.__dbgMsg("Modules to be installed for %s are: %s" % (project, ", ".join(modNameVer))) + for modName in modNames: + result = self.getTarsLocation(project, modName) + if not result['OK']: + return result + tarsURL = result['Value'] + modVersion = modVersions[modName] + defLoc = self.getModuleVersionFromLocalCfg(modName) + if defLoc: + modVersion = defLoc # this overwrite the version which are defined in the release.cfg + modsToInstall[modName] = (tarsURL, modVersion) + modsOrder.insert(0, modName) + + for modName in extensions: + if modName.split(":")[0] not in extraFound: + return S_ERROR("No module %s defined. You sure it's defined for this release?" % modName) + + return S_OK((modsOrder, modsToInstall)) + + +################################################################################# +# End of ReleaseConfig +################################################################################# + + +# platformAlias = { 'Darwin_i386_10.6' : 'Darwin_i386_10.5' } +platformAlias = {} + +#### +# Start of helper functions +#### + + +def logDEBUG(msg): + """ + :param str msg: debug message + """ + if cliParams.debug: + for line in msg.split("\n"): + print("%s UTC dirac-install [DEBUG] %s" % (time.strftime('%Y-%m-%d %H:%M:%S', + time.gmtime()), + line)) + sys.stdout.flush() + + +def logERROR(msg): + """ + :param str msg: error message + """ + for line in msg.split("\n"): + print("%s UTC dirac-install [ERROR] %s" % (time.strftime('%Y-%m-%d %H:%M:%S', + time.gmtime()), + line)) + sys.stdout.flush() + + +def logWARN(msg): + """ + :param str msg: warning message + """ + for line in msg.split("\n"): + print("%s UTC dirac-install [WARN] %s" % (time.strftime('%Y-%m-%d %H:%M:%S', + time.gmtime()), + line)) + sys.stdout.flush() + + +def logNOTICE(msg): + """ + :param str msg: notice message + """ + for line in msg.split("\n"): + print("%s UTC dirac-install [NOTICE] %s" % (time.strftime('%Y-%m-%d %H:%M:%S', + time.gmtime()), + line)) + sys.stdout.flush() + + +def alarmTimeoutHandler(*args): + """ + When a connection time out then raise and exception + """ + raise Exception('Timeout') + + +def urlretrieveTimeout(url, fileName='', timeout=0, retries=3): + """ + Retrieve remote url to local file, with timeout wrapper + + :param str fileName: file name + :param int timeout: time out in second used for downloading the files. + """ + if fileName: + # This can be a local file + if os.path.exists(url): # we do not download from web, use locally + logDEBUG('Local file used: "%s"' % url) + shutil.copy(url, fileName) + return True + localFD = open(fileName, "wb") + + # NOTE: Not thread-safe, since all threads will catch same alarm. + # This is OK for dirac-install, since there are no threads. + logDEBUG('Retrieving remote file "%s"' % url) + + urlData = '' + if timeout: + signal.signal(signal.SIGALRM, alarmTimeoutHandler) + # set timeout alarm + signal.alarm(timeout + 5) + try: + # if "http_proxy" in os.environ and os.environ['http_proxy']: + # proxyIP = os.environ['http_proxy'] + # proxy = urllib2.ProxyHandler( {'http': proxyIP} ) + # opener = urllib2.build_opener( proxy ) + # #opener = urllib2.build_opener() + # urllib2.install_opener( opener ) + + # Try to use insecure context explicitly, needed for python >= 2.7.9 + try: + context = ssl._create_unverified_context() + remoteFD = urlopen(url, context=context) # pylint: disable=unexpected-keyword-arg + # the keyword 'context' is present from 2.7.9+ + except AttributeError: + remoteFD = urlopen(url) + + expectedBytes = 0 + # Sometimes repositories do not return Content-Length parameter + try: + expectedBytes = int(remoteFD.info()['Content-Length']) + except Exception: + logWARN('Content-Length parameter not returned, skipping expectedBytes check') + + receivedBytes = 0 + data = remoteFD.read(16384) + count = 1 + progressBar = False + while data: + receivedBytes += len(data) + if fileName: + localFD.write(data) + else: + urlData += data.decode('utf8', 'ignore') + data = remoteFD.read(16384) + if count % 20 == 0 and sys.stdout.isatty(): + print(u'\033[1D' + ".", end=" ") + sys.stdout.flush() + progressBar = True + count += 1 + if progressBar and sys.stdout.isatty(): + # return cursor to the beginning of the line + print('\033[1K', end=" ") + print('\033[1A') + if fileName: + localFD.close() + remoteFD.close() + if receivedBytes != expectedBytes and expectedBytes > 0: + logERROR("File should be %s bytes but received %s" % (expectedBytes, receivedBytes)) + return False + except HTTPError as x: + if x.code == 404: + logERROR("%s does not exist" % url) + if timeout: + signal.alarm(0) + return False + else: + logWARN('Status code accessing URL %s was %s' % (url, x.getcode())) + if retries: + logWARN('Will retry after 30 seconds, %s retries remaining' % retries) + time.sleep(30) + return urlretrieveTimeout(url, fileName=fileName, timeout=timeout, retries=retries - 1) + else: + logERROR("No retries remaining for %s" % url) + sys.exit(1) + except URLError: + logERROR('Timeout after %s seconds on transfer request for "%s"' % (str(timeout), url)) + except Exception as x: + if x == 'Timeout': + logERROR('Timeout after %s seconds on transfer request for "%s"' % (str(timeout), url)) + if timeout: + signal.alarm(0) + raise x + if timeout: + signal.alarm(0) + + if fileName: + return True + else: + return urlData + + +def downloadAndExtractTarball(tarsURL, pkgName, pkgVer, checkHash=True, cache=False): + """ + It downloads and extracts a given tarball from a given destination: file system, + web server or code repository. + + :param str tarsURL: the location of the source repository + :param str pkgName: the name of the package to be installed + :param str pkgVer: the version of the package + :param bool checkHash: check the sanity of the file + :param bool cache: use local cache for the tarballs + + """ + tarName = "%s-%s.tar.gz" % (pkgName, pkgVer) + tarPath = os.path.join(cliParams.targetPath, tarName) + tarFileURL = "%s/%s" % (tarsURL, tarName) + tarFileCVMFS = "/cvmfs/dirac.egi.eu/installSource/%s" % tarName + cacheDir = os.path.join(cliParams.basePath, ".installCache") + tarCachePath = os.path.join(cacheDir, tarName) + isSource = False + if cache and os.path.isfile(tarCachePath): + logNOTICE("Using cached copy of %s" % tarName) + shutil.copy(tarCachePath, tarPath) + elif os.path.exists(tarFileCVMFS): + logNOTICE("Using CVMFS copy of %s" % tarName) + tarPath = tarFileCVMFS + checkHash = False + cache = False + else: + logNOTICE("Retrieving %s" % tarFileURL) + try: + if not urlretrieveTimeout(tarFileURL, tarPath, cliParams.timeout): + if os.path.exists(tarPath): + os.unlink(tarPath) + retVal = checkoutFromGit(pkgName, tarsURL, pkgVer) + if not retVal['OK']: + logERROR("Cannot download %s" % tarName) + logERROR("Cannot download %s" % retVal['Message']) + return False + else: + isSource = True + except Exception as e: + logERROR("Cannot download %s: %s" % (tarName, str(e))) + sys.exit(1) + if not isSource and checkHash: + md5Name = "%s-%s.md5" % (pkgName, pkgVer) + md5Path = os.path.join(cliParams.targetPath, md5Name) + md5FileURL = "%s/%s" % (tarsURL, md5Name) + md5CachePath = os.path.join(cacheDir, md5Name) + if cache and os.path.isfile(md5CachePath): + logNOTICE("Using cached copy of %s" % md5Name) + shutil.copy(md5CachePath, md5Path) + else: + logNOTICE("Retrieving %s" % md5FileURL) + try: + if not urlretrieveTimeout(md5FileURL, md5Path, 60): + logERROR("Cannot download %s" % tarName) + return False + except Exception as e: + logERROR("Cannot download %s: %s" % (md5Name, str(e))) + return False + # Read md5 + fd = open(os.path.join(cliParams.targetPath, md5Name), "r") + md5Expected = fd.read().strip() + fd.close() + # Calculate md5 + md5Calculated = hashlib.md5() + with open(os.path.join(cliParams.targetPath, tarName), "rb") as fd: + buf = fd.read(4096) + while buf: + md5Calculated.update(buf) + buf = fd.read(4096) + + # Check + if md5Expected != md5Calculated.hexdigest(): + logERROR("Oops... md5 for package %s failed!" % pkgVer) + sys.exit(1) + # Delete md5 file + if cache: + if not os.path.isdir(cacheDir): + os.makedirs(cacheDir) + os.rename(md5Path, md5CachePath) + else: + os.unlink(md5Path) + # Extract + # cwd = os.getcwd() + # os.chdir(cliParams.targetPath) + # tf = tarfile.open( tarPath, "r" ) + # for member in tf.getmembers(): + # tf.extract( member ) + # os.chdir(cwd) + if not isSource: + logNOTICE("Extract using system tar: %s" % tarPath) + tarCmd = "tar xzf '%s' -C '%s'" % (tarPath, cliParams.targetPath) + if os.system(tarCmd): + logERROR("Extraction of tarball %s failed" % tarPath) + raise RuntimeError("Failed to extract tarball") + # Delete tar + if cache: + if not os.path.isdir(cacheDir): + os.makedirs(cacheDir) + os.rename(tarPath, tarCachePath) + else: + if tarPath != tarFileCVMFS: + os.unlink(tarPath) + + postInstallScript = os.path.join(cliParams.targetPath, pkgName, 'dirac-postInstall.py') + if os.path.isfile(postInstallScript): + os.chmod(postInstallScript, executablePerms) + logNOTICE("Executing %s..." % postInstallScript) + if os.system("python '%s' > '%s.out' 2> '%s.err'" % (postInstallScript, + postInstallScript, + postInstallScript)): + logERROR("Post installation script %s failed. Check %s.err" % (postInstallScript, + postInstallScript)) + return True + + +def fixBuildPaths(): + """ + At compilation time many scripts get the building directory inserted, + this needs to be changed to point to the current installation path: + cliParams.targetPath +""" + + # Locate build path (from header of pydoc) + binaryPath = os.path.join(cliParams.targetPath, cliParams.platform) + pydocPath = os.path.join(binaryPath, 'bin', 'pydoc') + try: + fd = open(pydocPath) + line = fd.readline() + fd.close() + buildPath = line[2:line.find(cliParams.platform) - 1] + replaceCmd = "grep -rIl '%s' %s | xargs sed -i'.org' 's:%s:%s:g'" % (buildPath, + binaryPath, + buildPath, + cliParams.targetPath) + os.system(replaceCmd) + + except BaseException: + pass + + +def fixPythonShebang(): + """ + Some scripts (like the gfal2 scripts) come with a shebang pointing to the system python. + We replace it with the environment one + """ + + binaryPath = os.path.join(cliParams.targetPath, cliParams.platform) + try: + replaceCmd = "grep -rIl '#!/usr/bin/python' %s/bin |\ + xargs sed -i'.org' 's:#!/usr/bin/python:#!/usr/bin/env python:g'" % binaryPath + os.system(replaceCmd) + except BaseException: + pass + + +def runExternalsPostInstall(): + """ + If there are any postInstall in externals, run them + """ + if cliParams.diracOS or cliParams.diracOSVersion: + postInstallPath = os.path.join(cliParams.targetPath, "postInstall") + else: + postInstallPath = os.path.join(cliParams.targetPath, cliParams.platform, "postInstall") + if not os.path.isdir(postInstallPath): + logDEBUG("There's no %s directory. Skipping postInstall step" % postInstallPath) + return + postInstallSuffix = "-postInstall" + for scriptName in os.listdir(postInstallPath): + if not scriptName.endswith(postInstallSuffix): + logDEBUG("%s does not have the %s suffix. Skipping.." % (scriptName, postInstallSuffix)) + continue + scriptPath = os.path.join(postInstallPath, scriptName) + os.chmod(scriptPath, executablePerms) + logNOTICE("Executing %s..." % scriptPath) + if os.system("'%s' > '%s.out' 2> '%s.err'" % (scriptPath, scriptPath, scriptPath)): + logERROR("Post installation script %s failed. Check %s.err" % (scriptPath, scriptPath)) + sys.exit(1) + + +def fixMySQLScript(): + """ + Update the mysql.server script (if installed) to point to the proper datadir + """ + scriptPath = os.path.join(cliParams.targetPath, 'scripts', 'dirac-fix-mysql-script') + bashrcFile = os.path.join(cliParams.targetPath, 'bashrc') + if cliParams.useVersionsDir: + bashrcFile = os.path.join(cliParams.basePath, 'bashrc') + command = 'source %s; %s > /dev/null' % (bashrcFile, scriptPath) + if os.path.exists(scriptPath): + logNOTICE("Executing %s..." % command) + os.system('bash -c "%s"' % command) + + +def checkPlatformAliasLink(): + """ + Make a link if there's an alias + """ + if cliParams.platform in platformAlias: + os.symlink(os.path.join(cliParams.targetPath, platformAlias[cliParams.platform]), + os.path.join(cliParams.targetPath, cliParams.platform)) + + +def installExternalRequirements(extType): + """ Install the extension requirements if any + """ + reqScript = os.path.join(cliParams.targetPath, "scripts", 'dirac-externals-requirements') + bashrcFile = os.path.join(cliParams.targetPath, 'bashrc') + if cliParams.useVersionsDir: + bashrcFile = os.path.join(cliParams.basePath, 'bashrc') + if os.path.isfile(reqScript): + os.chmod(reqScript, executablePerms) + logNOTICE("Executing %s..." % reqScript) + command = "%s -t '%s' > '%s.out' 2> '%s.err'" % (reqScript, extType, reqScript, reqScript) + if os.system('bash -c "source %s; %s"' % (bashrcFile, command)): + logERROR("Requirements installation script %s failed. Check %s.err" % (reqScript, + reqScript)) + return True + + +def discoverModules(modules): + """ + Created the dictionary which contains all modules, which can be installed + for example: {"DIRAC:{"sourceUrl":"https://github.com/zmathe/DIRAC.git","Vesrion:v6r20p11"}} + + :param: str modules: it contains meta information for the module, + which will be installed: https://github.com/zmathe/DIRAC.git:::DIRAC:::dev_main_branch + """ + + projects = {} + + for module in modules.split(","): + s = m = v = None + try: + s, m, v = module.split(":::") + except ValueError: + m = module.split(":::")[0] # the source and version is not provided + + projects[m] = {} + if s and v: + projects[m] = {"sourceUrl": s, "Version": v} + else: + logWARN('Unable to parse module: %s' % module) + return projects + +#### +# End of helper functions +#### + + +cmdOpts = (('r:', 'release=', 'Release version to install'), + ('l:', 'project=', 'Project to install'), + ('e:', 'extensions=', 'Extensions to install (comma separated)'), + ('t:', 'installType=', 'Installation type (client/server)'), + ('i:', 'pythonVersion=', 'Python version to compile (27/26)'), + ('p:', 'platform=', 'Platform to install'), + ('P:', 'installationPath=', 'Path where to install (default current working dir)'), + ('b', 'build', 'Force local compilation'), + ('g:', 'grid=', 'lcg tools package version'), + (' ', 'no-lcg-bundle', 'lcg tools not to be installed'), + ('B', 'noAutoBuild', 'Do not build if not available'), + ('v', 'useVersionsDir', 'Use versions directory'), + ('u:', 'baseURL=', "Use URL as the source for installation tarballs"), + ('d', 'debug', 'Show debug messages'), + ('V:', 'installation=', 'Installation from which to extract parameter values'), + ('X', 'externalsOnly', 'Only install external binaries'), + ('M:', 'defaultsURL=', 'Where to retrieve the global defaults from'), + ('h', 'help', 'Show this help'), + ('T:', 'Timeout=', 'Timeout for downloads (default = %s)'), + (' ', 'dirac-os-version=', 'the version of the DIRAC OS'), + (' ', 'dirac-os', 'Enable installation of DIRAC OS'), + (' ', 'tag=', 'release version to install from git, http or local'), + ('m:', 'module=', + 'Module to be installed. for example: -m DIRAC or -m git://github.com/DIRACGrid/DIRAC.git:DIRAC'), + ('x:', 'external=', 'external version'), + (' ', 'createLink', 'create version symbolic link from the versions directory. This is equivalent to the \ + following command: ln -s /opt/dirac/versions/vArBpC vArBpC'), + (' ', 'scriptSymlink', 'Symlink the scripts instead of creating wrapper'), + (' ', 'userEnvVariables=', + 'User-requested environment variables (comma-separated, name and value separated by ":::")') + ) + + +def usage(): + print("\nUsage:\n\n %s " % os.path.basename(sys.argv[0])) + print("\nOptions:") + for cmdOpt in cmdOpts: + print(" %s %s : %s" % (cmdOpt[0].ljust(3), cmdOpt[1].ljust(20), cmdOpt[2])) + print() + print("Known options and default values from /defaults section of releases file:") + for options in [('Release', cliParams.release), + ('Project', cliParams.project), + ('ModulesToInstall', []), + ('ExternalsType', cliParams.externalsType), + ('PythonVersion', cliParams.pythonVersion), + ('LcgVer', cliParams.lcgVer), + ('UseVersionsDir', cliParams.useVersionsDir), + ('BuildExternals', cliParams.buildExternals), + ('NoAutoBuild', cliParams.noAutoBuild), + ('Debug', cliParams.debug), + ('Timeout', cliParams.timeout)]: + print(" %s = %s" % options) + + sys.exit(0) + + +def loadConfiguration(): + """ + It loads the configuration file + """ + optList, args = getopt.getopt(sys.argv[1:], + "".join([opt[0] for opt in cmdOpts]), + [opt[1] for opt in cmdOpts]) + + # First check if the name is defined + for o, v in optList: + if o in ('-h', '--help'): + usage() + elif o in ('-V', '--installation'): + cliParams.installation = v + elif o in ("-d", "--debug"): + cliParams.debug = True + elif o in ("-M", "--defaultsURL"): + cliParams.globalDefaults = v + + releaseConfig = ReleaseConfig( + instName=cliParams.installation, + globalDefaultsURL=cliParams.globalDefaults) + if cliParams.debug: + releaseConfig.debugCB = logDEBUG + + result = releaseConfig.loadInstallationDefaults() + if not result['OK']: + logERROR("Could not load defaults: %s" % result['Message']) + + releaseConfig.loadInstallationLocalDefaults(args) + + for opName in ('release', 'externalsType', 'installType', 'pythonVersion', + 'buildExternals', 'noAutoBuild', 'debug', 'globalDefaults', + 'lcgVer', 'useVersionsDir', 'targetPath', + 'project', 'release', 'extensions', 'timeout'): + try: + opVal = releaseConfig.getInstallationConfig( + "LocalInstallation/%s" % (opName[0].upper() + opName[1:])) + except KeyError: + continue + + if opName == 'installType': + opName = 'externalsType' + + if isinstance(getattr(cliParams, opName), str_type): + setattr(cliParams, opName, opVal) + elif isinstance(getattr(cliParams, opName), bool): + setattr(cliParams, opName, opVal.lower() in ("y", "yes", "true", "1")) + elif isinstance(getattr(cliParams, opName), list): + setattr(cliParams, opName, [opV.strip() for opV in opVal.split(",") if opV]) + + # Now parse the ops + for o, v in optList: + if o in ('-r', '--release'): + cliParams.release = v + elif o in ('-l', '--project'): + cliParams.project = v + elif o in ('-e', '--extensions'): + for pkg in [p.strip() for p in v.split(",") if p.strip()]: + if pkg not in cliParams.extensions: + cliParams.extensions.append(pkg) + elif o in ('-t', '--installType'): + cliParams.externalsType = v + elif o in ('-i', '--pythonVersion'): + cliParams.pythonVersion = v + elif o in ('-p', '--platform'): + cliParams.platform = v + elif o in ('-d', '--debug'): + cliParams.debug = True + elif o in ('-g', '--grid'): + cliParams.lcgVer = v + elif o in ('--no-lcg-bundle'): + cliParams.noLcg = True + elif o in ('-u', '--baseURL'): + cliParams.installSource = v + elif o in ('-P', '--installationPath'): + cliParams.targetPath = v + try: + os.makedirs(v) + except BaseException: + pass + elif o in ('-v', '--useVersionsDir'): + cliParams.useVersionsDir = True + elif o in ('-b', '--build'): + cliParams.buildExternals = True + elif o in ("-B", '--noAutoBuild'): + cliParams.noAutoBuild = True + elif o in ('-X', '--externalsOnly'): + cliParams.externalsOnly = True + elif o in ('-T', '--Timeout'): + try: + cliParams.timeout = max(cliParams.timeout, int(v)) + cliParams.timeout = min(cliParams.timeout, 3600) + except ValueError: + pass + elif o == '--dirac-os-version': + cliParams.diracOSVersion = v + elif o == '--dirac-os': + cliParams.diracOS = True + elif o == '--tag': + cliParams.tag = v + elif o in ('-m', '--module'): + cliParams.modules = discoverModules(v) + elif o in ('-x', '--external'): + cliParams.externalVersion = v + elif o == '--createLink': + cliParams.createLink = True + elif o == '--scriptSymlink': + cliParams.scriptSymlink = True + elif o == '--userEnvVariables': + cliParams.userEnvVariables = dict(zip([name.split(':::')[0] for name in v.replace(' ', '').split(',')], + [value.split(':::')[1] for value in v.replace(' ', '').split(',')])) + + if not cliParams.release and not cliParams.modules: + logERROR("Missing release to install") + usage() + + cliParams.basePath = cliParams.targetPath + if cliParams.useVersionsDir: + # install under /versions/_ + cliParams.targetPath = os.path.join( + cliParams.targetPath, 'versions', '%s_%s' % (cliParams.release, int(time.time()))) + try: + os.makedirs(cliParams.targetPath) + except BaseException: + pass + + # If we are running an update, DIRACOS will be set in the environment + if not cliParams.diracOS and 'DIRACOS' in os.environ: + logWARN("Forcing to install DIRACOS, because it is already installed!") + cliParams.diracOS = True + + logNOTICE("Destination path for installation is %s" % cliParams.targetPath) + releaseConfig.projectName = cliParams.project + + result = releaseConfig.loadProjectRelease(cliParams.release, + project=cliParams.project, + sourceURL=cliParams.installSource) + if not result['OK']: + return result + + if not releaseConfig.isProjectLoaded("DIRAC"): + return S_ERROR("DIRAC is not depended by this installation. Aborting") + + # Reload the local configuration to ensure it takes prescience + releaseConfig.loadInstallationLocalDefaults(args) + + return S_OK(releaseConfig) + + +def compileExternals(extVersion): + """ + It is used to compile the external for a given platform + + :param str extVersion: the external version + """ + logNOTICE("Compiling externals %s" % extVersion) + buildCmd = os.path.join( + cliParams.targetPath, + "DIRAC", + "Core", + "scripts", + "dirac-compile-externals.py") + buildCmd = "%s -t '%s' -D '%s' -v '%s' -i '%s'" % (buildCmd, + cliParams.externalsType, + os.path.join( + cliParams.targetPath, + cliParams.platform), + extVersion, + cliParams.pythonVersion) + if os.system(buildCmd): + logERROR("Could not compile binaries") + return False + return True + + +def getPlatform(): + """ + It returns the platform, where this script is running using Platform.py + """ + platformPath = os.path.join(cliParams.targetPath, "DIRAC", "Core", "Utilities", "Platform.py") + try: + platFD = open(platformPath, "r") + except IOError: + logERROR("Cannot open Platform.py. Is DIRAC installed?") + return '' + + Platform = imp.load_module("Platform", platFD, platformPath, ("", "r", imp.PY_SOURCE)) + platFD.close() + return Platform.getPlatformString() + + +def installExternals(releaseConfig): + """ + It install the DIRAC external. The version of the external is provided by + the cmd or in the configuration file. + + :param object releaseConfig: + """ + if not releaseConfig: + externalsVersion = cliParams.externalVersion + else: + externalsVersion = releaseConfig.getExternalsVersion() + if not externalsVersion: + logERROR("No externals defined") + return False + + if not cliParams.platform: + cliParams.platform = getPlatform() + if not cliParams.platform: + return False + + if cliParams.installSource: + tarsURL = cliParams.installSource + else: + tarsURL = releaseConfig.getTarsLocation('DIRAC')['Value'] + + if cliParams.buildExternals: + compileExternals(externalsVersion) + else: + logDEBUG("Using platform: %s" % cliParams.platform) + extVer = "%s-%s-%s-python%s" % (cliParams.externalsType, externalsVersion, + cliParams.platform, cliParams.pythonVersion) + logDEBUG("Externals %s are to be installed" % extVer) + if not downloadAndExtractTarball(tarsURL, "Externals", extVer, cache=True): + return (not cliParams.noAutoBuild) and compileExternals(externalsVersion) + logNOTICE("Fixing externals paths...") + fixBuildPaths() + logNOTICE("Running externals post install...") + checkPlatformAliasLink() + return True + + +def installLCGutils(releaseConfig): + """ + DIRAC uses various tools from LCG area. This method install a given + lcg version. + :param object releaseConfig: the configuration file object (class ReleaseConfig) + """ + if not cliParams.platform: + cliParams.platform = getPlatform() + if not cliParams.platform: + return False + + if cliParams.installSource: + tarsURL = cliParams.installSource + else: + tarsURL = releaseConfig.getTarsLocation('DIRAC')['Value'] + + # lcg utils? + # LCG utils if required + if not releaseConfig: + lcgVer = cliParams.lcgVer + else: + lcgVer = releaseConfig.getLCGVersion(cliParams.lcgVer) + if lcgVer: + verString = "%s-%s-python%s" % (lcgVer, cliParams.platform, cliParams.pythonVersion) + # HACK: try to find a more elegant solution for the lcg bundles location + if not downloadAndExtractTarball(tarsURL + "/../lcgBundles", "DIRAC-lcg", verString, False, cache=True): + logERROR( + "\nThe requested LCG software version %s for the local operating system could not be downloaded." % + verString) + logERROR("Please, check the availability of the LCG software bindings for you \ + platform 'DIRAC-lcg-%s' \n in the repository %s/lcgBundles/." % + (verString, os.path.dirname(tarsURL))) + logERROR( + "\nIf you would like to skip the installation of the LCG software, redo the installation with \ + adding the option --no-lcg-bundle to the command line.") + return False + + logNOTICE("Fixing Python Shebang...") + fixPythonShebang() + return True + + +def createPermanentDirLinks(): + """ Create links to permanent directories from within the version directory + """ + if cliParams.useVersionsDir: + try: + # Directories + for directory in ['startup', 'runit', 'data', 'work', 'control', 'sbin', 'etc', 'webRoot']: + fake = os.path.join(cliParams.targetPath, directory) + real = os.path.join(cliParams.basePath, directory) + if not os.path.exists(real): + os.makedirs(real) + if os.path.exists(fake): + # Try to reproduce the directory structure to avoid lacking directories + fakeDirs = os.listdir(fake) + for fd in fakeDirs: + if os.path.isdir(os.path.join(fake, fd)): + if not os.path.exists(os.path.join(real, fd)): + os.makedirs(os.path.join(real, fd)) + os.rename(fake, fake + '.bak') + os.symlink(real, fake) + + # Files + for filename in ['bashrc']: + fake = os.path.join(cliParams.targetPath, filename) + real = os.path.join(cliParams.basePath, filename) + os.symlink(real, fake) + except Exception as x: + logERROR(str(x)) + return False + + return True + + +def createOldProLinks(): + """ Create links to permanent directories from within the version directory + """ + proPath = cliParams.targetPath + if cliParams.useVersionsDir: + oldPath = os.path.join(cliParams.basePath, 'old') + proPath = os.path.join(cliParams.basePath, 'pro') + try: + if os.path.exists(proPath) or os.path.islink(proPath): + if os.path.exists(oldPath) or os.path.islink(oldPath): + os.unlink(oldPath) + os.rename(proPath, oldPath) + os.symlink(cliParams.targetPath, proPath) + except Exception as x: + logERROR(str(x)) + return False + + return True + + +def createBashrc(): + """ Create DIRAC environment setting script for the bash shell + """ + + proPath = cliParams.targetPath + # Now create bashrc at basePath + try: + bashrcFile = os.path.join(cliParams.targetPath, 'bashrc') + if cliParams.useVersionsDir: + bashrcFile = os.path.join(cliParams.basePath, 'bashrc') + proPath = os.path.join(cliParams.basePath, 'pro') + logNOTICE('Creating %s' % bashrcFile) + if not os.path.exists(bashrcFile): + lines = ['# DIRAC bashrc file, used by service and agent run scripts to set environment', + 'export PYTHONUNBUFFERED=yes', + 'export PYTHONOPTIMIZE=x'] + if 'HOME' in os.environ: + lines.append('[ -z "$HOME" ] && export HOME=%s' % os.environ['HOME']) + + # Determining where the CAs are... + if 'X509_CERT_DIR' in os.environ: + certDir = os.environ['X509_CERT_DIR'] + else: + if os.path.isdir('/etc/grid-security/certificates') and \ + os.listdir('/etc/grid-security/certificates'): + # Assuming that, if present, it is not empty, and has correct CAs + certDir = '/etc/grid-security/certificates' + else: + # But this will have to be created at some point (dirac-configure) + certDir = '%s/etc/grid-security/certificates' % proPath + lines.extend(['# CAs path for SSL verification', + 'export X509_CERT_DIR=${X509_CERT_DIR:-%s}' % certDir, + 'export SSL_CERT_DIR=${SSL_CERT_DIR:-%s}' % certDir]) + + lines.append( + 'export X509_VOMS_DIR=${X509_VOMS_DIR:-%s}' % + os.path.join( + proPath, + 'etc', + 'grid-security', + 'vomsdir')) + lines.append( + 'export X509_VOMSES=${X509_VOMSES:-%s}' % + os.path.join( + proPath, + 'etc', + 'grid-security', + 'vomses')) + lines.extend( + [ + '# Some DIRAC locations', + '[ -z "$DIRAC" ] && export DIRAC=%s' % + proPath, + 'export DIRACBIN=%s' % + os.path.join( + "$DIRAC", + cliParams.platform, + 'bin'), + 'export DIRACSCRIPTS=%s' % + os.path.join( + "$DIRAC", + 'scripts'), + 'export DIRACLIB=%s' % + os.path.join( + "$DIRAC", + cliParams.platform, + 'lib'), + 'export TERMINFO=%s' % + __getTerminfoLocations( + os.path.join( + "$DIRAC", + cliParams.platform, + 'share', + 'terminfo')), + 'export RRD_DEFAULT_FONT=%s' % + os.path.join( + "$DIRAC", + cliParams.platform, + 'share', + 'rrdtool', + 'fonts', + 'DejaVuSansMono-Roman.ttf')]) + + lines.extend(['# Prepend the PYTHONPATH, the LD_LIBRARY_PATH, and the DYLD_LIBRARY_PATH']) + lines.extend(['( echo $PATH | grep -q $DIRACBIN ) || export PATH=$DIRACBIN:$PATH', + '( echo $PATH | grep -q $DIRACSCRIPTS ) || export PATH=$DIRACSCRIPTS:$PATH', + '( echo $LD_LIBRARY_PATH | grep -q $DIRACLIB ) || \ + export LD_LIBRARY_PATH=$DIRACLIB:$LD_LIBRARY_PATH', + '( echo $LD_LIBRARY_PATH | grep -q $DIRACLIB/mysql ) || \ + export LD_LIBRARY_PATH=$DIRACLIB/mysql:$LD_LIBRARY_PATH', + '( echo $DYLD_LIBRARY_PATH | grep -q $DIRACLIB ) || \ + export DYLD_LIBRARY_PATH=$DIRACLIB:$DYLD_LIBRARY_PATH', + '( echo $DYLD_LIBRARY_PATH | grep -q $DIRACLIB/mysql ) || \ + export DYLD_LIBRARY_PATH=$DIRACLIB/mysql:$DYLD_LIBRARY_PATH']) + + lines.extend(['export PYTHONPATH=$DIRAC']) + + lines.extend(['# new OpenSSL version require OPENSSL_CONF to point to some accessible location', + 'export OPENSSL_CONF=/tmp']) + + # gfal2 requires some environment variables to be set + lines.extend(['# Gfal2 configuration and plugins', 'export GFAL_CONFIG_DIR=%s' % + os.path.join("$DIRAC", cliParams.platform, 'etc/gfal2.d'), 'export GFAL_PLUGIN_DIR=%s' % + os.path.join("$DIRACLIB", 'gfal2-plugins')]) + # add DIRACPLAT environment variable for client installations + if cliParams.externalsType == 'client': + lines.extend(['# DIRAC platform', + '[ -z "$DIRACPLAT" ] && export DIRACPLAT=`$DIRAC/scripts/dirac-platform`']) + # Add the lines required for globus-* tools to use IPv6 + lines.extend(['# IPv6 support', + 'export GLOBUS_IO_IPV6=TRUE', + 'export GLOBUS_FTP_CLIENT_IPV6=TRUE']) + # Add the lines required for ARC CE support + lines.extend(['# ARC Computing Element', + 'export ARC_PLUGIN_PATH=$DIRACLIB/arc']) + + # Add the lines required for fork support for xrootd + lines.extend(['# Fork support for xrootd', + 'export XRD_RUNFORKHANDLER=1']) + + # Add the lines required for further env variables requested + if cliParams.userEnvVariables: + lines.extend(['# User-requested variables']) + for envName, envValue in cliParams.userEnvVariables.items(): + lines.extend(['( echo $%s | grep -q $%s ) || export %s=$%s:$%s' % ( + envName, envValue, + envName, envName, envValue)]) + + # Add possible DIRAC environment variables + lines.append('') + lines.append('# before enabling any of these variables, please see the documentation ') + lines.append('# https://dirac.readthedocs.io/en/latest/AdministratorGuide/' + + 'ServerInstallations/environment_variable_configuration.html') + lines.append('# export DIRAC_DEBUG_DENCODE_CALLSTACK=1') + lines.append('# export DIRAC_DEBUG_STOMP=1') + lines.append('# export DIRAC_DEPRECATED_FAIL=1') + lines.append('# export DIRAC_GFAL_GRIDFTP_SESSION_REUSE=true') + lines.append('# export DIRAC_USE_JSON_DECODE=no') + lines.append('# export DIRAC_USE_JSON_ENCODE=no') + lines.append('# export DIRAC_USE_M2CRYPTO=true') + lines.append('# export DIRAC_USE_NEWTHREADPOOL=yes') + lines.append('# export DIRAC_NO_CFG=true') + lines.append('') + f = open(bashrcFile, 'w') + f.write('\n'.join(lines)) + f.close() + except Exception as x: + logERROR(str(x)) + return False + + return True + + +def createCshrc(): + """ Create DIRAC environment setting script for the (t)csh shell + """ + proPath = cliParams.targetPath + # Now create cshrc at basePath + try: + cshrcFile = os.path.join(cliParams.targetPath, 'cshrc') + if cliParams.useVersionsDir: + cshrcFile = os.path.join(cliParams.basePath, 'cshrc') + proPath = os.path.join(cliParams.basePath, 'pro') + logNOTICE('Creating %s' % cshrcFile) + if not os.path.exists(cshrcFile): + lines = ['# DIRAC cshrc file, used by clients to set up the environment', + 'setenv PYTHONUNBUFFERED yes', + 'setenv PYTHONOPTIMIZE x'] + + # Determining where the CAs are... + if 'X509_CERT_DIR' in os.environ: + certDir = os.environ['X509_CERT_DIR'] + else: + if os.path.isdir('/etc/grid-security/certificates') and \ + os.listdir('/etc/grid-security/certificates'): + # Assuming that, if present, it is not empty, and has correct CAs + certDir = '/etc/grid-security/certificates' + else: + # But this will have to be created at some point (dirac-configure) + certDir = '%s/etc/grid-security/certificates' % proPath + lines.extend(['# CAs path for SSL verification', + 'setenv X509_CERT_DIR %s' % certDir, + 'setenv SSL_CERT_DIR %s' % certDir]) + + lines.append( + 'setenv X509_VOMS_DIR %s' % + os.path.join( + proPath, + 'etc', + 'grid-security', + 'vomsdir')) + lines.append( + 'setenv X509_VOMSES %s' % + os.path.join( + proPath, + 'etc', + 'grid-security', + 'vomses')) + lines.extend(['# Some DIRAC locations', + '( test $?DIRAC -eq 1 ) || setenv DIRAC %s' % proPath, + 'setenv DIRACBIN %s' % os.path.join("$DIRAC", cliParams.platform, 'bin'), + 'setenv DIRACSCRIPTS %s' % os.path.join("$DIRAC", 'scripts'), + 'setenv DIRACLIB %s' % os.path.join("$DIRAC", cliParams.platform, 'lib'), + 'setenv TERMINFO %s' % __getTerminfoLocations(os.path.join("$DIRAC", + cliParams.platform, + 'share', + 'terminfo'))]) + + lines.extend(['# Prepend the PYTHONPATH, the LD_LIBRARY_PATH, and the DYLD_LIBRARY_PATH']) + + lines.extend(['( test $?PATH -eq 1 ) || setenv PATH ""', + '( test $?LD_LIBRARY_PATH -eq 1 ) || setenv LD_LIBRARY_PATH ""', + '( test $?DY_LD_LIBRARY_PATH -eq 1 ) || setenv DYLD_LIBRARY_PATH ""', + '( test $?PYTHONPATH -eq 1 ) || setenv PYTHONPATH ""', + '( echo $PATH | grep -q $DIRACBIN ) || setenv PATH ${DIRACBIN}:$PATH', + '( echo $PATH | grep -q $DIRACSCRIPTS ) || setenv PATH ${DIRACSCRIPTS}:$PATH', + '( echo $LD_LIBRARY_PATH | grep -q $DIRACLIB ) || \ + setenv LD_LIBRARY_PATH ${DIRACLIB}:$LD_LIBRARY_PATH', + '( echo $LD_LIBRARY_PATH | grep -q $DIRACLIB/mysql ) || \ + setenv LD_LIBRARY_PATH ${DIRACLIB}/mysql:$LD_LIBRARY_PATH', + '( echo $DYLD_LIBRARY_PATH | grep -q $DIRACLIB ) || \ + setenv DYLD_LIBRARY_PATH ${DIRACLIB}:$DYLD_LIBRARY_PATH', + '( echo $DYLD_LIBRARY_PATH | grep -q $DIRACLIB/mysql ) || \ + setenv DYLD_LIBRARY_PATH ${DIRACLIB}/mysql:$DYLD_LIBRARY_PATH']) + + lines.extend(['setenv PYTHONPATH ${DIRAC}']) + + lines.extend(['# new OpenSSL version require OPENSSL_CONF to point to some accessible location', + 'setenv OPENSSL_CONF /tmp']) + lines.extend(['# IPv6 support', + 'setenv GLOBUS_IO_IPV6 TRUE', + 'setenv GLOBUS_FTP_CLIENT_IPV6 TRUE']) + # gfal2 requires some environment variables to be set + lines.extend(['# Gfal2 configuration and plugins', 'setenv GFAL_CONFIG_DIR %s' % + os.path.join("$DIRAC", cliParams.platform, 'etc/gfal2.d'), 'setenv GFAL_PLUGIN_DIR %s' % + os.path.join("$DIRACLIB", 'gfal2-plugins')]) + # add DIRACPLAT environment variable for client installations + if cliParams.externalsType == 'client': + lines.extend(['# DIRAC platform', + 'test $?DIRACPLAT -eq 1 || setenv DIRACPLAT `$DIRAC/scripts/dirac-platform`']) + # Add the lines required for ARC CE support + lines.extend(['# ARC Computing Element', + 'setenv ARC_PLUGIN_PATH $DIRACLIB/arc']) + + # Add the lines required for fork support for xrootd + lines.extend(['# Fork support for xrootd', + 'setenv XRD_RUNFORKHANDLER 1']) + + # Add the lines required for further env variables requested + if cliParams.userEnvVariables: + lines.extend(['# User-requested variables']) + for envName, envValue in cliParams.userEnvVariables.items(): + lines.extend(['setenv %s %s' % (envName, envValue)]) + + lines.append('') + f = open(cshrcFile, 'w') + f.write('\n'.join(lines)) + f.close() + except Exception as x: + logERROR(str(x)) + return False + + return True + + +def writeDefaultConfiguration(): + """ + After DIRAC is installed a default configuration file is created, + which contains a minimal setup. + """ + if not releaseConfig: + return + instCFG = releaseConfig.getInstallationCFG() + if not instCFG: + return + for opName in instCFG.getOptions(): + instCFG.delPath(opName) + + # filePath = os.path.join( cliParams.targetPath, "defaults-%s.cfg" % cliParams.installation ) + # Keep the default configuration file in the working directory + filePath = "defaults-%s.cfg" % cliParams.installation + try: + fd = open(filePath, "w") + fd.write(instCFG.toString()) + fd.close() + except Exception as excp: + logERROR("Could not write %s: %s" % (filePath, excp)) + logNOTICE("Defaults written to %s" % filePath) + + +def __getTerminfoLocations(defaultLocation=None): + """returns the terminfo locations as a colon separated string""" + + terminfoLocations = [] + if defaultLocation: + terminfoLocations = [defaultLocation] + + for termpath in ['/usr/share/terminfo', '/etc/terminfo']: + if os.path.exists(termpath): + terminfoLocations.append(termpath) + + return ":".join(terminfoLocations) + + +def installDiracOS(releaseConfig): + """ + Install DIRAC OS. + + :param ReleaseConfig releaseConfig: The ReleaseConfig object for configuring the installation + """ + diracos, diracOSVersion = releaseConfig.getDiracOSVersion(cliParams.diracOSVersion) + if not diracOSVersion: + logERROR("No diracos defined") + return False + tarsURL = cliParams.installSource + if not tarsURL: + tarsURL = releaseConfig.getDiracOsLocation(useVanillaDiracOS=(diracos.lower() == 'diracos')) + if not tarsURL: + tarsURL = releaseConfig.getTarsLocation('DIRAC')['Value'] + logWARN("DIRACOS location is not specified using %s" % tarsURL) + if not downloadAndExtractTarball(tarsURL, diracos, diracOSVersion, cache=True): + return False + logNOTICE("Fixing externals paths...") + fixBuildPaths() + logNOTICE("Running externals post install...") + checkPlatformAliasLink() + return True + + +def installDiracOSPython3(releaseConfig): + """ + Install DIRAC OS for Python 3. + + :param ReleaseConfig releaseConfig: The ReleaseConfig object for configuring the installation + """ + url = "https://github.com/chrisburr/DIRACOS2/releases/latest/download/DIRACOS-Linux-x86_64.sh" + installerFn = os.path.join(cliParams.basePath, "DIRACOS-Linux-x86_64.sh") + if not urlretrieveTimeout(url, installerFn, cliParams.timeout): + raise Exception("Failed to download DIRACOS from " + url) + subprocess.check_call(["bash", installerFn, "-b", "-p", os.path.join(cliParams.basePath, "diracos")]) + return True + + +def createBashrcForDiracOS(): + """ Create DIRAC environment setting script for the bash shell + """ + + proPath = cliParams.targetPath + # Now create bashrc at basePath + try: + bashrcFile = os.path.join(cliParams.targetPath, 'bashrc') + if cliParams.useVersionsDir: + bashrcFile = os.path.join(cliParams.basePath, 'bashrc') + proPath = os.path.join(cliParams.basePath, 'pro') + logNOTICE('Creating %s' % bashrcFile) + if not os.path.exists(bashrcFile): + lines = ['# DIRAC bashrc file, used by service and agent run scripts to set environment', + 'export PYTHONUNBUFFERED=yes', + 'export PYTHONOPTIMIZE=x', + '[ -z "$DIRAC" ] && export DIRAC=%s' % proPath, + '[ -z "$DIRACOS" ] && export DIRACOS=$DIRAC/diracos', + '. $DIRACOS/diracosrc'] + if 'HOME' in os.environ: + lines.append('[ -z "$HOME" ] && export HOME=%s' % os.environ['HOME']) + + # Determining where the CAs are... + if 'X509_CERT_DIR' in os.environ: + certDir = os.environ['X509_CERT_DIR'] + else: + if os.path.isdir('/etc/grid-security/certificates') and \ + os.listdir('/etc/grid-security/certificates'): + # Assuming that, if present, it is not empty, and has correct CAs + certDir = '/etc/grid-security/certificates' + else: + # But this will have to be created at some point (dirac-configure) + certDir = '%s/etc/grid-security/certificates' % proPath + lines.extend(['# CAs path for SSL verification', + 'export X509_CERT_DIR=${X509_CERT_DIR:-%s}' % certDir, + 'export SSL_CERT_DIR=${SSL_CERT_DIR:-%s}' % certDir]) + + lines.append( + 'export X509_VOMS_DIR=${X509_VOMS_DIR:-%s}' % + os.path.join( + proPath, + 'etc', + 'grid-security', + 'vomsdir')) + lines.append( + 'export X509_VOMSES=${X509_VOMSES:-%s}' % + os.path.join( + proPath, + 'etc', + 'grid-security', + 'vomses')) + lines.extend( + [ + '# Some DIRAC locations', + 'export DIRACSCRIPTS=%s' % + os.path.join( + "$DIRAC", + 'scripts')]) + + lines.extend(['# Prepend the PATH and set the PYTHONPATH']) + + lines.extend(['( echo $PATH | grep -q $DIRACSCRIPTS ) || export PATH=$DIRACSCRIPTS:$PATH']) + + lines.extend(['export PYTHONPATH=$DIRAC']) + + lines.extend(['# new OpenSSL version require OPENSSL_CONF to point to some accessible location', + 'export OPENSSL_CONF=/tmp']) + + # add DIRACPLAT environment variable for client installations + if cliParams.externalsType == 'client': + lines.extend(['# DIRAC platform', + '[ -z "$DIRACPLAT" ] && export DIRACPLAT=`$DIRAC/scripts/dirac-platform`']) + # Add the lines required for globus-* tools to use IPv6 + lines.extend(['# IPv6 support', + 'export GLOBUS_IO_IPV6=TRUE', + 'export GLOBUS_FTP_CLIENT_IPV6=TRUE']) + + # Add the lines required for fork support for xrootd + lines.extend(['# Fork support for xrootd', + 'export XRD_RUNFORKHANDLER=1']) + + # Add possible DIRAC environment variables + lines.append('') + lines.append('# before enabling any of these variables, please see the documentation ') + lines.append('# https://dirac.readthedocs.io/en/latest/AdministratorGuide/' + + 'ServerInstallations/environment_variable_configuration.html') + lines.append('# export DIRAC_DEBUG_DENCODE_CALLSTACK=1') + lines.append('# export DIRAC_DEBUG_STOMP=1') + lines.append('# export DIRAC_DEPRECATED_FAIL=1') + lines.append('# export DIRAC_GFAL_GRIDFTP_SESSION_REUSE=true') + lines.append('# export DIRAC_USE_JSON_DECODE=no') + lines.append('# export DIRAC_USE_JSON_ENCODE=no') + lines.append('# export DIRAC_USE_M2CRYPTO=true') + lines.append('# export DIRAC_USE_NEWTHREADPOOL=yes') + + # Add the lines required for further env variables requested + if cliParams.userEnvVariables: + lines.extend(['# User-requested variables']) + for envName, envValue in cliParams.userEnvVariables.items(): + lines.extend(['export %s=%s' % (envName, envValue)]) + + lines.append('') + with open(bashrcFile, 'w') as f: + f.write('\n'.join(lines)) + except Exception as x: + logERROR(str(x)) + return False + + return True + + +def checkoutFromGit(moduleName, sourceURL, tagVersion, destinationDir=None): + """ + This method checkout a given tag from a git repository. + Note: we can checkout any project form a git repository. + + :param str moduleName: The name of the Module: for example: LHCbWebDIRAC + :param str sourceURL: The code repository: https://github.com/DIRACGrid/WebAppDIRAC.git + :param str tagVersion: the tag for example: v3r1p10 + + """ + + codeRepo = moduleName + 'Repo' + + fDirName = os.path.join(cliParams.targetPath, codeRepo) + if os.path.isdir(sourceURL): + logNOTICE("Using local copy for source: %s" % sourceURL) + shutil.copytree(sourceURL, fDirName) + else: + cmd = "git clone '%s' '%s'" % (sourceURL, fDirName) + + logNOTICE("Executing: %s" % cmd) + if os.system(cmd): + return S_ERROR("Error while retrieving sources from git") + + branchName = "%s-%s" % (tagVersion, os.getpid()) + + isTagCmd = "( cd '%s'; git tag -l | grep '%s' )" % (fDirName, tagVersion) + if os.system(isTagCmd): + # No tag found, assume branch + branchSource = 'origin/%s' % tagVersion + else: + branchSource = tagVersion + + cmd = "( cd '%s'; git checkout -b '%s' '%s' )" % (fDirName, branchName, branchSource) + + logNOTICE("Executing: %s" % cmd) + exportRes = os.system(cmd) + + if exportRes: + return S_ERROR("Error while exporting from git") + + # replacing the code + if os.path.exists(fDirName + '/' + moduleName): + cmd = "ln -s %s/%s %s" % (codeRepo, moduleName, os.path.join(cliParams.targetPath, moduleName)) + else: + cmd = "mv %s %s" % (fDirName, os.path.join(cliParams.targetPath, moduleName)) + logNOTICE("Executing: %s" % cmd) + retVal = os.system(cmd) + + if retVal: + return S_ERROR("Error while creating module: %s" % (moduleName)) + + return S_OK() + + +def createSymbolicLink(): + """ + It creates a symbolic link to the actual directory from versions + directory. + """ + + cmd = "ln -s %s %s" % (cliParams.targetPath, cliParams.release) + logNOTICE("Executing: %s" % cmd) + retVal = os.system(cmd) + if retVal: + return S_ERROR("Error while creating symbolic link!") + + return S_OK() + + +if __name__ == "__main__": + logNOTICE("Processing installation requirements") + result = loadConfiguration() + releaseConfig = None + modsToInstall = {} + modsOrder = [] + if not result['OK']: + # the configuration files does not exists, which means the module is not released. + if cliParams.modules: + logNOTICE(str(cliParams.modules)) + for i in cliParams.modules: + modsOrder.append(i) + modsToInstall[i] = (cliParams.modules[i]['sourceUrl'], cliParams.modules[i]['Version']) + else: + # there is no module provided which can be deployed + logERROR(result['Message']) + sys.exit(1) + else: + releaseConfig = result['Value'] + if not createPermanentDirLinks(): + sys.exit(1) + + if not cliParams.externalsOnly: + logNOTICE("Discovering modules to install") + if releaseConfig: + result = releaseConfig.getModulesToInstall(cliParams.release, cliParams.extensions) + if not result['OK']: + logERROR(result['Message']) + sys.exit(1) + modsOrder, modsToInstall = result['Value'] + if cliParams.debug and releaseConfig: + logNOTICE("Writing down the releases files") + releaseConfig.dumpReleasesToPath() + logNOTICE("Installing modules...") + for modName in set(modsOrder): + tarsURL, modVersion = modsToInstall[modName] + if cliParams.installSource and not cliParams.modules: + # we install not release version of DIRAC + tarsURL = cliParams.installSource + if modName in cliParams.modules: + sourceURL = cliParams.modules[modName].get('sourceUrl') + if 'Version' in cliParams.modules[modName]: + modVersion = cliParams.modules[modName]['Version'] + if not sourceURL: + retVal = releaseConfig.getModSource(cliParams.release, modName) + if retVal['OK']: + tarsURL = retVal['Value'][1] # this is the git repository url + modVersion = cliParams.tag + else: + tarsURL = sourceURL + retVal = checkoutFromGit(modName, tarsURL, modVersion) + if not retVal['OK']: + logERROR("Cannot checkout %s" % retVal['Message']) + sys.exit(1) + else: + logNOTICE("Installing %s:%s" % (modName, modVersion)) + if not downloadAndExtractTarball(tarsURL, modName, modVersion): + sys.exit(1) + logNOTICE("Deploying scripts...") + ddeLocation = os.path.join(cliParams.targetPath, "DIRAC", "Core", + "scripts", "dirac-deploy-scripts.py") + if not os.path.isfile(ddeLocation): + ddeLocation = os.path.join(cliParams.targetPath, "DIRAC", "Core", + "scripts", "dirac_deploy_scripts.py") + if not os.path.isfile(ddeLocation): + ddeLocation = os.path.join(cliParams.targetPath, "DIRAC", "src", "DIRAC", "Core", + "scripts", "dirac_deploy_scripts.py") + if os.path.isfile(ddeLocation): + cmd = ddeLocation + + # if specified, create symlink instead of wrapper. + if cliParams.scriptSymlink: + cmd += ' --symlink' + + # In MacOS /usr/bin/env does not find python in the $PATH, passing binary path + # as an argument to the dirac-deploy-scripts + if not cliParams.platform: + cliParams.platform = getPlatform() + if "Darwin" in cliParams.platform: + binaryPath = os.path.join(cliParams.targetPath, cliParams.platform) + logNOTICE("For MacOS (Darwin) use explicit binary path %s" % binaryPath) + cmd += ' %s' % binaryPath + + os.system(cmd) + else: + logERROR("No dirac-deploy-scripts found. This doesn't look good") + else: + logNOTICE("Skipping installing DIRAC") + + # we install with DIRACOS from v7rX DIRAC release + if cliParams.diracOS \ + or list(releaseConfig.prjRelCFG['DIRAC'])[0][1] not in '0123456789' \ + or int(list(releaseConfig.prjRelCFG['DIRAC'])[0][1]) > 6: + logNOTICE("Installing DIRAC OS %s..." % cliParams.diracOSVersion) + if cliParams.pythonVersion.startswith("3"): + if not installDiracOSPython3(releaseConfig): + sys.exit(1) + else: + if not installDiracOS(releaseConfig): + sys.exit(1) + if not createBashrcForDiracOS(): + sys.exit(1) + else: + logNOTICE("Installing %s externals..." % cliParams.externalsType) + if not installExternals(releaseConfig): + sys.exit(1) + if cliParams.noLcg: + logNOTICE("Skipping installation of LCG software...") + else: + logNOTICE("Installing LCG software...") + if not installLCGutils(releaseConfig): + sys.exit(1) + if not createBashrc(): + sys.exit(1) + if not createCshrc(): + sys.exit(1) + if not createOldProLinks(): + sys.exit(1) + runExternalsPostInstall() + writeDefaultConfiguration() + if cliParams.externalsType == "server": + fixMySQLScript() + installExternalRequirements(cliParams.externalsType) + if cliParams.createLink: + createSymbolicLink() + logNOTICE("%s properly installed" % cliParams.installation) + sys.exit(0) From f541e2b06a126c94fc17aed444951db260591f96 Mon Sep 17 00:00:00 2001 From: Chris Burr Date: Thu, 25 Feb 2021 08:13:12 +0100 Subject: [PATCH 2/2] Stop supporting Python 3 with dirac-install.py --- dirac-install.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/dirac-install.py b/dirac-install.py index 6fe5e5d..6b87149 100755 --- a/dirac-install.py +++ b/dirac-install.py @@ -2406,12 +2406,10 @@ def installDiracOSPython3(releaseConfig): :param ReleaseConfig releaseConfig: The ReleaseConfig object for configuring the installation """ - url = "https://github.com/chrisburr/DIRACOS2/releases/latest/download/DIRACOS-Linux-x86_64.sh" - installerFn = os.path.join(cliParams.basePath, "DIRACOS-Linux-x86_64.sh") - if not urlretrieveTimeout(url, installerFn, cliParams.timeout): - raise Exception("Failed to download DIRACOS from " + url) - subprocess.check_call(["bash", installerFn, "-b", "-p", os.path.join(cliParams.basePath, "diracos")]) - return True + raise NotImplementedError( + "Creating a python 3 based installation of DIRAC is not supported by dirac-install.py\n" + "See https://github.com/DIRACGrid/DIRAC/#install for details." + ) def createBashrcForDiracOS():