From a38051e69b5bcc24c9616e0dc7bb02be89676b46 Mon Sep 17 00:00:00 2001 From: Andrei Tsaregorodtsev Date: Thu, 15 Mar 2012 21:20:43 +0100 Subject: [PATCH 1/8] Added possibility to define a JDL attribute as list of lists of strings --- Core/Utilities/ClassAd/ClassAdLight.py | 51 ++++++++++++++++++++++++-- 1 file changed, 47 insertions(+), 4 deletions(-) diff --git a/Core/Utilities/ClassAd/ClassAdLight.py b/Core/Utilities/ClassAd/ClassAdLight.py index f26fae4b62d..0c03a2f06bf 100755 --- a/Core/Utilities/ClassAd/ClassAdLight.py +++ b/Core/Utilities/ClassAd/ClassAdLight.py @@ -127,6 +127,17 @@ def insertAttributeVectorString( self, name, attributelist ): tmp = map ( lambda x : '"' + x + '"', attributelist ) tmpstr = ','.join( tmp ) self.contents[name] = '{' + tmpstr + '}' + + def insertAttributeVectorStringList( self, name, attributelist ): + """Insert a named list of string lists + """ + + listOfLists = [] + for stringList in attributelist: + tmp = map ( lambda x : '"' + x + '"', stringList ) + tmpstr = ','.join( tmp ) + listOfLists.append('{' + tmpstr + '}') + self.contents[name] = '{' + ','.join(listOfLists) + '}' def lookupAttribute( self, name ): """Check the presence of the given attribute @@ -159,13 +170,45 @@ def isAttributeList( self, name ): return attribute.startswith( '{' ) def getListFromExpression( self, name ): - """ Get a list of values from a given expression + """ Get a list of strings from a given expression """ - tempString = self.get_expression( name ) - tempString = tempString.replace( "{", "" ).replace( "}", "" ).replace( "\"", "" ).replace( " ", "" ) + tempString = self.get_expression( name ).strip() + listMode = False + if tempString.startswith('{'): + tempString = tempString[1:-1] + listMode = True + + tempString = tempString.replace( " ", "" ).replace( '\n','' ) + if tempString.find('{') < 0: + if not listMode: + tempString = tempString.replace( "\"", "" ) + return tempString.split( ',' ) + + resultList = [] + while tempString: + if tempString.find( '{' ) == 0 : + end = tempString.find( '}' ) + resultList.append(tempString[:end+1]) + tempString = tempString[end+1:] + if tempString.startswith(','): + tempString = tempString[1:] + elif tempString.find( '"' ) == 0 : + end = tempString[1:].find( '"' ) + resultList.append( tempString[1:end+1] ) + tempString = tempString[end+2:] + if tempString.startswith(','): + tempString = tempString[1:] + else: + end = tempString.find( ',' ) + if end < 0: + resultList.append( tempString.replace( "\"", "" ).replace( " ", "" ) ) + break + else: + resultList.append( tempString[:end-1].replace( "\"", "" ).replace( " ", "" ) ) + tempString = tempString[end+1:] - return tempString.split( ',' ) + return resultList def getDictionaryFromSubJDL( self, name ): """ Get a dictionary of the JDL attributes from a subsection From bb7bba2a729ced18c132e58215b0c6e2d40475ab Mon Sep 17 00:00:00 2001 From: Andrei Tsaregorodtsev Date: Fri, 16 Mar 2012 16:37:07 +0100 Subject: [PATCH 2/8] Enable parameteric jobs with a list of parameter lists --- Interfaces/API/Job.py | 31 +++++++++++-------- .../scripts/dirac-jobexec.py | 6 ++++ 2 files changed, 24 insertions(+), 13 deletions(-) diff --git a/Interfaces/API/Job.py b/Interfaces/API/Job.py index b1759f983b6..23de767aee0 100755 --- a/Interfaces/API/Job.py +++ b/Interfaces/API/Job.py @@ -259,15 +259,13 @@ def setParametricInputSandbox( self, files ): if not fileName.lower().count( "lfn:" ): return self._reportError( 'All files should be LFNs', **kwargs ) resolvedFiles = self._resolveInputSandbox( files ) - fileList = ";".join( resolvedFiles ) - self.parametric['InputSandbox'] = fileList + self.parametric['InputSandbox'] = resolvedFiles #self.sandboxFiles=resolvedFiles elif type( files ) == type( " " ): if not files.lower().count( "lfn:" ): return self._reportError( 'All files should be LFNs', **kwargs ) resolvedFiles = self._resolveInputSandbox( [files] ) - fileList = ";".join( resolvedFiles ) - self.parametric['InputSandbox'] = fileList + self.parametric['InputSandbox'] = resolvedFiles #self.sandboxFiles = [files] else: return self._reportError( 'Expected file string or list of files for input sandbox contents', **kwargs ) @@ -350,10 +348,12 @@ def setParametricInputData( self, lfns ): """ if type( lfns ) == list and len( lfns ): for i in xrange( len( lfns ) ): - lfns[i] = lfns[i].replace( 'LFN:', '' ) - inputData = map( lambda x: 'LFN:' + x, lfns ) - inputDataStr = ';'.join( inputData ) - self.parametric['InputData'] = inputDataStr + if type( lfns[i] ) == list and len( lfns[i] ): + for k in xrange( len( lfns[i] ) ): + lfns[i][k] = 'LFN:' + lfns[i][k].replace( 'LFN:', '' ) + else: + lfns[i] = 'LFN:' + lfns[i].replace( 'LFN:', '' ) + self.parametric['InputData'] = inputData elif type( lfns ) == type( ' ' ): #single LFN self.parametric['InputData'] = lfns else: @@ -1126,7 +1126,7 @@ def _toJDL( self, xmlFile = '' ): #messy but need to account for xml file being paramsDict['InputData']['value'] = "%s" paramsDict['InputData']['type'] = 'JDL' self.parametric['files'] = self.parametric['InputData'] - arguments.append( ' -p ParametricInputData=%s' ) + arguments.append( ' -p ParametricInputData="%s"' ) elif self.parametric.has_key( 'InputSandbox' ): if paramsDict.has_key( 'InputSandbox' ): currentFiles = paramsDict['InputSandbox']['value'] + ";%s" @@ -1135,15 +1135,15 @@ def _toJDL( self, xmlFile = '' ): #messy but need to account for xml file being paramsDict['InputSandbox'] = {} paramsDict['InputSandbox']['value'] = '%s' paramsDict['InputSandbox']['type'] = 'JDL' - self.parametric['files']= self.parametric['InputSandbox'] + self.parametric['files'] = self.parametric['InputSandbox'] arguments.append(' -p ParametricInputSandbox="%s"') if self.parametric.has_key('files'): paramsDict['Parameters']={} - paramsDict['Parameters']['value']=";".join(self.parametric['files']) + paramsDict['Parameters']['value'] = self.parametric['files'] paramsDict['Parameters']['type'] = 'JDL' if self.parametric.has_key('GenericParameters'): paramsDict['Parameters']={} - paramsDict['Parameters']['value']=";".join(self.parametric['GenericParameters']) + paramsDict['Parameters']['value'] = self.parametric['GenericParameters'] paramsDict['Parameters']['type'] = 'JDL' arguments.append(' -p ParametricParameters="%s"') ##This needs to be put here so that the InputData and/or InputSandbox parameters for parametric jobs are processed @@ -1159,7 +1159,12 @@ def _toJDL( self, xmlFile = '' ): #messy but need to account for xml file being requirements = True if re.search( '^JDL', ptype ): - if not re.search( ';', value ) or name == 'GridRequirements': #not a nice fix... + if type( value ) == list: + if type( value[0] ) == list: + classadJob.insertAttributeVectorStringList( name, value ) + else: + classadJob.insertAttributeVectorString( name, value ) + elif not re.search( ';', value ) or name == 'GridRequirements': #not a nice fix... classadJob.insertAttributeString( name, value ) else: classadJob.insertAttributeVectorString( name, value.split( ';' ) ) diff --git a/WorkloadManagementSystem/scripts/dirac-jobexec.py b/WorkloadManagementSystem/scripts/dirac-jobexec.py index f35808fde82..a41472fcc5b 100755 --- a/WorkloadManagementSystem/scripts/dirac-jobexec.py +++ b/WorkloadManagementSystem/scripts/dirac-jobexec.py @@ -75,6 +75,12 @@ def jobexec( jobxml, wfParameters = {} ): for switch, parameter in parList: if switch == "p": name, value = parameter.split( '=' ) + value = value.strip() + + # The comma separated list in curly brackets is interpreted as a list + if value.startswith("{"): + value = value[1:-1].replace('"','').replace(" ",'').split(',') + parDict[name] = value gLogger.verbose( 'PYTHONPATH:\n%s' % ( string.join( sys.path, '\n' ) ) ) result = jobexec( jobXMLfile, parDict ) From 17f4439378e7804ee48177c24fd99be55b73d000 Mon Sep 17 00:00:00 2001 From: Andrei Tsaregorodtsev Date: Fri, 16 Mar 2012 20:05:06 +0100 Subject: [PATCH 3/8] Clean-up after expanding parameters in the submitJob() --- .../Service/JobManagerHandler.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/WorkloadManagementSystem/Service/JobManagerHandler.py b/WorkloadManagementSystem/Service/JobManagerHandler.py index 1d2091bfe74..e16b8429795 100755 --- a/WorkloadManagementSystem/Service/JobManagerHandler.py +++ b/WorkloadManagementSystem/Service/JobManagerHandler.py @@ -132,7 +132,17 @@ def export_submitJob( self, jobDesc ): jobDescList = [] for n,p in enumerate(parameterList): - jobDescList.append( jobDesc.replace('%s',str(p)).replace('%n',str(n)) ) + newJobDesc = jobDesc.replace('%s',str(p)).replace('%n',str(n)) + newClassAd = ClassAd(newJobDesc) + for attr in ['Parameters','ParameterStep','ParameterFactor']: + newClassAd.deleteAttribute(attr) + if p.startswith('{'): + newClassAd.insertAttributeInt( 'Parameter',str(p) ) + else: + newClassAd.insertAttributeString( 'Parameter',str(p) ) + newClassAd.insertAttributeInt('ParameterNumber',n) + newJDL = newClassAd.asJDL() + jobDescList.append( newJDL ) else: jobDescList = [ jobDesc ] From 18e8704754e8ba9ebc86583072c386493cabdf94 Mon Sep 17 00:00:00 2001 From: Andrei Tsaregorodtsev Date: Fri, 16 Mar 2012 20:06:10 +0100 Subject: [PATCH 4/8] Never interpret certain JDL attributes as lists --- Core/Utilities/JDL.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Core/Utilities/JDL.py b/Core/Utilities/JDL.py index a311a68b56b..222bbc12f57 100644 --- a/Core/Utilities/JDL.py +++ b/Core/Utilities/JDL.py @@ -137,7 +137,8 @@ def dumpCFGAsJDL( cfg, level = 1, tab = " " ): contents.append( "%s;" % dumpCFGAsJDL( cfg[ key ], level + 1, tab ) ) else: val = List.fromChar( cfg[ key ] ) - if len( val ) < 2: + # Some attributes are never lists + if len( val ) < 2 or key in ['Arguments','Executable','StdOutput','StdError']: value = cfg[ key ] try: try_value = float( value ) From 7c28f5fef3d1bce5074e7bd33976b12cc88f27b1 Mon Sep 17 00:00:00 2001 From: Andrei Tsaregorodtsev Date: Fri, 16 Mar 2012 20:07:35 +0100 Subject: [PATCH 5/8] Do not use double quotes in a "list of lists" attributes --- Core/Utilities/ClassAd/ClassAdLight.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Core/Utilities/ClassAd/ClassAdLight.py b/Core/Utilities/ClassAd/ClassAdLight.py index 0c03a2f06bf..99258474b05 100755 --- a/Core/Utilities/ClassAd/ClassAdLight.py +++ b/Core/Utilities/ClassAd/ClassAdLight.py @@ -134,8 +134,8 @@ def insertAttributeVectorStringList( self, name, attributelist ): listOfLists = [] for stringList in attributelist: - tmp = map ( lambda x : '"' + x + '"', stringList ) - tmpstr = ','.join( tmp ) + #tmp = map ( lambda x : '"' + x + '"', stringList ) + tmpstr = ','.join( stringList ) listOfLists.append('{' + tmpstr + '}') self.contents[name] = '{' + ','.join(listOfLists) + '}' From d194a0ba429b6980ec0f03f690fe65e25450c82d Mon Sep 17 00:00:00 2001 From: Andrei Tsaregorodtsev Date: Fri, 16 Mar 2012 20:08:06 +0100 Subject: [PATCH 6/8] Few fixes after debugging --- Interfaces/API/Job.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/Interfaces/API/Job.py b/Interfaces/API/Job.py index 23de767aee0..c2ec7fc16bf 100755 --- a/Interfaces/API/Job.py +++ b/Interfaces/API/Job.py @@ -353,7 +353,7 @@ def setParametricInputData( self, lfns ): lfns[i][k] = 'LFN:' + lfns[i][k].replace( 'LFN:', '' ) else: lfns[i] = 'LFN:' + lfns[i].replace( 'LFN:', '' ) - self.parametric['InputData'] = inputData + self.parametric['InputData'] = lfns elif type( lfns ) == type( ' ' ): #single LFN self.parametric['InputData'] = lfns else: @@ -1126,7 +1126,7 @@ def _toJDL( self, xmlFile = '' ): #messy but need to account for xml file being paramsDict['InputData']['value'] = "%s" paramsDict['InputData']['type'] = 'JDL' self.parametric['files'] = self.parametric['InputData'] - arguments.append( ' -p ParametricInputData="%s"' ) + arguments.append( ' -p ParametricInputData=%s' ) elif self.parametric.has_key( 'InputSandbox' ): if paramsDict.has_key( 'InputSandbox' ): currentFiles = paramsDict['InputSandbox']['value'] + ";%s" @@ -1136,7 +1136,7 @@ def _toJDL( self, xmlFile = '' ): #messy but need to account for xml file being paramsDict['InputSandbox']['value'] = '%s' paramsDict['InputSandbox']['type'] = 'JDL' self.parametric['files'] = self.parametric['InputSandbox'] - arguments.append(' -p ParametricInputSandbox="%s"') + arguments.append(' -p ParametricInputSandbox=%s') if self.parametric.has_key('files'): paramsDict['Parameters']={} paramsDict['Parameters']['value'] = self.parametric['files'] @@ -1145,7 +1145,7 @@ def _toJDL( self, xmlFile = '' ): #messy but need to account for xml file being paramsDict['Parameters']={} paramsDict['Parameters']['value'] = self.parametric['GenericParameters'] paramsDict['Parameters']['type'] = 'JDL' - arguments.append(' -p ParametricParameters="%s"') + arguments.append(' -p ParametricParameters=%s') ##This needs to be put here so that the InputData and/or InputSandbox parameters for parametric jobs are processed classadJob.insertAttributeString( 'Arguments', ' '.join( arguments ) ) @@ -1164,9 +1164,11 @@ def _toJDL( self, xmlFile = '' ): #messy but need to account for xml file being classadJob.insertAttributeVectorStringList( name, value ) else: classadJob.insertAttributeVectorString( name, value ) + elif value == "%s": + classadJob.insertAttributeInt( name, value ) elif not re.search( ';', value ) or name == 'GridRequirements': #not a nice fix... classadJob.insertAttributeString( name, value ) - else: + else: classadJob.insertAttributeVectorString( name, value.split( ';' ) ) if not requirements: From 0c07819dfd76e1f24dfeca1ee3aec9ef455e89ed Mon Sep 17 00:00:00 2001 From: Andrei Tsaregorodtsev Date: Fri, 16 Mar 2012 21:07:16 +0100 Subject: [PATCH 7/8] Consider list JDL parameters only those that start with { and not have it inside the body --- WorkloadManagementSystem/Agent/JobAgent.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/WorkloadManagementSystem/Agent/JobAgent.py b/WorkloadManagementSystem/Agent/JobAgent.py index a6e01d9a71f..4c0d1e69a91 100755 --- a/WorkloadManagementSystem/Agent/JobAgent.py +++ b/WorkloadManagementSystem/Agent/JobAgent.py @@ -544,7 +544,7 @@ def __getJDLParameters( self, jdl ): classAdJob = ClassAd( jdl ) paramsDict = classAdJob.contents for param, value in paramsDict.items(): - if re.search( '{', value ): + if value.strip().startswith('{'): self.log.debug( 'Found list type parameter %s' % ( param ) ) rawValues = value.replace( '{', '' ).replace( '}', '' ).replace( '"', '' ).split() valueList = [] From 55c0574bd2d3b1f7426dfa30bf8c148a0aac7a10 Mon Sep 17 00:00:00 2001 From: Andrei Tsaregorodtsev Date: Fri, 16 Mar 2012 21:08:31 +0100 Subject: [PATCH 8/8] Do not treat the ParametricXXX workflow parameters as JDL type --- Interfaces/API/Job.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Interfaces/API/Job.py b/Interfaces/API/Job.py index c2ec7fc16bf..e9796abb229 100755 --- a/Interfaces/API/Job.py +++ b/Interfaces/API/Job.py @@ -904,11 +904,11 @@ def __setJobDefaults( self ): self._addParameter( self.workflow, 'InputData', 'JDL', '', 'Default null input data value' ) self._addParameter( self.workflow, 'LogLevel', 'JDL', self.logLevel, 'Job Logging Level' ) #Those 2 below are need for on-site resolution - self._addParameter( self.workflow, 'ParametricInputData', 'JDL', '', + self._addParameter( self.workflow, 'ParametricInputData', 'WF', '', 'Default null parametric input data value' ) - self._addParameter( self.workflow, 'ParametricInputSandbox', 'JDL', '', + self._addParameter( self.workflow, 'ParametricInputSandbox', 'WF', '', 'Default null parametric input sandbox value' ) - self._addParameter( self.workflow, 'ParametricParameters', 'JDL', '', + self._addParameter( self.workflow, 'ParametricParameters', 'WF', '', 'Default null parametric input parameters value' ) #############################################################################