diff --git a/Core/Utilities/ClassAd/ClassAdLight.py b/Core/Utilities/ClassAd/ClassAdLight.py index f26fae4b62d..99258474b05 100755 --- a/Core/Utilities/ClassAd/ClassAdLight.py +++ b/Core/Utilities/ClassAd/ClassAdLight.py @@ -127,6 +127,17 @@ def insertAttributeVectorString( self, name, attributelist ): tmp = map ( lambda x : '"' + x + '"', attributelist ) tmpstr = ','.join( tmp ) self.contents[name] = '{' + tmpstr + '}' + + def insertAttributeVectorStringList( self, name, attributelist ): + """Insert a named list of string lists + """ + + listOfLists = [] + for stringList in attributelist: + #tmp = map ( lambda x : '"' + x + '"', stringList ) + tmpstr = ','.join( stringList ) + listOfLists.append('{' + tmpstr + '}') + self.contents[name] = '{' + ','.join(listOfLists) + '}' def lookupAttribute( self, name ): """Check the presence of the given attribute @@ -159,13 +170,45 @@ def isAttributeList( self, name ): return attribute.startswith( '{' ) def getListFromExpression( self, name ): - """ Get a list of values from a given expression + """ Get a list of strings from a given expression """ - tempString = self.get_expression( name ) - tempString = tempString.replace( "{", "" ).replace( "}", "" ).replace( "\"", "" ).replace( " ", "" ) + tempString = self.get_expression( name ).strip() + listMode = False + if tempString.startswith('{'): + tempString = tempString[1:-1] + listMode = True + + tempString = tempString.replace( " ", "" ).replace( '\n','' ) + if tempString.find('{') < 0: + if not listMode: + tempString = tempString.replace( "\"", "" ) + return tempString.split( ',' ) + + resultList = [] + while tempString: + if tempString.find( '{' ) == 0 : + end = tempString.find( '}' ) + resultList.append(tempString[:end+1]) + tempString = tempString[end+1:] + if tempString.startswith(','): + tempString = tempString[1:] + elif tempString.find( '"' ) == 0 : + end = tempString[1:].find( '"' ) + resultList.append( tempString[1:end+1] ) + tempString = tempString[end+2:] + if tempString.startswith(','): + tempString = tempString[1:] + else: + end = tempString.find( ',' ) + if end < 0: + resultList.append( tempString.replace( "\"", "" ).replace( " ", "" ) ) + break + else: + resultList.append( tempString[:end-1].replace( "\"", "" ).replace( " ", "" ) ) + tempString = tempString[end+1:] - return tempString.split( ',' ) + return resultList def getDictionaryFromSubJDL( self, name ): """ Get a dictionary of the JDL attributes from a subsection diff --git a/Core/Utilities/JDL.py b/Core/Utilities/JDL.py index a311a68b56b..222bbc12f57 100644 --- a/Core/Utilities/JDL.py +++ b/Core/Utilities/JDL.py @@ -137,7 +137,8 @@ def dumpCFGAsJDL( cfg, level = 1, tab = " " ): contents.append( "%s;" % dumpCFGAsJDL( cfg[ key ], level + 1, tab ) ) else: val = List.fromChar( cfg[ key ] ) - if len( val ) < 2: + # Some attributes are never lists + if len( val ) < 2 or key in ['Arguments','Executable','StdOutput','StdError']: value = cfg[ key ] try: try_value = float( value ) diff --git a/Interfaces/API/Job.py b/Interfaces/API/Job.py index b1759f983b6..e9796abb229 100755 --- a/Interfaces/API/Job.py +++ b/Interfaces/API/Job.py @@ -259,15 +259,13 @@ def setParametricInputSandbox( self, files ): if not fileName.lower().count( "lfn:" ): return self._reportError( 'All files should be LFNs', **kwargs ) resolvedFiles = self._resolveInputSandbox( files ) - fileList = ";".join( resolvedFiles ) - self.parametric['InputSandbox'] = fileList + self.parametric['InputSandbox'] = resolvedFiles #self.sandboxFiles=resolvedFiles elif type( files ) == type( " " ): if not files.lower().count( "lfn:" ): return self._reportError( 'All files should be LFNs', **kwargs ) resolvedFiles = self._resolveInputSandbox( [files] ) - fileList = ";".join( resolvedFiles ) - self.parametric['InputSandbox'] = fileList + self.parametric['InputSandbox'] = resolvedFiles #self.sandboxFiles = [files] else: return self._reportError( 'Expected file string or list of files for input sandbox contents', **kwargs ) @@ -350,10 +348,12 @@ def setParametricInputData( self, lfns ): """ if type( lfns ) == list and len( lfns ): for i in xrange( len( lfns ) ): - lfns[i] = lfns[i].replace( 'LFN:', '' ) - inputData = map( lambda x: 'LFN:' + x, lfns ) - inputDataStr = ';'.join( inputData ) - self.parametric['InputData'] = inputDataStr + if type( lfns[i] ) == list and len( lfns[i] ): + for k in xrange( len( lfns[i] ) ): + lfns[i][k] = 'LFN:' + lfns[i][k].replace( 'LFN:', '' ) + else: + lfns[i] = 'LFN:' + lfns[i].replace( 'LFN:', '' ) + self.parametric['InputData'] = lfns elif type( lfns ) == type( ' ' ): #single LFN self.parametric['InputData'] = lfns else: @@ -904,11 +904,11 @@ def __setJobDefaults( self ): self._addParameter( self.workflow, 'InputData', 'JDL', '', 'Default null input data value' ) self._addParameter( self.workflow, 'LogLevel', 'JDL', self.logLevel, 'Job Logging Level' ) #Those 2 below are need for on-site resolution - self._addParameter( self.workflow, 'ParametricInputData', 'JDL', '', + self._addParameter( self.workflow, 'ParametricInputData', 'WF', '', 'Default null parametric input data value' ) - self._addParameter( self.workflow, 'ParametricInputSandbox', 'JDL', '', + self._addParameter( self.workflow, 'ParametricInputSandbox', 'WF', '', 'Default null parametric input sandbox value' ) - self._addParameter( self.workflow, 'ParametricParameters', 'JDL', '', + self._addParameter( self.workflow, 'ParametricParameters', 'WF', '', 'Default null parametric input parameters value' ) ############################################################################# @@ -1135,17 +1135,17 @@ def _toJDL( self, xmlFile = '' ): #messy but need to account for xml file being paramsDict['InputSandbox'] = {} paramsDict['InputSandbox']['value'] = '%s' paramsDict['InputSandbox']['type'] = 'JDL' - self.parametric['files']= self.parametric['InputSandbox'] - arguments.append(' -p ParametricInputSandbox="%s"') + self.parametric['files'] = self.parametric['InputSandbox'] + arguments.append(' -p ParametricInputSandbox=%s') if self.parametric.has_key('files'): paramsDict['Parameters']={} - paramsDict['Parameters']['value']=";".join(self.parametric['files']) + paramsDict['Parameters']['value'] = self.parametric['files'] paramsDict['Parameters']['type'] = 'JDL' if self.parametric.has_key('GenericParameters'): paramsDict['Parameters']={} - paramsDict['Parameters']['value']=";".join(self.parametric['GenericParameters']) + paramsDict['Parameters']['value'] = self.parametric['GenericParameters'] paramsDict['Parameters']['type'] = 'JDL' - arguments.append(' -p ParametricParameters="%s"') + arguments.append(' -p ParametricParameters=%s') ##This needs to be put here so that the InputData and/or InputSandbox parameters for parametric jobs are processed classadJob.insertAttributeString( 'Arguments', ' '.join( arguments ) ) @@ -1159,9 +1159,16 @@ def _toJDL( self, xmlFile = '' ): #messy but need to account for xml file being requirements = True if re.search( '^JDL', ptype ): - if not re.search( ';', value ) or name == 'GridRequirements': #not a nice fix... + if type( value ) == list: + if type( value[0] ) == list: + classadJob.insertAttributeVectorStringList( name, value ) + else: + classadJob.insertAttributeVectorString( name, value ) + elif value == "%s": + classadJob.insertAttributeInt( name, value ) + elif not re.search( ';', value ) or name == 'GridRequirements': #not a nice fix... classadJob.insertAttributeString( name, value ) - else: + else: classadJob.insertAttributeVectorString( name, value.split( ';' ) ) if not requirements: diff --git a/WorkloadManagementSystem/Agent/JobAgent.py b/WorkloadManagementSystem/Agent/JobAgent.py index a6e01d9a71f..4c0d1e69a91 100755 --- a/WorkloadManagementSystem/Agent/JobAgent.py +++ b/WorkloadManagementSystem/Agent/JobAgent.py @@ -544,7 +544,7 @@ def __getJDLParameters( self, jdl ): classAdJob = ClassAd( jdl ) paramsDict = classAdJob.contents for param, value in paramsDict.items(): - if re.search( '{', value ): + if value.strip().startswith('{'): self.log.debug( 'Found list type parameter %s' % ( param ) ) rawValues = value.replace( '{', '' ).replace( '}', '' ).replace( '"', '' ).split() valueList = [] diff --git a/WorkloadManagementSystem/Service/JobManagerHandler.py b/WorkloadManagementSystem/Service/JobManagerHandler.py index 1d2091bfe74..e16b8429795 100755 --- a/WorkloadManagementSystem/Service/JobManagerHandler.py +++ b/WorkloadManagementSystem/Service/JobManagerHandler.py @@ -132,7 +132,17 @@ def export_submitJob( self, jobDesc ): jobDescList = [] for n,p in enumerate(parameterList): - jobDescList.append( jobDesc.replace('%s',str(p)).replace('%n',str(n)) ) + newJobDesc = jobDesc.replace('%s',str(p)).replace('%n',str(n)) + newClassAd = ClassAd(newJobDesc) + for attr in ['Parameters','ParameterStep','ParameterFactor']: + newClassAd.deleteAttribute(attr) + if p.startswith('{'): + newClassAd.insertAttributeInt( 'Parameter',str(p) ) + else: + newClassAd.insertAttributeString( 'Parameter',str(p) ) + newClassAd.insertAttributeInt('ParameterNumber',n) + newJDL = newClassAd.asJDL() + jobDescList.append( newJDL ) else: jobDescList = [ jobDesc ] diff --git a/WorkloadManagementSystem/scripts/dirac-jobexec.py b/WorkloadManagementSystem/scripts/dirac-jobexec.py index f35808fde82..a41472fcc5b 100755 --- a/WorkloadManagementSystem/scripts/dirac-jobexec.py +++ b/WorkloadManagementSystem/scripts/dirac-jobexec.py @@ -75,6 +75,12 @@ def jobexec( jobxml, wfParameters = {} ): for switch, parameter in parList: if switch == "p": name, value = parameter.split( '=' ) + value = value.strip() + + # The comma separated list in curly brackets is interpreted as a list + if value.startswith("{"): + value = value[1:-1].replace('"','').replace(" ",'').split(',') + parDict[name] = value gLogger.verbose( 'PYTHONPATH:\n%s' % ( string.join( sys.path, '\n' ) ) ) result = jobexec( jobXMLfile, parDict )