NEW: support for parametric jobs with lists of lists

Core/Utilities/ClassAd/ClassAdLight.py

@@ -127,6 +127,17 @@ def insertAttributeVectorString( self, name, attributelist ):
     tmp = map ( lambda x : '"' + x + '"', attributelist )
     tmpstr = ','.join( tmp )
     self.contents[name] = '{' + tmpstr + '}'
+
+  def insertAttributeVectorStringList( self, name, attributelist ):
+    """Insert a named list of string lists 
+    """
+
+    listOfLists = []
+    for stringList in attributelist:
+      #tmp = map ( lambda x : '"' + x + '"', stringList )
+      tmpstr = ','.join( stringList )
+      listOfLists.append('{' + tmpstr + '}')
+    self.contents[name] = '{' + ','.join(listOfLists) + '}'    
 
   def lookupAttribute( self, name ):
     """Check the presence of the given attribute
@@ -159,13 +170,45 @@ def isAttributeList( self, name ):
     return attribute.startswith( '{' )
 
   def getListFromExpression( self, name ):
-    """ Get a list of values from a given expression
+    """ Get a list of strings from a given expression
     """
 
-    tempString = self.get_expression( name )
-    tempString = tempString.replace( "{", "" ).replace( "}", "" ).replace( "\"", "" ).replace( " ", "" )
+    tempString = self.get_expression( name ).strip()
+    listMode = False
+    if tempString.startswith('{'):
+      tempString = tempString[1:-1]
+      listMode = True
+
+    tempString = tempString.replace( " ", "" ).replace( '\n','' )  
+    if tempString.find('{') < 0:
+      if not listMode:
+        tempString = tempString.replace( "\"", "" )
+        return tempString.split( ',' ) 
+
+    resultList = []    
+    while tempString:      
+      if tempString.find( '{' ) == 0 :
+        end = tempString.find( '}' )
+        resultList.append(tempString[:end+1])
+        tempString = tempString[end+1:]
+        if tempString.startswith(','):
+          tempString = tempString[1:]
+      elif tempString.find( '"' ) == 0 :
+        end = tempString[1:].find( '"' )
+        resultList.append( tempString[1:end+1] )
+        tempString = tempString[end+2:]    
+        if tempString.startswith(','):
+          tempString = tempString[1:]          
+      else:
+        end = tempString.find( ',' )
+        if end < 0:
+          resultList.append( tempString.replace( "\"", "" ).replace( " ", "" ) )
+          break
+        else:  
+          resultList.append( tempString[:end-1].replace( "\"", "" ).replace( " ", "" ) )
+          tempString = tempString[end+1:]
 
-    return tempString.split( ',' )
+    return resultList
 
   def getDictionaryFromSubJDL( self, name ):
     """ Get a dictionary of the JDL attributes from a subsection

Core/Utilities/JDL.py

@@ -137,7 +137,8 @@ def dumpCFGAsJDL( cfg, level = 1, tab = "  " ):
       contents.append( "%s;" % dumpCFGAsJDL( cfg[ key ], level + 1, tab ) )
     else:
       val = List.fromChar( cfg[ key ] )
-      if len( val ) < 2:
+      # Some attributes are never lists
+      if len( val ) < 2 or key in ['Arguments','Executable','StdOutput','StdError']:
         value = cfg[ key ]
         try:
           try_value = float( value )

Interfaces/API/Job.py

@@ -259,15 +259,13 @@ def setParametricInputSandbox( self, files ):
         if not fileName.lower().count( "lfn:" ):
           return self._reportError( 'All files should be LFNs', **kwargs )
       resolvedFiles = self._resolveInputSandbox( files )
-      fileList = ";".join( resolvedFiles )
-      self.parametric['InputSandbox'] = fileList
+      self.parametric['InputSandbox'] = resolvedFiles
       #self.sandboxFiles=resolvedFiles
     elif type( files ) == type( " " ):
       if not files.lower().count( "lfn:" ):
         return self._reportError( 'All files should be LFNs', **kwargs )
       resolvedFiles = self._resolveInputSandbox( [files] )
-      fileList = ";".join( resolvedFiles )
-      self.parametric['InputSandbox'] = fileList
+      self.parametric['InputSandbox'] = resolvedFiles
       #self.sandboxFiles = [files]
     else:
       return self._reportError( 'Expected file string or list of files for input sandbox contents', **kwargs )
@@ -350,10 +348,12 @@ def setParametricInputData( self, lfns ):
     """
     if type( lfns ) == list and len( lfns ):
       for i in xrange( len( lfns ) ):
-        lfns[i] = lfns[i].replace( 'LFN:', '' )
-      inputData = map( lambda x: 'LFN:' + x, lfns )
-      inputDataStr = ';'.join( inputData )
-      self.parametric['InputData'] = inputDataStr
+        if type( lfns[i] ) == list and len( lfns[i] ):
+          for k in xrange( len( lfns[i] ) ):
+            lfns[i][k] = 'LFN:' + lfns[i][k].replace( 'LFN:', '' )
+        else:  
+          lfns[i] = 'LFN:' + lfns[i].replace( 'LFN:', '' )
+      self.parametric['InputData'] = lfns
     elif type( lfns ) == type( ' ' ):  #single LFN
       self.parametric['InputData'] = lfns
     else:
@@ -904,11 +904,11 @@ def __setJobDefaults( self ):
     self._addParameter( self.workflow, 'InputData', 'JDL', '', 'Default null input data value' )
     self._addParameter( self.workflow, 'LogLevel', 'JDL', self.logLevel, 'Job Logging Level' )
     #Those 2 below are need for on-site resolution
-    self._addParameter( self.workflow, 'ParametricInputData', 'JDL', '',
+    self._addParameter( self.workflow, 'ParametricInputData', 'WF', '',
                         'Default null parametric input data value' )
-    self._addParameter( self.workflow, 'ParametricInputSandbox', 'JDL', '',
+    self._addParameter( self.workflow, 'ParametricInputSandbox', 'WF', '',
                         'Default null parametric input sandbox value' )
-    self._addParameter( self.workflow, 'ParametricParameters', 'JDL', '', 
+    self._addParameter( self.workflow, 'ParametricParameters', 'WF', '', 
                         'Default null parametric input parameters value' )
 
   #############################################################################
@@ -1135,17 +1135,17 @@ def _toJDL( self, xmlFile = '' ): #messy but need to account for xml file being
           paramsDict['InputSandbox'] = {}
           paramsDict['InputSandbox']['value'] = '%s'
           paramsDict['InputSandbox']['type'] = 'JDL'
-        self.parametric['files']=  self.parametric['InputSandbox']
-        arguments.append(' -p ParametricInputSandbox="%s"')
+        self.parametric['files'] = self.parametric['InputSandbox']
+        arguments.append(' -p ParametricInputSandbox=%s')
       if self.parametric.has_key('files'):   
         paramsDict['Parameters']={}
-        paramsDict['Parameters']['value']=";".join(self.parametric['files'])
+        paramsDict['Parameters']['value'] = self.parametric['files']
         paramsDict['Parameters']['type'] = 'JDL'
       if self.parametric.has_key('GenericParameters'):
         paramsDict['Parameters']={}
-        paramsDict['Parameters']['value']=";".join(self.parametric['GenericParameters'])
+        paramsDict['Parameters']['value'] = self.parametric['GenericParameters']
         paramsDict['Parameters']['type'] = 'JDL'
-        arguments.append(' -p ParametricParameters="%s"')
+        arguments.append(' -p ParametricParameters=%s')
     ##This needs to be put here so that the InputData and/or InputSandbox parameters for parametric jobs are processed
     classadJob.insertAttributeString( 'Arguments', ' '.join( arguments ) )
 
@@ -1159,9 +1159,16 @@ def _toJDL( self, xmlFile = '' ): #messy but need to account for xml file being
         requirements = True
 
       if re.search( '^JDL', ptype ):
-        if not re.search( ';', value ) or name == 'GridRequirements': #not a nice fix...
+        if type( value ) == list:
+          if type( value[0] ) == list:
+            classadJob.insertAttributeVectorStringList( name, value )
+          else:
+            classadJob.insertAttributeVectorString( name, value )  
+        elif value == "%s":
+          classadJob.insertAttributeInt( name, value )    
+        elif not re.search( ';', value ) or name == 'GridRequirements': #not a nice fix...
           classadJob.insertAttributeString( name, value )
-        else:
+        else:  
           classadJob.insertAttributeVectorString( name, value.split( ';' ) )
 
     if not requirements:

WorkloadManagementSystem/Agent/JobAgent.py

@@ -544,7 +544,7 @@ def __getJDLParameters( self, jdl ):
       classAdJob = ClassAd( jdl )
       paramsDict = classAdJob.contents
       for param, value in paramsDict.items():
-        if re.search( '{', value ):
+        if value.strip().startswith('{'):
           self.log.debug( 'Found list type parameter %s' % ( param ) )
           rawValues = value.replace( '{', '' ).replace( '}', '' ).replace( '"', '' ).split()
           valueList = []

WorkloadManagementSystem/Service/JobManagerHandler.py

@@ -132,7 +132,17 @@ def export_submitJob( self, jobDesc ):
 
       jobDescList = []
       for n,p in enumerate(parameterList):
-        jobDescList.append( jobDesc.replace('%s',str(p)).replace('%n',str(n)) )
+        newJobDesc = jobDesc.replace('%s',str(p)).replace('%n',str(n))
+        newClassAd = ClassAd(newJobDesc)
+        for attr in ['Parameters','ParameterStep','ParameterFactor']:
+          newClassAd.deleteAttribute(attr)
+        if p.startswith('{'):
+          newClassAd.insertAttributeInt( 'Parameter',str(p) )
+        else:
+          newClassAd.insertAttributeString( 'Parameter',str(p) )
+        newClassAd.insertAttributeInt('ParameterNumber',n)
+        newJDL = newClassAd.asJDL()
+        jobDescList.append( newJDL )
     else:
       jobDescList = [ jobDesc ]     
 

WorkloadManagementSystem/scripts/dirac-jobexec.py

@@ -75,6 +75,12 @@ def jobexec( jobxml, wfParameters = {} ):
 for switch, parameter in parList:
   if switch == "p":
     name, value = parameter.split( '=' )
+    value = value.strip()
+
+    # The comma separated list in curly brackets is interpreted as a list
+    if value.startswith("{"):
+      value = value[1:-1].replace('"','').replace(" ",'').split(',')
+
     parDict[name] = value
 gLogger.verbose( 'PYTHONPATH:\n%s' % ( string.join( sys.path, '\n' ) ) )
 result = jobexec( jobXMLfile, parDict )