Sparse row optimizer

src/main/java/org/apache/sysds/api/DMLOptions.java

@@ -85,6 +85,7 @@ public class DMLOptions {
 	public boolean              federatedCompilation = false;     // Compile federated instructions based on input federation state and privacy constraints.
 	public boolean              noFedRuntimeConversion = false;   // If activated, no runtime conversion of CP instructions to FED instructions will be performed.
 	public int                  seed          = -1;               // The general seed for the execution, if -1 random (system time).
+	public boolean				sparseIntermediate = false;       // whether SparseRowIntermediates should be used for rowwise operations
 
 	public final static DMLOptions defaultOptions = new DMLOptions(null);
 
@@ -117,7 +118,8 @@ public String toString() {
 			", w=" + fedWorker +
 			", federatedCompilation=" + federatedCompilation +
 			", noFedRuntimeConversion=" + noFedRuntimeConversion +
-			", seed=" + seed + 
+			", seed=" + seed +
+			", sparseIntermediate=" + sparseIntermediate +
 			'}';
 	}
 
@@ -350,6 +352,10 @@ else if (lineageType.equalsIgnoreCase("debugger"))
 			dmlOptions.seed = Integer.parseInt(line.getOptionValue("seed"));
 		}
 
+		if(line.hasOption("sparseIntermediate")){
+			dmlOptions.sparseIntermediate = true;
+		}
+
 		return dmlOptions;
 	}
 
@@ -431,7 +437,10 @@ private static Options createCLIOptions() {
 		Option commandlineSeed = OptionBuilder
 			.withDescription("A general seed for the execution through the commandline")
 			.hasArg().create("seed");
-
+		Option sparseRowIntermediates = OptionBuilder
+			.withDescription("If activated, sparseRowVector intermediates will be used to calculate rowwise operations.")
+			.create("sparseIntermediate");
+
 		options.addOption(configOpt);
 		options.addOption(cleanOpt);
 		options.addOption(statsOpt);
@@ -451,6 +460,7 @@ private static Options createCLIOptions() {
 		options.addOption(federatedCompilation);
 		options.addOption(noFedRuntimeConversion);
 		options.addOption(commandlineSeed);
+		options.addOption(sparseRowIntermediates);
 
 		// Either a clean(-clean), a file(-f), a script(-s) or help(-help) needs to be specified
 		OptionGroup fileOrScriptOpt = new OptionGroup()

src/main/java/org/apache/sysds/api/DMLScript.java

@@ -153,6 +153,9 @@ public class DMLScript
 	// Global seed 
 	public static int               SEED                 = -1;
 
+	// Sparse row flag
+	public static boolean 			SPARSE_INTERMEDIATE = false;
+
 	public static String MONITORING_ADDRESS = null;
 
 	// flag that indicates whether or not to suppress any prints to stdout
@@ -275,6 +278,7 @@ public static boolean executeScript( String[] args )
 			LINEAGE_ESTIMATE      = dmlOptions.lineage_estimate;
 			LINEAGE_DEBUGGER      = dmlOptions.lineage_debugger;
 			SEED                  = dmlOptions.seed;
+			SPARSE_INTERMEDIATE	  = dmlOptions.sparseIntermediate;
 
 
 			String fnameOptConfig = dmlOptions.configFile;

src/main/java/org/apache/sysds/hops/codegen/SpoofCompiler.java

@@ -470,7 +470,7 @@ public static Hop optimize( Hop root, boolean recompile ) {
 	 * @param recompile true if invoked during dynamic recompilation
 	 * @return dag root nodes of modified dag 
 	 */
-	public static ArrayList<Hop> optimize(ArrayList<Hop> roots, boolean recompile) 
+	public static ArrayList<Hop> optimize(ArrayList<Hop> roots, boolean recompile)
 	{
 		if( roots == null || roots.isEmpty() )
 			return roots;

src/main/java/org/apache/sysds/hops/codegen/cplan/CNode.java

@@ -19,6 +19,7 @@
 
 package org.apache.sysds.hops.codegen.cplan;
 
+import org.apache.sysds.api.DMLScript;
 import org.apache.sysds.common.Types.DataType;
 import org.apache.sysds.hops.codegen.SpoofCompiler.GeneratorAPI;
 import org.apache.sysds.hops.codegen.template.TemplateUtils;
@@ -77,6 +78,14 @@ public String createVarname() {
 			_genVar = "TMP"+_seqVar.getNextID();
 		return _genVar; 
 	}
+
+	public String createVarname(boolean sparse) {
+		if(!sparse) {
+			return createVarname();
+		} else {
+			return _genVar = "S" + createVarname();
+		}
+	}
 
 	public String getVarname() {
 		return _genVar;
@@ -98,6 +107,8 @@ public String getVectorLength(GeneratorAPI api) {
 				return "len";
 			if(getVarname().startsWith("b"))
 				return getVarname() + ".clen";
+			else if(getVarname().startsWith("STMP"))
+				return "len";
 			else if(_dataType == DataType.MATRIX)
 				return getVarname() + ".length";
 		}
@@ -222,8 +233,13 @@ public boolean equals(Object that) {
 
 	protected String replaceUnaryPlaceholders(String tmp, String varj, boolean vectIn, GeneratorAPI api) {
 		//replace sparse and dense inputs
-		tmp = tmp.replace("%IN1v%", varj+"vals");
-		tmp = tmp.replace("%IN1i%", varj+"ix");
+		if(DMLScript.SPARSE_INTERMEDIATE) {
+			tmp = tmp.replace("%IN1v%", varj.startsWith("STMP") ? varj+".values()" : varj+"vals");
+			tmp = tmp.replace("%IN1i%", varj.startsWith("STMP") ? varj+".indexes()" :varj+"ix");
+		} else {
+			tmp = tmp.replace("%IN1v%", varj+"vals");
+			tmp = tmp.replace("%IN1i%", varj+"ix");
+		}
 		tmp = tmp.replace("%IN1%", 
 			(vectIn && TemplateUtils.isMatrix(_inputs.get(0))) ? 
 				((api == GeneratorAPI.JAVA) ? varj + ".values(rix)" : varj + ".vals(0)" ) :

src/main/java/org/apache/sysds/hops/codegen/cplan/CNodeBinary.java

@@ -22,6 +22,7 @@
 import java.util.Arrays;
 
 import org.apache.commons.lang3.StringUtils;
+import org.apache.sysds.api.DMLScript;
 import org.apache.sysds.common.Opcodes;
 import org.apache.sysds.hops.codegen.template.TemplateUtils;
 import org.apache.sysds.common.Types.DataType;
@@ -126,7 +127,8 @@ public boolean isNotSupportedBySpoofCUDA() {
 	}
 
 	private final BinType _type;
-
+	private boolean sparseTemplate;
+
 	public CNodeBinary( CNode in1, CNode in2, BinType type ) {
 		//canonicalize commutative matrix-scalar operations
 		//to increase reuse potential
@@ -143,6 +145,23 @@ public CNodeBinary( CNode in1, CNode in2, BinType type ) {
 		setOutputDims();
 	}
 
+	public CNodeBinary( CNode in1, CNode in2, BinType type, double sparsityEst, double scalarVal ) {
+		//canonicalize commutative matrix-scalar operations
+		//to increase reuse potential
+		if( type.isCommutative() && in1 instanceof CNodeData
+			&& in1.getDataType()==DataType.SCALAR ) {
+			CNode tmp = in1;
+			in1 = in2;
+			in2 = tmp;
+		}
+
+		_inputs.add(in1);
+		_inputs.add(in2);
+		_type = type;
+		setOutputDims();
+		sparseTemplate = getTemplateType(sparsityEst, scalarVal);
+	}
+
 	public BinType getType() {
 		return _type;
 	}
@@ -157,60 +176,63 @@ public String codegen(boolean sparse, GeneratorAPI api) {
 		//generate children
 		sb.append(_inputs.get(0).codegen(sparse, api));
 		sb.append(_inputs.get(1).codegen(sparse, api));
-		
+
 		//generate binary operation (use sparse template, if data input)
-		boolean lsparseLhs = sparse && _inputs.get(0) instanceof CNodeData 
-			&& _inputs.get(0).getVarname().startsWith("a");
-		boolean lsparseRhs = sparse && _inputs.get(1) instanceof CNodeData 
-			&& _inputs.get(1).getVarname().startsWith("a");
+		boolean lsparseLhs = sparse ? _inputs.get(0) instanceof CNodeData
+			&& _inputs.get(0).getVarname().startsWith("a") ||
+			_inputs.get(0).getVarname().startsWith("STMP") : false;
+		boolean lsparseRhs = sparse ? _inputs.get(1) instanceof CNodeData
+			&& _inputs.get(1).getVarname().startsWith("a") ||
+			_inputs.get(1).getVarname().startsWith("STMP") : false;
 		boolean scalarInput = _inputs.get(0).getDataType().isScalar();
 		boolean scalarVector = (_inputs.get(0).getDataType().isScalar()
 			&& _inputs.get(1).getDataType().isMatrix());
 		boolean vectorVector = _inputs.get(0).getDataType().isMatrix()
 			&& _inputs.get(1).getDataType().isMatrix();
-		String var = createVarname();
+		String var = createVarname(sparse && sparseTemplate && getOutputType(scalarVector, lsparseLhs, lsparseRhs));
 		String tmp = getLanguageTemplateClass(this, api)
-			.getTemplate(_type, lsparseLhs, lsparseRhs, scalarVector, scalarInput, vectorVector);
+			.getTemplate(_type, lsparseLhs, lsparseRhs, scalarVector, scalarInput, vectorVector, sparseTemplate);
 
 		tmp = tmp.replace("%TMP%", var);
-		
+
 		//replace input references and start indexes
 		for( int j=0; j<2; j++ ) {
 			String varj = _inputs.get(j).getVarname(api);
-
 			//replace sparse and dense inputs
-			tmp = tmp.replace("%IN"+(j+1)+"v%", varj+"vals");
-			tmp = tmp.replace("%IN"+(j+1)+"i%", varj+"ix");
+			tmp = tmp.replace("%IN"+(j+1)+"v%", varj.startsWith("STMP") ? varj+".values()" : varj+"vals");
+			tmp = tmp.replace("%IN"+(j+1)+"i%", varj.startsWith("STMP") ? varj+".indexes()" : varj+"ix");
 			tmp = tmp.replace("%IN"+(j+1)+"%",
-					varj.startsWith("a") ? (api == GeneratorAPI.JAVA ? varj : 
-						(_inputs.get(j).getDataType() == DataType.MATRIX ? varj + ".vals(0)" : varj)) :
-						varj.startsWith("b") ? (api == GeneratorAPI.JAVA ? varj + ".values(rix)" : 
-								(_type == BinType.VECT_MATRIXMULT ? varj : varj + ".vals(0)")) :
-							_inputs.get(j).getDataType() == DataType.MATRIX ? (api == GeneratorAPI.JAVA ? varj : varj + ".vals(0)") : varj);
-
+				varj.startsWith("a") ? (api == GeneratorAPI.JAVA ? varj :
+					(_inputs.get(j).getDataType() == DataType.MATRIX ? varj + ".vals(0)" : varj)) :
+					varj.startsWith("b") ? (api == GeneratorAPI.JAVA ? varj + ".values(rix)" :
+						(_type == BinType.VECT_MATRIXMULT ? varj : varj + ".vals(0)")) :
+						_inputs.get(j).getDataType() == DataType.MATRIX ? (api == GeneratorAPI.JAVA ? varj : varj + ".vals(0)") : varj);
+
+				tmp = tmp.replace("%SLEN"+(j+1)+"%", varj.startsWith("STMP") ? varj+".size()" : varj.startsWith("a") ? "alen" : "blen");
+
 			//replace start position of main input
-			tmp = tmp.replace("%POS"+(j+1)+"%", (_inputs.get(j) instanceof CNodeData 
-					&& _inputs.get(j).getDataType().isMatrix()) ? (!varj.startsWith("b")) ? varj+"i" : 
-					((TemplateUtils.isMatrix(_inputs.get(j)) || (_type.isElementwise()
-						&& TemplateUtils.isColVector(_inputs.get(j)))) && _type!=BinType.VECT_MATRIXMULT) ?
+			tmp = tmp.replace("%POS"+(j+1)+"%", (_inputs.get(j) instanceof CNodeData
+				&& _inputs.get(j).getDataType().isMatrix()) ? (!varj.startsWith("b")) ? varj+"i" :
+				((TemplateUtils.isMatrix(_inputs.get(j)) || (_type.isElementwise()
+					&& TemplateUtils.isColVector(_inputs.get(j)))) && _type!=BinType.VECT_MATRIXMULT) ?
 					varj + ".pos(rix)" : "0" : "0");
 		}
 		//replace length information (e.g., after matrix mult)
-		if( _type == BinType.VECT_OUTERMULT_ADD || (_type == BinType.VECT_CBIND && vectorVector) ) {
+		if( _type == BinType.VECT_OUTERMULT_ADD || (_type == BinType.VECT_CBIND && vectorVector)) {
 			for( int j=0; j<2; j++ )
 				tmp = tmp.replace("%LEN"+(j+1)+"%", _inputs.get(j).getVectorLength(api));
 		}
-		else { //general case 
+		else { //general case
 			CNode mInput = getIntermediateInputVector();
 			if( mInput != null )
 				tmp = tmp.replace("%LEN%", mInput.getVectorLength(api));
 		}
-		
+
 		sb.append(tmp);
-		
+
 		//mark as generated
 		_generated = true;
-		
+
 		return sb.toString();
 	}
 
@@ -219,7 +241,126 @@ private CNode getIntermediateInputVector() {
 			if( getInput().get(i).getDataType().isMatrix() )
 				return getInput().get(i);
 		return null;
-	} 
+	}
+
+	private boolean getTemplateType(double sparsityEst, double scalarVal) {
+		if(!DMLScript.SPARSE_INTERMEDIATE)
+			return false;
+		else {
+			switch(_type) {
+				case VECT_MULT:
+				case VECT_DIV:
+				case VECT_LESS:
+				case VECT_MINUS:
+				case VECT_PLUS:
+				case VECT_XOR:
+				case VECT_BITWAND:
+				case VECT_BIASADD:
+				case VECT_BIASMULT:
+				case VECT_MIN:
+				case VECT_MAX:
+				case VECT_NOTEQUAL:
+				case VECT_GREATER:
+				case VECT_EQUAL:
+				case VECT_LESSEQUAL:
+				case VECT_GREATEREQUAL: return sparsityEst < 0.1;
+			 	case VECT_MULT_SCALAR:
+				case VECT_DIV_SCALAR:
+				case VECT_XOR_SCALAR:
+				case VECT_BITWAND_SCALAR: return sparsityEst < 0.3;
+				case VECT_GREATER_SCALAR: {
+					if(scalarVal != Double.NaN) {
+						return _inputs.get(1).getDataType().isScalar() ? scalarVal >= 0 && sparsityEst < 0.2
+							: _inputs.get(0).getDataType().isScalar() && scalarVal < 0 && sparsityEst < 0.2;
+					} else
+						return false;
+				}
+				case VECT_GREATEREQUAL_SCALAR: {
+					if(scalarVal != Double.NaN) {
+						return _inputs.get(1).getDataType().isScalar() ? scalarVal > 0 && sparsityEst < 0.2
+							: _inputs.get(0).getDataType().isScalar() && scalarVal <= 0 && sparsityEst < 0.2;
+					} else
+						return false;
+				}
+				case VECT_MIN_SCALAR: {
+					if(scalarVal != Double.NaN) {
+						return _inputs.get(1).getDataType().isScalar() ? scalarVal >= 0 && sparsityEst < 0.2
+							: _inputs.get(0).getDataType().isScalar() && scalarVal >= 0 && sparsityEst < 0.2;
+					} else
+						return false;
+				}
+				case VECT_LESS_SCALAR: {
+					if(scalarVal != Double.NaN) {
+						return _inputs.get(1).getDataType().isScalar() ? scalarVal <= 0 && sparsityEst < 0.2
+							: _inputs.get(0).getDataType().isScalar() && scalarVal > 0 && sparsityEst < 0.2;
+					} else
+						return false;
+				}
+				case VECT_LESSEQUAL_SCALAR: {
+					if(scalarVal != Double.NaN) {
+						return _inputs.get(1).getDataType().isScalar() ? scalarVal < 0 && sparsityEst < 0.2
+							: _inputs.get(0).getDataType().isScalar() && scalarVal >= 0 && sparsityEst < 0.2;
+					} else
+						return false;
+				}
+				case VECT_MAX_SCALAR: {
+					if(scalarVal != Double.NaN) {
+						return _inputs.get(1).getDataType().isScalar() ? scalarVal <= 0 && sparsityEst < 0.2
+							: _inputs.get(0).getDataType().isScalar() && scalarVal <= 0 && sparsityEst < 0.2;
+					} else
+						return false;
+				}
+				case VECT_POW_SCALAR:
+				case VECT_EQUAL_SCALAR:{
+					if(scalarVal != Double.NaN) {
+						return _inputs.get(1).getDataType().isScalar() ? scalarVal != 0 && sparsityEst < 0.2
+							: _inputs.get(0).getDataType().isScalar() && scalarVal != 0 && sparsityEst < 0.2;
+					} else
+						return false;
+				}
+				case VECT_NOTEQUAL_SCALAR:{
+					if(scalarVal != Double.NaN) {
+						return _inputs.get(1).getDataType().isScalar() ? scalarVal == 0 && sparsityEst < 0.2
+							: _inputs.get(0).getDataType().isScalar() && scalarVal == 0 && sparsityEst < 0.2;
+					} else
+						return false;
+				}
+				default: return sparsityEst < 0.3;
+			}
+		}
+	}
+
+	public boolean getOutputType(boolean scalarVector, boolean lsparseLhs, boolean lsparseRhs) {
+		switch(_type) {
+			case VECT_POW_SCALAR: return !scalarVector && lsparseLhs;
+			case VECT_MULT_SCALAR:
+			case VECT_DIV_SCALAR:
+			case VECT_XOR_SCALAR:
+			case VECT_MIN_SCALAR:
+			case VECT_MAX_SCALAR:
+			case VECT_EQUAL_SCALAR:
+			case VECT_NOTEQUAL_SCALAR:
+			case VECT_LESS_SCALAR:
+			case VECT_LESSEQUAL_SCALAR:
+			case VECT_GREATER_SCALAR:
+			case VECT_GREATEREQUAL_SCALAR:
+			case VECT_BITWAND_SCALAR: return lsparseLhs || lsparseRhs;
+			case VECT_MULT:
+			case VECT_DIV:
+			case VECT_MINUS:
+			case VECT_PLUS:
+			case VECT_XOR:
+			case VECT_BITWAND:
+			case VECT_BIASADD:
+			case VECT_BIASMULT:
+			case VECT_MIN:
+			case VECT_MAX:
+			case VECT_NOTEQUAL:
+			case VECT_LESS:
+			case VECT_GREATER: return lsparseLhs && lsparseRhs;
+			default: return false;
+		}
+	}
 
 	@Override
 	public String toString() {

src/main/java/org/apache/sysds/hops/codegen/cplan/CNodeNary.java

@@ -60,7 +60,10 @@ public String getTemplate(boolean sparseGen, long len, ArrayList<CNode> inputs,
 							sb.append( sparseInput ?
 								"    LibSpoofPrimitives.vectWrite("+varj+"vals, %TMP%, "
 									+varj+"ix, "+pos+", "+off+", "+input._cols+");\n" :
-								"    LibSpoofPrimitives.vectWrite("+(varj.startsWith("b")?varj+".values(rix)":varj)
+								varj.startsWith("STMP") ?
+									"    LibSpoofPrimitives.vectWrite("+varj+".values(), %TMP%, "
+										+varj+".indexes(), "+pos+", "+off+", "+varj+".size());\n" :
+									"    LibSpoofPrimitives.vectWrite("+(varj.startsWith("b")?varj+".values(rix)":varj)
 									+", %TMP%, "+pos+", "+off+", "+input._cols+");\n");
 							off += input._cols;	
 						}