diff --git a/src/main/java/org/apache/sysds/api/DMLOptions.java b/src/main/java/org/apache/sysds/api/DMLOptions.java index 763ac7b9388..b5b0908432b 100644 --- a/src/main/java/org/apache/sysds/api/DMLOptions.java +++ b/src/main/java/org/apache/sysds/api/DMLOptions.java @@ -85,6 +85,7 @@ public class DMLOptions { public boolean federatedCompilation = false; // Compile federated instructions based on input federation state and privacy constraints. public boolean noFedRuntimeConversion = false; // If activated, no runtime conversion of CP instructions to FED instructions will be performed. public int seed = -1; // The general seed for the execution, if -1 random (system time). + public boolean sparseIntermediate = false; // whether SparseRowIntermediates should be used for rowwise operations public final static DMLOptions defaultOptions = new DMLOptions(null); @@ -117,7 +118,8 @@ public String toString() { ", w=" + fedWorker + ", federatedCompilation=" + federatedCompilation + ", noFedRuntimeConversion=" + noFedRuntimeConversion + - ", seed=" + seed + + ", seed=" + seed + + ", sparseIntermediate=" + sparseIntermediate + '}'; } @@ -350,6 +352,10 @@ else if (lineageType.equalsIgnoreCase("debugger")) dmlOptions.seed = Integer.parseInt(line.getOptionValue("seed")); } + if(line.hasOption("sparseIntermediate")){ + dmlOptions.sparseIntermediate = true; + } + return dmlOptions; } @@ -431,7 +437,10 @@ private static Options createCLIOptions() { Option commandlineSeed = OptionBuilder .withDescription("A general seed for the execution through the commandline") .hasArg().create("seed"); - + Option sparseRowIntermediates = OptionBuilder + .withDescription("If activated, sparseRowVector intermediates will be used to calculate rowwise operations.") + .create("sparseIntermediate"); + options.addOption(configOpt); options.addOption(cleanOpt); options.addOption(statsOpt); @@ -451,6 +460,7 @@ private static Options createCLIOptions() { options.addOption(federatedCompilation); options.addOption(noFedRuntimeConversion); options.addOption(commandlineSeed); + options.addOption(sparseRowIntermediates); // Either a clean(-clean), a file(-f), a script(-s) or help(-help) needs to be specified OptionGroup fileOrScriptOpt = new OptionGroup() diff --git a/src/main/java/org/apache/sysds/api/DMLScript.java b/src/main/java/org/apache/sysds/api/DMLScript.java index d6853891e24..44dc35ee75e 100644 --- a/src/main/java/org/apache/sysds/api/DMLScript.java +++ b/src/main/java/org/apache/sysds/api/DMLScript.java @@ -153,6 +153,9 @@ public class DMLScript // Global seed public static int SEED = -1; + // Sparse row flag + public static boolean SPARSE_INTERMEDIATE = false; + public static String MONITORING_ADDRESS = null; // flag that indicates whether or not to suppress any prints to stdout @@ -275,6 +278,7 @@ public static boolean executeScript( String[] args ) LINEAGE_ESTIMATE = dmlOptions.lineage_estimate; LINEAGE_DEBUGGER = dmlOptions.lineage_debugger; SEED = dmlOptions.seed; + SPARSE_INTERMEDIATE = dmlOptions.sparseIntermediate; String fnameOptConfig = dmlOptions.configFile; diff --git a/src/main/java/org/apache/sysds/runtime/codegen/LibSpoofPrimitives.java b/src/main/java/org/apache/sysds/runtime/codegen/LibSpoofPrimitives.java index 6c0dc395c3f..bc6ba198951 100644 --- a/src/main/java/org/apache/sysds/runtime/codegen/LibSpoofPrimitives.java +++ b/src/main/java/org/apache/sysds/runtime/codegen/LibSpoofPrimitives.java @@ -23,6 +23,7 @@ import org.apache.commons.math3.util.FastMath; import org.apache.sysds.runtime.data.DenseBlockFP64; +import org.apache.sysds.runtime.data.SparseRowVector; import org.apache.sysds.runtime.functionobjects.BitwAnd; import org.apache.sysds.runtime.functionobjects.IntegerDivide; import org.apache.sysds.runtime.functionobjects.Modulus; @@ -51,6 +52,10 @@ public class LibSpoofPrimitives @Override protected VectorBuffer initialValue() { return new VectorBuffer(0,0,0); } }; + private static ThreadLocal sparseMemPool = new ThreadLocal<>() { + @Override protected SparseVectorBuffer initialValue() { return new SparseVectorBuffer(0,0,0); } + }; + public static double rowMaxsVectMult(double[] a, double[] b, int ai, int bi, int len) { double val = Double.NEGATIVE_INFINITY; int j=0; @@ -2164,6 +2169,1155 @@ public static double vectVar(double[] avals, int[] aix, int ai, int alen, int le return (vectSum(avalsSqr, 0, len)-len*meanVal)/(len-1); } + /** + * Vector primitives with SparseRowVector intermediates + * Changes: + * - Changed method signature to avoid method duplicate conflicts + * e.g. (double[], double, int[], int, int, int) --> (int, double[], double, int[], int, int) + * - Added blen for vector - vector calculations to be able to use both vectors as SparseRowVectors + * - Implemented a new SparseVectorBuffer class that creates a ring buffer for SparseRowVectors in different sizes + */ + + public static SparseRowVector vectMultWrite(int len, double[] a, double bval, int[] aix, int ai, int alen) { + SparseRowVector c = allocSparseVector(alen); + if( a == null ) return c; + int[] indexes = c.indexes(); + double[] values = c.values(); + for(int j = 0; j < alen; j++) { + indexes[j] = aix[ai+j]; + values[j] = a[ai+j]*bval; + } + c.setSize(alen); + return c; + } + + public static SparseRowVector vectMultWrite(int len, double bval, double[] a, int[] aix, int ai, int alen) { + return vectMultWrite(len, a, bval, aix, ai, alen); + } + + //old version with branching (not used) + public static SparseRowVector vectMultWriteB(int len, double[] a, double[] b, int[] aix, int[] bix, int ai, int bi, int alen, int blen) { + SparseRowVector c = allocSparseVector(Math.min(alen, blen)); + if( a == null || b == null ) return c; + int aItr = ai; + int bItr = bi; + int index = 0; + int[] indexes = c.indexes(); + double[] values = c.values(); + while(aItr < ai+alen && bItr < bi+blen) { + int aIdx = aix[aItr]; + int bIdx = bix[bItr]; + if(aIdx == bIdx) { + indexes[index] = aIdx; + values[index] = a[aItr] * b[bItr]; + aItr++; + bItr++; + index++; + } else if(aIdx < bIdx) + aItr++; + else + bItr++; + } + c.setSize(index); + return c; + } + + //version without branching + public static SparseRowVector vectMultWrite(int len, double[] a, double[] b, int[] aix, int[] bix, int ai, int bi, int alen, int blen) { + SparseRowVector c = allocSparseVector(Math.min(alen, blen)); + int index = 0; + int aItr = ai; + int bItr = bi; + int[] indexes = c.indexes(); + double[] values = c.values(); + while(aItr < ai+alen && bItr < bi+blen) { + int aIdx = aix[aItr]; + int bIdx = bix[bItr]; + indexes[index] = aIdx; + values[index] = a[aItr] * b[bItr]; + index += aIdx == bIdx ? 1 : 0; + aItr += aIdx <= bIdx ? 1 : 0; + bItr += aIdx >= bIdx ? 1 : 0; + } + c.setSize(index); + return c; + } + + public static void vectWrite(double[] a, int[] aix, double[] c, int ci, int len) { + if( a == null ) return; + for(int j = 0; j < len; j++) + c[ci+aix[j]] = a[j]; + } + + public static void vectWrite(double[] a, double[] c, int[] aix, int ai, int ci, int alen) { + if( a == null ) return; + for(int j = 0; j < alen; j++) + c[ci+aix[ai+j]] = a[ai+j]; + } + + public static SparseRowVector vectDivWrite(int len, double[] a, double bval, int[] aix, int ai, int alen) { + SparseRowVector c = allocSparseVector(alen); + int[] indexes = c.indexes(); + double[] values = c.values(); + for( int j = 0; j < alen; j++ ) { + indexes[j] = aix[ai+j]; + values[j] = a[ai+j] / bval; + } + c.setSize(alen); + return c; + } + + public static SparseRowVector vectDivWrite(int len, double bval, double[] a, int[] aix, int ai, int alen) { + SparseRowVector c = allocSparseVector(alen); + int[] indexes = c.indexes(); + double[] values = c.values(); + for(int j = 0; j < alen; j++) { + indexes[j] = aix[ai+j]; + values[j] = bval / a[ai+j]; + } + c.setSize(alen); + return c; + } + + //old version with branching (not used) + public static SparseRowVector vectDivWriteB(int len, double[] a, double[] b, int[] aix, int[] bix, int ai, int bi, int alen, int blen) { + SparseRowVector c = allocSparseVector(alen); + int aItr = ai; + int bItr = bi; + int index = 0; + int[] indexes = c.indexes(); + double[] values = c.values(); + while(aItr < ai+alen && bItr < bi+blen) { + int aIdx = aix[aItr]; + int bIdx = bix[bItr]; + if(aIdx == bIdx) { + indexes[index] = aIdx; + values[index] = a[aItr] / b[bItr]; + aItr++; + bItr++; + index++; + } else if(aIdx < bIdx) { + indexes[index] = aIdx; + values[index] = (a[aItr]>0) ? Double.POSITIVE_INFINITY : Double.NEGATIVE_INFINITY; + aItr++; + index++; + } else { + bItr++; + } + } + c.setSize(index); + return c; + } + + //version without branching + public static SparseRowVector vectDivWrite(int len, double[] a, double[] b, int[] aix, int[] bix, int ai, int bi, int alen, int blen) { + SparseRowVector c = allocSparseVector(Math.min(alen, blen)); + int index = 0; + int aItr = ai; + int bItr = bi; + int[] indexes = c.indexes(); + double[] values = c.values(); + while(aItr < ai+alen && bItr < bi+blen) { + int aIdx = aix[aItr]; + int bIdx = bix[bItr]; + indexes[index] = aIdx; + values[index] = a[aItr] / b[bItr]; + index += aIdx == bIdx ? 1 : 0; + aItr += aIdx <= bIdx ? 1 : 0; + bItr += aIdx >= bIdx ? 1 : 0; + } + c.setSize(index); + return c; + } + + public static SparseRowVector vectMinusWriteB(int len, double[] a, double[] b, int[] aix, int[] bix, int ai, int bi, int alen, int blen) { + SparseRowVector c = allocSparseVector(alen+blen); + int aItr = ai; + int bItr = bi; + int index = 0; + int[] indexes = c.indexes(); + double[] values = c.values(); + while(aItr < ai+alen && bItr < bi+blen) { + int aIdx = aix[aItr]; + int bIdx = bix[bItr]; + if(aIdx == bIdx) { + indexes[index] = aIdx; + values[index] = a[aItr] - b[bItr]; + aItr++; + bItr++; + index++; + } else if(aIdx < bIdx) { + indexes[index] = aIdx; + values[index] = a[aItr]; + aItr++; + index++; + } else { + indexes[index] = bIdx; + values[index] = -b[bItr]; + bItr++; + index++; + } + } + for (; aItr < ai+alen; aItr++) { + indexes[index] = aix[aItr]; + values[index] = a[aItr]; + index++; + } + for (; bItr < bi+blen; bItr++) { + indexes[index] = bix[bItr]; + values[index] = -b[bItr]; + index++; + } + c.setSize(index); + return c; + } + + public static SparseRowVector vectMinusWrite(int len, double[] a, double[] b, int[] aix, int[] bix, int ai, int bi, int alen, int blen) { + SparseRowVector c = allocSparseVector(alen+blen); + int aEnd = ai+alen; + int bEnd = bi+blen; + int aItr = ai; + int bItr = bi; + int index = 0; + int[] indexes = c.indexes(); + double[] values = c.values(); + while (aItr < aEnd && bItr < bEnd) { + int aIdx = aix[aItr]; + int bIdx = bix[bItr]; + int useA = (aIdx <= bIdx) ? 1 : 0; + int useB = (aIdx >= bIdx) ? 1 : 0; + indexes[index] = (useA == 1) ? aIdx : bIdx; + double av = (useA == 1) ? a[aItr] : 0.0; + double bv = (useB == 1) ? b[bItr] : 0.0; + + values[index] = av - bv; + aItr += useA; + bItr += useB; + index++; + } + for (; aItr < aEnd; aItr++, index++) { + indexes[index] = aix[aItr]; + values[index] = a[aItr]; + } + for (; bItr < bEnd; bItr++, index++) { + indexes[index] = bix[bItr]; + values[index] = -b[bItr]; + } + c.setSize(index); + return c; + } + + public static SparseRowVector vectPlusWrite(int len, double[] a, double[] b, int[] aix, int[] bix, int ai, int bi, int alen, int blen) { + SparseRowVector c = allocSparseVector(alen+blen); + int aEnd = ai+alen; + int bEnd = bi+blen; + int aItr = ai; + int bItr = bi; + int index = 0; + int[] indexes = c.indexes(); + double[] values = c.values(); + while (aItr < aEnd && bItr < bEnd) { + int aIdx = aix[aItr]; + int bIdx = bix[bItr]; + int useA = (aIdx <= bIdx) ? 1 : 0; + int useB = (aIdx >= bIdx) ? 1 : 0; + indexes[index] = (useA == 1) ? aIdx : bIdx; + double av = (useA == 1) ? a[aItr] : 0.0; + double bv = (useB == 1) ? b[bItr] : 0.0; + + values[index] = av + bv; + aItr += useA; + bItr += useB; + index++; + } + for (; aItr < aEnd; aItr++) { + indexes[index] = aix[aItr]; + values[index] = a[aItr]; + index++; + } + for (; bItr < bEnd; bItr++) { + indexes[index] = bix[bItr]; + values[index] = b[bItr]; + index++; + } + c.setSize(index); + return c; + } + + public static SparseRowVector vectXorWrite(int len, double[] a, double bval, int[] aix, int ai, int alen) { + if(bval != 0) { + SparseRowVector c = allocSparseVector(len); + int[] indexes = c.indexes(); + double[] values = c.values(); + int index = 0; + int aItr = 0; + while(aItr < ai+alen && index < len) { + indexes[index] = index; + if(aix[aItr] == index) { + values[index] = !(a[aItr] != 0) ? 1 : 0; + aItr++; + } else { + values[index] = 1; + } + index++; + } + for(; index < len; index++) { + indexes[index] = index; + values[index] = 1; + } + c.setSize(len); + return c; + } else { + SparseRowVector c = allocSparseVector(alen); + int[] indexes = c.indexes(); + double[] values = c.values(); + for(int j = 0; j < alen; j++) { + indexes[j] = aix[ai+j]; + values[j] = (a[ai+j] != 0) ? 1 : 0; + } + c.setSize(alen); + return c; + } + } + + public static SparseRowVector vectXorWrite(int len, double bval, double[] a, int[] aix, int ai, int alen) { + return vectXorWrite(len, a, bval, aix, ai, alen); + } + + public static SparseRowVector vectXorWrite(int len, double[] a, double[] b, int[] aix, int[] bix, int ai, int bi, int alen, int blen) { + SparseRowVector c = allocSparseVector(alen+blen); + int aEnd = ai+alen; + int bEnd = bi+blen; + int aItr = ai; + int bItr = bi; + int index = 0; + int[] indexes = c.indexes(); + double[] values = c.values(); + while (aItr < aEnd && bItr < bEnd) { + int aIdx = aix[aItr]; + int bIdx = bix[bItr]; + int useA = (aIdx <= bIdx) ? 1 : 0; + int useB = (aIdx >= bIdx) ? 1 : 0; + indexes[index] = (useA == 1) ? aIdx : bIdx; + double av = (useA == 1) ? a[aItr] : 0.0; + double bv = (useB == 1) ? b[bItr] : 0.0; + + values[index] = ((av != 0) != (bv != 0)) ? 1 : 0; + aItr += useA; + bItr += useB; + index++; + } + for (; aItr < aEnd; aItr++) { + indexes[index] = aix[aItr]; + values[index] = (a[aItr] != 0) ? 1 : 0; + index++; + } + for (; bItr < bEnd; bItr++) { + indexes[index] = bix[bItr]; + values[index] = (b[bItr] != 0) ? 1 : 0; + index++; + } + c.setSize(index); + return c; + } + + public static SparseRowVector vectPowWrite(int len, double[] a, double bval, int[] aix, int ai, int alen) { + if(bval == 0) { + SparseRowVector c = allocSparseVector(len); + int[] indexes = c.indexes(); + double[] values = c.values(); + int index = 0; + int aItr = 0; + while(aItr < ai+alen && index < len) { + indexes[index] = index; + if(aix[aItr] == index) { + values[index] = Math.pow(a[aItr], bval) - 1; + aItr++; + } else { + values[index] = 1; + } + index++; + } + for(; index < len; index++) { + indexes[index] = index; + values[index] = 1; + } + c.setSize(len); + return c; + } else { + SparseRowVector c = allocSparseVector(alen); + int[] indexes = c.indexes(); + double[] values = c.values(); + for(int j = 0; j < alen; j++) { + indexes[j] = aix[ai+j]; + values[j] = Math.pow(a[ai+j], bval); + } + c.setSize(alen); + return c; + } + } + + public static SparseRowVector vectMinWrite(int len, double[] a, double bval, int[] aix, int ai, int alen) { + if(bval < 0) { + SparseRowVector c = allocSparseVector(len); + int[] indexes = c.indexes(); + double[] values = c.values(); + int index = 0; + int aItr = 0; + while(aItr < ai+alen && index < len) { + indexes[index] = index; + if(aix[aItr] == index) { + values[index] = Math.min(a[aItr], bval); + aItr++; + } else { + values[index] = bval; + } + index++; + } + for(; index < len; index++) { + indexes[index] = index; + values[index] = bval; + } + c.setSize(len); + return c; + } else { + SparseRowVector c = allocSparseVector(alen); + int[] indexes = c.indexes(); + double[] values = c.values(); + for(int j = 0; j < alen; j++) { + indexes[j] = aix[ai+j]; + values[j] = Math.min(a[ai+j], bval); + } + c.setSize(alen); + return c; + } + } + + public static SparseRowVector vectMinWrite(int len, double bval, double[] a, int[] aix, int ai, int alen) { + return vectMinWrite(len, a, bval, aix, ai, alen); + } + + public static SparseRowVector vectMinWrite(int len, double[] a, double[] b, int[] aix, int[] bix, int ai, int bi, int alen, int blen) { + SparseRowVector c = allocSparseVector(alen+blen); + int aEnd = ai+alen; + int bEnd = bi+blen; + int aItr = ai; + int bItr = bi; + int index = 0; + int[] indexes = c.indexes(); + double[] values = c.values(); + while (aItr < aEnd && bItr < bEnd) { + int aIdx = aix[aItr]; + int bIdx = bix[bItr]; + int useA = (aIdx <= bIdx) ? 1 : 0; + int useB = (aIdx >= bIdx) ? 1 : 0; + indexes[index] = (useA == 1) ? aIdx : bIdx; + double av = (useA == 1) ? a[aItr] : 0.0; + double bv = (useB == 1) ? b[bItr] : 0.0; + + values[index] = Math.min(av, bv); + aItr += useA; + bItr += useB; + index++; + } + for (; aItr < aEnd; aItr++) { + indexes[index] = aix[aItr]; + values[index] = Math.min(a[aItr], 0); + index++; + } + for (; bItr < bEnd; bItr++) { + indexes[index] = bix[bItr]; + values[index] = Math.min(b[bItr], 0); + index++; + } + c.setSize(index); + return c; + } + + public static SparseRowVector vectMaxWrite(int len, double[] a, double bval, int[] aix, int ai, int alen) { + if(bval > 0) { + SparseRowVector c = allocSparseVector(len); + int[] indexes = c.indexes(); + double[] values = c.values(); + int index = 0; + int aItr = 0; + while(aItr < ai+alen && index < len) { + indexes[index] = index; + if(aix[aItr] == index) { + values[index] = Math.max(a[aItr], bval); + aItr++; + } else { + values[index] = bval; + } + index++; + } + for(; index < len; index++) { + indexes[index] = index; + values[index] = bval; + } + c.setSize(len); + return c; + } else { + SparseRowVector c = allocSparseVector(alen); + int[] indexes = c.indexes(); + double[] values = c.values(); + for(int j = 0; j < alen; j++) { + indexes[j] = aix[ai+j]; + values[j] = Math.max(a[ai+j], bval); + } + c.setSize(alen); + return c; + } + } + + public static SparseRowVector vectMaxWrite(int len, double bval, double[] a, int[] aix, int ai, int alen) { + return vectMaxWrite(len, a, bval, aix, ai, alen); + } + + public static SparseRowVector vectMaxWrite(int len, double[] a, double[] b, int[] aix, int[] bix, int ai, int bi, int alen, int blen) { + SparseRowVector c = allocSparseVector(alen+blen); + int aEnd = ai+alen; + int bEnd = bi+blen; + int aItr = ai; + int bItr = bi; + int index = 0; + int[] indexes = c.indexes(); + double[] values = c.values(); + while (aItr < aEnd && bItr < bEnd) { + int aIdx = aix[aItr]; + int bIdx = bix[bItr]; + int useA = (aIdx <= bIdx) ? 1 : 0; + int useB = (aIdx >= bIdx) ? 1 : 0; + indexes[index] = (useA == 1) ? aIdx : bIdx; + double av = (useA == 1) ? a[aItr] : 0.0; + double bv = (useB == 1) ? b[bItr] : 0.0; + + values[index] = Math.max(av, bv); + aItr += useA; + bItr += useB; + index++; + } + for (; aItr < aEnd; aItr++) { + indexes[index] = aix[aItr]; + values[index] = Math.max(a[aItr], 0); + index++; + } + for (; bItr < bEnd; bItr++) { + indexes[index] = bix[bItr]; + values[index] = Math.max(b[bItr], 0); + index++; + } + c.setSize(index); + return c; + } + + public static SparseRowVector vectEqualWrite(int len, double[] a, double bval, int[] aix, int ai, int alen) { + if(bval == 0) { + SparseRowVector c = allocSparseVector(len); + int[] indexes = c.indexes(); + double[] values = c.values(); + int index = 0; + int aItr = 0; + while(aItr < ai+alen && index < len) { + indexes[index] = index; + if(aix[aItr] == index) { + values[index] = a[aItr] == bval ? 1 : 0; + aItr++; + } else { + values[index] = 1; + } + index++; + } + for(; index < len; index++) { + indexes[index] = index; + values[index] = 1; + } + c.setSize(len); + return c; + } else { + SparseRowVector c = allocSparseVector(alen); + int[] indexes = c.indexes(); + double[] values = c.values(); + for(int j = 0; j < alen; j++) { + indexes[j] = aix[ai+j]; + values[j] = a[ai+j] == bval ? 1 : 0; + } + c.setSize(alen); + return c; + } + } + + public static SparseRowVector vectEqualWrite(int len, double bval, double[] a, int[] aix, int ai, int alen) { + return vectEqualWrite(len, a, bval, aix, ai, alen); + } + + //doesn't return SparseRowVector, but still uses two sparse vectors as inputs + public static double[] vectEqualWrite(int len, double[] a, double[] b, int[] aix, int[] bix, int ai, int bi, int alen, int blen) { + double[] c = allocVector(len, true, 1); + int aEnd = ai+alen; + int bEnd = bi+blen; + int aItr = ai; + int bItr = bi; + while(aItr < aEnd && bItr < bEnd) { + int aIdx = aix[aItr]; + int bIdx = bix[bItr]; + int useA = (aIdx <= bIdx) ? 1 : 0; + int useB = (aIdx >= bIdx) ? 1 : 0; + int index = (useA == 1) ? aIdx : bIdx; + double av = (useA == 1) ? a[aItr] : 0.0; + double bv = (useB == 1) ? b[bItr] : 0.0; + + c[index] = av == bv ? 1 : 0; + aItr += useA; + bItr += useB; + } + for (; aItr < aEnd; aItr++) c[aix[aItr]] = 0; + for (; bItr < bEnd; bItr++) c[bix[bItr]] = 0; + return c; + } + + public static SparseRowVector vectNotequalWrite(int len, double[] a, double bval, int[] aix, int ai, int alen) { + if(bval != 0) { + SparseRowVector c = allocSparseVector(len); + int[] indexes = c.indexes(); + double[] values = c.values(); + int index = 0; + int aItr = 0; + while(aItr < ai+alen && index < len) { + indexes[index] = index; + if(aix[aItr] == index) { + values[index] = a[aItr] != bval ? 1 : 0; + aItr++; + } else { + values[index] = 1; + } + index++; + } + for(; index < len; index++) { + indexes[index] = index; + values[index] = 1; + } + c.setSize(len); + return c; + } else { + SparseRowVector c = allocSparseVector(alen); + int[] indexes = c.indexes(); + double[] values = c.values(); + for(int j = 0; j < alen; j++) { + indexes[j] = aix[ai+j]; + values[j] = a[ai+j] != bval ? 1 : 0; + } + c.setSize(alen); + return c; + } + } + + public static SparseRowVector vectNotequalWrite(int len, double bval, double[] a, int[] aix, int ai, int alen) { + return vectNotequalWrite(len, a, bval, aix, ai, alen); + } + + public static SparseRowVector vectNotequalWrite(int len, double[] a, double[] b, int[] aix, int[] bix, int ai, int bi, int alen, int blen) { + SparseRowVector c = allocSparseVector(alen+blen); + int aEnd = ai+alen; + int bEnd = bi+blen; + int aItr = ai; + int bItr = bi; + int index = 0; + int[] indexes = c.indexes(); + double[] values = c.values(); + while (aItr < aEnd && bItr < bEnd) { + int aIdx = aix[aItr]; + int bIdx = bix[bItr]; + int useA = (aIdx <= bIdx) ? 1 : 0; + int useB = (aIdx >= bIdx) ? 1 : 0; + indexes[index] = (useA == 1) ? aIdx : bIdx; + double av = (useA == 1) ? a[aItr] : 0.0; + double bv = (useB == 1) ? b[bItr] : 0.0; + + values[index] = (av != bv) ? 1 : 0; + aItr += useA; + bItr += useB; + index++; + } + for (; aItr < aEnd; aItr++) { + indexes[index] = aix[aItr]; + values[index] = a[aItr] != 0 ? 1 : 0; + index++; + } + for (; bItr < bEnd; bItr++) { + indexes[index] = bix[bItr]; + values[index] = b[bItr] != 0 ? 1 : 0; + index++; + } + c.setSize(index); + return c; + } + + public static SparseRowVector vectLessWrite(int len, double[] a, double bval, int[] aix, int ai, int alen) { + if(bval > 0) { + SparseRowVector c = allocSparseVector(len); + int[] indexes = c.indexes(); + double[] values = c.values(); + int index = 0; + int aItr = 0; + while(aItr < ai+alen && index < len) { + indexes[index] = index; + if(aix[aItr] == index) { + values[index] = a[aItr] < bval ? 1 : 0; + aItr++; + } else { + values[index] = 1; + } + index++; + } + for(; index < len; index++) { + indexes[index] = index; + values[index] = 1; + } + c.setSize(len); + return c; + } else { + SparseRowVector c = allocSparseVector(alen); + int[] indexes = c.indexes(); + double[] values = c.values(); + for(int j = 0; j < alen; j++) { + indexes[j] = aix[ai+j]; + values[j] = a[ai+j] < bval ? 1 : 0; + } + c.setSize(alen); + return c; + } + } + + public static SparseRowVector vectLessWrite(int len, double bval, double[] a, int[] aix, int ai, int alen) { + return vectGreaterequalWrite(len, a, bval, aix, ai, alen); + } + + public static SparseRowVector vectLessWrite(int len, double[] a, double[] b, int[] aix, int[] bix, int ai, int bi, int alen, int blen) { + SparseRowVector c = allocSparseVector(alen+blen); + int aEnd = ai+alen; + int bEnd = bi+blen; + int aItr = ai; + int bItr = bi; + int index = 0; + int[] indexes = c.indexes(); + double[] values = c.values(); + while (aItr < aEnd && bItr < bEnd) { + int aIdx = aix[aItr]; + int bIdx = bix[bItr]; + int useA = (aIdx <= bIdx) ? 1 : 0; + int useB = (aIdx >= bIdx) ? 1 : 0; + indexes[index] = (useA == 1) ? aIdx : bIdx; + double av = (useA == 1) ? a[aItr] : 0.0; + double bv = (useB == 1) ? b[bItr] : 0.0; + + values[index] = av < bv ? 1 : 0; + aItr += useA; + bItr += useB; + index++; + } + for (; aItr < aEnd; aItr++) { + indexes[index] = aix[aItr]; + values[index] = a[aItr] < 0? 1 : 0; + index++; + } + for (; bItr < bEnd; bItr++) { + indexes[index] = bix[bItr]; + values[index] = 0 < b[bItr] ? 1 : 0; + index++; + } + c.setSize(index); + return c; + } + + public static SparseRowVector vectLessequalWrite(int len, double[] a, double bval, int[] aix, int ai, int alen) { + if(bval >= 0) { + SparseRowVector c = allocSparseVector(len); + int[] indexes = c.indexes(); + double[] values = c.values(); + int index = 0; + int aItr = 0; + while(aItr < ai+alen && index < len) { + indexes[index] = index; + if(aix[aItr] == index) { + values[index] = a[aItr] <= bval ? 1 : 0; + aItr++; + } else { + values[index] = 1; + } + index++; + } + for(; index < len; index++) { + indexes[index] = index; + values[index] = 1; + } + c.setSize(len); + return c; + } else { + SparseRowVector c = allocSparseVector(alen); + int[] indexes = c.indexes(); + double[] values = c.values(); + for(int j = 0; j < alen; j++) { + indexes[j] = aix[ai+j]; + values[j] = a[ai+j] <= bval ? 1 : 0; + } + c.setSize(alen); + return c; + } + } + + public static SparseRowVector vectLessequalWrite(int len, double bval, double[] a, int[] aix, int ai, int alen) { + return vectGreaterWrite(len, a, bval, aix, ai, alen); + } + + //doesn't return SparseRowVector, but still uses two sparse vectors as inputs + public static double[] vectLessequalWrite(int len, double[] a, double[] b, int[] aix, int[] bix, int ai, int bi, int alen, int blen) { + double[] c = allocVector(len, true, 1); + int aEnd = ai+alen; + int bEnd = bi+blen; + int aItr = ai; + int bItr = bi; + while(aItr < aEnd && bItr < bEnd) { + int aIdx = aix[aItr]; + int bIdx = bix[bItr]; + int useA = (aIdx <= bIdx) ? 1 : 0; + int useB = (aIdx >= bIdx) ? 1 : 0; + int index = (useA == 1) ? aIdx : bIdx; + double av = (useA == 1) ? a[aItr] : 0.0; + double bv = (useB == 1) ? b[bItr] : 0.0; + + c[index] = av <= bv ? 1 : 0; + aItr += useA; + bItr += useB; + } + for(; aItr < ai+alen; aItr++) c[aix[aItr]] = (a[aItr] <= 0) ? 1 : 0; + for(; bItr < bi+blen; bItr++) c[bix[bItr]] = (0 <= b[bItr]) ? 1 : 0; + return c; + } + + public static SparseRowVector vectGreaterWrite(int len, double[] a, double bval, int[] aix, int ai, int alen) { + if(bval < 0) { + SparseRowVector c = allocSparseVector(len); + int[] indexes = c.indexes(); + double[] values = c.values(); + int index = 0; + int aItr = 0; + while(aItr < ai+alen && index < len) { + indexes[index] = index; + if(aix[aItr] == index) { + values[index] = a[aItr] > bval ? 1 : 0; + aItr++; + } else { + values[index] = 1; + } + index++; + } + for(; index < len; index++) { + indexes[index] = index; + values[index] = 1; + } + c.setSize(len); + return c; + } else { + SparseRowVector c = allocSparseVector(alen); + int[] indexes = c.indexes(); + double[] values = c.values(); + for(int j = 0; j < alen; j++) { + indexes[j] = aix[ai+j]; + values[j] = a[ai+j] > bval ? 1 : 0; + } + c.setSize(alen); + return c; + } + } + + public static SparseRowVector vectGreaterWrite(int len, double bval, double[] a, int[] aix, int ai, int alen) { + return vectLessequalWrite(len, a, bval, aix, ai, alen); + } + + public static SparseRowVector vectGreaterWrite(int len, double[] a, double[] b, int[] aix, int[] bix, int ai, int bi, int alen, int blen) { + SparseRowVector c = allocSparseVector(alen+blen); + int aEnd = ai+alen; + int bEnd = bi+blen; + int aItr = ai; + int bItr = bi; + int index = 0; + int[] indexes = c.indexes(); + double[] values = c.values(); + while (aItr < aEnd && bItr < bEnd) { + int aIdx = aix[aItr]; + int bIdx = bix[bItr]; + int useA = (aIdx <= bIdx) ? 1 : 0; + int useB = (aIdx >= bIdx) ? 1 : 0; + indexes[index] = (useA == 1) ? aIdx : bIdx; + double av = (useA == 1) ? a[aItr] : 0.0; + double bv = (useB == 1) ? b[bItr] : 0.0; + + values[index] = av >bv ? 1 : 0; + aItr += useA; + bItr += useB; + index++; + } + for (; aItr < ai+alen; aItr++) { + indexes[index] = aix[aItr]; + values[index] = a[aItr] > 0 ? 1 : 0; + index++; + } + for (; bItr < bi+blen; bItr++) { + indexes[index] = bix[bItr]; + values[index] = 0 > b[bItr] ? 1 : 0; + index++; + } + c.setSize(index); + return c; + } + + public static SparseRowVector vectGreaterequalWrite(int len, double[] a, double bval, int[] aix, int ai, int alen) { + if(bval <= 0) { + SparseRowVector c = allocSparseVector(len); + int[] indexes = c.indexes(); + double[] values = c.values(); + int index = 0; + int aItr = 0; + while(aItr < ai+alen && index < len) { + indexes[index] = index; + if(aix[aItr] == index) { + values[index] = a[aItr] >= bval ? 1 : 0; + aItr++; + } else { + values[index] = 1; + } + index++; + } + for(; index < len; index++) { + indexes[index] = index; + values[index] = 1; + } + c.setSize(len); + return c; + } else { + SparseRowVector c = allocSparseVector(alen); + int[] indexes = c.indexes(); + double[] values = c.values(); + for(int j = 0; j < alen; j++) { + indexes[j] = aix[ai+j]; + values[j] = a[ai+j] >= bval ? 1 : 0; + } + c.setSize(alen); + return c; + } + } + + public static SparseRowVector vectGreaterequalWrite(int len, double bval, double[] a, int[] aix, int ai, int alen) { + return vectLessWrite(len, a, bval, aix, ai, alen); + } + + //doesn't return SparseRowVector, but still uses two sparse vectors as inputs + public static double[] vectGreaterequalWrite(int len, double[] a, double[] b, int[] aix, int[] bix, int ai, int bi, int alen, int blen) { + double[] c = allocVector(len, true, 1); + int aEnd = ai+alen; + int bEnd = bi+blen; + int aItr = ai; + int bItr = bi; + while(aItr < aEnd && bItr < bEnd) { + int aIdx = aix[aItr]; + int bIdx = bix[bItr]; + int useA = (aIdx <= bIdx) ? 1 : 0; + int useB = (aIdx >= bIdx) ? 1 : 0; + int index = (useA == 1) ? aIdx : bIdx; + double av = (useA == 1) ? a[aItr] : 0.0; + double bv = (useB == 1) ? b[bItr] : 0.0; + + c[index] = av >= bv ? 1 : 0; + aItr += useA; + bItr += useB; + } + for(; aItr < ai+alen; aItr++) c[aix[aItr]] = (a[aItr] >= 0) ? 1 : 0; + for(; bItr < bi+blen; bItr++) c[bix[bItr]] = (0 >= b[bItr]) ? 1 : 0; + return c; + } + + public static SparseRowVector vectBitwandWrite(int len, double[] a, double bval, int[] aix, int ai, int alen) { + SparseRowVector c = allocSparseVector(alen); + int[] indexes = c.indexes(); + double[] values = c.values(); + int bval1 = (int) bval; + for( int j = 0; j < alen; j++ ) { + indexes[j] = aix[ai+j]; + values[j] = bwAnd(a[ai+j], bval1); + } + c.setSize(alen); + return c; + } + + public static SparseRowVector vectBitwandWrite(int len, double bval, double[] a, int[] aix, int ai, int alen) { + return vectBitwandWrite(len, a, bval, aix, ai, alen); + } + + public static SparseRowVector vectBitwandWrite(int len, double[] a, double[] b, int[] aix, int[] bix, int ai, int bi, int alen, int blen) { + SparseRowVector c = allocSparseVector(Math.min(alen, blen)); + int index = 0; + int aItr = ai; + int bItr = bi; + int[] indexes = c.indexes(); + double[] values = c.values(); + while(aItr < ai+alen && bItr < bi+blen) { + int aIdx = aix[aItr]; + int bIdx = bix[bItr]; + indexes[index] = aIdx; + values[index] = bwAnd(a[aItr], b[bItr]); + index += aIdx == bIdx ? 1 : 0; + aItr += aIdx <= bIdx ? 1 : 0; + bItr += aIdx >= bIdx ? 1 : 0; + } + c.setSize(index); + return c; + } + + public static SparseRowVector vectSqrtWrite(int len, double[] a, int[] aix, int ai, int alen) { + SparseRowVector c = allocSparseVector(alen); + int[] indexes = c.indexes(); + double[] values = c.values(); + for(int j = 0; j < alen; j++) { + indexes[j] = aix[ai+j]; + values[j] = Math.sqrt(a[ai+j]); + } + c.setSize(alen); + return c; + } + + public static SparseRowVector vectAbsWrite(int len, double[] a, int[] aix, int ai, int alen) { + SparseRowVector c = allocSparseVector(alen); + int[] indexes = c.indexes(); + double[] values = c.values(); + for(int j = 0; j < alen; j++) { + indexes[j] = aix[ai+j]; + values[j] = Math.abs(a[ai+j]); + } + c.setSize(alen); + return c; + } + + public static SparseRowVector vectRoundWrite(int len, double[] a, int[] aix, int ai, int alen) { + SparseRowVector c = allocSparseVector(alen); + int[] indexes = c.indexes(); + double[] values = c.values(); + for(int j = 0; j < alen; j++) { + indexes[j] = aix[ai+j]; + values[j] = Math.round(a[ai+j]); + } + c.setSize(alen); + return c; + } + + public static SparseRowVector vectCeilWrite(int len, double[] a, int[] aix, int ai, int alen) { + SparseRowVector c = allocSparseVector(alen); + int[] indexes = c.indexes(); + double[] values = c.values(); + for(int j = 0; j < alen; j++) { + indexes[j] = aix[ai+j]; + values[j] = Math.ceil(a[ai+j]); + } + c.setSize(alen); + return c; + } + + public static SparseRowVector vectFloorWrite(int len, double[] a, int[] aix, int ai, int alen) { + SparseRowVector c = allocSparseVector(alen); + int[] indexes = c.indexes(); + double[] values = c.values(); + for(int j = 0; j < alen; j++) { + indexes[j] = aix[ai+j]; + values[j] = Math.floor(a[ai+j]); + } + c.setSize(alen); + return c; + } + + public static SparseRowVector vectSinWrite(int len, double[] a, int[] aix, int ai, int alen) { + SparseRowVector c = allocSparseVector(alen); + int[] indexes = c.indexes(); + double[] values = c.values(); + for(int j = 0; j < alen; j++) { + indexes[j] = aix[ai+j]; + values[j] = Math.sin(a[ai+j]); + } + c.setSize(alen); + return c; + } + + public static SparseRowVector vectTanWrite(int len, double[] a, int[] aix, int ai, int alen) { + SparseRowVector c = allocSparseVector(alen); + int[] indexes = c.indexes(); + double[] values = c.values(); + for(int j = 0; j < alen; j++) { + indexes[j] = aix[ai+j]; + values[j] = Math.tan(a[ai+j]); + } + c.setSize(alen); + return c; + } + + public static SparseRowVector vectAsinWrite(int len, double[] a, int[] aix, int ai, int alen) { + SparseRowVector c = allocSparseVector(alen); + int[] indexes = c.indexes(); + double[] values = c.values(); + for(int j = 0; j < alen; j++) { + indexes[j] = aix[ai+j]; + values[j] = Math.asin(a[ai+j]); + } + c.setSize(alen); + return c; + } + + public static SparseRowVector vectAtanWrite(int len, double[] a, int[] aix, int ai, int alen) { + SparseRowVector c = allocSparseVector(alen); + int[] indexes = c.indexes(); + double[] values = c.values(); + for(int j = 0; j < alen; j++) { + indexes[j] = aix[ai+j]; + values[j] = Math.atan(a[ai+j]); + } + c.setSize(alen); + return c; + } + + public static SparseRowVector vectSinhWrite(int len, double[] a, int[] aix, int ai, int alen) { + SparseRowVector c = allocSparseVector(alen); + int[] indexes = c.indexes(); + double[] values = c.values(); + for(int j = 0; j < alen; j++) { + indexes[j] = aix[ai+j]; + values[j] = Math.sinh(a[ai+j]); + } + c.setSize(alen); + return c; + } + + public static SparseRowVector vectTanhWrite(int len, double[] a, int[] aix, int ai, int alen) { + SparseRowVector c = allocSparseVector(alen); + int[] indexes = c.indexes(); + double[] values = c.values(); + for(int j = 0; j < alen; j++) { + indexes[j] = aix[ai+j]; + values[j] = Math.tanh(a[ai+j]); + } + c.setSize(alen); + return c; + } + + public static SparseRowVector vectSignWrite(int len, double[] a, int[] aix, int ai, int alen) { + SparseRowVector c = allocSparseVector(alen); + int[] indexes = c.indexes(); + double[] values = c.values(); + for(int j = 0; j < alen; j++) { + indexes[j] = aix[ai+j]; + values[j] = Math.signum(a[ai+j]); + } + c.setSize(alen); + return c; + } + //complex builtin functions that are not directly generated //(included here in order to reduce the number of imports) @@ -2194,10 +3348,19 @@ public static void setupThreadLocalMemory(int numVectors, int len, int len2) { if( numVectors > 0 ) memPool.set(new VectorBuffer(numVectors, len, len2)); } + + public static void setupSparseThreadLocalMemory(int numVectors, int len, int len2) { + if( numVectors > 0 ) + sparseMemPool.set(new SparseVectorBuffer(numVectors, len, len2)); + } public static void cleanupThreadLocalMemory() { memPool.remove(); } + + public static void cleanupSparseThreadLocalMemory() { + sparseMemPool.remove(); + } public static double[] allocVector(int len, boolean reset) { return allocVector(len, reset, 0); @@ -2217,6 +3380,21 @@ protected static double[] allocVector(int len, boolean reset, double resetVal) { Arrays.fill(vect, resetVal); return vect; } + + public static SparseRowVector allocSparseVector(int len) { + SparseVectorBuffer buff = sparseMemPool.get(); + + //find next matching vector in ring buffer or + //allocate new vector if no vector was returned + SparseRowVector vect = buff.next(len); + if(vect == null) + vect = new SparseRowVector(len); + //reset vector for normal outputs + else if(vect.size() != 0) + vect.reset(len, len); + + return vect; + } /** * Simple ring buffer of allocated vectors, where @@ -2265,4 +3443,52 @@ public boolean isReusable(int num, int len1, int len2) { && _data.length == lnum); } } + + /** + * Simple ring buffer of allocated SparseRowVectors, where + * vectors of different sizes are interspersed. + */ + private static class SparseVectorBuffer { + private static final int MAX_SIZE = 512*1024; //4MB + private final SparseRowVector[] _data; + private int _pos; + private int _len1; + private int _len2; + + public SparseVectorBuffer(int num, int len1, int len2) { + //best effort size restriction since large intermediates + //not necessarily used (num refers to the total number) + len1 = Math.min(len1, MAX_SIZE); + len2 = Math.min(len2, MAX_SIZE); + //pre-allocate ring buffer + int lnum = (len2>0 && len1!=len2) ? 2*num : num; + _data = new SparseRowVector[lnum]; + for( int i=0; i num ) { + _data[2*i] = new SparseRowVector(len1); + _data[2*i+1] = new SparseRowVector(len2); + } + else { + _data[i] = new SparseRowVector(len1); + } + } + _pos = -1; + _len1 = len1; + _len2 = len2; + } + public SparseRowVector next(int len) { + if( _len1=_data.length) ? 0 : _pos+1; + } while( _data[_pos].values().length0 && len1!=len2) ? 2*num : num; + return (_len1 == len1 && _len2 == len2 + && _data.length == lnum); + } + } } diff --git a/src/main/java/org/apache/sysds/runtime/codegen/SpoofRowwise.java b/src/main/java/org/apache/sysds/runtime/codegen/SpoofRowwise.java index eab223bc970..e48d44f33fb 100644 --- a/src/main/java/org/apache/sysds/runtime/codegen/SpoofRowwise.java +++ b/src/main/java/org/apache/sysds/runtime/codegen/SpoofRowwise.java @@ -27,6 +27,7 @@ import java.util.concurrent.Future; import java.util.stream.IntStream; +import org.apache.sysds.api.DMLScript; import org.apache.sysds.runtime.DMLRuntimeException; import org.apache.sysds.runtime.compress.CompressedMatrixBlock; import org.apache.sysds.runtime.controlprogram.caching.MatrixObject; @@ -187,7 +188,12 @@ public MatrixBlock execute(ArrayList inputs, ArrayList 0 ) - LibSpoofPrimitives.setupThreadLocalMemory(_reqVectMem, n, n2); + if(inputs.get(0).isInSparseFormat() && DMLScript.SPARSE_INTERMEDIATE) { + LibSpoofPrimitives.setupSparseThreadLocalMemory(_reqVectMem, n, n2); + LibSpoofPrimitives.setupThreadLocalMemory(_reqVectMem, n, n2); + } else { + LibSpoofPrimitives.setupThreadLocalMemory(_reqVectMem, n, n2); + } //core sequential execute MatrixBlock a = inputs.get(0); @@ -201,7 +207,12 @@ public MatrixBlock execute(ArrayList inputs, ArrayList 0 ) - LibSpoofPrimitives.cleanupThreadLocalMemory(); + if(inputs.get(0).isInSparseFormat() && DMLScript.SPARSE_INTERMEDIATE) { + LibSpoofPrimitives.cleanupSparseThreadLocalMemory(); + LibSpoofPrimitives.cleanupThreadLocalMemory(); + } else { + LibSpoofPrimitives.cleanupThreadLocalMemory(); + } if( flipOut ) { fixTransposeDimensions(out); out = LibMatrixReorg.transpose(out, new MatrixBlock( @@ -431,7 +442,12 @@ public DenseBlock call() { //allocate vector intermediates and partial output if( _reqVectMem > 0 ) - LibSpoofPrimitives.setupThreadLocalMemory(_reqVectMem, _clen, _clen2); + if(_a.isInSparseFormat() && DMLScript.SPARSE_INTERMEDIATE) { + LibSpoofPrimitives.setupSparseThreadLocalMemory(_reqVectMem, _clen, _clen2); + LibSpoofPrimitives.setupThreadLocalMemory(_reqVectMem, _clen, _clen2); + } else { + LibSpoofPrimitives.setupThreadLocalMemory(_reqVectMem, _clen, _clen2); + } DenseBlock c = DenseBlockFactory.createDenseBlock(1, _outLen); if( !_a.isInSparseFormat() ) @@ -440,7 +456,12 @@ public DenseBlock call() { executeSparse(_a.getSparseBlock(), _b, _scalars, c, _clen, _rl, _ru, 0); if( _reqVectMem > 0 ) - LibSpoofPrimitives.cleanupThreadLocalMemory(); + if(_a.isInSparseFormat() && DMLScript.SPARSE_INTERMEDIATE) { + LibSpoofPrimitives.cleanupSparseThreadLocalMemory(); + LibSpoofPrimitives.cleanupThreadLocalMemory(); + } else { + LibSpoofPrimitives.cleanupThreadLocalMemory(); + } return c; } } @@ -474,15 +495,25 @@ protected ParExecTask( MatrixBlock a, SideInput[] b, MatrixBlock c, double[] sca public Long call() { //allocate vector intermediates if( _reqVectMem > 0 ) - LibSpoofPrimitives.setupThreadLocalMemory(_reqVectMem, _clen, _clen2); + if(_a.isInSparseFormat() && DMLScript.SPARSE_INTERMEDIATE) { + LibSpoofPrimitives.setupSparseThreadLocalMemory(_reqVectMem, _clen, _clen2); + LibSpoofPrimitives.setupThreadLocalMemory(_reqVectMem, _clen, _clen2); + } else { + LibSpoofPrimitives.setupThreadLocalMemory(_reqVectMem, _clen, _clen2); + } if( !_a.isInSparseFormat() ) executeDense(_a.getDenseBlock(), _b, _scalars, _c.getDenseBlock(), _clen, _rl, _ru, 0); else executeSparse(_a.getSparseBlock(), _b, _scalars, _c.getDenseBlock(), _clen, _rl, _ru, 0); - + if( _reqVectMem > 0 ) - LibSpoofPrimitives.cleanupThreadLocalMemory(); + if(_a.isInSparseFormat() && DMLScript.SPARSE_INTERMEDIATE) { + LibSpoofPrimitives.cleanupSparseThreadLocalMemory(); + LibSpoofPrimitives.cleanupThreadLocalMemory(); + } else { + LibSpoofPrimitives.cleanupThreadLocalMemory(); + } //maintain nnz for row partition return _c.recomputeNonZeros(_rl, _ru-1, 0, _c.getNumColumns()-1); diff --git a/src/test/java/org/apache/sysds/test/component/codegen/CPlanVectorPrimitivesTest.java b/src/test/java/org/apache/sysds/test/component/codegen/CPlanVectorPrimitivesTest.java index 7d8b4c9096d..bae22b1c384 100644 --- a/src/test/java/org/apache/sysds/test/component/codegen/CPlanVectorPrimitivesTest.java +++ b/src/test/java/org/apache/sysds/test/component/codegen/CPlanVectorPrimitivesTest.java @@ -21,6 +21,7 @@ import java.lang.reflect.Method; +import org.apache.sysds.runtime.data.SparseRowVector; import org.junit.Test; import org.apache.sysds.common.Types.OpOp2; import org.apache.sysds.hops.codegen.cplan.CNodeBinary.BinType; @@ -716,6 +717,287 @@ public void testVectorVectorBitwAndSparseDense() { testVectorBinaryPrimitive(BinType.VECT_BITWAND, InputType.VECTOR_SPARSE, InputType.VECTOR_DENSE); } + //********************testing with sparse intermediates********************// + //vector - scalar + + @Test + public void testVectorScalarMultSparseToSparse() { + testVectorBinarySparsePrimitive(BinType.VECT_MULT_SCALAR, InputType.VECTOR_SPARSE, InputType.SCALAR); + } + + @Test + public void testVectorScalarDivSparseToSparse() { + testVectorBinarySparsePrimitive(BinType.VECT_DIV_SCALAR, InputType.VECTOR_SPARSE, InputType.SCALAR); + } + + @Test + public void testVectorScalarMinSparseToSparse() { + testVectorBinarySparsePrimitive(BinType.VECT_MIN_SCALAR, InputType.VECTOR_SPARSE, InputType.SCALAR); + } + + @Test + public void testVectorScalarMaxSparseToSparse() { + testVectorBinarySparsePrimitive(BinType.VECT_MAX_SCALAR, InputType.VECTOR_SPARSE, InputType.SCALAR); + } + + @Test + public void testVectorScalarPowSparseToSparse() { + testVectorBinarySparsePrimitive(BinType.VECT_POW_SCALAR, InputType.VECTOR_SPARSE, InputType.SCALAR); + } + + @Test + public void testVectorScalarEqualSparseToSparse() { + testVectorBinarySparsePrimitive(BinType.VECT_EQUAL_SCALAR, InputType.VECTOR_SPARSE, InputType.SCALAR); + } + + @Test + public void testVectorScalarNotEqualSparseToSparse() { + testVectorBinarySparsePrimitive(BinType.VECT_NOTEQUAL_SCALAR, InputType.VECTOR_SPARSE, InputType.SCALAR); + } + + @Test + public void testVectorScalarLessSparseToSparse() { + testVectorBinarySparsePrimitive(BinType.VECT_LESS_SCALAR, InputType.VECTOR_SPARSE, InputType.SCALAR); + } + + @Test + public void testVectorScalarLessEqualSparseToSparse() { + testVectorBinarySparsePrimitive(BinType.VECT_LESSEQUAL_SCALAR, InputType.VECTOR_SPARSE, InputType.SCALAR); + } + + @Test + public void testVectorScalarGreaterSparseToSparse() { + testVectorBinarySparsePrimitive(BinType.VECT_GREATER_SCALAR, InputType.VECTOR_SPARSE, InputType.SCALAR); + } + + @Test + public void testVectorScalarGreaterEqualSparseToSparse() { + testVectorBinarySparsePrimitive(BinType.VECT_GREATEREQUAL_SCALAR, InputType.VECTOR_SPARSE, InputType.SCALAR); + } + + //todo: this only makes sense, when bval is 0 + @Test + public void testVectorScalarXorSparseToSparse() { + testVectorBinarySparsePrimitive(BinType.VECT_XOR_SCALAR, InputType.VECTOR_SPARSE, InputType.SCALAR); + } + + @Test + public void testVectorScalarBitwAndSparseToSparse() { + testVectorBinarySparsePrimitive(BinType.VECT_BITWAND_SCALAR, InputType.VECTOR_SPARSE, InputType.SCALAR); + } + + //scalar - vector + + @Test + public void testScalarVectorMultSparseToSparse() { + testVectorBinarySparsePrimitive(BinType.VECT_MULT_SCALAR, InputType.SCALAR, InputType.VECTOR_SPARSE); + } + + @Test + public void testScalarVectorDivSparseToSparse() { + testVectorBinarySparsePrimitive(BinType.VECT_DIV_SCALAR, InputType.SCALAR, InputType.VECTOR_SPARSE); + } + + @Test + public void testScalarVectorMinSparseToSparse() { + testVectorBinarySparsePrimitive(BinType.VECT_MIN_SCALAR, InputType.SCALAR, InputType.VECTOR_SPARSE); + } + + @Test + public void testScalarVectorMaxSparseToSparse() { + testVectorBinarySparsePrimitive(BinType.VECT_MAX_SCALAR, InputType.SCALAR, InputType.VECTOR_SPARSE); + } + + @Test + public void testScalarVectorEqualSparseToSparse() { + testVectorBinarySparsePrimitive(BinType.VECT_EQUAL_SCALAR, InputType.SCALAR, InputType.VECTOR_SPARSE); + } + + @Test + public void testScalarVectorNotEqualSparseToSparse() { + testVectorBinarySparsePrimitive(BinType.VECT_NOTEQUAL_SCALAR, InputType.SCALAR, InputType.VECTOR_SPARSE); + } + + @Test + public void testScalarVectorLessSparseToSparse() { + testVectorBinarySparsePrimitive(BinType.VECT_LESS_SCALAR, InputType.SCALAR, InputType.VECTOR_SPARSE); + } + + @Test + public void testScalarVectorLessEqualSparseToSparse() { + testVectorBinarySparsePrimitive(BinType.VECT_LESSEQUAL_SCALAR, InputType.SCALAR, InputType.VECTOR_SPARSE); + } + + @Test + public void testScalarVectorGreaterSparseToSparse() { + testVectorBinarySparsePrimitive(BinType.VECT_GREATER_SCALAR, InputType.SCALAR, InputType.VECTOR_SPARSE); + } + + @Test + public void testScalarVectorGreaterEqualSparseToSparse() { + testVectorBinarySparsePrimitive(BinType.VECT_GREATEREQUAL_SCALAR, InputType.SCALAR, InputType.VECTOR_SPARSE); + } + + @Test + public void testScalarVectorXorSparseToSparse() { + testVectorBinarySparsePrimitive(BinType.VECT_XOR_SCALAR, InputType.SCALAR, InputType.VECTOR_SPARSE); + } + + @Test + public void testScalarVectorBitwAndSparseToSparse() { + testVectorBinarySparsePrimitive(BinType.VECT_BITWAND_SCALAR, InputType.SCALAR, InputType.VECTOR_SPARSE); + } + + //special binary + + // @Test + // public void testVectorPow2SparseToSparse() { + // testVectorUnarySparsePrimitive(UnaryType.VECT_POW2, InputType.VECTOR_SPARSE); + // } + // + // @Test + // public void testVectorMult2SparseToSparse() { + // testVectorUnarySparsePrimitive(UnaryType.VECT_MULT2, InputType.VECTOR_SPARSE); + // } + + //vector - vector + + @Test + public void testVectorVectorMultSparseToSparse() { + testVectorBinarySparsePrimitive(BinType.VECT_MULT, InputType.VECTOR_SPARSE, InputType.VECTOR_SPARSE); + } + + @Test + public void testVectorVectorDivSparseToSparse() { + testVectorBinarySparsePrimitive(BinType.VECT_DIV, InputType.VECTOR_SPARSE, InputType.VECTOR_SPARSE); + } + + @Test + public void testVectorVectorPlusSparseToSparse() { + testVectorBinarySparsePrimitive(BinType.VECT_PLUS, InputType.VECTOR_SPARSE, InputType.VECTOR_SPARSE); + } + + @Test + public void testVectorVectorMinusSparseToSparse() { + testVectorBinarySparsePrimitive(BinType.VECT_MINUS, InputType.VECTOR_SPARSE, InputType.VECTOR_SPARSE); + } + + @Test + public void testVectorVectorMinSparseToSparse() { + testVectorBinarySparsePrimitive(BinType.VECT_MIN, InputType.VECTOR_SPARSE, InputType.VECTOR_SPARSE); + } + @Test + public void testVectorVectorMaxSparseToSparse() { + testVectorBinarySparsePrimitive(BinType.VECT_MAX, InputType.VECTOR_SPARSE, InputType.VECTOR_SPARSE); + } + + @Test + public void testVectorVectorEqualSparseToSparse() { + testVectorBinarySparsePrimitive(BinType.VECT_EQUAL, InputType.VECTOR_SPARSE, InputType.VECTOR_SPARSE); + } + + @Test + public void testVectorVectorNotEqualSparseToSparse() { + testVectorBinarySparsePrimitive(BinType.VECT_NOTEQUAL, InputType.VECTOR_SPARSE, InputType.VECTOR_SPARSE); + } + + @Test + public void testVectorVectorLessSparseToSparse() { + testVectorBinarySparsePrimitive(BinType.VECT_LESS, InputType.VECTOR_SPARSE, InputType.VECTOR_SPARSE); + } + + @Test + public void testVectorVectorLessEqualSparseToSparse() { + testVectorBinarySparsePrimitive(BinType.VECT_LESSEQUAL, InputType.VECTOR_SPARSE, InputType.VECTOR_SPARSE); + } + + @Test + public void testVectorVectorGreaterSparseToSparse() { + testVectorBinarySparsePrimitive(BinType.VECT_GREATER, InputType.VECTOR_SPARSE, InputType.VECTOR_SPARSE); + } + + @Test + public void testVectorVectorGreaterEqualSparseToSparse() { + testVectorBinarySparsePrimitive(BinType.VECT_GREATEREQUAL, InputType.VECTOR_SPARSE, InputType.VECTOR_SPARSE); + } + + @Test + public void testVectorVectorXorSparseToSparse() { + testVectorBinarySparsePrimitive(BinType.VECT_XOR, InputType.VECTOR_SPARSE, InputType.VECTOR_SPARSE); + } + + @Test + public void testVectorVectorBitwAndSparseToSparse() { + testVectorBinarySparsePrimitive(BinType.VECT_BITWAND, InputType.VECTOR_SPARSE, InputType.VECTOR_SPARSE); + } + + // @Test + // public void testVectorVectorMatrixMultSparseToSparse() { + // testVectorBinarySparsePrimitive(BinType.VECT_MATRIXMULT, InputType.VECTOR_SPARSE, InputType.VECTOR_SPARSE); + // } + + //unary primitives with sparse intermediates + + @Test + public void testVectorSqrtSparseToSparse() { + testVectorUnarySparsePrimitive(UnaryType.VECT_SQRT, InputType.VECTOR_SPARSE); + } + + @Test + public void testVectorAbsSparseToSparse() { + testVectorUnarySparsePrimitive(UnaryType.VECT_ABS, InputType.VECTOR_SPARSE); + } + + @Test + public void testVectorRoundSparseToSparse() { + testVectorUnarySparsePrimitive(UnaryType.VECT_ROUND, InputType.VECTOR_SPARSE); + } + + @Test + public void testVectorCeilSparseToSparse() { + testVectorUnarySparsePrimitive(UnaryType.VECT_CEIL, InputType.VECTOR_SPARSE); + } + + @Test + public void testVectorFloorSparseToSparse() { + testVectorUnarySparsePrimitive(UnaryType.VECT_FLOOR, InputType.VECTOR_SPARSE); + } + + @Test + public void testVectorSinSparseToSparse() { + testVectorUnarySparsePrimitive(UnaryType.VECT_SIN, InputType.VECTOR_SPARSE); + } + + @Test + public void testVectorTanSparseToSparse() { + testVectorUnarySparsePrimitive(UnaryType.VECT_TAN, InputType.VECTOR_SPARSE); + } + + @Test + public void testVectorAsinSparseToSparse() { + testVectorUnarySparsePrimitive(UnaryType.VECT_ASIN, InputType.VECTOR_SPARSE); + } + + @Test + public void testVectorAtanSparseToSparse() { + testVectorUnarySparsePrimitive(UnaryType.VECT_ATAN, InputType.VECTOR_SPARSE); + } + + @Test + public void testVectorSinhSparseToSparse() { + testVectorUnarySparsePrimitive(UnaryType.VECT_SINH, InputType.VECTOR_SPARSE); + } + + @Test + public void testVectorTanhSparseToSparse() { + testVectorUnarySparsePrimitive(UnaryType.VECT_TANH, InputType.VECTOR_SPARSE); + } + + @Test + public void testVectorSignSparseToSparse() { + testVectorUnarySparsePrimitive(UnaryType.VECT_SIGN, InputType.VECTOR_SPARSE); + } + @SuppressWarnings("incomplete-switch") private static void testVectorAggPrimitive(UnaryType aggtype, InputType type1) { @@ -792,6 +1074,45 @@ private static void testVectorUnaryPrimitive(UnaryType utype, InputType type1) throw new RuntimeException(ex); } } + + private static void testVectorUnarySparsePrimitive(UnaryType utype, InputType type1) + { + try { + //generate input data + double sparsity = (type1 == InputType.VECTOR_DENSE) ? sparsity1 : sparsity2; + MatrixBlock in = MatrixBlock.randOperations(m, n, sparsity, -1, 1, "uniform", 7); + + //get vector primitive via reflection + String meName = "vect"+StringUtils.camelize(utype.name().split("_")[1])+"Write"; + Method me = LibSpoofPrimitives.class.getMethod(meName, new Class[]{int.class, double[].class, int[].class, int.class, int.class}); + + + for( int i=0; i= 0 && i == m-1) { + if(testType == 0) { + retX = (SparseRowVector) me.invoke(null, n, inA.getSparseBlock().values(i), 0, inA.getSparseBlock().indexes(i), + inA.getSparseBlock().pos(i), inA.getSparseBlock().size(i)); + } else { + retX = (SparseRowVector) me.invoke(null, n, inA.getSparseBlock().values(i), inB.min(), inA.getSparseBlock().indexes(i), + inA.getSparseBlock().pos(i), inA.getSparseBlock().size(i)); + } + } else { + retX = (SparseRowVector) me.invoke(null, n, inA.getSparseBlock().values(i), inB.max(), inA.getSparseBlock().indexes(i), + inA.getSparseBlock().pos(i), inA.getSparseBlock().size(i)); + } + else if( type1==InputType.SCALAR && type2==InputType.VECTOR_SPARSE ) + if(testType >= 0 && i == m-1) { + if(testType == 0) { + retX = (SparseRowVector) me.invoke(null, n, 0, inB.getSparseBlock().values(i), + inB.getSparseBlock().indexes(i), inB.getSparseBlock().pos(i), inB.getSparseBlock().size(i)); + } else { + retX = (SparseRowVector) me.invoke(null, n, inA.min(), inB.getSparseBlock().values(i), + inB.getSparseBlock().indexes(i), inB.getSparseBlock().pos(i), inB.getSparseBlock().size(i)); + } + } else { + retX = (SparseRowVector) me.invoke(null, n, inA.max(), inB.getSparseBlock().values(i), + inB.getSparseBlock().indexes(i), inB.getSparseBlock().pos(i), inB.getSparseBlock().size(i)); + } + else if( type1==InputType.VECTOR_SPARSE && type2==InputType.VECTOR_SPARSE ) + if(sparse) + retX = (SparseRowVector) me.invoke(null, n, inA.getSparseBlock().values(i), inB.getSparseBlock().values(i), + inA.getSparseBlock().indexes(i), inB.getSparseBlock().indexes(i), inA.getSparseBlock().pos(i), inB.getSparseBlock().pos(i), inA.getSparseBlock().size(i), inB.getSparseBlock().size(i)); + else + ret1 = (double[]) me.invoke(null, n, inA.getSparseBlock().values(i), inB.getSparseBlock().values(i), + inA.getSparseBlock().indexes(i), inB.getSparseBlock().indexes(i), inA.getSparseBlock().pos(i), inB.getSparseBlock().pos(i), inA.getSparseBlock().size(i), inB.getSparseBlock().size(i)); + + if(sparse) { + int[] indexes = retX.indexes(); + for (int j = 0; j < retX.size(); j++) { + ret1[indexes[j]] = retX.get(indexes[j]); + } + } + + //execute comparison operation + String opcode = OpOp2.valueOf(bintype.name().split("_")[1]).toString(); + MatrixBlock in1 = inA.slice(i, i, 0, n-1, new MatrixBlock()); + MatrixBlock in2 = inB.slice(i, i, 0, n-1, new MatrixBlock()); + double[] ret2 = null; + if( type1 == InputType.SCALAR ) { + ScalarOperator bop = InstructionUtils.parseScalarBinaryOperator(opcode, true); + bop = bop.setConstant(testType >= 0 && i == m-1 ? testType == 0 ? 0 : inA.min() : inA.max()); + ret2 = DataConverter.convertToDoubleVector( + in2.scalarOperations(bop, new MatrixBlock()), false); + } + else if( type2 == InputType.SCALAR ) { + ScalarOperator bop = InstructionUtils.parseScalarBinaryOperator(opcode, false); + bop = bop.setConstant(testType >= 0 && i == m-1 ? testType == 0 ? 0 : inB.min() : inB.max()); + ret2 = DataConverter.convertToDoubleVector( + in1.scalarOperations(bop, new MatrixBlock()), false); + } + else { //vector-vector + BinaryOperator bop = InstructionUtils.parseBinaryOperator(opcode); + ret2 = DataConverter.convertToDoubleVector( + in1.binaryOperations(bop, in2, new MatrixBlock()), false); + } + + //compare results + TestUtils.compareMatrices(ret2, ret1, eps); + } + } + catch( Exception ex ) { + throw new RuntimeException(ex); + } + } + + + /** + * @param type + * @return {@code true}, when the matching method has a sparse return + */ + private static boolean getOutputType(BinType type) { + switch(type) { + case VECT_EQUAL: + case VECT_LESSEQUAL: + case VECT_GREATEREQUAL: + return false; + default: + return true; + } + } + + /** + * @param type + * @return returns {@code -1}, for normal testing;
+ * returns {@code 0}, for testing with 0 and non-zeros;
+ * returns {@code 1}, for testing with negative and positive numbers; + */ + private static int getTestType(BinType type) { + switch(type) { + case VECT_DIV_SCALAR: + case VECT_EQUAL_SCALAR: + case VECT_NOTEQUAL_SCALAR: + case VECT_XOR_SCALAR: + // case VECT_POW_SCALAR: + return 0; + case VECT_GREATER_SCALAR: + case VECT_GREATEREQUAL_SCALAR: + case VECT_LESS_SCALAR: + case VECT_LESSEQUAL_SCALAR: + case VECT_MIN_SCALAR: + case VECT_MAX_SCALAR: + return 1; + default: + return -1; + } + } } diff --git a/src/test/java/org/apache/sysds/test/component/codegen/SparseVectorAllocTest.java b/src/test/java/org/apache/sysds/test/component/codegen/SparseVectorAllocTest.java new file mode 100644 index 00000000000..c03dd9adb1b --- /dev/null +++ b/src/test/java/org/apache/sysds/test/component/codegen/SparseVectorAllocTest.java @@ -0,0 +1,115 @@ +package org.apache.sysds.test.component.codegen; + +import org.apache.sysds.runtime.codegen.LibSpoofPrimitives; +import org.apache.sysds.runtime.data.SparseRowVector; +import org.apache.sysds.test.AutomatedTestBase; +import org.apache.sysds.test.TestUtils; +import org.junit.Assert; +import org.junit.Test; + +/** + * This is the component test for the ring buffer used in LibSpoofPrimitives.java, + * that allocates a vector with a certain size. + * Every allocation method is tested to achieve the needed coverage. + */ +public class SparseVectorAllocTest extends AutomatedTestBase +{ + double[] val1 = new double[]{1.5, 5.7, 9.1, 3.7, 5.3}; + double[] val2 = new double[]{9.6, 7.1, 2.7}; + int[] indexes1 = new int[]{3, 7, 14, 20, 81}; + int[] indexes2 = new int[]{20, 30, 90}; + + @Override + public void setUp() { + TestUtils.clearAssertionInformation(); + } + + @Test + public void testBasicAllocationSameLen() { + testBasicSparseVectorAllocation(1, 10, 10); + } + + @Test + public void testBasicAllocationLongerExp() { + testBasicSparseVectorAllocation(1, 10, 15); + } + + @Test + public void testBasicAllocationShorterExp() { + testBasicSparseVectorAllocation(1, 10, 7); + } + + @Test + public void testVectorReuse1() { + testBufferReuse(3, 10, 5, 5); + } + + @Test + public void testVectorReuse2() { + testBufferReuse(3, 10, -1, 5); + } + + /** tests the allocation of an empty vector + * @param numVectors number of vectors that should be pre-allocated + * @param len the length of the vector + * @param expLen the expected length of the allocated vector + */ + public void testBasicSparseVectorAllocation(int numVectors, int len, int expLen) { + //test the basic allocation of an empty vector + LibSpoofPrimitives.setupSparseThreadLocalMemory(numVectors, len, -1); + SparseRowVector sparseVec = LibSpoofPrimitives.allocSparseVector(expLen); + + Assert.assertTrue("Vector capacity should be initialized correctly", expLen <= sparseVec.capacity()); + Assert.assertEquals("Vector size should be initialized with 0", 0, sparseVec.size()); + + LibSpoofPrimitives.cleanupSparseThreadLocalMemory(); + } + + /** tests the allocation of a vector that is reused multiple times + * @param numVectors number of vectors that should be pre-allocated + * @param len1 length of the first vector + * @param len2 length of the second vector + * @param expLen expected length of allocated vector + */ + public void testBufferReuse(int numVectors, int len1, int len2, int expLen) { + //test the reuse of the vectors in the ring buffer + LibSpoofPrimitives.setupSparseThreadLocalMemory(numVectors, len1, len2); + + //allocate first vector + SparseRowVector vec1 = LibSpoofPrimitives.allocSparseVector(expLen); + vec1.set(0, 1.0); + vec1.set(2, 2.0); + + //allocate second vector + SparseRowVector vec2 = LibSpoofPrimitives.allocSparseVector(expLen); + + Assert.assertEquals("Reused vector should be reset to size 0", 0, vec2.size()); + + for(int j = 0; j < vec2.size(); j++) { + vec2.set(vec2.indexes()[j], vec2.get(vec2.indexes()[j]) * 32); + } + + SparseRowVector vec3 = LibSpoofPrimitives.allocSparseVector(expLen); + + Assert.assertEquals("Reused vector should be reset to size 0", 0, vec3.size()); + + SparseRowVector vec4 = LibSpoofPrimitives.allocSparseVector(expLen); + + for(int j = 0; j < vec4.size(); j++) { + vec4.set(vec4.indexes()[j], vec4.get(vec3.indexes()[j]) * 32); + } + + Assert.assertEquals("Reused vector should be reset to size 0", 0, vec4.size()); + + SparseRowVector vec5 = LibSpoofPrimitives.allocSparseVector(expLen); + + for(int j = 0; j < vec5.size(); j++) { + vec2.set(vec5.indexes()[j], vec5.get(vec5.indexes()[j]) * 32); + } + + Assert.assertEquals("Reused vector should be reset to size 0", 0, vec5.size()); + + LibSpoofPrimitives.cleanupSparseThreadLocalMemory(); + } + +}