From 8d74800cf308bcb53110664a7aa1edb00a7bd36f Mon Sep 17 00:00:00 2001 From: Frxms Date: Sat, 31 May 2025 17:16:44 +0200 Subject: [PATCH 01/60] carry over changes from sparseRows branch --- .../java/org/apache/sysds/api/DMLOptions.java | 14 +- .../java/org/apache/sysds/api/DMLScript.java | 4 + .../runtime/codegen/LibSpoofPrimitives.java | 733 ++++++++++++++++++ .../sysds/runtime/codegen/SpoofRowwise.java | 16 +- .../codegen/CPlanVectorPrimitivesTest.java | 445 +++++++++++ .../codegen/SparseVectorAllocTest.java | 158 ++++ 6 files changed, 1366 insertions(+), 4 deletions(-) create mode 100644 src/test/java/org/apache/sysds/test/component/codegen/SparseVectorAllocTest.java diff --git a/src/main/java/org/apache/sysds/api/DMLOptions.java b/src/main/java/org/apache/sysds/api/DMLOptions.java index 763ac7b9388..b5b0908432b 100644 --- a/src/main/java/org/apache/sysds/api/DMLOptions.java +++ b/src/main/java/org/apache/sysds/api/DMLOptions.java @@ -85,6 +85,7 @@ public class DMLOptions { public boolean federatedCompilation = false; // Compile federated instructions based on input federation state and privacy constraints. public boolean noFedRuntimeConversion = false; // If activated, no runtime conversion of CP instructions to FED instructions will be performed. public int seed = -1; // The general seed for the execution, if -1 random (system time). + public boolean sparseIntermediate = false; // whether SparseRowIntermediates should be used for rowwise operations public final static DMLOptions defaultOptions = new DMLOptions(null); @@ -117,7 +118,8 @@ public String toString() { ", w=" + fedWorker + ", federatedCompilation=" + federatedCompilation + ", noFedRuntimeConversion=" + noFedRuntimeConversion + - ", seed=" + seed + + ", seed=" + seed + + ", sparseIntermediate=" + sparseIntermediate + '}'; } @@ -350,6 +352,10 @@ else if (lineageType.equalsIgnoreCase("debugger")) dmlOptions.seed = Integer.parseInt(line.getOptionValue("seed")); } + if(line.hasOption("sparseIntermediate")){ + dmlOptions.sparseIntermediate = true; + } + return dmlOptions; } @@ -431,7 +437,10 @@ private static Options createCLIOptions() { Option commandlineSeed = OptionBuilder .withDescription("A general seed for the execution through the commandline") .hasArg().create("seed"); - + Option sparseRowIntermediates = OptionBuilder + .withDescription("If activated, sparseRowVector intermediates will be used to calculate rowwise operations.") + .create("sparseIntermediate"); + options.addOption(configOpt); options.addOption(cleanOpt); options.addOption(statsOpt); @@ -451,6 +460,7 @@ private static Options createCLIOptions() { options.addOption(federatedCompilation); options.addOption(noFedRuntimeConversion); options.addOption(commandlineSeed); + options.addOption(sparseRowIntermediates); // Either a clean(-clean), a file(-f), a script(-s) or help(-help) needs to be specified OptionGroup fileOrScriptOpt = new OptionGroup() diff --git a/src/main/java/org/apache/sysds/api/DMLScript.java b/src/main/java/org/apache/sysds/api/DMLScript.java index d6853891e24..44dc35ee75e 100644 --- a/src/main/java/org/apache/sysds/api/DMLScript.java +++ b/src/main/java/org/apache/sysds/api/DMLScript.java @@ -153,6 +153,9 @@ public class DMLScript // Global seed public static int SEED = -1; + // Sparse row flag + public static boolean SPARSE_INTERMEDIATE = false; + public static String MONITORING_ADDRESS = null; // flag that indicates whether or not to suppress any prints to stdout @@ -275,6 +278,7 @@ public static boolean executeScript( String[] args ) LINEAGE_ESTIMATE = dmlOptions.lineage_estimate; LINEAGE_DEBUGGER = dmlOptions.lineage_debugger; SEED = dmlOptions.seed; + SPARSE_INTERMEDIATE = dmlOptions.sparseIntermediate; String fnameOptConfig = dmlOptions.configFile; diff --git a/src/main/java/org/apache/sysds/runtime/codegen/LibSpoofPrimitives.java b/src/main/java/org/apache/sysds/runtime/codegen/LibSpoofPrimitives.java index 6c0dc395c3f..baccf53dc7f 100644 --- a/src/main/java/org/apache/sysds/runtime/codegen/LibSpoofPrimitives.java +++ b/src/main/java/org/apache/sysds/runtime/codegen/LibSpoofPrimitives.java @@ -23,6 +23,7 @@ import org.apache.commons.math3.util.FastMath; import org.apache.sysds.runtime.data.DenseBlockFP64; +import org.apache.sysds.runtime.data.SparseRowVector; import org.apache.sysds.runtime.functionobjects.BitwAnd; import org.apache.sysds.runtime.functionobjects.IntegerDivide; import org.apache.sysds.runtime.functionobjects.Modulus; @@ -51,6 +52,10 @@ public class LibSpoofPrimitives @Override protected VectorBuffer initialValue() { return new VectorBuffer(0,0,0); } }; + private static ThreadLocal sparseMemPool = new ThreadLocal<>() { + @Override protected SparseVectorBuffer initialValue() { return new SparseVectorBuffer(0,0,0); } + }; + public static double rowMaxsVectMult(double[] a, double[] b, int ai, int bi, int len) { double val = Double.NEGATIVE_INFINITY; int j=0; @@ -2164,6 +2169,636 @@ public static double vectVar(double[] avals, int[] aix, int ai, int alen, int le return (vectSum(avalsSqr, 0, len)-len*meanVal)/(len-1); } + /** + * Vector primitives with SparseRowVector intermediates + * Changes: + * - Changed method signature to avoid method duplicate conflicts + * e.g. (double[], double, int[], int, int, int) --> (int, double[], double, int[], int, int) + * - Added blen for vector - vector calculations to be able to use both vectors as SparseRowVectors + * - Implemented a new SparseVectorBuffer class that creates a ring buffer for SparseRowVectors in different sizes + */ + + public static SparseRowVector vectMultWrite(int len, double[] a, double bval, int[] aix, int ai, int alen) { + if( a == null ) return allocSparseVector(alen); + SparseRowVector c = allocSparseVector(alen, aix); + LibMatrixMult.vectMultiplyWrite(bval, a, c.values(), ai, 0, alen); + return c; + } + + public static SparseRowVector vectMultWrite(int len, double[] a, double[] b, int[] aix, int[] bix, int ai, int bi, int alen, int blen) { + SparseRowVector c = allocSparseVector(alen); + if( a == null || b == null ) return c; + SparseRowVector d = allocSparseVector(blen, b, bix); + for( int j = ai; j < ai+alen; j++ ) { + c.set(aix[j], a[j] * d.get(bi+aix[j])); + } + return c; + } + + public static void vectWrite(SparseRowVector a, double[] c, int ci, int len) { + if( a == null ) return; + int[] aix = a.indexes(); + for(int j = 0; j < a.size(); j++) + c[ci+aix[j]] = a.get(aix[j]); + } + + public static SparseRowVector vectDivWrite(int len, double[] a, double bval, int[] aix, int ai, int alen) { + //todo: can i just do nothing, when deviding through zero? + // if (bval != 0) { + SparseRowVector c = allocSparseVector(alen); + for( int j = ai; j < ai+alen; j++ ) + c.set(aix[j], a[j] / bval); + return c; + // } else { + // double init = Double.NaN; + // double[] c = allocVector(len, true, init); + // for( int j = ai; j < ai+alen; j++ ) + // c[aix[j]] = a[j] / bval; + // return new SparseRowVector(c); + // } + } + + public static SparseRowVector vectDivWrite(int len, double[] a, double[] b, int[] aix, int[] bix, int ai, int bi, int alen, int blen) { + SparseRowVector c = allocSparseVector(alen); + if(alen != 0 && blen != 0) { + //decide between two versions. First is smaller for smaller alen and the other for smaller blen + if(alen < blen) { + SparseRowVector bSparse = allocSparseVector(blen, b, bix); + for (int j = ai; j < ai+alen; j++) { + c.set(aix[j], a[j] / bSparse.get(aix[j])); + } + } else { + SparseRowVector aSparse = allocSparseVector(alen, a, aix); + for (int j = ai; j < ai+alen; j++) { + double aval = a[j]; + c.set(aix[j], (aval>0) ? + Double.POSITIVE_INFINITY : Double.NEGATIVE_INFINITY); + } + for (int j = bi; j < bi+blen; j++) { + c.set(bix[j], aSparse.get(bix[j]) / b[j]); + } + } + } + return c; + } + + public static SparseRowVector vectMinusWrite(int len, double[] a, double[] b, int[] aix, int[] bix, int ai, int bi, int alen, int blen) { + SparseRowVector c = allocSparseVector(alen, a, aix); + for (int j = bi; j < bi+blen; j++) { + c.add(bix[j], -b[j]); + } + return c; + } + + public static SparseRowVector vectPlusWrite(int len, double[] a, double[] b, int[] aix, int[] bix, int ai, int bi, int alen, int blen) { + if(alen < blen) { + SparseRowVector c = allocSparseVector(blen, b, bix); + for (int j = ai; j < ai+alen; j++) { + c.add(aix[j], a[j]); + } + return c; + } else { + SparseRowVector c = allocSparseVector(alen, a, aix); + for (int j = bi; j < bi+blen; j++) { + c.add(bix[j], b[j]); + } + return c; + } + } + + public static SparseRowVector vectXorWrite(int len, double[] a, double bval, int[] aix, int ai, int alen) { + if(bval != 0) { + double[] c = allocVector(len, true, 1); + for(int j = ai; j < ai+alen; j++) + c[aix[j]] = (a[j] != 0) ? 0 : 1; + return new SparseRowVector(c); + } else { + SparseRowVector c = allocSparseVector(alen); + for(int j = ai; j < ai+alen; j++) + c.set(aix[j], (a[j] != 0) ? 1 : 0); + return c; + } + } + + public static SparseRowVector vectXorWrite(int len, double bval, double[] a, int[] aix, int ai, int alen) { + return vectXorWrite(len, a, bval, aix, ai, alen); + } + + public static SparseRowVector vectXorWrite(int len, double[] a, double[] b, int[] aix, int[] bix, int ai, int bi, int alen, int blen) { + SparseRowVector c = allocSparseVector(alen); + for (int j = bi; j < bi+blen; j++) + c.set(bix[j], (b[j] != 0) ? 1 : 0); + for (int j = ai; j < ai+alen; j++) + c.set(aix[j], ((a[j] != 0) != (c.get(aix[j]) != 0)) ? 1 : 0); + return c; + } + + public static SparseRowVector vectPowWrite(int len, double[] a, double bval, int[] aix, int ai, int alen) { + if(bval == 0) { + //Todo: why -1 here, even though the result should be 1? + double[] c = allocVector(len, true, 1); + for(int j = ai; j < ai+alen; j++) + c[aix[j]] = Math.pow(a[j], bval) - 1; + return new SparseRowVector(c); + } else { + SparseRowVector c = allocSparseVector(alen); + for(int j = ai; j < ai+alen; j++) + c.set(aix[j], Math.pow(a[j], bval)); + return c; + } + } + + public static SparseRowVector vectMinWrite(int len, double[] a, double bval, int[] aix, int ai, int alen) { + if(bval < 0) { + double[] c = allocVector(len, true, bval); + for(int j = ai; j < ai+alen; j++) + c[aix[j]] = Math.min(a[j], bval); + return new SparseRowVector(c); + } else { + SparseRowVector c = allocSparseVector(alen); + for(int j = ai; j < ai+alen; j++) + c.set(aix[j], Math.min(a[j], bval)); + return c; + } + } + + public static SparseRowVector vectMinWrite(int len, double bval, double[] a, int[] aix, int ai, int alen) { + return vectMinWrite(len, a, bval, aix, ai, alen); + } + + public static SparseRowVector vectMinWrite(int len, double[] a, double[] b, int[] aix, int[] bix, int ai, int bi, int alen, int blen) { + SparseRowVector c = allocSparseVector(alen); + int aIndex = ai; + int bIndex = bi; + while(aIndex < ai+alen && bIndex < bi+blen) { + int aIdx = aix[aIndex]; + int bIdx = bix[bIndex]; + if(aIdx == bIdx) { + c.set(aIdx, Math.min(a[aIndex], b[bIndex])); + aIndex++; + bIndex++; + } else if(aIdx < bIdx) { + c.set(aIdx, Math.min(a[aIndex], 0)); + aIndex++; + } else { + c.set(bIdx, Math.min(b[bIndex], 0)); + bIndex++; + } + } + for (; aIndex < ai+alen; aIndex++) c.set(aix[aIndex], Math.min(a[aIndex], 0)); + for (; bIndex < bi+blen; bIndex++) c.set(bix[bIndex], Math.min(b[bIndex], 0)); + return c; + } + + public static SparseRowVector vectMaxWrite(int len, double[] a, double bval, int[] aix, int ai, int alen) { + if(bval > 0) { + double[] c = allocVector(len, true, bval); + for(int j = ai; j < ai+alen; j++) + c[aix[j]] = Math.max(a[j], bval); + return new SparseRowVector(c); + } else { + SparseRowVector c = allocSparseVector(alen); + for(int j = ai; j < ai+alen; j++) + c.set(aix[j], Math.max(a[j], bval)); + return c; + } + } + + public static SparseRowVector vectMaxWrite(int len, double bval, double[] a, int[] aix, int ai, int alen) { + return vectMaxWrite(len, a, bval, aix, ai, alen); + } + + public static SparseRowVector vectMaxWrite(int len, double[] a, double[] b, int[] aix, int[] bix, int ai, int bi, int alen, int blen) { + SparseRowVector c = allocSparseVector(alen); + int aIndex = ai; + int bIndex = bi; + while(aIndex < ai+alen && bIndex < bi+blen) { + int aIdx = aix[aIndex]; + int bIdx = bix[bIndex]; + if(aIdx == bIdx) { + c.set(aIdx, Math.max(a[aIndex], b[bIndex])); + aIndex++; + bIndex++; + } else if(aIdx < bIdx) { + c.set(aIdx, Math.max(a[aIndex], 0)); + aIndex++; + } else { + c.set(bIdx, Math.max(b[bIndex], 0)); + bIndex++; + } + } + for (; aIndex < ai+alen; aIndex++) c.set(aix[aIndex], Math.max(a[aIndex], 0)); + for (; bIndex < bi+blen; bIndex++) c.set(bix[bIndex], Math.max(b[bIndex], 0)); + return c; + } + + public static SparseRowVector vectEqualWrite(int len, double[] a, double bval, int[] aix, int ai, int alen) { + if(bval == 0) { + double[] c = allocVector(len, true, 1); + for( int j = ai; j < ai+alen; j++ ) + c[aix[j]] = (a[j] == bval) ? 1 : 0; + return new SparseRowVector(c); + } else { + SparseRowVector c = allocSparseVector(alen); + for(int j = ai; j < ai+alen; j++) + c.set(aix[j], (a[j] == bval) ? 1 : 0); + return c; + } + } + + public static SparseRowVector vectEqualWrite(int len, double bval, double[] a, int[] aix, int ai, int alen) { + return vectEqualWrite(len, a, bval, aix, ai, alen); + } + + //doesn't return SparseRowVector, but still uses two sparse vectors as inputs + public static double[] vectEqualWrite(int len, double[] a, double[] b, int[] aix, int[] bix, int ai, int bi, int alen, int blen) { + double[] c = allocVector(len, true, 1); + int aIndex = ai; + int bIndex = bi; + while(aIndex < ai+alen && bIndex < bi+blen) { + int aIdx = aix[aIndex]; + int bIdx = bix[bIndex]; + if (aIdx == bIdx) { + c[aIdx] = (a[aIndex] == b[bIndex]) ? 1 : 0; + aIndex++; + bIndex++; + } else if(aIdx < bIdx) { + //todo: this might be too unsafe + c[aIdx] = a[aIndex] == 0 ? 1 : 0; + // c[aIdx] = 0; + aIndex++; + } else { + c[bIdx] = b[bIndex] == 0 ? 1 : 0; + // c[bIdx] = 0; + bIndex++; + } + } + for (; aIndex < ai+alen; aIndex++) c[aix[aIndex]] = 0; + for (; bIndex < bi+blen; bIndex++) c[bix[bIndex]] = 0; + return c; + } + + public static SparseRowVector vectNotequalWrite(int len, double[] a, double bval, int[] aix, int ai, int alen) { + if(bval != 0) { + double[] c = allocVector(len, true, 1); + for(int j = ai; j < ai+alen; j++) + c[aix[j]] = ((a[j] != bval) ? 1 : 0); + return new SparseRowVector(c); + } else { + SparseRowVector c = allocSparseVector(alen); + for(int j = 0; j < ai+alen; j++) { + c.set(aix[j], (a[j] != bval) ? 1 : 0); + } + return c; + } + } + + public static SparseRowVector vectNotequalWrite(int len, double bval, double[] a, int[] aix, int ai, int alen) { + return vectNotequalWrite(len, a, bval, aix, ai, alen); + } + + public static SparseRowVector vectNotequalWrite(int len, double[] a, double[] b, int[] aix, int[] bix, int ai, int bi, int alen, int blen) { + SparseRowVector c = allocSparseVector(alen); + int aIndex = ai; + int bIndex = bi; + while(aIndex < ai+alen && bIndex < bi+blen) { + int aIdx = aix[aIndex]; + int bIdx = bix[bIndex]; + if(aIdx == bIdx) { + c.set(aIdx, (a[aIndex] != b[bIndex]) ? 1 : 0); + aIndex++; + bIndex++; + } else if(aIdx < bIdx) { + //todo: this might be too unsafe + c.set(aIdx, (a[aIndex] != 0) ? 1 : 0); + // c.set(aIdx, 1); + aIndex++; + } else { + c.set(bIdx, (b[bIndex] != 0) ? 1 : 0); + // c.set(bIdx, 1); + bIndex++; + } + } + for (; aIndex < ai+alen; aIndex++) c.set(aix[aIndex], (a[aIndex] != 0) ? 1 : 0); + for (; bIndex < bi+blen; bIndex++) c.set(bix[bIndex], (b[bIndex] != 0) ? 1 : 0); + return c; + } + + public static SparseRowVector vectLessWrite(int len, double[] a, double bval, int[] aix, int ai, int alen) { + if(bval > 0) { + double[] c = allocVector(len, true, 1); + for(int j = ai; j < ai+alen; j++) + c[aix[j]] = (a[j] < bval) ? 1 : 0; + return new SparseRowVector(c); + } else { + SparseRowVector c = allocSparseVector(alen); + for(int j = ai; j < ai+alen; j++) { + c.set(aix[j], (a[j] < bval) ? 1 : 0); + } + return c; + } + } + + public static SparseRowVector vectLessWrite(int len, double bval, double[] a, int[] aix, int ai, int alen) { + return vectGreaterequalWrite(len, a, bval, aix, ai, alen); + } + + public static SparseRowVector vectLessWrite(int len, double[] a, double[] b, int[] aix, int[] bix, int ai, int bi, int alen, int blen) { + SparseRowVector c = allocSparseVector(alen); + int aIndex = ai; + int bIndex = bi; + while(aIndex < ai+alen && bIndex < bi+blen) { + int aIdx = aix[aIndex]; + int bIdx = bix[bIndex]; + if(aIdx == bIdx) { + c.set(aIdx, (a[aIndex] < b[bIndex]) ? 1 : 0); + aIndex++; + bIndex++; + } else if(aIdx < bIdx) { + c.set(aIdx, (a[aIndex] < 0) ? 1 : 0); + aIndex++; + } else { + c.set(bIdx, (0 < b[bIndex]) ? 1 : 0); + bIndex++; + } + } + for(; aIndex < ai+alen; aIndex++) c.set(aix[aIndex], (a[aIndex] < 0) ? 1 : 0); + for(; bIndex < bi+blen; bIndex++) c.set(bix[bIndex], (0 < b[bIndex]) ? 1 : 0); + return c; + } + + public static SparseRowVector vectLessequalWrite(int len, double[] a, double bval, int[] aix, int ai, int alen) { + if(bval >= 0) { + double[] c = allocVector(len, true, 1); + for(int j = ai; j < ai+alen; j++) + c[aix[j]] = (a[j] <= bval) ? 1 : 0; + return new SparseRowVector(c); + } else { + SparseRowVector c = allocSparseVector(alen); + for(int j = ai; j < ai+alen; j++) { + c.set(aix[j], (a[j] <= bval) ? 1 : 0); + } + return c; + } + } + + public static SparseRowVector vectLessequalWrite(int len, double bval, double[] a, int[] aix, int ai, int alen) { + return vectGreaterWrite(len, a, bval, aix, ai, alen); + } + + //doesn't return SparseRowVector, but still uses two sparse vectors as inputs + public static double[] vectLessequalWrite(int len, double[] a, double[] b, int[] aix, int[] bix, int ai, int bi, int alen, int blen) { + double[] c = allocVector(len, true, 1); + int aIndex = ai; + int bIndex = bi; + while(aIndex < ai+alen && bIndex < bi+blen) { + int aIdx = aix[aIndex]; + int bIdx = bix[bIndex]; + if(aIdx == bIdx) { + c[aIdx] = (a[aIndex] <= b[bIndex]) ? 1 : 0; + aIndex++; + bIndex++; + } else if(aIdx < bIdx) { + c[aIdx] = (a[aIndex] <= 0) ? 1 : 0; + aIndex++; + } else { + c[bIdx] = (0 <= b[bIndex]) ? 1 : 0; + bIndex++; + } + } + for(; aIndex < ai+alen; aIndex++) c[aix[aIndex]] = (a[aIndex] <= 0) ? 1 : 0; + for(; bIndex < bi+blen; bIndex++) c[bix[bIndex]] = (0 <= b[bIndex]) ? 1 : 0; + return c; + } + + public static SparseRowVector vectGreaterWrite(int len, double[] a, double bval, int[] aix, int ai, int alen) { + if(bval < 0) { + double[] c = allocVector(len, true, 1); + for(int j = ai; j < ai+alen; j++) + c[aix[j]] = (a[j] > bval) ? 1 : 0; + return new SparseRowVector(c); + } else { + SparseRowVector c = allocSparseVector(alen); + for(int j = ai; j < ai+alen; j++) { + c.set(aix[j], (a[j] > bval) ? 1 : 0); + } + return c; + } + } + + public static SparseRowVector vectGreaterWrite(int len, double bval, double[] a, int[] aix, int ai, int alen) { + return vectLessequalWrite(len, a, bval, aix, ai, alen); + } + + public static SparseRowVector vectGreaterWrite(int len, double[] a, double[] b, int[] aix, int[] bix, int ai, int bi, int alen, int blen) { + SparseRowVector c = allocSparseVector(alen); + int aIndex = ai; + int bIndex = bi; + while(aIndex < ai+alen && bIndex < bi+blen) { + int aIdx = aix[aIndex]; + int bIdx = bix[bIndex]; + if(aIdx == bIdx) { + c.set(aIdx, (a[aIndex] > b[bIndex]) ? 1 : 0); + aIndex++; + bIndex++; + } else if(aIdx < bIdx) { + c.set(aIdx, (a[aIndex] > 0) ? 1 : 0); + aIndex++; + } else { + c.set(bIdx, (0 > b[bIndex]) ? 1 : 0); + bIndex++; + } + } + for(; aIndex < ai+alen; aIndex++) c.set(aix[aIndex], (a[aIndex] > 0) ? 1 : 0); + for(; bIndex < bi+blen; bIndex++) c.set(bix[bIndex], (0 > b[bIndex]) ? 1 : 0); + return c; + } + + public static SparseRowVector vectGreaterequalWrite(int len, double[] a, double bval, int[] aix, int ai, int alen) { + if(bval <= 0) { + double[] c = allocVector(len, true, 1); + for(int j = ai; j < ai+alen; j++) + c[aix[j]] = (a[j] >= bval) ? 1 : 0; + return new SparseRowVector(c); + } else { + SparseRowVector c = allocSparseVector(alen); + for(int j = ai; j < ai+alen; j++) { + c.set(aix[j], (a[j] >= bval) ? 1 : 0); + } + return c; + } + } + + public static SparseRowVector vectGreaterequalWrite(int len, double bval, double[] a, int[] aix, int ai, int alen) { + return vectLessWrite(len, a, bval, aix, ai, alen); + } + + //doesn't return SparseRowVector, but still uses two sparse vectors as inputs + public static double[] vectGreaterequalWrite(int len, double[] a, double[] b, int[] aix, int[] bix, int ai, int bi, int alen, int blen) { + double[] c = allocVector(len, true, 1); + int aIndex = ai; + int bIndex = bi; + while(aIndex < ai+alen && bIndex < bi+blen) { + int aIdx = aix[aIndex]; + int bIdx = bix[bIndex]; + if(aIdx == bIdx) { + c[aIdx] = (a[aIndex] >= b[bIndex]) ? 1 : 0; + aIndex++; + bIndex++; + } else if(aIdx < bIdx) { + c[aIdx] = (a[aIndex] >= 0) ? 1 : 0; + aIndex++; + } else { + c[bIdx] = (0 >= b[bIndex]) ? 1 : 0; + bIndex++; + } + } + for(; aIndex < ai+alen; aIndex++) c[aix[aIndex]] = (a[aIndex] >= 0) ? 1 : 0; + for(; bIndex < bi+blen; bIndex++) c[bix[bIndex]] = (0 >= b[bIndex]) ? 1 : 0; + return c; + } + + public static SparseRowVector vectBitwandWrite(int len, double[] a, double bval, int[] aix, int ai, int alen) { + SparseRowVector c = allocSparseVector(alen); + int bval1 = (int) bval; + for(int j = ai; j < ai+alen; j++) + c.set(aix[j], bwAnd(a[j], bval1)); + return c; + } + + public static SparseRowVector vectBitwandWrite(int len, double bval, double[] a, int[] aix, int ai, int alen) { + return vectBitwandWrite(len, a, bval, aix, ai, alen); + } + + public static SparseRowVector vectBitwandWrite(int len, double[] a, double[] b, int[] aix, int[] bix, int ai, int bi, int alen, int blen) { + SparseRowVector c = allocSparseVector(alen); + int aIndex = ai; + int bIndex = bi; + while(aIndex < ai+alen && bIndex < bi+blen) { + int aIdx = aix[aIndex]; + int bIdx = bix[bIndex]; + if(aIdx == bIdx) { + c.set(aIdx, bwAnd(a[aIndex], b[bIndex])); + aIndex++; + bIndex++; + } else if(aIdx < bIdx) { + aIndex++; + } else { + bIndex++; + } + } + return c; + } + + public static SparseRowVector vectSqrtWrite(int len, double[] a, int[] aix, int ai, int alen) { + SparseRowVector c = allocSparseVector(alen); + for(int j = ai; j < ai+alen; j++) + c.set(aix[j], Math.sqrt(a[j])); + return c; + } + + public static SparseRowVector vectAbsWrite(int len, double[] a, int[] aix, int ai, int alen) { + SparseRowVector c = allocSparseVector(alen); + for(int j = ai; j < ai+alen; j++) + c.set(aix[j], Math.abs(a[j])); + return c; + } + + public static SparseRowVector vectRoundWrite(int len, double[] a, int[] aix, int ai, int alen) { + SparseRowVector c = allocSparseVector(alen); + for(int j = ai; j < ai+alen; j++) + c.set(aix[j], Math.round(a[j])); + return c; + } + + public static SparseRowVector vectCeilWrite(int len, double[] a, int[] aix, int ai, int alen) { + SparseRowVector c = allocSparseVector(alen); + for(int j = ai; j < ai+alen; j++) + c.set(aix[j], FastMath.ceil(a[j])); + return c; + } + + public static SparseRowVector vectFloorWrite(int len, double[] a, int[] aix, int ai, int alen) { + SparseRowVector c = allocSparseVector(alen); + for(int j = ai; j < ai+alen; j++) + c.set(aix[j], FastMath.floor(a[j])); + return c; + } + + public static SparseRowVector vectSinWrite(int len, double[] a, int[] aix, int ai, int alen) { + SparseRowVector c = allocSparseVector(alen); + for(int j = ai; j < ai+alen; j++) + c.set(aix[j], FastMath.sin(a[j])); + return c; + } + + public static SparseRowVector vectTanWrite(int len, double[] a, int[] aix, int ai, int alen) { + SparseRowVector c = allocSparseVector(alen); + for(int j = ai; j < ai+alen; j++) + c.set(aix[j], FastMath.tan(a[j])); + return c; + } + + public static SparseRowVector vectAsinWrite(int len, double[] a, int[] aix, int ai, int alen) { + SparseRowVector c = allocSparseVector(alen); + for(int j = ai; j < ai+alen; j++) + c.set(aix[j], FastMath.asin(a[j])); + return c; + } + + public static SparseRowVector vectAtanWrite(int len, double[] a, int[] aix, int ai, int alen) { + SparseRowVector c = allocSparseVector(alen); + for(int j = ai; j < ai+alen; j++) + c.set(aix[j], FastMath.atan(a[j])); + return c; + } + + public static SparseRowVector vectSinhWrite(int len, double[] a, int[] aix, int ai, int alen) { + SparseRowVector c = allocSparseVector(alen); + for(int j = ai; j < ai+alen; j++) + c.set(aix[j], FastMath.sinh(a[j])); + return c; + } + + public static SparseRowVector vectTanhWrite(int len, double[] a, int[] aix, int ai, int alen) { + SparseRowVector c = allocSparseVector(alen); + for(int j = ai; j < ai+alen; j++) + c.set(aix[j], FastMath.tanh(a[j])); + return c; + } + + public static SparseRowVector vectSignWrite(int len, double[] a, int[] aix, int ai, int alen) { + SparseRowVector c = allocSparseVector(alen); + for(int j = ai; j < ai+alen; j++) + c.set(aix[j], FastMath.signum(a[j])); + return c; + } + + //todo MatrixMult, pow2 and mult2 drafts +// public static SparseRowVector vectMatrixMult(int len, double[] a, double[] b, int[] aix, int[] bix, int ai, int bi, int alen, int blen) { +// //note: assumption b is already transposed for efficient dot products +// int m2clen = b.length / len; +// SparseRowVector c = allocSparseVector(m2clen); +// for(int i = 0; i < m2clen; i++) { +// c.set(bix[i], LibMatrixMult.dotProduct(a, aix, ai, alen, b, bix, bi, blen)); +// } +// return c; +// } +// +// public static SparseRowVector vectPow2Write(int len, double[] a, int[] aix, int ai, int alen) { +// SparseRowVector c = allocSparseVector(len); +// for(int j = 0; j < ai+alen; j++) +// c.set(aix[j], a[j] * a[j]); +// return c; +// } +// +// public static SparseRowVector vectMult2Write(int len, double[] a, int[] aix, int ai, int alen) { +// SparseRowVector c = allocSparseVector(len); +// for(int j = 0; j < ai+alen; j++) +// c.set(aix[j], a[j] + a[j]); +// return c; +// } + //complex builtin functions that are not directly generated //(included here in order to reduce the number of imports) @@ -2194,10 +2829,19 @@ public static void setupThreadLocalMemory(int numVectors, int len, int len2) { if( numVectors > 0 ) memPool.set(new VectorBuffer(numVectors, len, len2)); } + + public static void setupSparseThreadLocalMemory(int numVectors, int len, int len2) { + if( numVectors > 0 ) + sparseMemPool.set(new SparseVectorBuffer(numVectors, len, len2)); + } public static void cleanupThreadLocalMemory() { memPool.remove(); } + + public static void cleanupSparseThreadLocalMemory() { + sparseMemPool.remove(); + } public static double[] allocVector(int len, boolean reset) { return allocVector(len, reset, 0); @@ -2217,6 +2861,47 @@ protected static double[] allocVector(int len, boolean reset, double resetVal) { Arrays.fill(vect, resetVal); return vect; } + + //allocate an empty vector + public static SparseRowVector allocSparseVector(int len) { + return allocSparseVector(len, null, null, false, false); + } + + //allocate a vector with the vales and indexes filled + public static SparseRowVector allocSparseVector(int len, double[] values,int[] indexes) { + return allocSparseVector(len, values, indexes, true, false); + } + + //allocate vector with only the indexes filled + public static SparseRowVector allocSparseVector(int len, int[] indexes) { + return allocSparseVector(len, null, indexes, false, true); + } + + protected static SparseRowVector allocSparseVector(int len, double[] values, int[] indexes, boolean fill, boolean indexCopy) { + SparseVectorBuffer buff = sparseMemPool.get(); + + //find next matching vector in ring buffer or + //allocate new vector if no vector was returned + SparseRowVector vect = buff.next(len); + if(vect == null) + vect = new SparseRowVector(len); + //reset vector for normal outputs + else + vect.reset(4, len); + + //fill vector + if(fill) { + System.arraycopy(values, 0, vect.values(), 0, len); + System.arraycopy(indexes, 0, vect.indexes(), 0, len); + vect.setSize(len); + //indexes copy + } else if(indexCopy) { + vect.setIndexes(indexes); + vect.setSize(len); + } + + return vect; + } /** * Simple ring buffer of allocated vectors, where @@ -2265,4 +2950,52 @@ public boolean isReusable(int num, int len1, int len2) { && _data.length == lnum); } } + + /** + * Simple ring buffer of allocated SparseRowVectors, where + * vectors of different sizes are interspersed. + */ + private static class SparseVectorBuffer { + private static final int MAX_SIZE = 512*1024; //4MB + private final SparseRowVector[] _data; + private int _pos; + private int _len1; + private int _len2; + + public SparseVectorBuffer(int num, int len1, int len2) { + //best effort size restriction since large intermediates + //not necessarily used (num refers to the total number) + len1 = Math.min(len1, MAX_SIZE); + len2 = Math.min(len2, MAX_SIZE); + //pre-allocate ring buffer + int lnum = (len2>0 && len1!=len2) ? 2*num : num; + _data = new SparseRowVector[lnum]; + for( int i=0; i num ) { + _data[2*i] = new SparseRowVector(len1); + _data[2*i+1] = new SparseRowVector(len2); + } + else { + _data[i] = new SparseRowVector(len1); + } + } + _pos = -1; + _len1 = len1; + _len2 = len2; + } + public SparseRowVector next(int len) { + if( _len1!=len && _len2!=len ) + return null; + do { + _pos = (_pos+1>=_data.length) ? 0 : _pos+1; + } while( _data[_pos].values().length!=len ); + return _data[_pos]; + } + @SuppressWarnings("unused") + public boolean isReusable(int num, int len1, int len2) { + int lnum = (len2>0 && len1!=len2) ? 2*num : num; + return (_len1 == len1 && _len2 == len2 + && _data.length == lnum); + } + } } diff --git a/src/main/java/org/apache/sysds/runtime/codegen/SpoofRowwise.java b/src/main/java/org/apache/sysds/runtime/codegen/SpoofRowwise.java index eab223bc970..f7ca5c02ef2 100644 --- a/src/main/java/org/apache/sysds/runtime/codegen/SpoofRowwise.java +++ b/src/main/java/org/apache/sysds/runtime/codegen/SpoofRowwise.java @@ -27,6 +27,7 @@ import java.util.concurrent.Future; import java.util.stream.IntStream; +import org.apache.sysds.api.DMLScript; import org.apache.sysds.runtime.DMLRuntimeException; import org.apache.sysds.runtime.compress.CompressedMatrixBlock; import org.apache.sysds.runtime.controlprogram.caching.MatrixObject; @@ -187,7 +188,13 @@ public MatrixBlock execute(ArrayList inputs, ArrayList 0 ) - LibSpoofPrimitives.setupThreadLocalMemory(_reqVectMem, n, n2); + if(inputs.get(0).isInSparseFormat() && DMLScript.SPARSE_INTERMEDIATE) { + //todo: look at wether n, n2 are correct for vector - vector calculations + LibSpoofPrimitives.setupSparseThreadLocalMemory(_reqVectMem, n, n2); + LibSpoofPrimitives.setupThreadLocalMemory(_reqVectMem, n, n2); + } else { + LibSpoofPrimitives.setupThreadLocalMemory(_reqVectMem, n, n2); + } //core sequential execute MatrixBlock a = inputs.get(0); @@ -201,7 +208,12 @@ public MatrixBlock execute(ArrayList inputs, ArrayList 0 ) - LibSpoofPrimitives.cleanupThreadLocalMemory(); + if(inputs.get(0).isInSparseFormat() && DMLScript.SPARSE_INTERMEDIATE) { + LibSpoofPrimitives.cleanupSparseThreadLocalMemory(); + LibSpoofPrimitives.cleanupThreadLocalMemory(); + } else { + LibSpoofPrimitives.cleanupThreadLocalMemory(); + } if( flipOut ) { fixTransposeDimensions(out); out = LibMatrixReorg.transpose(out, new MatrixBlock( diff --git a/src/test/java/org/apache/sysds/test/component/codegen/CPlanVectorPrimitivesTest.java b/src/test/java/org/apache/sysds/test/component/codegen/CPlanVectorPrimitivesTest.java index 7d8b4c9096d..eaf750c01cd 100644 --- a/src/test/java/org/apache/sysds/test/component/codegen/CPlanVectorPrimitivesTest.java +++ b/src/test/java/org/apache/sysds/test/component/codegen/CPlanVectorPrimitivesTest.java @@ -21,6 +21,7 @@ import java.lang.reflect.Method; +import org.apache.sysds.runtime.data.SparseRowVector; import org.junit.Test; import org.apache.sysds.common.Types.OpOp2; import org.apache.sysds.hops.codegen.cplan.CNodeBinary.BinType; @@ -716,6 +717,277 @@ public void testVectorVectorBitwAndSparseDense() { testVectorBinaryPrimitive(BinType.VECT_BITWAND, InputType.VECTOR_SPARSE, InputType.VECTOR_DENSE); } + //********************testing with sparse intermediates********************// + //vector - scalar + + @Test + public void testVectorScalarMultSparseToSparse() { + testVectorBinarySparsePrimitive(BinType.VECT_MULT_SCALAR, InputType.VECTOR_SPARSE, InputType.SCALAR); + } + + @Test + public void testVectorScalarDivSparseToSparse() { + testVectorBinarySparsePrimitive(BinType.VECT_DIV_SCALAR, InputType.VECTOR_SPARSE, InputType.SCALAR); + } + + @Test + public void testVectorScalarMinSparseToSparse() { + testVectorBinarySparsePrimitive(BinType.VECT_MIN_SCALAR, InputType.VECTOR_SPARSE, InputType.SCALAR); + } + + @Test + public void testVectorScalarMaxSparseToSparse() { + testVectorBinarySparsePrimitive(BinType.VECT_MAX_SCALAR, InputType.VECTOR_SPARSE, InputType.SCALAR); + } + + @Test + public void testVectorScalarPowSparseToSparse() { + testVectorBinarySparsePrimitive(BinType.VECT_POW_SCALAR, InputType.VECTOR_SPARSE, InputType.SCALAR); + } + + @Test + public void testVectorScalarEqualSparseToSparse() { + testVectorBinarySparsePrimitive(BinType.VECT_EQUAL_SCALAR, InputType.VECTOR_SPARSE, InputType.SCALAR); + } + + @Test + public void testVectorScalarNotEqualSparseToSparse() { + testVectorBinarySparsePrimitive(BinType.VECT_NOTEQUAL_SCALAR, InputType.VECTOR_SPARSE, InputType.SCALAR); + } + + @Test + public void testVectorScalarLessSparseToSparse() { + testVectorBinarySparsePrimitive(BinType.VECT_LESS_SCALAR, InputType.VECTOR_SPARSE, InputType.SCALAR); + } + + @Test + public void testVectorScalarLessEqualSparseToSparse() { + testVectorBinarySparsePrimitive(BinType.VECT_LESSEQUAL_SCALAR, InputType.VECTOR_SPARSE, InputType.SCALAR); + } + + @Test + public void testVectorScalarGreaterSparseToSparse() { + testVectorBinarySparsePrimitive(BinType.VECT_GREATER_SCALAR, InputType.VECTOR_SPARSE, InputType.SCALAR); + } + + @Test + public void testVectorScalarGreaterEqualSparseToSparse() { + testVectorBinarySparsePrimitive(BinType.VECT_GREATEREQUAL_SCALAR, InputType.VECTOR_SPARSE, InputType.SCALAR); + } + + //todo: this only makes sense, when bval is 0 + @Test + public void testVectorScalarXorSparseToSparse() { + testVectorBinarySparsePrimitive(BinType.VECT_XOR_SCALAR, InputType.VECTOR_SPARSE, InputType.SCALAR); + } + + @Test + public void testVectorScalarBitwAndSparseToSparse() { + testVectorBinarySparsePrimitive(BinType.VECT_BITWAND_SCALAR, InputType.VECTOR_SPARSE, InputType.SCALAR); + } + + //scalar - vector + + @Test + public void testScalarVectorMinSparseToSparse() { + testVectorBinarySparsePrimitive(BinType.VECT_MIN_SCALAR, InputType.SCALAR, InputType.VECTOR_SPARSE); + } + + @Test + public void testScalarVectorMaxSparseToSparse() { + testVectorBinarySparsePrimitive(BinType.VECT_MAX_SCALAR, InputType.SCALAR, InputType.VECTOR_SPARSE); + } + + @Test + public void testScalarVectorEqualSparseToSparse() { + testVectorBinarySparsePrimitive(BinType.VECT_EQUAL_SCALAR, InputType.SCALAR, InputType.VECTOR_SPARSE); + } + + @Test + public void testScalarVectorNotEqualSparseToSparse() { + testVectorBinarySparsePrimitive(BinType.VECT_NOTEQUAL_SCALAR, InputType.SCALAR, InputType.VECTOR_SPARSE); + } + + @Test + public void testScalarVectorLessSparseToSparse() { + testVectorBinarySparsePrimitive(BinType.VECT_LESS_SCALAR, InputType.SCALAR, InputType.VECTOR_SPARSE); + } + + @Test + public void testScalarVectorLessEqualSparseToSparse() { + testVectorBinarySparsePrimitive(BinType.VECT_LESSEQUAL_SCALAR, InputType.SCALAR, InputType.VECTOR_SPARSE); + } + + @Test + public void testScalarVectorGreaterSparseToSparse() { + testVectorBinarySparsePrimitive(BinType.VECT_GREATER_SCALAR, InputType.SCALAR, InputType.VECTOR_SPARSE); + } + + @Test + public void testScalarVectorGreaterEqualSparseToSparse() { + testVectorBinarySparsePrimitive(BinType.VECT_GREATEREQUAL_SCALAR, InputType.SCALAR, InputType.VECTOR_SPARSE); + } + + @Test + public void testScalarVectorXorSparseToSparse() { + testVectorBinarySparsePrimitive(BinType.VECT_XOR_SCALAR, InputType.SCALAR, InputType.VECTOR_SPARSE); + } + + @Test + public void testScalarVectorBitwAndSparseToSparse() { + testVectorBinarySparsePrimitive(BinType.VECT_BITWAND_SCALAR, InputType.SCALAR, InputType.VECTOR_SPARSE); + } + + //special binary + + // @Test + // public void testVectorPow2SparseToSparse() { + // testVectorUnarySparsePrimitive(UnaryType.VECT_POW2, InputType.VECTOR_SPARSE); + // } + // + // @Test + // public void testVectorMult2SparseToSparse() { + // testVectorUnarySparsePrimitive(UnaryType.VECT_MULT2, InputType.VECTOR_SPARSE); + // } + + //vector - vector + + @Test + public void testVectorVectorMultSparseToSparse() { + testVectorBinarySparsePrimitive(BinType.VECT_MULT, InputType.VECTOR_SPARSE, InputType.VECTOR_SPARSE); + } + + @Test + public void testVectorVectorDivSparseToSparse() { + testVectorBinarySparsePrimitive(BinType.VECT_DIV, InputType.VECTOR_SPARSE, InputType.VECTOR_SPARSE); + } + + @Test + public void testVectorVectorPlusSparseToSparse() { + testVectorBinarySparsePrimitive(BinType.VECT_PLUS, InputType.VECTOR_SPARSE, InputType.VECTOR_SPARSE); + } + + @Test + public void testVectorVectorMinusSparseToSparse() { + testVectorBinarySparsePrimitive(BinType.VECT_MINUS, InputType.VECTOR_SPARSE, InputType.VECTOR_SPARSE); + } + + @Test + public void testVectorVectorMinSparseToSparse() { + testVectorBinarySparsePrimitive(BinType.VECT_MIN, InputType.VECTOR_SPARSE, InputType.VECTOR_SPARSE); + } + @Test + public void testVectorVectorMaxSparseToSparse() { + testVectorBinarySparsePrimitive(BinType.VECT_MAX, InputType.VECTOR_SPARSE, InputType.VECTOR_SPARSE); + } + + @Test + public void testVectorVectorEqualSparseToSparse() { + testVectorBinarySparsePrimitive(BinType.VECT_EQUAL, InputType.VECTOR_SPARSE, InputType.VECTOR_SPARSE); + } + + @Test + public void testVectorVectorNotEqualSparseToSparse() { + testVectorBinarySparsePrimitive(BinType.VECT_NOTEQUAL, InputType.VECTOR_SPARSE, InputType.VECTOR_SPARSE); + } + + @Test + public void testVectorVectorLessSparseToSparse() { + testVectorBinarySparsePrimitive(BinType.VECT_LESS, InputType.VECTOR_SPARSE, InputType.VECTOR_SPARSE); + } + + @Test + public void testVectorVectorLessEqualSparseToSparse() { + testVectorBinarySparsePrimitive(BinType.VECT_LESSEQUAL, InputType.VECTOR_SPARSE, InputType.VECTOR_SPARSE); + } + + @Test + public void testVectorVectorGreaterSparseToSparse() { + testVectorBinarySparsePrimitive(BinType.VECT_GREATER, InputType.VECTOR_SPARSE, InputType.VECTOR_SPARSE); + } + + @Test + public void testVectorVectorGreaterEqualSparseToSparse() { + testVectorBinarySparsePrimitive(BinType.VECT_GREATEREQUAL, InputType.VECTOR_SPARSE, InputType.VECTOR_SPARSE); + } + + @Test + public void testVectorVectorXorSparseToSparse() { + testVectorBinarySparsePrimitive(BinType.VECT_XOR, InputType.VECTOR_SPARSE, InputType.VECTOR_SPARSE); + } + + @Test + public void testVectorVectorBitwAndSparseToSparse() { + testVectorBinarySparsePrimitive(BinType.VECT_BITWAND, InputType.VECTOR_SPARSE, InputType.VECTOR_SPARSE); + } + + // @Test + // public void testVectorVectorMatrixMultSparseToSparse() { + // testVectorBinarySparsePrimitive(BinType.VECT_MATRIXMULT, InputType.VECTOR_SPARSE, InputType.VECTOR_SPARSE); + // } + + //unary primitives with sparse intermediates + + @Test + public void testVectorSqrtSparseToSparse() { + testVectorUnarySparsePrimitive(UnaryType.VECT_SQRT, InputType.VECTOR_SPARSE); + } + + @Test + public void testVectorAbsSparseToSparse() { + testVectorUnarySparsePrimitive(UnaryType.VECT_ABS, InputType.VECTOR_SPARSE); + } + + @Test + public void testVectorRoundSparseToSparse() { + testVectorUnarySparsePrimitive(UnaryType.VECT_ROUND, InputType.VECTOR_SPARSE); + } + + @Test + public void testVectorCeilSparseToSparse() { + testVectorUnarySparsePrimitive(UnaryType.VECT_CEIL, InputType.VECTOR_SPARSE); + } + + @Test + public void testVectorFloorSparseToSparse() { + testVectorUnarySparsePrimitive(UnaryType.VECT_FLOOR, InputType.VECTOR_SPARSE); + } + + @Test + public void testVectorSinSparseToSparse() { + testVectorUnarySparsePrimitive(UnaryType.VECT_SIN, InputType.VECTOR_SPARSE); + } + + @Test + public void testVectorTanSparseToSparse() { + testVectorUnarySparsePrimitive(UnaryType.VECT_TAN, InputType.VECTOR_SPARSE); + } + + @Test + public void testVectorAsinSparseToSparse() { + testVectorUnarySparsePrimitive(UnaryType.VECT_ASIN, InputType.VECTOR_SPARSE); + } + + @Test + public void testVectorAtanSparseToSparse() { + testVectorUnarySparsePrimitive(UnaryType.VECT_ATAN, InputType.VECTOR_SPARSE); + } + + @Test + public void testVectorSinhSparseToSparse() { + testVectorUnarySparsePrimitive(UnaryType.VECT_SINH, InputType.VECTOR_SPARSE); + } + + @Test + public void testVectorTanhSparseToSparse() { + testVectorUnarySparsePrimitive(UnaryType.VECT_TANH, InputType.VECTOR_SPARSE); + } + + @Test + public void testVectorSignSparseToSparse() { + testVectorUnarySparsePrimitive(UnaryType.VECT_SIGN, InputType.VECTOR_SPARSE); + } + @SuppressWarnings("incomplete-switch") private static void testVectorAggPrimitive(UnaryType aggtype, InputType type1) { @@ -792,6 +1064,45 @@ private static void testVectorUnaryPrimitive(UnaryType utype, InputType type1) throw new RuntimeException(ex); } } + + private static void testVectorUnarySparsePrimitive(UnaryType utype, InputType type1) + { + try { + //generate input data + double sparsity = (type1 == InputType.VECTOR_DENSE) ? sparsity1 : sparsity2; + MatrixBlock in = MatrixBlock.randOperations(m, n, sparsity, -1, 1, "uniform", 7); + + //get vector primitive via reflection + String meName = "vect"+StringUtils.camelize(utype.name().split("_")[1])+"Write"; + Method me = LibSpoofPrimitives.class.getMethod(meName, new Class[]{int.class, double[].class, int[].class, int.class, int.class}); + + + for( int i=0; i= 0 && i == m-1) { + if(testType == 0) { + retX = (SparseRowVector) me.invoke(null, n, inA.getSparseBlock().values(i), 0, inA.getSparseBlock().indexes(i), + inA.getSparseBlock().pos(i), inA.getSparseBlock().size(i)); + } else { + retX = (SparseRowVector) me.invoke(null, n, inA.getSparseBlock().values(i), inB.min(), inA.getSparseBlock().indexes(i), + inA.getSparseBlock().pos(i), inA.getSparseBlock().size(i)); + } + } else { + retX = (SparseRowVector) me.invoke(null, n, inA.getSparseBlock().values(i), inB.max(), inA.getSparseBlock().indexes(i), + inA.getSparseBlock().pos(i), inA.getSparseBlock().size(i)); + } + else if( type1==InputType.SCALAR && type2==InputType.VECTOR_SPARSE ) + if(testType >= 0 && i == m-1) { + if(testType == 0) { + retX = (SparseRowVector) me.invoke(null, n, 0, inB.getSparseBlock().values(i), + inB.getSparseBlock().indexes(i), inB.getSparseBlock().pos(i), inB.getSparseBlock().size(i)); + } else { + retX = (SparseRowVector) me.invoke(null, n, inA.min(), inB.getSparseBlock().values(i), + inB.getSparseBlock().indexes(i), inB.getSparseBlock().pos(i), inB.getSparseBlock().size(i)); + } + } else { + retX = (SparseRowVector) me.invoke(null, n, inA.max(), inB.getSparseBlock().values(i), + inB.getSparseBlock().indexes(i), inB.getSparseBlock().pos(i), inB.getSparseBlock().size(i)); + } + else if( type1==InputType.VECTOR_SPARSE && type2==InputType.VECTOR_SPARSE ) + if(sparse) + retX = (SparseRowVector) me.invoke(null, n, inA.getSparseBlock().values(i), inB.getSparseBlock().values(i), + inA.getSparseBlock().indexes(i), inB.getSparseBlock().indexes(i), inA.getSparseBlock().pos(i), inB.getSparseBlock().pos(i), inA.getSparseBlock().size(i), inB.getSparseBlock().size(i)); + else + ret1 = (double[]) me.invoke(null, n, inA.getSparseBlock().values(i), inB.getSparseBlock().values(i), + inA.getSparseBlock().indexes(i), inB.getSparseBlock().indexes(i), inA.getSparseBlock().pos(i), inB.getSparseBlock().pos(i), inA.getSparseBlock().size(i), inB.getSparseBlock().size(i)); + + if(sparse) { + int[] indexes = retX.indexes(); + for (int j = 0; j < retX.size(); j++) { + ret1[indexes[j]] = retX.get(indexes[j]); + } + } + + //execute comparison operation + String opcode = OpOp2.valueOf(bintype.name().split("_")[1]).toString(); + MatrixBlock in1 = inA.slice(i, i, 0, n-1, new MatrixBlock()); + MatrixBlock in2 = inB.slice(i, i, 0, n-1, new MatrixBlock()); + double[] ret2 = null; + if( type1 == InputType.SCALAR ) { + ScalarOperator bop = InstructionUtils.parseScalarBinaryOperator(opcode, true); + bop = bop.setConstant(testType >= 0 && i == m-1 ? testType == 0 ? 0 : inA.min() : inA.max()); + ret2 = DataConverter.convertToDoubleVector( + in2.scalarOperations(bop, new MatrixBlock()), false); + } + else if( type2 == InputType.SCALAR ) { + ScalarOperator bop = InstructionUtils.parseScalarBinaryOperator(opcode, false); + bop = bop.setConstant(testType >= 0 && i == m-1 ? testType == 0 ? 0 : inB.min() : inB.max()); + ret2 = DataConverter.convertToDoubleVector( + in1.scalarOperations(bop, new MatrixBlock()), false); + } + else { //vector-vector + BinaryOperator bop = InstructionUtils.parseBinaryOperator(opcode); + ret2 = DataConverter.convertToDoubleVector( + in1.binaryOperations(bop, in2, new MatrixBlock()), false); + } + + //compare results + TestUtils.compareMatrices(ret1, ret2, eps); + } + } + catch( Exception ex ) { + throw new RuntimeException(ex); + } + } + + private static boolean sparseOutput(BinType type) { + switch(type) { + case VECT_EQUAL: + case VECT_LESSEQUAL: + case VECT_GREATEREQUAL: + return false; + default: + return true; + } + } + + /** + * returns -1, for normal testing; + * returns 0, for testing with 0 and non-zeros; + * returns 1, for testing with negative and positive numbers; + */ + private static int testType(BinType type) { + switch(type) { + case VECT_DIV_SCALAR: + case VECT_EQUAL_SCALAR: + case VECT_NOTEQUAL_SCALAR: + case VECT_XOR_SCALAR: + // case VECT_POW_SCALAR: + return 0; + case VECT_GREATER_SCALAR: + case VECT_GREATEREQUAL_SCALAR: + case VECT_LESS_SCALAR: + case VECT_LESSEQUAL_SCALAR: + case VECT_MIN_SCALAR: + case VECT_MAX_SCALAR: + return 1; + default: + return -1; + } + } } diff --git a/src/test/java/org/apache/sysds/test/component/codegen/SparseVectorAllocTest.java b/src/test/java/org/apache/sysds/test/component/codegen/SparseVectorAllocTest.java new file mode 100644 index 00000000000..8f1c354b2a3 --- /dev/null +++ b/src/test/java/org/apache/sysds/test/component/codegen/SparseVectorAllocTest.java @@ -0,0 +1,158 @@ +package org.apache.sysds.test.component.codegen; + +import org.apache.sysds.runtime.codegen.LibSpoofPrimitives; +import org.apache.sysds.runtime.data.SparseRowVector; +import org.apache.sysds.test.AutomatedTestBase; +import org.apache.sysds.test.TestUtils; +import org.junit.Assert; +import org.junit.Test; + +/** + * This is the component test for the ring buffer used in LibSpoofPrimitives.java, + * that allocates a vector with a certain size. + * Every allocation method is tested to achieve the needed coverage. + */ +public class SparseVectorAllocTest extends AutomatedTestBase +{ + double[] val1 = new double[]{1.5, 5.7, 9.1, 3.7, 5.3}; + double[] val2 = new double[]{9.6, 7.1, 2.7}; + int[] indexes1 = new int[]{3, 7, 14, 20, 81}; + int[] indexes2 = new int[]{20, 30, 90}; + + @Override + public void setUp() { + TestUtils.clearAssertionInformation(); + } + + @Test + public void testBasicAllocationSameLen() { + testBasicSparseVectorAllocation(1, 10, 10); + } + + @Test + public void testBasicAllocationLongerExp() { + testBasicSparseVectorAllocation(1, 10, 15); + } + + @Test + public void testBasicAllocationShorterExp() { + testBasicSparseVectorAllocation(1, 10, 7); + } + + @Test + public void testAllocationWithValues1() { + testSparseVectorWithValues(1, 10,val1, indexes1); + } + + @Test + public void testAllocationWithValues2() { + testSparseVectorWithValues(1, 10, val2, indexes2); + } + + @Test + public void testAllocationWithIndexes1() { + testSparseVectorWithIndexes(1, 10, indexes1); + } + + @Test + public void testAllocationWithIndexes2() { + testSparseVectorWithIndexes(1, 10, indexes2); + } + + @Test + public void testVectorReuse1() { + testBufferReuse(1, 10, -1, 5, val1, indexes1); + } + + @Test + public void testVectorReuse2() { + testBufferReuse(1, 10, -1, 5, val2, indexes2); + } + + /** tests the allocation of an empty vector + * @param numVectors number of vectors that should be pre-allocated + * @param len the length of the vector + * @param expLen the expected length of the allocated vector + */ + public void testBasicSparseVectorAllocation(int numVectors, int len, int expLen) { + //test the basic allocation of an empty vector + LibSpoofPrimitives.setupSparseThreadLocalMemory(numVectors, len, -1); + SparseRowVector sparseVec = LibSpoofPrimitives.allocSparseVector(expLen); + + Assert.assertEquals("Vector capacity should be initialized correctly", expLen, sparseVec.capacity()); + Assert.assertEquals("Vector size should be initialized with 0", 0, sparseVec.size()); + + LibSpoofPrimitives.cleanupSparseThreadLocalMemory(); + } + + /** tests the allocation of a vector with certain values and indexes + * @param numVectors number of vectors that should be pre-allocated + * @param len the length of the vector + * @param values expected values + * @param indexes expected indexes + */ + public void testSparseVectorWithValues(int numVectors, int len, double[] values, int[] indexes) { + //test the allocation of a vector with preset values + LibSpoofPrimitives.setupSparseThreadLocalMemory(numVectors, len, -1); + + SparseRowVector sparseVec = LibSpoofPrimitives.allocSparseVector(values.length, values, indexes); + + Assert.assertEquals("Vector size should match input array length", values.length, sparseVec.size()); + for(int j = 0; j < sparseVec.size(); j++) { + Assert.assertEquals("Value array should match input array", values[j], sparseVec.get(indexes[j]), 0.001); + } + + LibSpoofPrimitives.cleanupSparseThreadLocalMemory(); + } + + /** tests the allocation of a vector with certain indexes + * @param numVectors number of vectors that should be pre-allocated + * @param len the length of the vector + * @param indexes expected indexes + */ + public void testSparseVectorWithIndexes(int numVectors, int len, int[] indexes) { + //test allocation of a vector with preset indexes + LibSpoofPrimitives.setupSparseThreadLocalMemory(numVectors, len, -1); + + SparseRowVector sparseVec = LibSpoofPrimitives.allocSparseVector(indexes.length, indexes); + + Assert.assertEquals("Vector size should match input array length", indexes.length, sparseVec.size()); + Assert.assertArrayEquals("Indexes array should match input array", indexes, sparseVec.indexes()); + + LibSpoofPrimitives.cleanupSparseThreadLocalMemory(); + } + + /** tests the allocation of a vector that is reused multiple times + * @param numVectors number of vectors that should be pre-allocated + * @param len1 length of the first vector + * @param len2 length of the second vector + * @param expLen expected length of allocated vector + * @param values expected values + * @param indexes expected indexes + */ + public void testBufferReuse(int numVectors, int len1, int len2, int expLen, double[] values, int[] indexes) { + //test the reuse of the vectors in the ring buffer + LibSpoofPrimitives.setupSparseThreadLocalMemory(numVectors, len1, len2); + + //allocate first vector + SparseRowVector vec1 = LibSpoofPrimitives.allocSparseVector(expLen); + vec1.set(0, 1.0); + vec1.set(2, 2.0); + + //allocate second vector + SparseRowVector vec2 = LibSpoofPrimitives.allocSparseVector(expLen); + + Assert.assertEquals("Reused vector should be reset to size 0", 0, vec2.size()); + + //allocate third vector with values + SparseRowVector vec3 = LibSpoofPrimitives.allocSparseVector(values.length, values, indexes); + + Assert.assertEquals("Vector size should match input", values.length, vec3.size()); + for(int j = 0; j < vec3.size(); j++) { + Assert.assertEquals("Value array should match input array", values[j], vec3.get(indexes[j]), 0.001); + } + + LibSpoofPrimitives.cleanupSparseThreadLocalMemory(); + } + +} From 1eb446e0095bb9c35e1a3e7296a22ea480b61c4d Mon Sep 17 00:00:00 2001 From: Frxms Date: Sat, 31 May 2025 17:46:02 +0200 Subject: [PATCH 02/60] carry over changes from sparseRows branch --- .../java/org/apache/sysds/api/DMLOptions.java | 14 +- .../java/org/apache/sysds/api/DMLScript.java | 4 + .../sysds/hops/codegen/cplan/CNode.java | 26 ++- .../sysds/hops/codegen/cplan/CNodeBinary.java | 197 +++++++++++++----- .../sysds/hops/codegen/cplan/CNodeRow.java | 4 +- .../sysds/hops/codegen/cplan/CNodeUnary.java | 29 ++- .../sysds/hops/codegen/cplan/java/Binary.java | 61 +++++- .../sysds/hops/codegen/cplan/java/Unary.java | 25 ++- 8 files changed, 281 insertions(+), 79 deletions(-) diff --git a/src/main/java/org/apache/sysds/api/DMLOptions.java b/src/main/java/org/apache/sysds/api/DMLOptions.java index 763ac7b9388..b5b0908432b 100644 --- a/src/main/java/org/apache/sysds/api/DMLOptions.java +++ b/src/main/java/org/apache/sysds/api/DMLOptions.java @@ -85,6 +85,7 @@ public class DMLOptions { public boolean federatedCompilation = false; // Compile federated instructions based on input federation state and privacy constraints. public boolean noFedRuntimeConversion = false; // If activated, no runtime conversion of CP instructions to FED instructions will be performed. public int seed = -1; // The general seed for the execution, if -1 random (system time). + public boolean sparseIntermediate = false; // whether SparseRowIntermediates should be used for rowwise operations public final static DMLOptions defaultOptions = new DMLOptions(null); @@ -117,7 +118,8 @@ public String toString() { ", w=" + fedWorker + ", federatedCompilation=" + federatedCompilation + ", noFedRuntimeConversion=" + noFedRuntimeConversion + - ", seed=" + seed + + ", seed=" + seed + + ", sparseIntermediate=" + sparseIntermediate + '}'; } @@ -350,6 +352,10 @@ else if (lineageType.equalsIgnoreCase("debugger")) dmlOptions.seed = Integer.parseInt(line.getOptionValue("seed")); } + if(line.hasOption("sparseIntermediate")){ + dmlOptions.sparseIntermediate = true; + } + return dmlOptions; } @@ -431,7 +437,10 @@ private static Options createCLIOptions() { Option commandlineSeed = OptionBuilder .withDescription("A general seed for the execution through the commandline") .hasArg().create("seed"); - + Option sparseRowIntermediates = OptionBuilder + .withDescription("If activated, sparseRowVector intermediates will be used to calculate rowwise operations.") + .create("sparseIntermediate"); + options.addOption(configOpt); options.addOption(cleanOpt); options.addOption(statsOpt); @@ -451,6 +460,7 @@ private static Options createCLIOptions() { options.addOption(federatedCompilation); options.addOption(noFedRuntimeConversion); options.addOption(commandlineSeed); + options.addOption(sparseRowIntermediates); // Either a clean(-clean), a file(-f), a script(-s) or help(-help) needs to be specified OptionGroup fileOrScriptOpt = new OptionGroup() diff --git a/src/main/java/org/apache/sysds/api/DMLScript.java b/src/main/java/org/apache/sysds/api/DMLScript.java index d6853891e24..44dc35ee75e 100644 --- a/src/main/java/org/apache/sysds/api/DMLScript.java +++ b/src/main/java/org/apache/sysds/api/DMLScript.java @@ -153,6 +153,9 @@ public class DMLScript // Global seed public static int SEED = -1; + // Sparse row flag + public static boolean SPARSE_INTERMEDIATE = false; + public static String MONITORING_ADDRESS = null; // flag that indicates whether or not to suppress any prints to stdout @@ -275,6 +278,7 @@ public static boolean executeScript( String[] args ) LINEAGE_ESTIMATE = dmlOptions.lineage_estimate; LINEAGE_DEBUGGER = dmlOptions.lineage_debugger; SEED = dmlOptions.seed; + SPARSE_INTERMEDIATE = dmlOptions.sparseIntermediate; String fnameOptConfig = dmlOptions.configFile; diff --git a/src/main/java/org/apache/sysds/hops/codegen/cplan/CNode.java b/src/main/java/org/apache/sysds/hops/codegen/cplan/CNode.java index 36cc8f49799..f5998b73bf5 100644 --- a/src/main/java/org/apache/sysds/hops/codegen/cplan/CNode.java +++ b/src/main/java/org/apache/sysds/hops/codegen/cplan/CNode.java @@ -19,6 +19,7 @@ package org.apache.sysds.hops.codegen.cplan; +import org.apache.sysds.api.DMLScript; import org.apache.sysds.common.Types.DataType; import org.apache.sysds.hops.codegen.SpoofCompiler.GeneratorAPI; import org.apache.sysds.hops.codegen.template.TemplateUtils; @@ -77,6 +78,14 @@ public String createVarname() { _genVar = "TMP"+_seqVar.getNextID(); return _genVar; } + + public String createVarname(boolean sparse) { + if(!sparse) { + return createVarname(); + } else { + return _genVar = "S" + createVarname(); + } + } public String getVarname() { return _genVar; @@ -103,6 +112,14 @@ else if(_dataType == DataType.MATRIX) } return ""; } + + public String getVectorLength(GeneratorAPI api, boolean sparse) { + if(sparse && getVarname().startsWith("STMP")) { + return "len"; + } else { + return getVectorLength(api); + } + } public String getClassname() { return getVarname(); @@ -222,8 +239,13 @@ public boolean equals(Object that) { protected String replaceUnaryPlaceholders(String tmp, String varj, boolean vectIn, GeneratorAPI api) { //replace sparse and dense inputs - tmp = tmp.replace("%IN1v%", varj+"vals"); - tmp = tmp.replace("%IN1i%", varj+"ix"); + if(DMLScript.SPARSE_INTERMEDIATE) { + tmp = tmp.replace("%IN1v%", varj.startsWith("STMP") ? varj+".values()" : varj+"vals"); + tmp = tmp.replace("%IN1i%", varj.startsWith("STMP") ? varj+".indexes()" :varj+"ix"); + } else { + tmp = tmp.replace("%IN1v%", varj+"vals"); + tmp = tmp.replace("%IN1i%", varj+"ix"); + } tmp = tmp.replace("%IN1%", (vectIn && TemplateUtils.isMatrix(_inputs.get(0))) ? ((api == GeneratorAPI.JAVA) ? varj + ".values(rix)" : varj + ".vals(0)" ) : diff --git a/src/main/java/org/apache/sysds/hops/codegen/cplan/CNodeBinary.java b/src/main/java/org/apache/sysds/hops/codegen/cplan/CNodeBinary.java index b29d586c38a..09e7f700ef9 100644 --- a/src/main/java/org/apache/sysds/hops/codegen/cplan/CNodeBinary.java +++ b/src/main/java/org/apache/sysds/hops/codegen/cplan/CNodeBinary.java @@ -22,6 +22,7 @@ import java.util.Arrays; import org.apache.commons.lang3.StringUtils; +import org.apache.sysds.api.DMLScript; import org.apache.sysds.common.Opcodes; import org.apache.sysds.hops.codegen.template.TemplateUtils; import org.apache.sysds.common.Types.DataType; @@ -157,61 +158,125 @@ public String codegen(boolean sparse, GeneratorAPI api) { //generate children sb.append(_inputs.get(0).codegen(sparse, api)); sb.append(_inputs.get(1).codegen(sparse, api)); - - //generate binary operation (use sparse template, if data input) - boolean lsparseLhs = sparse && _inputs.get(0) instanceof CNodeData - && _inputs.get(0).getVarname().startsWith("a"); - boolean lsparseRhs = sparse && _inputs.get(1) instanceof CNodeData - && _inputs.get(1).getVarname().startsWith("a"); - boolean scalarInput = _inputs.get(0).getDataType().isScalar(); - boolean scalarVector = (_inputs.get(0).getDataType().isScalar() - && _inputs.get(1).getDataType().isMatrix()); - boolean vectorVector = _inputs.get(0).getDataType().isMatrix() - && _inputs.get(1).getDataType().isMatrix(); - String var = createVarname(); - String tmp = getLanguageTemplateClass(this, api) - .getTemplate(_type, lsparseLhs, lsparseRhs, scalarVector, scalarInput, vectorVector); - tmp = tmp.replace("%TMP%", var); - - //replace input references and start indexes - for( int j=0; j<2; j++ ) { - String varj = _inputs.get(j).getVarname(api); - - //replace sparse and dense inputs - tmp = tmp.replace("%IN"+(j+1)+"v%", varj+"vals"); - tmp = tmp.replace("%IN"+(j+1)+"i%", varj+"ix"); - tmp = tmp.replace("%IN"+(j+1)+"%", - varj.startsWith("a") ? (api == GeneratorAPI.JAVA ? varj : + /** + * todo: remember that only certain primitives will be called through this method, + * because the optimizer will choose which primitive functions should be calculated sparse and which not + */ + if(DMLScript.SPARSE_INTERMEDIATE) { + //generate binary operation (use sparse template, if data input) + boolean lsparseLhs = sparse ? _inputs.get(0) instanceof CNodeData + && _inputs.get(0).getVarname().startsWith("a") || + (_inputs.get(0).getVarname().startsWith("STMP")) + && _inputs.get(0).getDataType().isMatrix() : false; + boolean lsparseRhs = sparse ? _inputs.get(1) instanceof CNodeData + && _inputs.get(1).getVarname().startsWith("a") || + (_inputs.get(1).getVarname().startsWith("STMP")) + && _inputs.get(1).getDataType().isMatrix() : false; + boolean scalarInput = _inputs.get(0).getDataType().isScalar(); + boolean scalarVector = (_inputs.get(0).getDataType().isScalar() + && _inputs.get(1).getDataType().isMatrix()); + boolean vectorVector = _inputs.get(0).getDataType().isMatrix() + && _inputs.get(1).getDataType().isMatrix(); + String var = createVarname(sparse && getOutputType(scalarVector, lsparseLhs, lsparseRhs)); + String tmp = getLanguageTemplateClass(this, api) + .getTemplate(_type, lsparseLhs, lsparseRhs, scalarVector, scalarInput, vectorVector); + + tmp = tmp.replace("%TMP%", var); + + //replace input references and start indexes + for( int j=0; j<2; j++ ) { + String varj = _inputs.get(j).getVarname(api); + //replace sparse and dense inputs + tmp = tmp.replace("%IN"+(j+1)+"v%", varj.startsWith("STMP") ? varj+".values()" : varj+"vals"); + tmp = tmp.replace("%IN"+(j+1)+"i%", varj.startsWith("STMP") ? varj+".indexes()" : varj+"ix"); + tmp = tmp.replace("%IN"+(j+1)+"%", + varj.startsWith("a") ? (api == GeneratorAPI.JAVA ? varj : (_inputs.get(j).getDataType() == DataType.MATRIX ? varj + ".vals(0)" : varj)) : - varj.startsWith("b") ? (api == GeneratorAPI.JAVA ? varj + ".values(rix)" : - (_type == BinType.VECT_MATRIXMULT ? varj : varj + ".vals(0)")) : + varj.startsWith("b") ? (api == GeneratorAPI.JAVA ? varj + ".values(rix)" : + (_type == BinType.VECT_MATRIXMULT ? varj : varj + ".vals(0)")) : _inputs.get(j).getDataType() == DataType.MATRIX ? (api == GeneratorAPI.JAVA ? varj : varj + ".vals(0)") : varj); - - //replace start position of main input - tmp = tmp.replace("%POS"+(j+1)+"%", (_inputs.get(j) instanceof CNodeData - && _inputs.get(j).getDataType().isMatrix()) ? (!varj.startsWith("b")) ? varj+"i" : + + tmp = tmp.replace("%LEN"+(j+1)+"%", varj.startsWith("STMP") ? varj+".size()" : j == 0 ? "alen" : "blen"); + + //replace start position of main input + tmp = tmp.replace("%POS"+(j+1)+"%", (_inputs.get(j) instanceof CNodeData + && _inputs.get(j).getDataType().isMatrix()) ? (!varj.startsWith("b")) ? varj+"i" : ((TemplateUtils.isMatrix(_inputs.get(j)) || (_type.isElementwise() && TemplateUtils.isColVector(_inputs.get(j)))) && _type!=BinType.VECT_MATRIXMULT) ? - varj + ".pos(rix)" : "0" : "0"); - } - //replace length information (e.g., after matrix mult) - if( _type == BinType.VECT_OUTERMULT_ADD || (_type == BinType.VECT_CBIND && vectorVector) ) { - for( int j=0; j<2; j++ ) - tmp = tmp.replace("%LEN"+(j+1)+"%", _inputs.get(j).getVectorLength(api)); - } - else { //general case - CNode mInput = getIntermediateInputVector(); - if( mInput != null ) - tmp = tmp.replace("%LEN%", mInput.getVectorLength(api)); + varj + ".pos(rix)" : "0" : "0"); + } + //todo: the following if else block could be simplified, because the first condition won't be true + //replace length information (e.g., after matrix mult) + if( _type == BinType.VECT_OUTERMULT_ADD || (_type == BinType.VECT_CBIND && vectorVector)) { + for( int j=0; j<2; j++ ) + tmp = tmp.replace("%LEN"+(j+1)+"%", _inputs.get(j).getVectorLength(api)); + } + else { //general case + CNode mInput = getIntermediateInputVector(); + if( mInput != null ) + tmp = tmp.replace("%LEN%", mInput.getVectorLength(api, sparse)); + } + + sb.append(tmp); + + //mark as generated + _generated = true; + + return sb.toString(); + } else { + boolean lsparseLhs = + sparse && _inputs.get(0) instanceof CNodeData && _inputs.get(0).getVarname().startsWith("a"); + boolean lsparseRhs = + sparse && _inputs.get(1) instanceof CNodeData && _inputs.get(1).getVarname().startsWith("a"); + boolean scalarInput = _inputs.get(0).getDataType().isScalar(); + boolean scalarVector = (_inputs.get(0).getDataType().isScalar() && _inputs.get(1).getDataType().isMatrix()); + boolean vectorVector = _inputs.get(0).getDataType().isMatrix() && _inputs.get(1).getDataType().isMatrix(); + String var = createVarname(); + String tmp = getLanguageTemplateClass(this, api).getTemplate(_type, lsparseLhs, lsparseRhs, scalarVector, + scalarInput, vectorVector); + + tmp = tmp.replace("%TMP%", var); + + //replace input references and start indexes + for(int j = 0; j < 2; j++) { + String varj = _inputs.get(j).getVarname(api); + + //replace sparse and dense inputs + tmp = tmp.replace("%IN" + (j + 1) + "v%", varj + "vals"); + tmp = tmp.replace("%IN" + (j + 1) + "i%", varj + "ix"); + tmp = tmp.replace("%IN" + (j + 1) + "%", varj.startsWith("a") ? ( + api == GeneratorAPI.JAVA ? varj : (_inputs.get(j).getDataType() == DataType.MATRIX ? varj + + ".vals(0)" : varj)) : varj.startsWith("b") ? (api == GeneratorAPI.JAVA ? varj + + ".values(rix)" : (_type == BinType.VECT_MATRIXMULT ? varj : varj + ".vals(0)")) : + _inputs.get(j).getDataType() == DataType.MATRIX ? ( + api == GeneratorAPI.JAVA ? varj : varj + ".vals(0)") : varj); + + //replace start position of main input + tmp = tmp.replace("%POS" + (j + 1) + "%", (_inputs.get(j) instanceof CNodeData && + _inputs.get(j).getDataType().isMatrix()) ? (!varj.startsWith("b")) ? varj + "i" : ( + (TemplateUtils.isMatrix(_inputs.get(j)) || + (_type.isElementwise() && TemplateUtils.isColVector(_inputs.get(j)))) && + _type != BinType.VECT_MATRIXMULT) ? varj + ".pos(rix)" : "0" : "0"); + } + //replace length information (e.g., after matrix mult) + if(_type == BinType.VECT_OUTERMULT_ADD || (_type == BinType.VECT_CBIND && vectorVector)) { + for(int j = 0; j < 2; j++) + tmp = tmp.replace("%LEN" + (j + 1) + "%", _inputs.get(j).getVectorLength(api)); + } + else { //general case + CNode mInput = getIntermediateInputVector(); + if(mInput != null) + tmp = tmp.replace("%LEN%", mInput.getVectorLength(api)); + } + + sb.append(tmp); + + //mark as generated + _generated = true; + + return sb.toString(); } - - sb.append(tmp); - - //mark as generated - _generated = true; - - return sb.toString(); } private CNode getIntermediateInputVector() { @@ -219,7 +284,39 @@ private CNode getIntermediateInputVector() { if( getInput().get(i).getDataType().isMatrix() ) return getInput().get(i); return null; - } + } + + public boolean getOutputType(boolean scalarVector, boolean lsparseLhs, boolean lsparseRhs) { + switch(_type) { + case VECT_MULT_SCALAR: + case VECT_DIV_SCALAR: + case VECT_POW_SCALAR: return !scalarVector && lsparseLhs; + case VECT_XOR_SCALAR: + case VECT_MIN_SCALAR: + case VECT_MAX_SCALAR: + case VECT_EQUAL_SCALAR: + case VECT_NOTEQUAL_SCALAR: + case VECT_LESS_SCALAR: + case VECT_LESSEQUAL_SCALAR: + case VECT_GREATER_SCALAR: + case VECT_GREATEREQUAL_SCALAR: + case VECT_BITWAND_SCALAR: return lsparseLhs || lsparseRhs; + case VECT_MULT: + case VECT_DIV: + case VECT_MINUS: + case VECT_PLUS: + case VECT_XOR: + case VECT_BITWAND: + case VECT_BIASADD: + case VECT_BIASMULT: + case VECT_MIN: + case VECT_MAX: + case VECT_NOTEQUAL: + case VECT_LESS: + case VECT_GREATER: return lsparseLhs && lsparseRhs; + default: return false; + } + } @Override public String toString() { diff --git a/src/main/java/org/apache/sysds/hops/codegen/cplan/CNodeRow.java b/src/main/java/org/apache/sysds/hops/codegen/cplan/CNodeRow.java index 77dec97cbe1..93e3585c378 100644 --- a/src/main/java/org/apache/sysds/hops/codegen/cplan/CNodeRow.java +++ b/src/main/java/org/apache/sysds/hops/codegen/cplan/CNodeRow.java @@ -37,6 +37,7 @@ public class CNodeRow extends CNodeTpl + "import org.apache.sysds.runtime.codegen.SpoofOperator.SideInput;\n" + "import org.apache.sysds.runtime.codegen.SpoofRowwise;\n" + "import org.apache.sysds.runtime.codegen.SpoofRowwise.RowType;\n" + + "import org.apache.sysds.runtime.data.SparseRowVector;\n" + "import org.apache.commons.math3.util.FastMath;\n" + "\n" + "public final class %TMP% extends SpoofRowwise { \n" @@ -162,7 +163,8 @@ private String getOutputStatement(String varName) { case NO_AGG_B1: case NO_AGG_CONST: if(api == GeneratorAPI.JAVA) - return TEMPLATE_NOAGG_OUT.replace("%IN%", varName).replace("%LEN%", _output.getVarname()+".length"); + return TEMPLATE_NOAGG_OUT.replace("%IN%", varName).replace("%LEN%", + _output.getVarname().startsWith("STMP") ? _output.getVarname()+".size()" : _output.getVarname()+".length"); else return TEMPLATE_NOAGG_CONST_OUT_CUDA.replace("%IN%", varName + ".vals(0)").replaceAll("%LEN%", _output.getVarname()+".length"); case FULL_AGG: diff --git a/src/main/java/org/apache/sysds/hops/codegen/cplan/CNodeUnary.java b/src/main/java/org/apache/sysds/hops/codegen/cplan/CNodeUnary.java index fe67995b6b5..f8d5c3eeb4c 100644 --- a/src/main/java/org/apache/sysds/hops/codegen/cplan/CNodeUnary.java +++ b/src/main/java/org/apache/sysds/hops/codegen/cplan/CNodeUnary.java @@ -23,6 +23,7 @@ import org.apache.commons.lang3.ArrayUtils; import org.apache.commons.lang3.StringUtils; +import org.apache.sysds.api.DMLScript; import org.apache.sysds.common.Opcodes; import org.apache.sysds.common.Types.DataType; import org.apache.sysds.runtime.util.UtilFunctions; @@ -111,10 +112,12 @@ public String codegen(boolean sparse, GeneratorAPI api) { sb.append(_inputs.get(0).codegen(sparse, api)); //generate unary operation - boolean lsparse = sparse && (_inputs.get(0) instanceof CNodeData - && _inputs.get(0).getVarname().startsWith("a") - && !_inputs.get(0).isLiteral()); - String var = createVarname(); + boolean lsparse = sparse && + ((_inputs.get(0) instanceof CNodeData + && _inputs.get(0).getVarname().startsWith("a") + && !_inputs.get(0).isLiteral()) + || (DMLScript.SPARSE_INTERMEDIATE && _inputs.get(0).getVarname().startsWith("STMP"))); + String var = createVarname(DMLScript.SPARSE_INTERMEDIATE && lsparse && getOutputType()); String tmp = getLanguageTemplateClass(this, api).getTemplate(_type, lsparse); tmp = tmp.replaceAll("%TMP%", var); @@ -130,6 +133,24 @@ public String codegen(boolean sparse, GeneratorAPI api) { return sb.toString(); } + + public boolean getOutputType() { + switch(_type) { + case VECT_SQRT: + case VECT_ABS: + case VECT_ROUND: + case VECT_CEIL: + case VECT_FLOOR: + case VECT_SIN: + case VECT_TAN: + case VECT_ASIN: + case VECT_ATAN: + case VECT_SINH: + case VECT_TANH: + case VECT_SIGN: return true; + default: return false; + } + } @Override public String toString() { diff --git a/src/main/java/org/apache/sysds/hops/codegen/cplan/java/Binary.java b/src/main/java/org/apache/sysds/hops/codegen/cplan/java/Binary.java index 40496249e52..a5b5766cf37 100644 --- a/src/main/java/org/apache/sysds/hops/codegen/cplan/java/Binary.java +++ b/src/main/java/org/apache/sysds/hops/codegen/cplan/java/Binary.java @@ -19,6 +19,7 @@ package org.apache.sysds.hops.codegen.cplan.java; +import org.apache.sysds.api.DMLScript; import org.apache.sysds.hops.codegen.cplan.CNodeBinary.BinType; import org.apache.sysds.hops.codegen.cplan.CodeTemplate; @@ -70,11 +71,20 @@ public String getTemplate(BinType type, boolean sparseLhs, boolean sparseRhs, //vector-scalar operations case VECT_MULT_SCALAR: case VECT_DIV_SCALAR: - case VECT_MINUS_SCALAR: - case VECT_PLUS_SCALAR: - case VECT_POW_SCALAR: + case VECT_POW_SCALAR: { + String vectName = type.getVectorPrimitiveName(); + if( scalarVector ) + return sparseRhs ? " double[] %TMP% = LibSpoofPrimitives.vect"+vectName+"Write(%IN1%, %IN2v%, %IN2i%, %POS2%, alen, %LEN%);\n" : + " double[] %TMP% = LibSpoofPrimitives.vect"+vectName+"Write(%IN1%, %IN2%, %POS2%, %LEN%);\n"; + else if(DMLScript.SPARSE_INTERMEDIATE) { + return sparseLhs ? " SparseRowVector %TMP% = LibSpoofPrimitives.vect"+vectName+"Write(%LEN%, %IN1v%, %IN2%, %IN1i%, %POS1%, %LEN1%);\n" : + " double[] %TMP% = LibSpoofPrimitives.vect"+vectName+"Write(%IN1%, %IN2%, %POS1%, %LEN%);\n"; + } else { + return sparseLhs ? " double[] %TMP% = LibSpoofPrimitives.vect"+vectName+"Write(%IN1v%, %IN2%, %IN1i%, %POS1%, alen, %LEN%);\n" : + " double[] %TMP% = LibSpoofPrimitives.vect"+vectName+"Write(%IN1%, %IN2%, %POS1%, %LEN%);\n"; + } + } case VECT_XOR_SCALAR: - case VECT_BITWAND_SCALAR: case VECT_MIN_SCALAR: case VECT_MAX_SCALAR: case VECT_EQUAL_SCALAR: @@ -82,7 +92,22 @@ public String getTemplate(BinType type, boolean sparseLhs, boolean sparseRhs, case VECT_LESS_SCALAR: case VECT_LESSEQUAL_SCALAR: case VECT_GREATER_SCALAR: - case VECT_GREATEREQUAL_SCALAR: { + case VECT_GREATEREQUAL_SCALAR: + case VECT_BITWAND_SCALAR: { + String vectName = type.getVectorPrimitiveName(); + if(scalarVector) { + if(sparseRhs) + return DMLScript.SPARSE_INTERMEDIATE ? " SparseRowVector %TMP% = LibSpoofPrimitives.vect"+vectName+"Write(%LEN%, %IN1%, %IN2v%, %IN2i%, %POS2%, %LEN1%);\n" : + " double[] %TMP% = LibSpoofPrimitives.vect"+vectName+"Write(%IN1%, %IN2v%, %IN2i%, %POS2%, alen, %LEN%);\n"; + } else { + if(sparseLhs) + return DMLScript.SPARSE_INTERMEDIATE ? " SparseRowVector %TMP% = LibSpoofPrimitives.vect"+vectName+"Write(%LEN%, %IN1v%, %IN2%, %IN1i%, %POS1%, %LEN1%);\n" : + " double[] %TMP% = LibSpoofPrimitives.vect"+vectName+"Write(%IN1v%, %IN2%, %IN1i%, %POS1%, alen, %LEN%);\n"; + } + return " double[] %TMP% = LibSpoofPrimitives.vect"+vectName+"Write(%IN1%, %IN2%, %POS1%, %LEN%);\n"; + } + case VECT_MINUS_SCALAR: + case VECT_PLUS_SCALAR: { String vectName = type.getVectorPrimitiveName(); if( scalarVector ) return sparseRhs ? " double[] %TMP% = LibSpoofPrimitives.vect"+vectName+"Write(%IN1%, %IN2v%, %IN2i%, %POS2%, alen, %LEN%);\n" : @@ -115,20 +140,34 @@ else if( !vectorVector ) case VECT_BIASMULT: case VECT_MIN: case VECT_MAX: - case VECT_EQUAL: case VECT_NOTEQUAL: case VECT_LESS: + case VECT_GREATER:{ + String vectName = type.getVectorPrimitiveName(); + if(DMLScript.SPARSE_INTERMEDIATE && sparseLhs && sparseRhs) { + return " SparseRowVector %TMP% = LibSpoofPrimitives.vect"+vectName+"Write(%LEN%, %IN1v%, %IN2v%, %IN1i%, %IN2i%, %POS1%, %POS2%, %LEN1%, %LEN2%);\n"; + } else { + return sparseLhs ? + " double[] %TMP% = LibSpoofPrimitives.vect"+vectName+"Write(%IN1v%, %IN2%, %IN1i%, %POS1%, %POS2%, alen, %LEN%);\n" : + sparseRhs ? + " double[] %TMP% = LibSpoofPrimitives.vect"+vectName+"Write(%IN1%, %IN2v%, %POS1%, %IN2i%, %POS2%, alen, %LEN%);\n" : + " double[] %TMP% = LibSpoofPrimitives.vect"+vectName+"Write(%IN1%, %IN2%, %POS1%, %POS2%, %LEN%);\n"; + } + } + case VECT_EQUAL: case VECT_LESSEQUAL: - case VECT_GREATER: case VECT_GREATEREQUAL: { String vectName = type.getVectorPrimitiveName(); - return sparseLhs ? + if(DMLScript.SPARSE_INTERMEDIATE && sparseLhs && sparseRhs) { + return " double[] %TMP% = LibSpoofPrimitives.vect"+vectName+"Write(%LEN%, %IN1v%, %IN2v%, %IN1i%, %IN2i%, %POS1%, %POS2%, %LEN1%, %LEN2%);\n"; + } else { + return sparseLhs ? " double[] %TMP% = LibSpoofPrimitives.vect"+vectName+"Write(%IN1v%, %IN2%, %IN1i%, %POS1%, %POS2%, alen, %LEN%);\n" : sparseRhs ? - " double[] %TMP% = LibSpoofPrimitives.vect"+vectName+"Write(%IN1%, %IN2v%, %POS1%, %IN2i%, %POS2%, alen, %LEN%);\n" : - " double[] %TMP% = LibSpoofPrimitives.vect"+vectName+"Write(%IN1%, %IN2%, %POS1%, %POS2%, %LEN%);\n"; + " double[] %TMP% = LibSpoofPrimitives.vect"+vectName+"Write(%IN1%, %IN2v%, %POS1%, %IN2i%, %POS2%, alen, %LEN%);\n" : + " double[] %TMP% = LibSpoofPrimitives.vect"+vectName+"Write(%IN1%, %IN2%, %POS1%, %POS2%, %LEN%);\n"; + } } - //scalar-scalar operations case MULT: return " double %TMP% = %IN1% * %IN2%;\n"; diff --git a/src/main/java/org/apache/sysds/hops/codegen/cplan/java/Unary.java b/src/main/java/org/apache/sysds/hops/codegen/cplan/java/Unary.java index d8a1085df58..a1d21228982 100644 --- a/src/main/java/org/apache/sysds/hops/codegen/cplan/java/Unary.java +++ b/src/main/java/org/apache/sysds/hops/codegen/cplan/java/Unary.java @@ -20,6 +20,7 @@ package org.apache.sysds.hops.codegen.cplan.java; import org.apache.commons.lang3.StringUtils; +import org.apache.sysds.api.DMLScript; import org.apache.sysds.hops.codegen.cplan.CNodeUnary.UnaryType; import org.apache.sysds.hops.codegen.cplan.CodeTemplate; @@ -32,31 +33,37 @@ public String getTemplate(UnaryType type, boolean sparse) { case ROW_MINS: case ROW_MAXS: case ROW_MEANS: - case ROW_VARS: case ROW_COUNTNNZS: { String vectName = StringUtils.capitalize(type.name().substring(4, type.name().length()-1).toLowerCase()); return sparse ? " double %TMP% = LibSpoofPrimitives.vect"+vectName+"(%IN1v%, %IN1i%, %POS1%, alen, len);\n": " double %TMP% = LibSpoofPrimitives.vect"+vectName+"(%IN1%, %POS1%, %LEN%);\n"; } - case VECT_EXP: - case VECT_POW2: - case VECT_MULT2: + case VECT_SQRT: - case VECT_LOG: case VECT_ABS: case VECT_ROUND: case VECT_CEIL: case VECT_FLOOR: - case VECT_SIGN: case VECT_SIN: - case VECT_COS: case VECT_TAN: case VECT_ASIN: - case VECT_ACOS: case VECT_ATAN: case VECT_SINH: - case VECT_COSH: case VECT_TANH: + case VECT_SIGN:{ + String vectName = type.getVectorPrimitiveName(); + return sparse ? DMLScript.SPARSE_INTERMEDIATE ? + " SparseRowVector %TMP% = LibSpoofPrimitives.vect"+vectName+"Write(len, %IN1v%, %IN1i%, %POS1%, alen);\n" : + " double[] %TMP% = LibSpoofPrimitives.vect"+vectName+"Write(%IN1%, %POS1%, %LEN%);\n" : + " double[] %TMP% = LibSpoofPrimitives.vect"+vectName+"Write(%IN1%, %POS1%, %LEN%);\n"; + } + case VECT_EXP: + case VECT_POW2: + case VECT_MULT2: + case VECT_LOG: + case VECT_COS: + case VECT_ACOS: + case VECT_COSH: case VECT_CUMSUM: case VECT_CUMMIN: case VECT_CUMMAX: From 7ebfd5ff8b7053d8223f5e0730577fd84e3c5891 Mon Sep 17 00:00:00 2001 From: Frxms Date: Tue, 3 Jun 2025 15:39:08 +0200 Subject: [PATCH 03/60] implement DIV for scalarVector --- .../apache/sysds/runtime/codegen/LibSpoofPrimitives.java | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/main/java/org/apache/sysds/runtime/codegen/LibSpoofPrimitives.java b/src/main/java/org/apache/sysds/runtime/codegen/LibSpoofPrimitives.java index baccf53dc7f..f2f367c98c3 100644 --- a/src/main/java/org/apache/sysds/runtime/codegen/LibSpoofPrimitives.java +++ b/src/main/java/org/apache/sysds/runtime/codegen/LibSpoofPrimitives.java @@ -2218,6 +2218,13 @@ public static SparseRowVector vectDivWrite(int len, double[] a, double bval, int // } } + public static SparseRowVector vectDivWrite(int len, double bval, double[] a, int[] aix, int ai, int alen) { + SparseRowVector c = allocSparseVector(alen); + for(int j = 0; j < ai+alen; j++) + c.set(aix[j], bval / a[j]); + return c; + } + public static SparseRowVector vectDivWrite(int len, double[] a, double[] b, int[] aix, int[] bix, int ai, int bi, int alen, int blen) { SparseRowVector c = allocSparseVector(alen); if(alen != 0 && blen != 0) { From 1a7570465e408ccf456e6c2a0d344152a72945e8 Mon Sep 17 00:00:00 2001 From: Frxms Date: Tue, 3 Jun 2025 15:42:03 +0200 Subject: [PATCH 04/60] adjust templates, so that DIV for scalarVector works --- .../java/org/apache/sysds/hops/codegen/cplan/CNodeBinary.java | 2 +- .../java/org/apache/sysds/hops/codegen/cplan/java/Binary.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/apache/sysds/hops/codegen/cplan/CNodeBinary.java b/src/main/java/org/apache/sysds/hops/codegen/cplan/CNodeBinary.java index 09e7f700ef9..e2bc0f47e9b 100644 --- a/src/main/java/org/apache/sysds/hops/codegen/cplan/CNodeBinary.java +++ b/src/main/java/org/apache/sysds/hops/codegen/cplan/CNodeBinary.java @@ -289,8 +289,8 @@ private CNode getIntermediateInputVector() { public boolean getOutputType(boolean scalarVector, boolean lsparseLhs, boolean lsparseRhs) { switch(_type) { case VECT_MULT_SCALAR: - case VECT_DIV_SCALAR: case VECT_POW_SCALAR: return !scalarVector && lsparseLhs; + case VECT_DIV_SCALAR: case VECT_XOR_SCALAR: case VECT_MIN_SCALAR: case VECT_MAX_SCALAR: diff --git a/src/main/java/org/apache/sysds/hops/codegen/cplan/java/Binary.java b/src/main/java/org/apache/sysds/hops/codegen/cplan/java/Binary.java index a5b5766cf37..81b2517fe1f 100644 --- a/src/main/java/org/apache/sysds/hops/codegen/cplan/java/Binary.java +++ b/src/main/java/org/apache/sysds/hops/codegen/cplan/java/Binary.java @@ -70,7 +70,6 @@ public String getTemplate(BinType type, boolean sparseLhs, boolean sparseRhs, //vector-scalar operations case VECT_MULT_SCALAR: - case VECT_DIV_SCALAR: case VECT_POW_SCALAR: { String vectName = type.getVectorPrimitiveName(); if( scalarVector ) @@ -84,6 +83,7 @@ else if(DMLScript.SPARSE_INTERMEDIATE) { " double[] %TMP% = LibSpoofPrimitives.vect"+vectName+"Write(%IN1%, %IN2%, %POS1%, %LEN%);\n"; } } + case VECT_DIV_SCALAR: case VECT_XOR_SCALAR: case VECT_MIN_SCALAR: case VECT_MAX_SCALAR: From f193ecb230753d5acfc7de872a388540d43461cf Mon Sep 17 00:00:00 2001 From: Frxms Date: Wed, 4 Jun 2025 18:05:54 +0200 Subject: [PATCH 05/60] fix sparse vector allocation --- .../runtime/codegen/LibSpoofPrimitives.java | 123 +++++++++--------- .../codegen/CPlanVectorPrimitivesTest.java | 20 ++- .../codegen/SparseVectorAllocTest.java | 91 ++++--------- 3 files changed, 101 insertions(+), 133 deletions(-) diff --git a/src/main/java/org/apache/sysds/runtime/codegen/LibSpoofPrimitives.java b/src/main/java/org/apache/sysds/runtime/codegen/LibSpoofPrimitives.java index f2f367c98c3..40c138a1dd5 100644 --- a/src/main/java/org/apache/sysds/runtime/codegen/LibSpoofPrimitives.java +++ b/src/main/java/org/apache/sysds/runtime/codegen/LibSpoofPrimitives.java @@ -2179,18 +2179,29 @@ public static double vectVar(double[] avals, int[] aix, int ai, int alen, int le */ public static SparseRowVector vectMultWrite(int len, double[] a, double bval, int[] aix, int ai, int alen) { - if( a == null ) return allocSparseVector(alen); - SparseRowVector c = allocSparseVector(alen, aix); - LibMatrixMult.vectMultiplyWrite(bval, a, c.values(), ai, 0, alen); + SparseRowVector c = allocSparseVector(alen); + if( a == null ) return c; + for(int j = 0; j < ai+alen; j++) + c.set(aix[j], a[j]*bval); return c; } public static SparseRowVector vectMultWrite(int len, double[] a, double[] b, int[] aix, int[] bix, int ai, int bi, int alen, int blen) { SparseRowVector c = allocSparseVector(alen); if( a == null || b == null ) return c; - SparseRowVector d = allocSparseVector(blen, b, bix); - for( int j = ai; j < ai+alen; j++ ) { - c.set(aix[j], a[j] * d.get(bi+aix[j])); + int aIndex = ai; + int bIndex = bi; + while(aIndex < ai+alen && bIndex < bi+blen) { + int aIdx = aix[aIndex]; + int bIdx = bix[bIndex]; + if(aIdx == bIdx) { + c.set(aIdx, a[aIndex] * b[bIndex]); + aIndex++; + bIndex++; + } else if(aIdx < bIdx) + aIndex++; + else + bIndex++; } return c; } @@ -2226,49 +2237,69 @@ public static SparseRowVector vectDivWrite(int len, double bval, double[] a, int } public static SparseRowVector vectDivWrite(int len, double[] a, double[] b, int[] aix, int[] bix, int ai, int bi, int alen, int blen) { +// SparseRowVector c = allocSparseVector(alen); +// if(alen != 0 && blen != 0) { +// //decide between two versions. First is smaller for smaller alen and the other for smaller blen +// if(alen < blen) { +// SparseRowVector bSparse = allocSparseVector(blen, b, bix); +// for (int j = ai; j < ai+alen; j++) { +// c.set(aix[j], a[j] / bSparse.get(aix[j])); +// } +// } else { +// SparseRowVector aSparse = allocSparseVector(alen, a, aix); +// for (int j = ai; j < ai+alen; j++) { +// double aval = a[j]; +// c.set(aix[j], (aval>0) ? +// Double.POSITIVE_INFINITY : Double.NEGATIVE_INFINITY); +// } +// for (int j = bi; j < bi+blen; j++) { +// c.set(bix[j], aSparse.get(bix[j]) / b[j]); +// } +// } +// } SparseRowVector c = allocSparseVector(alen); - if(alen != 0 && blen != 0) { - //decide between two versions. First is smaller for smaller alen and the other for smaller blen - if(alen < blen) { - SparseRowVector bSparse = allocSparseVector(blen, b, bix); - for (int j = ai; j < ai+alen; j++) { - c.set(aix[j], a[j] / bSparse.get(aix[j])); - } + int aIndex = ai; + int bIndex = bi; + while(aIndex < ai+alen && bIndex < bi+blen) { + int aIdx = aix[aIndex]; + int bIdx = bix[bIndex]; + if(aIdx == bIdx) { + c.set(aIdx, a[aIndex] / b[bIndex]); + aIndex++; + bIndex++; + } else if(aIdx < bIdx) { + c.set(aIdx, (a[aIndex]>0) ? Double.POSITIVE_INFINITY : Double.NEGATIVE_INFINITY); + aIndex++; } else { - SparseRowVector aSparse = allocSparseVector(alen, a, aix); - for (int j = ai; j < ai+alen; j++) { - double aval = a[j]; - c.set(aix[j], (aval>0) ? - Double.POSITIVE_INFINITY : Double.NEGATIVE_INFINITY); - } - for (int j = bi; j < bi+blen; j++) { - c.set(bix[j], aSparse.get(bix[j]) / b[j]); - } + bIndex++; } } return c; } public static SparseRowVector vectMinusWrite(int len, double[] a, double[] b, int[] aix, int[] bix, int ai, int bi, int alen, int blen) { - SparseRowVector c = allocSparseVector(alen, a, aix); - for (int j = bi; j < bi+blen; j++) { + SparseRowVector c = allocSparseVector(alen); + for(int j = 0; j < ai+alen; j++) + c.set(aix[j], a[j]); + for (int j = bi; j < bi+blen; j++) c.add(bix[j], -b[j]); - } return c; } public static SparseRowVector vectPlusWrite(int len, double[] a, double[] b, int[] aix, int[] bix, int ai, int bi, int alen, int blen) { if(alen < blen) { - SparseRowVector c = allocSparseVector(blen, b, bix); - for (int j = ai; j < ai+alen; j++) { + SparseRowVector c = allocSparseVector(blen); + for(int j = 0; j < bi+blen; j++) + c.set(bix[j], b[j]); + for(int j = ai; j < ai+alen; j++) c.add(aix[j], a[j]); - } return c; } else { - SparseRowVector c = allocSparseVector(alen, a, aix); - for (int j = bi; j < bi+blen; j++) { + SparseRowVector c = allocSparseVector(alen); + for(int j = 0; j < ai+alen; j++) + c.set(aix[j], a[j]); + for(int j = bi; j < bi+blen; j++) c.add(bix[j], b[j]); - } return c; } } @@ -2869,22 +2900,7 @@ protected static double[] allocVector(int len, boolean reset, double resetVal) { return vect; } - //allocate an empty vector public static SparseRowVector allocSparseVector(int len) { - return allocSparseVector(len, null, null, false, false); - } - - //allocate a vector with the vales and indexes filled - public static SparseRowVector allocSparseVector(int len, double[] values,int[] indexes) { - return allocSparseVector(len, values, indexes, true, false); - } - - //allocate vector with only the indexes filled - public static SparseRowVector allocSparseVector(int len, int[] indexes) { - return allocSparseVector(len, null, indexes, false, true); - } - - protected static SparseRowVector allocSparseVector(int len, double[] values, int[] indexes, boolean fill, boolean indexCopy) { SparseVectorBuffer buff = sparseMemPool.get(); //find next matching vector in ring buffer or @@ -2893,19 +2909,8 @@ protected static SparseRowVector allocSparseVector(int len, double[] values, int if(vect == null) vect = new SparseRowVector(len); //reset vector for normal outputs - else - vect.reset(4, len); - - //fill vector - if(fill) { - System.arraycopy(values, 0, vect.values(), 0, len); - System.arraycopy(indexes, 0, vect.indexes(), 0, len); - vect.setSize(len); - //indexes copy - } else if(indexCopy) { - vect.setIndexes(indexes); - vect.setSize(len); - } + else if(vect.size() != 0) + vect.reset(len, len); return vect; } diff --git a/src/test/java/org/apache/sysds/test/component/codegen/CPlanVectorPrimitivesTest.java b/src/test/java/org/apache/sysds/test/component/codegen/CPlanVectorPrimitivesTest.java index eaf750c01cd..dfc79a627ea 100644 --- a/src/test/java/org/apache/sysds/test/component/codegen/CPlanVectorPrimitivesTest.java +++ b/src/test/java/org/apache/sysds/test/component/codegen/CPlanVectorPrimitivesTest.java @@ -1188,8 +1188,8 @@ private static void testVectorBinarySparsePrimitive(BinType bintype, InputType t double sparsityB = (type2 == InputType.VECTOR_DENSE) ? sparsity1 : sparsity2; MatrixBlock inB = MatrixBlock.randOperations(m, n, sparsityB, -5, 5, "uniform", 7); - boolean sparse = sparseOutput(bintype); - int testType = testType(bintype); + boolean sparse = getOutputType(bintype); + int testType = getTestType(bintype); //get vector primitive via reflection String meName = "vect"+StringUtils.camelize(bintype.name().split("_")[1])+"Write"; @@ -1278,7 +1278,12 @@ else if( type2 == InputType.SCALAR ) { } } - private static boolean sparseOutput(BinType type) { + + /** + * @param type + * @return {@code true}, when the matching method has a sparse return + */ + private static boolean getOutputType(BinType type) { switch(type) { case VECT_EQUAL: case VECT_LESSEQUAL: @@ -1290,11 +1295,12 @@ private static boolean sparseOutput(BinType type) { } /** - * returns -1, for normal testing; - * returns 0, for testing with 0 and non-zeros; - * returns 1, for testing with negative and positive numbers; + * @param type + * @return returns {@code -1}, for normal testing;
+ * returns {@code 0}, for testing with 0 and non-zeros;
+ * returns {@code 1}, for testing with negative and positive numbers; */ - private static int testType(BinType type) { + private static int getTestType(BinType type) { switch(type) { case VECT_DIV_SCALAR: case VECT_EQUAL_SCALAR: diff --git a/src/test/java/org/apache/sysds/test/component/codegen/SparseVectorAllocTest.java b/src/test/java/org/apache/sysds/test/component/codegen/SparseVectorAllocTest.java index 8f1c354b2a3..c6a2b10f031 100644 --- a/src/test/java/org/apache/sysds/test/component/codegen/SparseVectorAllocTest.java +++ b/src/test/java/org/apache/sysds/test/component/codegen/SparseVectorAllocTest.java @@ -39,34 +39,14 @@ public void testBasicAllocationShorterExp() { testBasicSparseVectorAllocation(1, 10, 7); } - @Test - public void testAllocationWithValues1() { - testSparseVectorWithValues(1, 10,val1, indexes1); - } - - @Test - public void testAllocationWithValues2() { - testSparseVectorWithValues(1, 10, val2, indexes2); - } - - @Test - public void testAllocationWithIndexes1() { - testSparseVectorWithIndexes(1, 10, indexes1); - } - - @Test - public void testAllocationWithIndexes2() { - testSparseVectorWithIndexes(1, 10, indexes2); - } - @Test public void testVectorReuse1() { - testBufferReuse(1, 10, -1, 5, val1, indexes1); + testBufferReuse(3, 10, 5, 5); } @Test public void testVectorReuse2() { - testBufferReuse(1, 10, -1, 5, val2, indexes2); + testBufferReuse(3, 10, -1, 5); } /** tests the allocation of an empty vector @@ -85,52 +65,13 @@ public void testBasicSparseVectorAllocation(int numVectors, int len, int expLen) LibSpoofPrimitives.cleanupSparseThreadLocalMemory(); } - /** tests the allocation of a vector with certain values and indexes - * @param numVectors number of vectors that should be pre-allocated - * @param len the length of the vector - * @param values expected values - * @param indexes expected indexes - */ - public void testSparseVectorWithValues(int numVectors, int len, double[] values, int[] indexes) { - //test the allocation of a vector with preset values - LibSpoofPrimitives.setupSparseThreadLocalMemory(numVectors, len, -1); - - SparseRowVector sparseVec = LibSpoofPrimitives.allocSparseVector(values.length, values, indexes); - - Assert.assertEquals("Vector size should match input array length", values.length, sparseVec.size()); - for(int j = 0; j < sparseVec.size(); j++) { - Assert.assertEquals("Value array should match input array", values[j], sparseVec.get(indexes[j]), 0.001); - } - - LibSpoofPrimitives.cleanupSparseThreadLocalMemory(); - } - - /** tests the allocation of a vector with certain indexes - * @param numVectors number of vectors that should be pre-allocated - * @param len the length of the vector - * @param indexes expected indexes - */ - public void testSparseVectorWithIndexes(int numVectors, int len, int[] indexes) { - //test allocation of a vector with preset indexes - LibSpoofPrimitives.setupSparseThreadLocalMemory(numVectors, len, -1); - - SparseRowVector sparseVec = LibSpoofPrimitives.allocSparseVector(indexes.length, indexes); - - Assert.assertEquals("Vector size should match input array length", indexes.length, sparseVec.size()); - Assert.assertArrayEquals("Indexes array should match input array", indexes, sparseVec.indexes()); - - LibSpoofPrimitives.cleanupSparseThreadLocalMemory(); - } - /** tests the allocation of a vector that is reused multiple times * @param numVectors number of vectors that should be pre-allocated * @param len1 length of the first vector * @param len2 length of the second vector * @param expLen expected length of allocated vector - * @param values expected values - * @param indexes expected indexes */ - public void testBufferReuse(int numVectors, int len1, int len2, int expLen, double[] values, int[] indexes) { + public void testBufferReuse(int numVectors, int len1, int len2, int expLen) { //test the reuse of the vectors in the ring buffer LibSpoofPrimitives.setupSparseThreadLocalMemory(numVectors, len1, len2); @@ -144,14 +85,30 @@ public void testBufferReuse(int numVectors, int len1, int len2, int expLen, doub Assert.assertEquals("Reused vector should be reset to size 0", 0, vec2.size()); - //allocate third vector with values - SparseRowVector vec3 = LibSpoofPrimitives.allocSparseVector(values.length, values, indexes); + for(int j = 0; j < vec2.size(); j++) { + vec2.set(vec2.indexes()[j], vec2.get(vec2.indexes()[j]) * 32); + } + + SparseRowVector vec3 = LibSpoofPrimitives.allocSparseVector(expLen); + + Assert.assertEquals("Reused vector should be reset to size 0", 0, vec3.size()); + + SparseRowVector vec4 = LibSpoofPrimitives.allocSparseVector(expLen); - Assert.assertEquals("Vector size should match input", values.length, vec3.size()); - for(int j = 0; j < vec3.size(); j++) { - Assert.assertEquals("Value array should match input array", values[j], vec3.get(indexes[j]), 0.001); + for(int j = 0; j < vec4.size(); j++) { + vec4.set(vec4.indexes()[j], vec4.get(vec3.indexes()[j]) * 32); } + Assert.assertEquals("Reused vector should be reset to size 0", 0, vec4.size()); + + SparseRowVector vec5 = LibSpoofPrimitives.allocSparseVector(expLen); + + for(int j = 0; j < vec5.size(); j++) { + vec2.set(vec5.indexes()[j], vec5.get(vec5.indexes()[j]) * 32); + } + + Assert.assertEquals("Reused vector should be reset to size 0", 0, vec5.size()); + LibSpoofPrimitives.cleanupSparseThreadLocalMemory(); } From 7646eaa41a5b5cb66c8ddcee5713f6608ed85b38 Mon Sep 17 00:00:00 2001 From: Frxms Date: Tue, 17 Jun 2025 18:39:21 +0200 Subject: [PATCH 06/60] add div to primitive tests and change primitives signature --- .../runtime/codegen/LibSpoofPrimitives.java | 109 +++++++++++------- .../codegen/CPlanVectorPrimitivesTest.java | 7 +- 2 files changed, 71 insertions(+), 45 deletions(-) diff --git a/src/main/java/org/apache/sysds/runtime/codegen/LibSpoofPrimitives.java b/src/main/java/org/apache/sysds/runtime/codegen/LibSpoofPrimitives.java index 40c138a1dd5..c93e0658571 100644 --- a/src/main/java/org/apache/sysds/runtime/codegen/LibSpoofPrimitives.java +++ b/src/main/java/org/apache/sysds/runtime/codegen/LibSpoofPrimitives.java @@ -2181,28 +2181,35 @@ public static double vectVar(double[] avals, int[] aix, int ai, int alen, int le public static SparseRowVector vectMultWrite(int len, double[] a, double bval, int[] aix, int ai, int alen) { SparseRowVector c = allocSparseVector(alen); if( a == null ) return c; - for(int j = 0; j < ai+alen; j++) - c.set(aix[j], a[j]*bval); + for(int j = 0; j < ai+alen; j++) { + c.indexes()[j] = aix[j]; + c.values()[j] = a[j]*bval; + } + c.setSize(alen); return c; } public static SparseRowVector vectMultWrite(int len, double[] a, double[] b, int[] aix, int[] bix, int ai, int bi, int alen, int blen) { - SparseRowVector c = allocSparseVector(alen); + SparseRowVector c = allocSparseVector(Math.min(alen, blen)); if( a == null || b == null ) return c; int aIndex = ai; int bIndex = bi; + int index = 0; while(aIndex < ai+alen && bIndex < bi+blen) { int aIdx = aix[aIndex]; int bIdx = bix[bIndex]; if(aIdx == bIdx) { - c.set(aIdx, a[aIndex] * b[bIndex]); + c.indexes()[index] = aIdx; + c.values()[index] = a[aIndex] * b[bIndex]; aIndex++; bIndex++; + index++; } else if(aIdx < bIdx) aIndex++; else bIndex++; } + c.setSize(index); return c; } @@ -2214,75 +2221,89 @@ public static void vectWrite(SparseRowVector a, double[] c, int ci, int len) { } public static SparseRowVector vectDivWrite(int len, double[] a, double bval, int[] aix, int ai, int alen) { - //todo: can i just do nothing, when deviding through zero? - // if (bval != 0) { SparseRowVector c = allocSparseVector(alen); - for( int j = ai; j < ai+alen; j++ ) - c.set(aix[j], a[j] / bval); + for( int j = ai; j < ai+alen; j++ ) { + c.indexes()[j] = aix[j]; + c.values()[j] = a[j] / bval; + } + c.setSize(alen); return c; - // } else { - // double init = Double.NaN; - // double[] c = allocVector(len, true, init); - // for( int j = ai; j < ai+alen; j++ ) - // c[aix[j]] = a[j] / bval; - // return new SparseRowVector(c); - // } } public static SparseRowVector vectDivWrite(int len, double bval, double[] a, int[] aix, int ai, int alen) { SparseRowVector c = allocSparseVector(alen); - for(int j = 0; j < ai+alen; j++) - c.set(aix[j], bval / a[j]); + for(int j = 0; j < ai+alen; j++) { + c.indexes()[j] = aix[j]; + c.values()[j] = bval / a[j]; + } + c.setSize(alen); return c; } public static SparseRowVector vectDivWrite(int len, double[] a, double[] b, int[] aix, int[] bix, int ai, int bi, int alen, int blen) { -// SparseRowVector c = allocSparseVector(alen); -// if(alen != 0 && blen != 0) { -// //decide between two versions. First is smaller for smaller alen and the other for smaller blen -// if(alen < blen) { -// SparseRowVector bSparse = allocSparseVector(blen, b, bix); -// for (int j = ai; j < ai+alen; j++) { -// c.set(aix[j], a[j] / bSparse.get(aix[j])); -// } -// } else { -// SparseRowVector aSparse = allocSparseVector(alen, a, aix); -// for (int j = ai; j < ai+alen; j++) { -// double aval = a[j]; -// c.set(aix[j], (aval>0) ? -// Double.POSITIVE_INFINITY : Double.NEGATIVE_INFINITY); -// } -// for (int j = bi; j < bi+blen; j++) { -// c.set(bix[j], aSparse.get(bix[j]) / b[j]); -// } -// } -// } SparseRowVector c = allocSparseVector(alen); int aIndex = ai; int bIndex = bi; + int index = 0; while(aIndex < ai+alen && bIndex < bi+blen) { int aIdx = aix[aIndex]; int bIdx = bix[bIndex]; if(aIdx == bIdx) { - c.set(aIdx, a[aIndex] / b[bIndex]); + c.indexes()[index] = aIdx; + c.values()[index] = a[aIndex] / b[bIndex]; aIndex++; bIndex++; + index++; } else if(aIdx < bIdx) { - c.set(aIdx, (a[aIndex]>0) ? Double.POSITIVE_INFINITY : Double.NEGATIVE_INFINITY); + c.indexes()[index] = aIdx; + c.values()[index] = (a[aIndex]>0) ? Double.POSITIVE_INFINITY : Double.NEGATIVE_INFINITY; aIndex++; + index++; } else { bIndex++; } } + c.setSize(index); return c; } public static SparseRowVector vectMinusWrite(int len, double[] a, double[] b, int[] aix, int[] bix, int ai, int bi, int alen, int blen) { - SparseRowVector c = allocSparseVector(alen); - for(int j = 0; j < ai+alen; j++) - c.set(aix[j], a[j]); - for (int j = bi; j < bi+blen; j++) - c.add(bix[j], -b[j]); + SparseRowVector c = allocSparseVector(alen+blen); + int aIndex = ai; + int bIndex = bi; + int index = 0; + while(aIndex < ai+alen && bIndex < bi+blen) { + int aIdx = aix[aIndex]; + int bIdx = bix[bIndex]; + if(aIdx == bIdx) { + c.indexes()[index] = aIdx; + c.values()[index] = a[aIndex] - b[bIndex]; + aIndex++; + bIndex++; + index++; + } else if(aIdx < bIdx) { + c.indexes()[index] = aIdx; + c.values()[index] = a[aIndex]; + aIndex++; + index++; + } else { + c.indexes()[index] = bIdx; + c.values()[index] = -b[bIndex]; + bIndex++; + index++; + } + } + for (; aIndex < ai+alen; aIndex++) { + c.indexes()[index] = aix[aIndex]; + c.values()[index] = a[aIndex]; + index++; + } + for (; bIndex < bi+blen; bIndex++) { + c.indexes()[index] = bix[bIndex]; + c.values()[index] = -b[bIndex]; + index++; + } + c.setSize(index); return c; } diff --git a/src/test/java/org/apache/sysds/test/component/codegen/CPlanVectorPrimitivesTest.java b/src/test/java/org/apache/sysds/test/component/codegen/CPlanVectorPrimitivesTest.java index dfc79a627ea..9b66c11371d 100644 --- a/src/test/java/org/apache/sysds/test/component/codegen/CPlanVectorPrimitivesTest.java +++ b/src/test/java/org/apache/sysds/test/component/codegen/CPlanVectorPrimitivesTest.java @@ -788,6 +788,11 @@ public void testVectorScalarBitwAndSparseToSparse() { //scalar - vector + @Test + public void testScalarVectorDivSparseToSparse() { + testVectorBinarySparsePrimitive(BinType.VECT_DIV_SCALAR, InputType.SCALAR, InputType.VECTOR_SPARSE); + } + @Test public void testScalarVectorMinSparseToSparse() { testVectorBinarySparsePrimitive(BinType.VECT_MIN_SCALAR, InputType.SCALAR, InputType.VECTOR_SPARSE); @@ -1270,7 +1275,7 @@ else if( type2 == InputType.SCALAR ) { } //compare results - TestUtils.compareMatrices(ret1, ret2, eps); + TestUtils.compareMatrices(ret2, ret1, eps); } } catch( Exception ex ) { From 05561f2b369da4e235437c754207e7d48fd371a9 Mon Sep 17 00:00:00 2001 From: Frxms Date: Wed, 18 Jun 2025 15:08:45 +0200 Subject: [PATCH 07/60] optimize primitives --- .../runtime/codegen/LibSpoofPrimitives.java | 248 +++++++++++++----- 1 file changed, 187 insertions(+), 61 deletions(-) diff --git a/src/main/java/org/apache/sysds/runtime/codegen/LibSpoofPrimitives.java b/src/main/java/org/apache/sysds/runtime/codegen/LibSpoofPrimitives.java index c93e0658571..c62e3c478f5 100644 --- a/src/main/java/org/apache/sysds/runtime/codegen/LibSpoofPrimitives.java +++ b/src/main/java/org/apache/sysds/runtime/codegen/LibSpoofPrimitives.java @@ -2181,9 +2181,11 @@ public static double vectVar(double[] avals, int[] aix, int ai, int alen, int le public static SparseRowVector vectMultWrite(int len, double[] a, double bval, int[] aix, int ai, int alen) { SparseRowVector c = allocSparseVector(alen); if( a == null ) return c; + int[] indexes = c.indexes(); + double[] values = c.values(); for(int j = 0; j < ai+alen; j++) { - c.indexes()[j] = aix[j]; - c.values()[j] = a[j]*bval; + indexes[j] = aix[j]; + values[j] = a[j]*bval; } c.setSize(alen); return c; @@ -2195,12 +2197,14 @@ public static SparseRowVector vectMultWrite(int len, double[] a, double[] b, int int aIndex = ai; int bIndex = bi; int index = 0; + int[] indexes = c.indexes(); + double[] values = c.values(); while(aIndex < ai+alen && bIndex < bi+blen) { int aIdx = aix[aIndex]; int bIdx = bix[bIndex]; if(aIdx == bIdx) { - c.indexes()[index] = aIdx; - c.values()[index] = a[aIndex] * b[bIndex]; + indexes[index] = aIdx; + values[index] = a[aIndex] * b[bIndex]; aIndex++; bIndex++; index++; @@ -2222,9 +2226,11 @@ public static void vectWrite(SparseRowVector a, double[] c, int ci, int len) { public static SparseRowVector vectDivWrite(int len, double[] a, double bval, int[] aix, int ai, int alen) { SparseRowVector c = allocSparseVector(alen); + int[] indexes = c.indexes(); + double[] values = c.values(); for( int j = ai; j < ai+alen; j++ ) { - c.indexes()[j] = aix[j]; - c.values()[j] = a[j] / bval; + indexes[j] = aix[j]; + values[j] = a[j] / bval; } c.setSize(alen); return c; @@ -2232,9 +2238,11 @@ public static SparseRowVector vectDivWrite(int len, double[] a, double bval, int public static SparseRowVector vectDivWrite(int len, double bval, double[] a, int[] aix, int ai, int alen) { SparseRowVector c = allocSparseVector(alen); + int[] indexes = c.indexes(); + double[] values = c.values(); for(int j = 0; j < ai+alen; j++) { - c.indexes()[j] = aix[j]; - c.values()[j] = bval / a[j]; + indexes[j] = aix[j]; + values[j] = bval / a[j]; } c.setSize(alen); return c; @@ -2245,18 +2253,20 @@ public static SparseRowVector vectDivWrite(int len, double[] a, double[] b, int[ int aIndex = ai; int bIndex = bi; int index = 0; + int[] indexes = c.indexes(); + double[] values = c.values(); while(aIndex < ai+alen && bIndex < bi+blen) { int aIdx = aix[aIndex]; int bIdx = bix[bIndex]; if(aIdx == bIdx) { - c.indexes()[index] = aIdx; - c.values()[index] = a[aIndex] / b[bIndex]; + indexes[index] = aIdx; + values[index] = a[aIndex] / b[bIndex]; aIndex++; bIndex++; index++; } else if(aIdx < bIdx) { - c.indexes()[index] = aIdx; - c.values()[index] = (a[aIndex]>0) ? Double.POSITIVE_INFINITY : Double.NEGATIVE_INFINITY; + indexes[index] = aIdx; + values[index] = (a[aIndex]>0) ? Double.POSITIVE_INFINITY : Double.NEGATIVE_INFINITY; aIndex++; index++; } else { @@ -2272,35 +2282,37 @@ public static SparseRowVector vectMinusWrite(int len, double[] a, double[] b, in int aIndex = ai; int bIndex = bi; int index = 0; + int[] indexes = c.indexes(); + double[] values = c.values(); while(aIndex < ai+alen && bIndex < bi+blen) { int aIdx = aix[aIndex]; int bIdx = bix[bIndex]; if(aIdx == bIdx) { - c.indexes()[index] = aIdx; - c.values()[index] = a[aIndex] - b[bIndex]; + indexes[index] = aIdx; + values[index] = a[aIndex] - b[bIndex]; aIndex++; bIndex++; index++; } else if(aIdx < bIdx) { - c.indexes()[index] = aIdx; - c.values()[index] = a[aIndex]; + indexes[index] = aIdx; + values[index] = a[aIndex]; aIndex++; index++; } else { - c.indexes()[index] = bIdx; - c.values()[index] = -b[bIndex]; + indexes[index] = bIdx; + values[index] = -b[bIndex]; bIndex++; index++; } } for (; aIndex < ai+alen; aIndex++) { - c.indexes()[index] = aix[aIndex]; - c.values()[index] = a[aIndex]; + indexes[index] = aix[aIndex]; + values[index] = a[aIndex]; index++; } for (; bIndex < bi+blen; bIndex++) { - c.indexes()[index] = bix[bIndex]; - c.values()[index] = -b[bIndex]; + indexes[index] = bix[bIndex]; + values[index] = -b[bIndex]; index++; } c.setSize(index); @@ -2308,24 +2320,49 @@ public static SparseRowVector vectMinusWrite(int len, double[] a, double[] b, in } public static SparseRowVector vectPlusWrite(int len, double[] a, double[] b, int[] aix, int[] bix, int ai, int bi, int alen, int blen) { - if(alen < blen) { - SparseRowVector c = allocSparseVector(blen); - for(int j = 0; j < bi+blen; j++) - c.set(bix[j], b[j]); - for(int j = ai; j < ai+alen; j++) - c.add(aix[j], a[j]); - return c; - } else { - SparseRowVector c = allocSparseVector(alen); - for(int j = 0; j < ai+alen; j++) - c.set(aix[j], a[j]); - for(int j = bi; j < bi+blen; j++) - c.add(bix[j], b[j]); - return c; + SparseRowVector c = allocSparseVector(alen+blen); + int aIndex = ai; + int bIndex = bi; + int index = 0; + int[] indexes = c.indexes(); + double[] values = c.values(); + while(aIndex < ai+alen && bIndex < bi+blen) { + int aIdx = aix[aIndex]; + int bIdx = bix[bIndex]; + if(aIdx == bIdx) { + indexes[index] = aIdx; + values[index] = a[aIndex] + b[bIndex]; + aIndex++; + bIndex++; + index++; + } else if(aIdx < bIdx) { + indexes[index] = aIdx; + values[index] = a[aIndex]; + aIndex++; + index++; + } else { + indexes[index] = bIdx; + values[index] = b[bIndex]; + bIndex++; + index++; + } + } + for (; aIndex < ai+alen; aIndex++) { + indexes[index] = aix[aIndex]; + values[index] = a[aIndex]; + index++; + } + for (; bIndex < bi+blen; bIndex++) { + indexes[index] = bix[bIndex]; + values[index] = b[bIndex]; + index++; } + c.setSize(index); + return c; } public static SparseRowVector vectXorWrite(int len, double[] a, double bval, int[] aix, int ai, int alen) { + //todo: test if this is faster, when SparseRowVector is allocated first if(bval != 0) { double[] c = allocVector(len, true, 1); for(int j = ai; j < ai+alen; j++) @@ -2333,8 +2370,13 @@ public static SparseRowVector vectXorWrite(int len, double[] a, double bval, int return new SparseRowVector(c); } else { SparseRowVector c = allocSparseVector(alen); - for(int j = ai; j < ai+alen; j++) - c.set(aix[j], (a[j] != 0) ? 1 : 0); + int[] indexes = c.indexes(); + double[] values = c.values(); + for(int j = 0; j < ai+alen; j++) { + indexes[j] = aix[j]; + values[j] = (a[j] != 0) ? 1 : 0; + } + c.setSize(alen); return c; } } @@ -2344,11 +2386,44 @@ public static SparseRowVector vectXorWrite(int len, double bval, double[] a, int } public static SparseRowVector vectXorWrite(int len, double[] a, double[] b, int[] aix, int[] bix, int ai, int bi, int alen, int blen) { - SparseRowVector c = allocSparseVector(alen); - for (int j = bi; j < bi+blen; j++) - c.set(bix[j], (b[j] != 0) ? 1 : 0); - for (int j = ai; j < ai+alen; j++) - c.set(aix[j], ((a[j] != 0) != (c.get(aix[j]) != 0)) ? 1 : 0); + SparseRowVector c = allocSparseVector(alen+blen); + int aIndex = ai; + int bIndex = bi; + int index = 0; + int[] indexes = c.indexes(); + double[] values = c.values(); + while(aIndex < ai+alen && bIndex < bi+blen) { + int aIdx = aix[aIndex]; + int bIdx = bix[bIndex]; + if(aIdx == bIdx) { + indexes[index] = aIdx; + values[index] = ((a[aIndex] != 0) != (b[bIndex] != 0)) ? 1 : 0; + aIndex++; + bIndex++; + index++; + } else if(aIdx < bIdx) { + indexes[index] = aIdx; + values[index] = (a[aIndex] != 0) ? 1 : 0; + aIndex++; + index++; + } else { + indexes[index] = bIdx; + values[index] = (b[bIndex] != 0) ? 1 : 0; + bIndex++; + index++; + } + } + for (; aIndex < ai+alen; aIndex++) { + indexes[index] = aix[aIndex]; + values[index] = (a[aIndex] != 0) ? 1 : 0; + index++; + } + for (; bIndex < bi+blen; bIndex++) { + indexes[index] = bix[bIndex]; + values[index] = (b[bIndex] != 0) ? 1 : 0; + index++; + } + c.setSize(index); return c; } @@ -2361,8 +2436,13 @@ public static SparseRowVector vectPowWrite(int len, double[] a, double bval, int return new SparseRowVector(c); } else { SparseRowVector c = allocSparseVector(alen); - for(int j = ai; j < ai+alen; j++) - c.set(aix[j], Math.pow(a[j], bval)); + int[] indexes = c.indexes(); + double[] values = c.values(); + for(int j = 0; j < ai+alen; j++) { + indexes[j] = aix[j]; + values[j] = Math.pow(a[j], bval); + } + c.setSize(alen); return c; } } @@ -2375,8 +2455,13 @@ public static SparseRowVector vectMinWrite(int len, double[] a, double bval, int return new SparseRowVector(c); } else { SparseRowVector c = allocSparseVector(alen); - for(int j = ai; j < ai+alen; j++) - c.set(aix[j], Math.min(a[j], bval)); + int[] indexes = c.indexes(); + double[] values = c.values(); + for(int j = 0; j < ai+alen; j++) { + indexes[j] = aix[j]; + values[j] = Math.min(a[j], bval); + } + c.setSize(alen); return c; } } @@ -2386,26 +2471,44 @@ public static SparseRowVector vectMinWrite(int len, double bval, double[] a, int } public static SparseRowVector vectMinWrite(int len, double[] a, double[] b, int[] aix, int[] bix, int ai, int bi, int alen, int blen) { - SparseRowVector c = allocSparseVector(alen); + SparseRowVector c = allocSparseVector(alen+blen); int aIndex = ai; int bIndex = bi; + int index = 0; + int[] indexes = c.indexes(); + double[] values = c.values(); while(aIndex < ai+alen && bIndex < bi+blen) { int aIdx = aix[aIndex]; int bIdx = bix[bIndex]; if(aIdx == bIdx) { - c.set(aIdx, Math.min(a[aIndex], b[bIndex])); + indexes[index] = aIdx; + values[index] = Math.min(a[aIndex], b[bIndex]); aIndex++; bIndex++; + index++; } else if(aIdx < bIdx) { - c.set(aIdx, Math.min(a[aIndex], 0)); + indexes[index] = aIdx; + values[index] = Math.min(a[aIndex], 0); aIndex++; + index++; } else { - c.set(bIdx, Math.min(b[bIndex], 0)); + indexes[index] = bIdx; + values[index] = Math.min(b[bIndex], 0); bIndex++; + index++; } } - for (; aIndex < ai+alen; aIndex++) c.set(aix[aIndex], Math.min(a[aIndex], 0)); - for (; bIndex < bi+blen; bIndex++) c.set(bix[bIndex], Math.min(b[bIndex], 0)); + for (; aIndex < ai+alen; aIndex++) { + indexes[index] = aix[aIndex]; + values[index] = Math.min(a[aIndex], 0); + index++; + } + for (; bIndex < bi+blen; bIndex++) { + indexes[index] = bix[bIndex]; + values[index] = Math.min(b[bIndex], 0); + index++; + } + c.setSize(index); return c; } @@ -2417,8 +2520,13 @@ public static SparseRowVector vectMaxWrite(int len, double[] a, double bval, int return new SparseRowVector(c); } else { SparseRowVector c = allocSparseVector(alen); - for(int j = ai; j < ai+alen; j++) - c.set(aix[j], Math.max(a[j], bval)); + int[] indexes = c.indexes(); + double[] values = c.values(); + for(int j = 0; j < ai+alen; j++) { + indexes[j] = aix[j]; + values[j] = Math.max(a[j], bval); + } + c.setSize(alen); return c; } } @@ -2428,26 +2536,44 @@ public static SparseRowVector vectMaxWrite(int len, double bval, double[] a, int } public static SparseRowVector vectMaxWrite(int len, double[] a, double[] b, int[] aix, int[] bix, int ai, int bi, int alen, int blen) { - SparseRowVector c = allocSparseVector(alen); + SparseRowVector c = allocSparseVector(alen+blen); int aIndex = ai; int bIndex = bi; + int index = 0; + int[] indexes = c.indexes(); + double[] values = c.values(); while(aIndex < ai+alen && bIndex < bi+blen) { int aIdx = aix[aIndex]; int bIdx = bix[bIndex]; if(aIdx == bIdx) { - c.set(aIdx, Math.max(a[aIndex], b[bIndex])); + indexes[index] = aIdx; + values[index] = Math.max(a[aIndex], b[bIndex]); aIndex++; bIndex++; + index++; } else if(aIdx < bIdx) { - c.set(aIdx, Math.max(a[aIndex], 0)); + indexes[index] = aIdx; + values[index] = Math.max(a[aIndex], 0); aIndex++; + index++; } else { - c.set(bIdx, Math.max(b[bIndex], 0)); + indexes[index] = bIdx; + values[index] = Math.max(b[bIndex], 0); bIndex++; + index++; } } - for (; aIndex < ai+alen; aIndex++) c.set(aix[aIndex], Math.max(a[aIndex], 0)); - for (; bIndex < bi+blen; bIndex++) c.set(bix[bIndex], Math.max(b[bIndex], 0)); + for (; aIndex < ai+alen; aIndex++) { + indexes[index] = aix[aIndex]; + values[index] = Math.max(a[aIndex], 0); + index++; + } + for (; bIndex < bi+blen; bIndex++) { + indexes[index] = bix[bIndex]; + values[index] = Math.max(b[bIndex], 0); + index++; + } + c.setSize(index); return c; } From 36d9e7bae6d087640fd12f9e4bc77d3825abd6e1 Mon Sep 17 00:00:00 2001 From: Frxms Date: Wed, 18 Jun 2025 18:05:23 +0200 Subject: [PATCH 08/60] xor signature change --- .../runtime/codegen/LibSpoofPrimitives.java | 26 +++++++++++++++---- 1 file changed, 21 insertions(+), 5 deletions(-) diff --git a/src/main/java/org/apache/sysds/runtime/codegen/LibSpoofPrimitives.java b/src/main/java/org/apache/sysds/runtime/codegen/LibSpoofPrimitives.java index c62e3c478f5..77829cf8cda 100644 --- a/src/main/java/org/apache/sysds/runtime/codegen/LibSpoofPrimitives.java +++ b/src/main/java/org/apache/sysds/runtime/codegen/LibSpoofPrimitives.java @@ -2362,12 +2362,28 @@ public static SparseRowVector vectPlusWrite(int len, double[] a, double[] b, int } public static SparseRowVector vectXorWrite(int len, double[] a, double bval, int[] aix, int ai, int alen) { - //todo: test if this is faster, when SparseRowVector is allocated first if(bval != 0) { - double[] c = allocVector(len, true, 1); - for(int j = ai; j < ai+alen; j++) - c[aix[j]] = (a[j] != 0) ? 0 : 1; - return new SparseRowVector(c); + SparseRowVector c = allocSparseVector(len); + int[] indexes = c.indexes(); + double[] values = c.values(); + int index = 0; + int aIndex = 0; + while(aIndex < ai+alen && index < len) { + indexes[index] = index; + if(aix[aIndex] == index) { + values[index] = !(a[aIndex] != 0) ? 1 : 0; + aIndex++; + } else { + values[index] = 1; + } + index++; + } + for(; index < len; index++) { + indexes[index] = index; + values[index] = 1; + } + c.setSize(len); + return c; } else { SparseRowVector c = allocSparseVector(alen); int[] indexes = c.indexes(); From 688778316e8012f98712c37727ce934a2c5ed9c8 Mon Sep 17 00:00:00 2001 From: Frxms Date: Wed, 18 Jun 2025 19:12:40 +0200 Subject: [PATCH 09/60] optimize Equal, Notequal, Max --- .../runtime/codegen/LibSpoofPrimitives.java | 183 +++++++++++++++--- 1 file changed, 153 insertions(+), 30 deletions(-) diff --git a/src/main/java/org/apache/sysds/runtime/codegen/LibSpoofPrimitives.java b/src/main/java/org/apache/sysds/runtime/codegen/LibSpoofPrimitives.java index 77829cf8cda..9d48524236e 100644 --- a/src/main/java/org/apache/sysds/runtime/codegen/LibSpoofPrimitives.java +++ b/src/main/java/org/apache/sysds/runtime/codegen/LibSpoofPrimitives.java @@ -2445,11 +2445,27 @@ public static SparseRowVector vectXorWrite(int len, double[] a, double[] b, int[ public static SparseRowVector vectPowWrite(int len, double[] a, double bval, int[] aix, int ai, int alen) { if(bval == 0) { - //Todo: why -1 here, even though the result should be 1? - double[] c = allocVector(len, true, 1); - for(int j = ai; j < ai+alen; j++) - c[aix[j]] = Math.pow(a[j], bval) - 1; - return new SparseRowVector(c); + SparseRowVector c = allocSparseVector(len); + int[] indexes = c.indexes(); + double[] values = c.values(); + int index = 0; + int aIndex = 0; + while(aIndex < ai+alen && index < len) { + indexes[index] = index; + if(aix[aIndex] == index) { + values[index] = Math.pow(a[aIndex], bval) - 1; + aIndex++; + } else { + values[index] = 1; + } + index++; + } + for(; index < len; index++) { + indexes[index] = index; + values[index] = 1; + } + c.setSize(len); + return c; } else { SparseRowVector c = allocSparseVector(alen); int[] indexes = c.indexes(); @@ -2465,10 +2481,27 @@ public static SparseRowVector vectPowWrite(int len, double[] a, double bval, int public static SparseRowVector vectMinWrite(int len, double[] a, double bval, int[] aix, int ai, int alen) { if(bval < 0) { - double[] c = allocVector(len, true, bval); - for(int j = ai; j < ai+alen; j++) - c[aix[j]] = Math.min(a[j], bval); - return new SparseRowVector(c); + SparseRowVector c = allocSparseVector(len); + int[] indexes = c.indexes(); + double[] values = c.values(); + int index = 0; + int aIndex = 0; + while(aIndex < ai+alen && index < len) { + indexes[index] = index; + if(aix[aIndex] == index) { + values[index] = Math.min(a[aIndex], bval); + aIndex++; + } else { + values[index] = bval; + } + index++; + } + for(; index < len; index++) { + indexes[index] = index; + values[index] = bval; + } + c.setSize(len); + return c; } else { SparseRowVector c = allocSparseVector(alen); int[] indexes = c.indexes(); @@ -2530,10 +2563,27 @@ public static SparseRowVector vectMinWrite(int len, double[] a, double[] b, int[ public static SparseRowVector vectMaxWrite(int len, double[] a, double bval, int[] aix, int ai, int alen) { if(bval > 0) { - double[] c = allocVector(len, true, bval); - for(int j = ai; j < ai+alen; j++) - c[aix[j]] = Math.max(a[j], bval); - return new SparseRowVector(c); + SparseRowVector c = allocSparseVector(len); + int[] indexes = c.indexes(); + double[] values = c.values(); + int index = 0; + int aIndex = 0; + while(aIndex < ai+alen && index < len) { + indexes[index] = index; + if(aix[aIndex] == index) { + values[index] = Math.max(a[aIndex], bval); + aIndex++; + } else { + values[index] = bval; + } + index++; + } + for(; index < len; index++) { + indexes[index] = index; + values[index] = bval; + } + c.setSize(len); + return c; } else { SparseRowVector c = allocSparseVector(alen); int[] indexes = c.indexes(); @@ -2595,14 +2645,36 @@ public static SparseRowVector vectMaxWrite(int len, double[] a, double[] b, int[ public static SparseRowVector vectEqualWrite(int len, double[] a, double bval, int[] aix, int ai, int alen) { if(bval == 0) { - double[] c = allocVector(len, true, 1); - for( int j = ai; j < ai+alen; j++ ) - c[aix[j]] = (a[j] == bval) ? 1 : 0; - return new SparseRowVector(c); + SparseRowVector c = allocSparseVector(len); + int[] indexes = c.indexes(); + double[] values = c.values(); + int index = 0; + int aIndex = 0; + while(aIndex < ai+alen && index < len) { + indexes[index] = index; + if(aix[aIndex] == index) { + values[index] = a[aIndex] == bval ? 1 : 0; + aIndex++; + } else { + values[index] = 1; + } + index++; + } + for(; index < len; index++) { + indexes[index] = index; + values[index] = 1; + } + c.setSize(len); + return c; } else { SparseRowVector c = allocSparseVector(alen); - for(int j = ai; j < ai+alen; j++) - c.set(aix[j], (a[j] == bval) ? 1 : 0); + int[] indexes = c.indexes(); + double[] values = c.values(); + for(int j = 0; j < ai+alen; j++) { + indexes[j] = aix[j]; + values[j] = a[j] == bval ? 1 : 0; + } + c.setSize(alen); return c; } } @@ -2641,16 +2713,46 @@ public static double[] vectEqualWrite(int len, double[] a, double[] b, int[] aix public static SparseRowVector vectNotequalWrite(int len, double[] a, double bval, int[] aix, int ai, int alen) { if(bval != 0) { - double[] c = allocVector(len, true, 1); - for(int j = ai; j < ai+alen; j++) - c[aix[j]] = ((a[j] != bval) ? 1 : 0); - return new SparseRowVector(c); + SparseRowVector c = allocSparseVector(len); + int[] indexes = c.indexes(); + double[] values = c.values(); + int index = 0; + int aIndex = 0; + while(aIndex < ai+alen && index < len) { + indexes[index] = index; + if(aix[aIndex] == index) { + values[index] = a[aIndex] != bval ? 1 : 0; + aIndex++; + } else { + values[index] = 1; + } + index++; + } + for(; index < len; index++) { + indexes[index] = index; + values[index] = 1; + } + c.setSize(len); + return c; +// double[] c = allocVector(len, true, 1); +// for(int j = ai; j < ai+alen; j++) +// c[aix[j]] = ((a[j] != bval) ? 1 : 0); +// return new SparseRowVector(c); } else { SparseRowVector c = allocSparseVector(alen); + int[] indexes = c.indexes(); + double[] values = c.values(); for(int j = 0; j < ai+alen; j++) { - c.set(aix[j], (a[j] != bval) ? 1 : 0); + indexes[j] = aix[j]; + values[j] = a[j] != bval ? 1 : 0; } + c.setSize(alen); return c; +// SparseRowVector c = allocSparseVector(alen); +// for(int j = 0; j < ai+alen; j++) { +// c.set(aix[j], (a[j] != bval) ? 1 : 0); +// } +// return c; } } @@ -2687,15 +2789,36 @@ public static SparseRowVector vectNotequalWrite(int len, double[] a, double[] b, public static SparseRowVector vectLessWrite(int len, double[] a, double bval, int[] aix, int ai, int alen) { if(bval > 0) { - double[] c = allocVector(len, true, 1); - for(int j = ai; j < ai+alen; j++) - c[aix[j]] = (a[j] < bval) ? 1 : 0; - return new SparseRowVector(c); + SparseRowVector c = allocSparseVector(len); + int[] indexes = c.indexes(); + double[] values = c.values(); + int index = 0; + int aIndex = 0; + while(aIndex < ai+alen && index < len) { + indexes[index] = index; + if(aix[aIndex] == index) { + values[index] = a[aIndex] < bval ? 1 : 0; + aIndex++; + } else { + values[index] = 1; + } + index++; + } + for(; index < len; index++) { + indexes[index] = index; + values[index] = 1; + } + c.setSize(len); + return c; } else { SparseRowVector c = allocSparseVector(alen); - for(int j = ai; j < ai+alen; j++) { - c.set(aix[j], (a[j] < bval) ? 1 : 0); + int[] indexes = c.indexes(); + double[] values = c.values(); + for(int j = 0; j < ai+alen; j++) { + indexes[j] = aix[j]; + values[j] = a[j] < bval ? 1 : 0; } + c.setSize(alen); return c; } } From c7deeb3085b8e7ce8e2defaf6547936c730a0489 Mon Sep 17 00:00:00 2001 From: Frxms Date: Wed, 18 Jun 2025 20:23:46 +0200 Subject: [PATCH 10/60] notEqual vec-vec implementation --- .../runtime/codegen/LibSpoofPrimitives.java | 54 +++++++++++++++---- 1 file changed, 45 insertions(+), 9 deletions(-) diff --git a/src/main/java/org/apache/sysds/runtime/codegen/LibSpoofPrimitives.java b/src/main/java/org/apache/sysds/runtime/codegen/LibSpoofPrimitives.java index 9d48524236e..0257e83c5ab 100644 --- a/src/main/java/org/apache/sysds/runtime/codegen/LibSpoofPrimitives.java +++ b/src/main/java/org/apache/sysds/runtime/codegen/LibSpoofPrimitives.java @@ -2761,30 +2761,66 @@ public static SparseRowVector vectNotequalWrite(int len, double bval, double[] a } public static SparseRowVector vectNotequalWrite(int len, double[] a, double[] b, int[] aix, int[] bix, int ai, int bi, int alen, int blen) { - SparseRowVector c = allocSparseVector(alen); + SparseRowVector c = allocSparseVector(alen+blen); int aIndex = ai; int bIndex = bi; + int index = 0; + int[] indexes = c.indexes(); + double[] values = c.values(); while(aIndex < ai+alen && bIndex < bi+blen) { int aIdx = aix[aIndex]; int bIdx = bix[bIndex]; if(aIdx == bIdx) { - c.set(aIdx, (a[aIndex] != b[bIndex]) ? 1 : 0); + indexes[index] = aIdx; + values[index] = (a[aIndex] != b[bIndex]) ? 1 : 0; aIndex++; bIndex++; + index++; } else if(aIdx < bIdx) { - //todo: this might be too unsafe - c.set(aIdx, (a[aIndex] != 0) ? 1 : 0); - // c.set(aIdx, 1); + indexes[index] = aIdx; + values[index] = a[aIndex] != 0 ? 1 : 0; aIndex++; + index++; } else { - c.set(bIdx, (b[bIndex] != 0) ? 1 : 0); - // c.set(bIdx, 1); + indexes[index] = bIdx; + values[index] = b[bIndex] != 0 ? 1 : 0; bIndex++; + index++; } } - for (; aIndex < ai+alen; aIndex++) c.set(aix[aIndex], (a[aIndex] != 0) ? 1 : 0); - for (; bIndex < bi+blen; bIndex++) c.set(bix[bIndex], (b[bIndex] != 0) ? 1 : 0); + for (; aIndex < ai+alen; aIndex++) { + indexes[index] = aix[aIndex]; + values[index] = a[aIndex] != 0 ? 1 : 0; + index++; + } + for (; bIndex < bi+blen; bIndex++) { + indexes[index] = bix[bIndex]; + values[index] = b[bIndex] != 0 ? 1 : 0; + index++; + } + c.setSize(index); return c; +// SparseRowVector c = allocSparseVector(alen); +// int aIndex = ai; +// int bIndex = bi; +// while(aIndex < ai+alen && bIndex < bi+blen) { +// int aIdx = aix[aIndex]; +// int bIdx = bix[bIndex]; +// if(aIdx == bIdx) { +// c.set(aIdx, (a[aIndex] != b[bIndex]) ? 1 : 0); +// aIndex++; +// bIndex++; +// } else if(aIdx < bIdx) { +// c.set(aIdx, (a[aIndex] != 0) ? 1 : 0); +// aIndex++; +// } else { +// c.set(bIdx, (b[bIndex] != 0) ? 1 : 0); +// bIndex++; +// } +// } +// for (; aIndex < ai+alen; aIndex++) c.set(aix[aIndex], (a[aIndex] != 0) ? 1 : 0); +// for (; bIndex < bi+blen; bIndex++) c.set(bix[bIndex], (b[bIndex] != 0) ? 1 : 0); +// return c; } public static SparseRowVector vectLessWrite(int len, double[] a, double bval, int[] aix, int ai, int alen) { From 6fdea69130ca820b3e04a9edd125290c51f0542d Mon Sep 17 00:00:00 2001 From: Frxms Date: Wed, 18 Jun 2025 22:09:51 +0200 Subject: [PATCH 11/60] change signature of all primitives --- .../runtime/codegen/LibSpoofPrimitives.java | 314 +++++++++++++----- 1 file changed, 227 insertions(+), 87 deletions(-) diff --git a/src/main/java/org/apache/sysds/runtime/codegen/LibSpoofPrimitives.java b/src/main/java/org/apache/sysds/runtime/codegen/LibSpoofPrimitives.java index 0257e83c5ab..69a07ecf4d6 100644 --- a/src/main/java/org/apache/sysds/runtime/codegen/LibSpoofPrimitives.java +++ b/src/main/java/org/apache/sysds/runtime/codegen/LibSpoofPrimitives.java @@ -2734,10 +2734,6 @@ public static SparseRowVector vectNotequalWrite(int len, double[] a, double bval } c.setSize(len); return c; -// double[] c = allocVector(len, true, 1); -// for(int j = ai; j < ai+alen; j++) -// c[aix[j]] = ((a[j] != bval) ? 1 : 0); -// return new SparseRowVector(c); } else { SparseRowVector c = allocSparseVector(alen); int[] indexes = c.indexes(); @@ -2748,11 +2744,6 @@ public static SparseRowVector vectNotequalWrite(int len, double[] a, double bval } c.setSize(alen); return c; -// SparseRowVector c = allocSparseVector(alen); -// for(int j = 0; j < ai+alen; j++) { -// c.set(aix[j], (a[j] != bval) ? 1 : 0); -// } -// return c; } } @@ -2800,27 +2791,6 @@ public static SparseRowVector vectNotequalWrite(int len, double[] a, double[] b, } c.setSize(index); return c; -// SparseRowVector c = allocSparseVector(alen); -// int aIndex = ai; -// int bIndex = bi; -// while(aIndex < ai+alen && bIndex < bi+blen) { -// int aIdx = aix[aIndex]; -// int bIdx = bix[bIndex]; -// if(aIdx == bIdx) { -// c.set(aIdx, (a[aIndex] != b[bIndex]) ? 1 : 0); -// aIndex++; -// bIndex++; -// } else if(aIdx < bIdx) { -// c.set(aIdx, (a[aIndex] != 0) ? 1 : 0); -// aIndex++; -// } else { -// c.set(bIdx, (b[bIndex] != 0) ? 1 : 0); -// bIndex++; -// } -// } -// for (; aIndex < ai+alen; aIndex++) c.set(aix[aIndex], (a[aIndex] != 0) ? 1 : 0); -// for (; bIndex < bi+blen; bIndex++) c.set(bix[bIndex], (b[bIndex] != 0) ? 1 : 0); -// return c; } public static SparseRowVector vectLessWrite(int len, double[] a, double bval, int[] aix, int ai, int alen) { @@ -2864,40 +2834,79 @@ public static SparseRowVector vectLessWrite(int len, double bval, double[] a, in } public static SparseRowVector vectLessWrite(int len, double[] a, double[] b, int[] aix, int[] bix, int ai, int bi, int alen, int blen) { - SparseRowVector c = allocSparseVector(alen); + SparseRowVector c = allocSparseVector(alen+blen); int aIndex = ai; int bIndex = bi; + int index = 0; + int[] indexes = c.indexes(); + double[] values = c.values(); while(aIndex < ai+alen && bIndex < bi+blen) { int aIdx = aix[aIndex]; int bIdx = bix[bIndex]; if(aIdx == bIdx) { - c.set(aIdx, (a[aIndex] < b[bIndex]) ? 1 : 0); + indexes[index] = aIdx; + values[index] = (a[aIndex] < b[bIndex]) ? 1 : 0; aIndex++; bIndex++; + index++; } else if(aIdx < bIdx) { - c.set(aIdx, (a[aIndex] < 0) ? 1 : 0); + indexes[index] = aIdx; + values[index] = a[aIndex] < 0 ? 1 : 0; aIndex++; + index++; } else { - c.set(bIdx, (0 < b[bIndex]) ? 1 : 0); + indexes[index] = bIdx; + values[index] = 0 < b[bIndex] ? 1 : 0; bIndex++; + index++; } } - for(; aIndex < ai+alen; aIndex++) c.set(aix[aIndex], (a[aIndex] < 0) ? 1 : 0); - for(; bIndex < bi+blen; bIndex++) c.set(bix[bIndex], (0 < b[bIndex]) ? 1 : 0); + for (; aIndex < ai+alen; aIndex++) { + indexes[index] = aix[aIndex]; + values[index] = a[aIndex] < 0? 1 : 0; + index++; + } + for (; bIndex < bi+blen; bIndex++) { + indexes[index] = bix[bIndex]; + values[index] = 0 < b[bIndex] ? 1 : 0; + index++; + } + c.setSize(index); return c; } public static SparseRowVector vectLessequalWrite(int len, double[] a, double bval, int[] aix, int ai, int alen) { if(bval >= 0) { - double[] c = allocVector(len, true, 1); - for(int j = ai; j < ai+alen; j++) - c[aix[j]] = (a[j] <= bval) ? 1 : 0; - return new SparseRowVector(c); + SparseRowVector c = allocSparseVector(len); + int[] indexes = c.indexes(); + double[] values = c.values(); + int index = 0; + int aIndex = 0; + while(aIndex < ai+alen && index < len) { + indexes[index] = index; + if(aix[aIndex] == index) { + values[index] = a[aIndex] <= bval ? 1 : 0; + aIndex++; + } else { + values[index] = 1; + } + index++; + } + for(; index < len; index++) { + indexes[index] = index; + values[index] = 1; + } + c.setSize(len); + return c; } else { SparseRowVector c = allocSparseVector(alen); - for(int j = ai; j < ai+alen; j++) { - c.set(aix[j], (a[j] <= bval) ? 1 : 0); + int[] indexes = c.indexes(); + double[] values = c.values(); + for(int j = 0; j < ai+alen; j++) { + indexes[j] = aix[j]; + values[j] = a[j] <= bval ? 1 : 0; } + c.setSize(alen); return c; } } @@ -2933,15 +2942,36 @@ public static double[] vectLessequalWrite(int len, double[] a, double[] b, int[] public static SparseRowVector vectGreaterWrite(int len, double[] a, double bval, int[] aix, int ai, int alen) { if(bval < 0) { - double[] c = allocVector(len, true, 1); - for(int j = ai; j < ai+alen; j++) - c[aix[j]] = (a[j] > bval) ? 1 : 0; - return new SparseRowVector(c); + SparseRowVector c = allocSparseVector(len); + int[] indexes = c.indexes(); + double[] values = c.values(); + int index = 0; + int aIndex = 0; + while(aIndex < ai+alen && index < len) { + indexes[index] = index; + if(aix[aIndex] == index) { + values[index] = a[aIndex] > bval ? 1 : 0; + aIndex++; + } else { + values[index] = 1; + } + index++; + } + for(; index < len; index++) { + indexes[index] = index; + values[index] = 1; + } + c.setSize(len); + return c; } else { SparseRowVector c = allocSparseVector(alen); - for(int j = ai; j < ai+alen; j++) { - c.set(aix[j], (a[j] > bval) ? 1 : 0); + int[] indexes = c.indexes(); + double[] values = c.values(); + for(int j = 0; j < ai+alen; j++) { + indexes[j] = aix[j]; + values[j] = a[j] > bval ? 1 : 0; } + c.setSize(alen); return c; } } @@ -2951,40 +2981,79 @@ public static SparseRowVector vectGreaterWrite(int len, double bval, double[] a, } public static SparseRowVector vectGreaterWrite(int len, double[] a, double[] b, int[] aix, int[] bix, int ai, int bi, int alen, int blen) { - SparseRowVector c = allocSparseVector(alen); + SparseRowVector c = allocSparseVector(alen+blen); int aIndex = ai; int bIndex = bi; + int index = 0; + int[] indexes = c.indexes(); + double[] values = c.values(); while(aIndex < ai+alen && bIndex < bi+blen) { int aIdx = aix[aIndex]; int bIdx = bix[bIndex]; if(aIdx == bIdx) { - c.set(aIdx, (a[aIndex] > b[bIndex]) ? 1 : 0); + indexes[index] = aIdx; + values[index] = (a[aIndex] > b[bIndex]) ? 1 : 0; aIndex++; bIndex++; + index++; } else if(aIdx < bIdx) { - c.set(aIdx, (a[aIndex] > 0) ? 1 : 0); + indexes[index] = aIdx; + values[index] = a[aIndex] > 0 ? 1 : 0; aIndex++; + index++; } else { - c.set(bIdx, (0 > b[bIndex]) ? 1 : 0); + indexes[index] = bIdx; + values[index] = 0 > b[bIndex] ? 1 : 0; bIndex++; + index++; } } - for(; aIndex < ai+alen; aIndex++) c.set(aix[aIndex], (a[aIndex] > 0) ? 1 : 0); - for(; bIndex < bi+blen; bIndex++) c.set(bix[bIndex], (0 > b[bIndex]) ? 1 : 0); + for (; aIndex < ai+alen; aIndex++) { + indexes[index] = aix[aIndex]; + values[index] = a[aIndex] > 0 ? 1 : 0; + index++; + } + for (; bIndex < bi+blen; bIndex++) { + indexes[index] = bix[bIndex]; + values[index] = 0 > b[bIndex] ? 1 : 0; + index++; + } + c.setSize(index); return c; } public static SparseRowVector vectGreaterequalWrite(int len, double[] a, double bval, int[] aix, int ai, int alen) { if(bval <= 0) { - double[] c = allocVector(len, true, 1); - for(int j = ai; j < ai+alen; j++) - c[aix[j]] = (a[j] >= bval) ? 1 : 0; - return new SparseRowVector(c); + SparseRowVector c = allocSparseVector(len); + int[] indexes = c.indexes(); + double[] values = c.values(); + int index = 0; + int aIndex = 0; + while(aIndex < ai+alen && index < len) { + indexes[index] = index; + if(aix[aIndex] == index) { + values[index] = a[aIndex] >= bval ? 1 : 0; + aIndex++; + } else { + values[index] = 1; + } + index++; + } + for(; index < len; index++) { + indexes[index] = index; + values[index] = 1; + } + c.setSize(len); + return c; } else { SparseRowVector c = allocSparseVector(alen); - for(int j = ai; j < ai+alen; j++) { - c.set(aix[j], (a[j] >= bval) ? 1 : 0); + int[] indexes = c.indexes(); + double[] values = c.values(); + for(int j = 0; j < ai+alen; j++) { + indexes[j] = aix[j]; + values[j] = a[j] >= bval ? 1 : 0; } + c.setSize(alen); return c; } } @@ -3020,9 +3089,14 @@ public static double[] vectGreaterequalWrite(int len, double[] a, double[] b, in public static SparseRowVector vectBitwandWrite(int len, double[] a, double bval, int[] aix, int ai, int alen) { SparseRowVector c = allocSparseVector(alen); + int[] indexes = c.indexes(); + double[] values = c.values(); int bval1 = (int) bval; - for(int j = ai; j < ai+alen; j++) - c.set(aix[j], bwAnd(a[j], bval1)); + for( int j = ai; j < ai+alen; j++ ) { + indexes[j] = aix[j]; + values[j] = bwAnd(a[j], bval1); + } + c.setSize(alen); return c; } @@ -3034,103 +3108,169 @@ public static SparseRowVector vectBitwandWrite(int len, double[] a, double[] b, SparseRowVector c = allocSparseVector(alen); int aIndex = ai; int bIndex = bi; + int index = 0; + int[] indexes = c.indexes(); + double[] values = c.values(); while(aIndex < ai+alen && bIndex < bi+blen) { int aIdx = aix[aIndex]; int bIdx = bix[bIndex]; if(aIdx == bIdx) { - c.set(aIdx, bwAnd(a[aIndex], b[bIndex])); + indexes[index] = aIdx; + values[index] = bwAnd(a[aIndex], b[bIndex]); aIndex++; bIndex++; + index++; } else if(aIdx < bIdx) { aIndex++; } else { bIndex++; } } + c.setSize(index); return c; } public static SparseRowVector vectSqrtWrite(int len, double[] a, int[] aix, int ai, int alen) { SparseRowVector c = allocSparseVector(alen); - for(int j = ai; j < ai+alen; j++) - c.set(aix[j], Math.sqrt(a[j])); + int[] indexes = c.indexes(); + double[] values = c.values(); + for(int j = 0; j < ai+alen; j++) { + indexes[j] = aix[j]; + values[j] = Math.sqrt(a[j]); + } + c.setSize(alen); return c; } public static SparseRowVector vectAbsWrite(int len, double[] a, int[] aix, int ai, int alen) { SparseRowVector c = allocSparseVector(alen); - for(int j = ai; j < ai+alen; j++) - c.set(aix[j], Math.abs(a[j])); + int[] indexes = c.indexes(); + double[] values = c.values(); + for(int j = 0; j < ai+alen; j++) { + indexes[j] = aix[j]; + values[j] = Math.abs(a[j]); + } + c.setSize(alen); return c; } public static SparseRowVector vectRoundWrite(int len, double[] a, int[] aix, int ai, int alen) { SparseRowVector c = allocSparseVector(alen); - for(int j = ai; j < ai+alen; j++) - c.set(aix[j], Math.round(a[j])); + int[] indexes = c.indexes(); + double[] values = c.values(); + for(int j = 0; j < ai+alen; j++) { + indexes[j] = aix[j]; + values[j] = Math.round(a[j]); + } + c.setSize(alen); return c; } public static SparseRowVector vectCeilWrite(int len, double[] a, int[] aix, int ai, int alen) { SparseRowVector c = allocSparseVector(alen); - for(int j = ai; j < ai+alen; j++) - c.set(aix[j], FastMath.ceil(a[j])); + int[] indexes = c.indexes(); + double[] values = c.values(); + for(int j = 0; j < ai+alen; j++) { + indexes[j] = aix[j]; + values[j] = Math.ceil(a[j]); + } + c.setSize(alen); return c; } public static SparseRowVector vectFloorWrite(int len, double[] a, int[] aix, int ai, int alen) { SparseRowVector c = allocSparseVector(alen); - for(int j = ai; j < ai+alen; j++) - c.set(aix[j], FastMath.floor(a[j])); + int[] indexes = c.indexes(); + double[] values = c.values(); + for(int j = 0; j < ai+alen; j++) { + indexes[j] = aix[j]; + values[j] = Math.floor(a[j]); + } + c.setSize(alen); return c; } public static SparseRowVector vectSinWrite(int len, double[] a, int[] aix, int ai, int alen) { SparseRowVector c = allocSparseVector(alen); - for(int j = ai; j < ai+alen; j++) - c.set(aix[j], FastMath.sin(a[j])); + int[] indexes = c.indexes(); + double[] values = c.values(); + for(int j = 0; j < ai+alen; j++) { + indexes[j] = aix[j]; + values[j] = Math.sin(a[j]); + } + c.setSize(alen); return c; } public static SparseRowVector vectTanWrite(int len, double[] a, int[] aix, int ai, int alen) { SparseRowVector c = allocSparseVector(alen); - for(int j = ai; j < ai+alen; j++) - c.set(aix[j], FastMath.tan(a[j])); + int[] indexes = c.indexes(); + double[] values = c.values(); + for(int j = 0; j < ai+alen; j++) { + indexes[j] = aix[j]; + values[j] = Math.tan(a[j]); + } + c.setSize(alen); return c; } public static SparseRowVector vectAsinWrite(int len, double[] a, int[] aix, int ai, int alen) { SparseRowVector c = allocSparseVector(alen); - for(int j = ai; j < ai+alen; j++) - c.set(aix[j], FastMath.asin(a[j])); + int[] indexes = c.indexes(); + double[] values = c.values(); + for(int j = 0; j < ai+alen; j++) { + indexes[j] = aix[j]; + values[j] = Math.asin(a[j]); + } + c.setSize(alen); return c; } public static SparseRowVector vectAtanWrite(int len, double[] a, int[] aix, int ai, int alen) { SparseRowVector c = allocSparseVector(alen); - for(int j = ai; j < ai+alen; j++) - c.set(aix[j], FastMath.atan(a[j])); + int[] indexes = c.indexes(); + double[] values = c.values(); + for(int j = 0; j < ai+alen; j++) { + indexes[j] = aix[j]; + values[j] = Math.atan(a[j]); + } + c.setSize(alen); return c; } public static SparseRowVector vectSinhWrite(int len, double[] a, int[] aix, int ai, int alen) { SparseRowVector c = allocSparseVector(alen); - for(int j = ai; j < ai+alen; j++) - c.set(aix[j], FastMath.sinh(a[j])); + int[] indexes = c.indexes(); + double[] values = c.values(); + for(int j = 0; j < ai+alen; j++) { + indexes[j] = aix[j]; + values[j] = Math.sinh(a[j]); + } + c.setSize(alen); return c; } public static SparseRowVector vectTanhWrite(int len, double[] a, int[] aix, int ai, int alen) { SparseRowVector c = allocSparseVector(alen); - for(int j = ai; j < ai+alen; j++) - c.set(aix[j], FastMath.tanh(a[j])); + int[] indexes = c.indexes(); + double[] values = c.values(); + for(int j = 0; j < ai+alen; j++) { + indexes[j] = aix[j]; + values[j] = Math.tanh(a[j]); + } + c.setSize(alen); return c; } public static SparseRowVector vectSignWrite(int len, double[] a, int[] aix, int ai, int alen) { SparseRowVector c = allocSparseVector(alen); - for(int j = ai; j < ai+alen; j++) - c.set(aix[j], FastMath.signum(a[j])); + int[] indexes = c.indexes(); + double[] values = c.values(); + for(int j = 0; j < ai+alen; j++) { + indexes[j] = aix[j]; + values[j] = Math.signum(a[j]); + } + c.setSize(alen); return c; } From d3c341ead6ba80e6dcaf8720251c1162f1e485d0 Mon Sep 17 00:00:00 2001 From: Frxms Date: Thu, 19 Jun 2025 22:52:46 +0200 Subject: [PATCH 12/60] add rowAggPattern49 to test files --- .../functions/codegen/RowAggTmplTest.java | 8 ++- .../functions/codegen/rowAggPattern49.R | 54 +++++++++++++++++++ .../functions/codegen/rowAggPattern49.dml | 48 +++++++++++++++++ 3 files changed, 108 insertions(+), 2 deletions(-) create mode 100644 src/test/scripts/functions/codegen/rowAggPattern49.R create mode 100644 src/test/scripts/functions/codegen/rowAggPattern49.dml diff --git a/src/test/java/org/apache/sysds/test/functions/codegen/RowAggTmplTest.java b/src/test/java/org/apache/sysds/test/functions/codegen/RowAggTmplTest.java index b89f3007b4a..99798988746 100644 --- a/src/test/java/org/apache/sysds/test/functions/codegen/RowAggTmplTest.java +++ b/src/test/java/org/apache/sysds/test/functions/codegen/RowAggTmplTest.java @@ -89,6 +89,7 @@ public class RowAggTmplTest extends AutomatedTestBase private static final String TEST_NAME46 = TEST_NAME+"46"; //conv2d(X - mean(X), F1) + conv2d(X - mean(X), F2); private static final String TEST_NAME47 = TEST_NAME+"47"; //sum(X + rowVars(X)) private static final String TEST_NAME48 = TEST_NAME+"48"; //sum(rowVars(X)) + private static final String TEST_NAME49 = TEST_NAME+"49"; private static final String TEST_DIR = "functions/codegen/"; private static final String TEST_CLASS_DIR = TEST_DIR + RowAggTmplTest.class.getSimpleName() + "/"; @@ -100,7 +101,7 @@ public class RowAggTmplTest extends AutomatedTestBase @Override public void setUp() { TestUtils.clearAssertionInformation(); - for(int i=1; i<=48; i++) + for(int i=1; i<=49; i++) addTestConfiguration( TEST_NAME+i, new TestConfiguration(TEST_CLASS_DIR, TEST_NAME+i, new String[] { String.valueOf(i) }) ); } @@ -827,6 +828,9 @@ public void testCodegenRowAgg48SP() { testCodegenIntegration( TEST_NAME48, false, ExecType.SPARK ); } + @Test + public void testCodegenRowAgg49CP() {testCodegenIntegration( TEST_NAME49, false, ExecType.CP );} + private void testCodegenIntegration( String testname, boolean rewrites, ExecType instType ) { boolean oldFlag = OptimizerUtils.ALLOW_ALGEBRAIC_SIMPLIFICATION; @@ -839,7 +843,7 @@ private void testCodegenIntegration( String testname, boolean rewrites, ExecType String HOME = SCRIPT_DIR + TEST_DIR; fullDMLScriptName = HOME + testname + ".dml"; - programArgs = new String[]{"-explain", "codegen", "-stats", "-args", output("S") }; + programArgs = new String[]{"-explain", "codegen", "-sparseIntermediate", "-stats", "-args", output("S") }; fullRScriptName = HOME + testname + ".R"; rCmd = getRCmd(inputDir(), expectedDir()); diff --git a/src/test/scripts/functions/codegen/rowAggPattern49.R b/src/test/scripts/functions/codegen/rowAggPattern49.R new file mode 100644 index 00000000000..7fc0c17aff0 --- /dev/null +++ b/src/test/scripts/functions/codegen/rowAggPattern49.R @@ -0,0 +1,54 @@ +#------------------------------------------------------------- +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +#------------------------------------------------------------- + +args<-commandArgs(TRUE) +options(digits=22) +library("Matrix") +# library("matrixStats") + +W = matrix(seq(28,29), 1, 2) +J = matrix(0, 1, 8) +Z = cbind(J, W, J) +Y = matrix(0, 10, 18) +X = rbind(Z, Y, Y, Y, Y, Y, Y, Y, Y) +v = seq(1,81) +v1 = seq(20, 37) +W = matrix(seq(13,14), 1, 2) +J = matrix(0, 1, 8) +Z= cbind(J, W, J) +Y = matrix(0, 10, 18) +K = rbind(Z, Y, Y, Y, Y, Y, Y, Y, Y) + +S = (X < rowSums(X*K)) +# S = X*rowMins(K)*X +# S = X*rowSums(K*v)*X +# S = (X*v)/rowSums(X*v) +# S = abs((X*v)/rowSums(X*v)) +# S = (X/v)+rowMeans(X-v) +# S = (X*v)+rowSums(X*v) +# S = (X*rowSums(X*v))/(X*v) + +# S = X*rowSums(X*K) +# S = rowSums((X*v)/K)*v +# S = (K*v)/(rowSums(X*v)) + + +writeMM(as(S, "CsparseMatrix"), paste(args[2], "S", sep="")); diff --git a/src/test/scripts/functions/codegen/rowAggPattern49.dml b/src/test/scripts/functions/codegen/rowAggPattern49.dml new file mode 100644 index 00000000000..9d31565fba7 --- /dev/null +++ b/src/test/scripts/functions/codegen/rowAggPattern49.dml @@ -0,0 +1,48 @@ +#------------------------------------------------------------- +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +#------------------------------------------------------------- + +W = matrix(seq(28,29), 1, 2) +J = matrix(0, 1, 8) +Z= cbind(J, W, J) +Y = matrix(0, 10, 18) +X = rbind(Z, Y, Y, Y, Y, Y, Y, Y, Y) +v = seq(1,81) +v1 = seq(20, 37) +W = matrix(seq(13,14), 1, 2) +J = matrix(0, 1, 8) +Z= cbind(J, W, J) +Y = matrix(0, 10, 18) +K = rbind(Z, Y, Y, Y, Y, Y, Y, Y, Y) + +S = (X < rowSums(X*K)) +# S = X*(k>1)*X +# S = X*rowSums(K*v)*X +# S = (X*v)/rowSums(X*v) +# S = abs((X*v)/rowSums(X*v)) +# S = (X/v)+rowMeans(X-v) +# S = (X*v)+rowSums(X*v) +# S = (X*rowSums(X*v))/(X*v) + +# S = X*rowSums(X*K) +# S = rowSums((X*v)/K)*v +# S = (K*v)/(rowSums(X*v)) + +write(S,$1) From 45fd365f08cbf08e5640516b77e1755a62d118c6 Mon Sep 17 00:00:00 2001 From: Frxms Date: Thu, 19 Jun 2025 23:10:30 +0200 Subject: [PATCH 13/60] work in progress --- .../sysds/hops/codegen/SpoofCompiler.java | 15 ++++++++++++- .../sysds/hops/codegen/cplan/CNodeBinary.java | 22 +++++++++++++++++++ .../sysds/hops/codegen/cplan/CNodeUnary.java | 12 ++++++++++ 3 files changed, 48 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/apache/sysds/hops/codegen/SpoofCompiler.java b/src/main/java/org/apache/sysds/hops/codegen/SpoofCompiler.java index 34329ca64dc..703810fcffb 100644 --- a/src/main/java/org/apache/sysds/hops/codegen/SpoofCompiler.java +++ b/src/main/java/org/apache/sysds/hops/codegen/SpoofCompiler.java @@ -470,7 +470,7 @@ public static Hop optimize( Hop root, boolean recompile ) { * @param recompile true if invoked during dynamic recompilation * @return dag root nodes of modified dag */ - public static ArrayList optimize(ArrayList roots, boolean recompile) + public static ArrayList optimize(ArrayList roots, boolean recompile) { if( roots == null || roots.isEmpty() ) return roots; @@ -503,6 +503,8 @@ public static ArrayList optimize(ArrayList roots, boolean recompile) //remove empty templates with single cnodedata input, remove spurious lookups, //perform common subexpression elimination) cplans = cleanupCPlans(memo, cplans); + + cPlanRowOptimizer(cplans); //explain before modification if( LOG.isTraceEnabled() && !cplans.isEmpty() ) { //existing cplans @@ -673,6 +675,17 @@ public static void setExecTypeSpecificJavaCompiler() { OptimizerUtils.isSparkExecutionMode() ? CompilerType.JANINO : CompilerType.JAVAC; } + + private static void cPlanRowOptimizer(HashMap> cplans) { + for( Entry> e : cplans.entrySet() ) { + CNodeTpl tpl = e.getValue().getValue(); + Hop[] inHops = e.getValue().getKey(); + + if(tpl.getOutput() instanceof CNodeBinary) { + ((CNodeBinary) tpl.getOutput()).setSparseRows(); + } + } + } //////////////////// // Codegen plan construction diff --git a/src/main/java/org/apache/sysds/hops/codegen/cplan/CNodeBinary.java b/src/main/java/org/apache/sysds/hops/codegen/cplan/CNodeBinary.java index e2bc0f47e9b..7a523759a6c 100644 --- a/src/main/java/org/apache/sysds/hops/codegen/cplan/CNodeBinary.java +++ b/src/main/java/org/apache/sysds/hops/codegen/cplan/CNodeBinary.java @@ -24,6 +24,7 @@ import org.apache.commons.lang3.StringUtils; import org.apache.sysds.api.DMLScript; import org.apache.sysds.common.Opcodes; +import org.apache.sysds.hops.OptimizerUtils; import org.apache.sysds.hops.codegen.template.TemplateUtils; import org.apache.sysds.common.Types.DataType; import org.apache.sysds.runtime.util.UtilFunctions; @@ -127,6 +128,7 @@ public boolean isNotSupportedBySpoofCUDA() { } private final BinType _type; + private boolean sparseRows = false; public CNodeBinary( CNode in1, CNode in2, BinType type ) { //canonicalize commutative matrix-scalar operations @@ -147,6 +149,19 @@ public CNodeBinary( CNode in1, CNode in2, BinType type ) { public BinType getType() { return _type; } + + public void setSparseRows() { + if(this._inputs == null) { + return; + } + for(CNode input : this._inputs) { + if(input instanceof CNodeBinary) + ((CNodeBinary)input).setSparseRows(); + else if(input instanceof CNodeUnary) + ((CNodeUnary)input).iterator(); + } + this.sparseRows = true; + } @Override public String codegen(boolean sparse, GeneratorAPI api) { @@ -286,6 +301,13 @@ private CNode getIntermediateInputVector() { return null; } + private boolean getSparsity() { +// double sparsity = OptimizerUtils.getBinaryOpSparsity(); + switch(_type) { + default: return true; + } + } + public boolean getOutputType(boolean scalarVector, boolean lsparseLhs, boolean lsparseRhs) { switch(_type) { case VECT_MULT_SCALAR: diff --git a/src/main/java/org/apache/sysds/hops/codegen/cplan/CNodeUnary.java b/src/main/java/org/apache/sysds/hops/codegen/cplan/CNodeUnary.java index f8d5c3eeb4c..773d6fb5121 100644 --- a/src/main/java/org/apache/sysds/hops/codegen/cplan/CNodeUnary.java +++ b/src/main/java/org/apache/sysds/hops/codegen/cplan/CNodeUnary.java @@ -101,6 +101,18 @@ public void setType(UnaryType type) { _type = type; } + public void iterator() { + if(this._inputs == null) { + return; + } + for(CNode input : this._inputs) { + if(input instanceof CNodeBinary) + ((CNodeBinary)input).setSparseRows(); + else if(input instanceof CNodeUnary) + ((CNodeUnary)input).iterator(); + } + } + @Override public String codegen(boolean sparse, GeneratorAPI api) { if( isGenerated() ) From b29b83195e70479c39ded9ca30702594280ebf81 Mon Sep 17 00:00:00 2001 From: Frxms Date: Fri, 20 Jun 2025 14:38:35 +0200 Subject: [PATCH 14/60] delete previous optimizer --- .../apache/sysds/hops/codegen/SpoofCompiler.java | 13 ------------- .../apache/sysds/hops/codegen/cplan/CNodeUnary.java | 12 ------------ 2 files changed, 25 deletions(-) diff --git a/src/main/java/org/apache/sysds/hops/codegen/SpoofCompiler.java b/src/main/java/org/apache/sysds/hops/codegen/SpoofCompiler.java index 703810fcffb..307205dbc89 100644 --- a/src/main/java/org/apache/sysds/hops/codegen/SpoofCompiler.java +++ b/src/main/java/org/apache/sysds/hops/codegen/SpoofCompiler.java @@ -503,8 +503,6 @@ public static ArrayList optimize(ArrayList roots, boolean recompile) //remove empty templates with single cnodedata input, remove spurious lookups, //perform common subexpression elimination) cplans = cleanupCPlans(memo, cplans); - - cPlanRowOptimizer(cplans); //explain before modification if( LOG.isTraceEnabled() && !cplans.isEmpty() ) { //existing cplans @@ -675,17 +673,6 @@ public static void setExecTypeSpecificJavaCompiler() { OptimizerUtils.isSparkExecutionMode() ? CompilerType.JANINO : CompilerType.JAVAC; } - - private static void cPlanRowOptimizer(HashMap> cplans) { - for( Entry> e : cplans.entrySet() ) { - CNodeTpl tpl = e.getValue().getValue(); - Hop[] inHops = e.getValue().getKey(); - - if(tpl.getOutput() instanceof CNodeBinary) { - ((CNodeBinary) tpl.getOutput()).setSparseRows(); - } - } - } //////////////////// // Codegen plan construction diff --git a/src/main/java/org/apache/sysds/hops/codegen/cplan/CNodeUnary.java b/src/main/java/org/apache/sysds/hops/codegen/cplan/CNodeUnary.java index 773d6fb5121..f8d5c3eeb4c 100644 --- a/src/main/java/org/apache/sysds/hops/codegen/cplan/CNodeUnary.java +++ b/src/main/java/org/apache/sysds/hops/codegen/cplan/CNodeUnary.java @@ -101,18 +101,6 @@ public void setType(UnaryType type) { _type = type; } - public void iterator() { - if(this._inputs == null) { - return; - } - for(CNode input : this._inputs) { - if(input instanceof CNodeBinary) - ((CNodeBinary)input).setSparseRows(); - else if(input instanceof CNodeUnary) - ((CNodeUnary)input).iterator(); - } - } - @Override public String codegen(boolean sparse, GeneratorAPI api) { if( isGenerated() ) From 8a9a3a0a0bcc0863c9fadc04c95d9c6702db0535 Mon Sep 17 00:00:00 2001 From: Frxms Date: Fri, 20 Jun 2025 14:42:18 +0200 Subject: [PATCH 15/60] implement new optimizer for the Binary template --- .../sysds/hops/codegen/cplan/CNodeBinary.java | 56 +++++++++++-------- .../hops/codegen/cplan/CodeTemplate.java | 4 +- .../sysds/hops/codegen/cplan/java/Binary.java | 12 ++-- .../hops/codegen/template/TemplateRow.java | 19 ++++++- 4 files changed, 58 insertions(+), 33 deletions(-) diff --git a/src/main/java/org/apache/sysds/hops/codegen/cplan/CNodeBinary.java b/src/main/java/org/apache/sysds/hops/codegen/cplan/CNodeBinary.java index 7a523759a6c..65d74827ee8 100644 --- a/src/main/java/org/apache/sysds/hops/codegen/cplan/CNodeBinary.java +++ b/src/main/java/org/apache/sysds/hops/codegen/cplan/CNodeBinary.java @@ -24,7 +24,6 @@ import org.apache.commons.lang3.StringUtils; import org.apache.sysds.api.DMLScript; import org.apache.sysds.common.Opcodes; -import org.apache.sysds.hops.OptimizerUtils; import org.apache.sysds.hops.codegen.template.TemplateUtils; import org.apache.sysds.common.Types.DataType; import org.apache.sysds.runtime.util.UtilFunctions; @@ -128,8 +127,8 @@ public boolean isNotSupportedBySpoofCUDA() { } private final BinType _type; - private boolean sparseRows = false; - + private boolean sparseTemplate; + public CNodeBinary( CNode in1, CNode in2, BinType type ) { //canonicalize commutative matrix-scalar operations //to increase reuse potential @@ -146,21 +145,25 @@ public CNodeBinary( CNode in1, CNode in2, BinType type ) { setOutputDims(); } - public BinType getType() { - return _type; + public CNodeBinary( CNode in1, CNode in2, BinType type, double sparsityEst ) { + //canonicalize commutative matrix-scalar operations + //to increase reuse potential + if( type.isCommutative() && in1 instanceof CNodeData + && in1.getDataType()==DataType.SCALAR ) { + CNode tmp = in1; + in1 = in2; + in2 = tmp; + } + + _inputs.add(in1); + _inputs.add(in2); + _type = type; + setOutputDims(); + sparseTemplate = getTemplateTpe(sparsityEst); } - public void setSparseRows() { - if(this._inputs == null) { - return; - } - for(CNode input : this._inputs) { - if(input instanceof CNodeBinary) - ((CNodeBinary)input).setSparseRows(); - else if(input instanceof CNodeUnary) - ((CNodeUnary)input).iterator(); - } - this.sparseRows = true; + public BinType getType() { + return _type; } @Override @@ -178,7 +181,7 @@ public String codegen(boolean sparse, GeneratorAPI api) { * todo: remember that only certain primitives will be called through this method, * because the optimizer will choose which primitive functions should be calculated sparse and which not */ - if(DMLScript.SPARSE_INTERMEDIATE) { + if(sparseTemplate) { //generate binary operation (use sparse template, if data input) boolean lsparseLhs = sparse ? _inputs.get(0) instanceof CNodeData && _inputs.get(0).getVarname().startsWith("a") || @@ -195,7 +198,7 @@ public String codegen(boolean sparse, GeneratorAPI api) { && _inputs.get(1).getDataType().isMatrix(); String var = createVarname(sparse && getOutputType(scalarVector, lsparseLhs, lsparseRhs)); String tmp = getLanguageTemplateClass(this, api) - .getTemplate(_type, lsparseLhs, lsparseRhs, scalarVector, scalarInput, vectorVector); + .getTemplate(_type, lsparseLhs, lsparseRhs, scalarVector, scalarInput, vectorVector, sparseTemplate); tmp = tmp.replace("%TMP%", var); @@ -249,7 +252,7 @@ public String codegen(boolean sparse, GeneratorAPI api) { boolean vectorVector = _inputs.get(0).getDataType().isMatrix() && _inputs.get(1).getDataType().isMatrix(); String var = createVarname(); String tmp = getLanguageTemplateClass(this, api).getTemplate(_type, lsparseLhs, lsparseRhs, scalarVector, - scalarInput, vectorVector); + scalarInput, vectorVector, false); tmp = tmp.replace("%TMP%", var); @@ -301,10 +304,17 @@ private CNode getIntermediateInputVector() { return null; } - private boolean getSparsity() { -// double sparsity = OptimizerUtils.getBinaryOpSparsity(); - switch(_type) { - default: return true; + private boolean getTemplateTpe(double sparsityEst) { + if(!DMLScript.SPARSE_INTERMEDIATE) { + return false; + } else { + switch(_type) { + case VECT_MULT: return sparsityEst < 0.08 ? true : false; + case VECT_MULT_SCALAR: return sparsityEst < 0.15 ? true : false; + case VECT_LESS_SCALAR: return false; + case VECT_LESS: return sparsityEst < 0.035 ? true : false; + default: return true; + } } } diff --git a/src/main/java/org/apache/sysds/hops/codegen/cplan/CodeTemplate.java b/src/main/java/org/apache/sysds/hops/codegen/cplan/CodeTemplate.java index e29594f5259..71639638b77 100644 --- a/src/main/java/org/apache/sysds/hops/codegen/cplan/CodeTemplate.java +++ b/src/main/java/org/apache/sysds/hops/codegen/cplan/CodeTemplate.java @@ -33,7 +33,7 @@ public String getTemplate() { } public String getTemplate(CNodeBinary.BinType type, boolean sparseLhs, boolean sparseRhs, boolean scalarVector, - boolean scalarInput, boolean vectorVector) { + boolean scalarInput, boolean vectorVector, boolean sparseTemplate) { throw new RuntimeException("Calling wrong getTemplate method on " + getClass().getCanonicalName()); } @@ -68,5 +68,5 @@ public static String getTemplate(String templateFileName) { return null; } } - + } diff --git a/src/main/java/org/apache/sysds/hops/codegen/cplan/java/Binary.java b/src/main/java/org/apache/sysds/hops/codegen/cplan/java/Binary.java index 81b2517fe1f..eb4864b8607 100644 --- a/src/main/java/org/apache/sysds/hops/codegen/cplan/java/Binary.java +++ b/src/main/java/org/apache/sysds/hops/codegen/cplan/java/Binary.java @@ -26,7 +26,7 @@ public class Binary extends CodeTemplate { public String getTemplate(BinType type, boolean sparseLhs, boolean sparseRhs, - boolean scalarVector, boolean scalarInput, boolean vectorVector) + boolean scalarVector, boolean scalarInput, boolean vectorVector, boolean sparseTemplate) { switch (type) { case ROWMAXS_VECTMULT: @@ -75,7 +75,7 @@ public String getTemplate(BinType type, boolean sparseLhs, boolean sparseRhs, if( scalarVector ) return sparseRhs ? " double[] %TMP% = LibSpoofPrimitives.vect"+vectName+"Write(%IN1%, %IN2v%, %IN2i%, %POS2%, alen, %LEN%);\n" : " double[] %TMP% = LibSpoofPrimitives.vect"+vectName+"Write(%IN1%, %IN2%, %POS2%, %LEN%);\n"; - else if(DMLScript.SPARSE_INTERMEDIATE) { + else if(sparseTemplate) { return sparseLhs ? " SparseRowVector %TMP% = LibSpoofPrimitives.vect"+vectName+"Write(%LEN%, %IN1v%, %IN2%, %IN1i%, %POS1%, %LEN1%);\n" : " double[] %TMP% = LibSpoofPrimitives.vect"+vectName+"Write(%IN1%, %IN2%, %POS1%, %LEN%);\n"; } else { @@ -97,11 +97,11 @@ else if(DMLScript.SPARSE_INTERMEDIATE) { String vectName = type.getVectorPrimitiveName(); if(scalarVector) { if(sparseRhs) - return DMLScript.SPARSE_INTERMEDIATE ? " SparseRowVector %TMP% = LibSpoofPrimitives.vect"+vectName+"Write(%LEN%, %IN1%, %IN2v%, %IN2i%, %POS2%, %LEN1%);\n" : + return sparseTemplate ? " SparseRowVector %TMP% = LibSpoofPrimitives.vect"+vectName+"Write(%LEN%, %IN1%, %IN2v%, %IN2i%, %POS2%, %LEN1%);\n" : " double[] %TMP% = LibSpoofPrimitives.vect"+vectName+"Write(%IN1%, %IN2v%, %IN2i%, %POS2%, alen, %LEN%);\n"; } else { if(sparseLhs) - return DMLScript.SPARSE_INTERMEDIATE ? " SparseRowVector %TMP% = LibSpoofPrimitives.vect"+vectName+"Write(%LEN%, %IN1v%, %IN2%, %IN1i%, %POS1%, %LEN1%);\n" : + return sparseTemplate ? " SparseRowVector %TMP% = LibSpoofPrimitives.vect"+vectName+"Write(%LEN%, %IN1v%, %IN2%, %IN1i%, %POS1%, %LEN1%);\n" : " double[] %TMP% = LibSpoofPrimitives.vect"+vectName+"Write(%IN1v%, %IN2%, %IN1i%, %POS1%, alen, %LEN%);\n"; } return " double[] %TMP% = LibSpoofPrimitives.vect"+vectName+"Write(%IN1%, %IN2%, %POS1%, %LEN%);\n"; @@ -144,7 +144,7 @@ else if( !vectorVector ) case VECT_LESS: case VECT_GREATER:{ String vectName = type.getVectorPrimitiveName(); - if(DMLScript.SPARSE_INTERMEDIATE && sparseLhs && sparseRhs) { + if(sparseTemplate && sparseLhs && sparseRhs) { return " SparseRowVector %TMP% = LibSpoofPrimitives.vect"+vectName+"Write(%LEN%, %IN1v%, %IN2v%, %IN1i%, %IN2i%, %POS1%, %POS2%, %LEN1%, %LEN2%);\n"; } else { return sparseLhs ? @@ -158,7 +158,7 @@ else if( !vectorVector ) case VECT_LESSEQUAL: case VECT_GREATEREQUAL: { String vectName = type.getVectorPrimitiveName(); - if(DMLScript.SPARSE_INTERMEDIATE && sparseLhs && sparseRhs) { + if(sparseTemplate && sparseLhs && sparseRhs) { return " double[] %TMP% = LibSpoofPrimitives.vect"+vectName+"Write(%LEN%, %IN1v%, %IN2v%, %IN1i%, %IN2i%, %POS1%, %POS2%, %LEN1%, %LEN2%);\n"; } else { return sparseLhs ? diff --git a/src/main/java/org/apache/sysds/hops/codegen/template/TemplateRow.java b/src/main/java/org/apache/sysds/hops/codegen/template/TemplateRow.java index 955bf778b8b..64c4c2abace 100644 --- a/src/main/java/org/apache/sysds/hops/codegen/template/TemplateRow.java +++ b/src/main/java/org/apache/sysds/hops/codegen/template/TemplateRow.java @@ -37,6 +37,7 @@ import org.apache.sysds.hops.ParameterizedBuiltinOp; import org.apache.sysds.hops.TernaryOp; import org.apache.sysds.hops.UnaryOp; +import org.apache.sysds.hops.OptimizerUtils; import org.apache.sysds.hops.codegen.cplan.CNode; import org.apache.sysds.hops.codegen.cplan.CNodeBinary; import org.apache.sysds.hops.codegen.cplan.CNodeData; @@ -440,7 +441,11 @@ else if(hop instanceof BinaryOp) cdata1 = new CNodeUnary(cdata1, UnaryType.LOOKUP_R); if( TemplateUtils.isColVector(cdata2) ) cdata2 = new CNodeUnary(cdata2, UnaryType.LOOKUP_R); - out = getVectorBinary(cdata1, cdata2, ((BinaryOp)hop).getOp().name()); + String opName = ((BinaryOp)hop).getOp().name(); + double sparsityEst = OptimizerUtils.getBinaryOpSparsity( + OptimizerUtils.getSparsity(hop.getInput(0)), + OptimizerUtils.getSparsity(hop.getInput(1)), OpOp2.valueOf(opName), true); + out = getVectorBinary(cdata1, cdata2, opName, sparsityEst); if( cdata1 instanceof CNodeData && !inHops2.containsKey("X") && !(cdata1.getDataType()==DataType.SCALAR) ) { inHops2.put("X", hop.getInput().get(0)); @@ -569,7 +574,17 @@ private static CNodeBinary getVectorBinary(CNode cdata1, CNode cdata2, String na return new CNodeBinary(cdata1, cdata2, BinType.valueOf("VECT_"+name+"_SCALAR")); } } - + + private static CNodeBinary getVectorBinary(CNode cdata1, CNode cdata2, String name, double sparsity) { + if( TemplateUtils.isMatrix(cdata1) && (TemplateUtils.isMatrix(cdata2) + || TemplateUtils.isRowVector(cdata2)) ) { + return new CNodeBinary(cdata1, cdata2, BinType.valueOf("VECT_"+name), sparsity); + } + else { + return new CNodeBinary(cdata1, cdata2, BinType.valueOf("VECT_"+name+"_SCALAR"), sparsity); + } + } + /** * Comparator to order input hops of the row aggregate template. We try * to order matrices-vectors-scalars via sorting by number of cells but From 5e5b665180d5fc760c53da9326735707fa87d145 Mon Sep 17 00:00:00 2001 From: Frxms Date: Fri, 20 Jun 2025 14:54:01 +0200 Subject: [PATCH 16/60] debug --- .../apache/sysds/hops/codegen/cplan/CodeTemplate.java | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/apache/sysds/hops/codegen/cplan/CodeTemplate.java b/src/main/java/org/apache/sysds/hops/codegen/cplan/CodeTemplate.java index 71639638b77..477ed765724 100644 --- a/src/main/java/org/apache/sysds/hops/codegen/cplan/CodeTemplate.java +++ b/src/main/java/org/apache/sysds/hops/codegen/cplan/CodeTemplate.java @@ -31,11 +31,19 @@ public abstract class CodeTemplate { public String getTemplate() { throw new RuntimeException("Calling wrong getTemplate method on " + getClass().getCanonicalName()); } - + + /** + * @param sparseTemplate added to turn SparseRowVector intermediates on and off + */ public String getTemplate(CNodeBinary.BinType type, boolean sparseLhs, boolean sparseRhs, boolean scalarVector, boolean scalarInput, boolean vectorVector, boolean sparseTemplate) { throw new RuntimeException("Calling wrong getTemplate method on " + getClass().getCanonicalName()); } + + public String getTemplate(CNodeBinary.BinType type, boolean sparseLhs, boolean sparseRhs, + boolean scalarVector, boolean scalarInput, boolean vectorVector) { + throw new RuntimeException("Calling wrong getTemplate method on " + getClass().getCanonicalName()); + } public String getTemplate(CNodeTernary.TernaryType type, boolean sparse) { throw new RuntimeException("Calling wrong getTemplate method on " + getClass().getCanonicalName()); From 0ca02960d5ae52353aabef93a823ad45992a408e Mon Sep 17 00:00:00 2001 From: Frxms Date: Mon, 23 Jun 2025 19:39:52 +0200 Subject: [PATCH 17/60] implement branchless vectDiv and vectMult --- .../runtime/codegen/LibSpoofPrimitives.java | 44 ++++++++++++++++++- 1 file changed, 42 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/apache/sysds/runtime/codegen/LibSpoofPrimitives.java b/src/main/java/org/apache/sysds/runtime/codegen/LibSpoofPrimitives.java index 69a07ecf4d6..35fa4cd2a3a 100644 --- a/src/main/java/org/apache/sysds/runtime/codegen/LibSpoofPrimitives.java +++ b/src/main/java/org/apache/sysds/runtime/codegen/LibSpoofPrimitives.java @@ -2191,7 +2191,7 @@ public static SparseRowVector vectMultWrite(int len, double[] a, double bval, in return c; } - public static SparseRowVector vectMultWrite(int len, double[] a, double[] b, int[] aix, int[] bix, int ai, int bi, int alen, int blen) { + public static SparseRowVector vectMultWriteB(int len, double[] a, double[] b, int[] aix, int[] bix, int ai, int bi, int alen, int blen) { SparseRowVector c = allocSparseVector(Math.min(alen, blen)); if( a == null || b == null ) return c; int aIndex = ai; @@ -2217,6 +2217,26 @@ public static SparseRowVector vectMultWrite(int len, double[] a, double[] b, int return c; } + public static SparseRowVector vectMultWrite(int len, double[] a, double[] b, int[] aix, int[] bix, int ai, int bi, int alen, int blen) { + SparseRowVector c = allocSparseVector(Math.min(alen, blen)); + int index = 0; + int aItr = ai; + int bItr = bi; + int[] indexes = c.indexes(); + double[] values = c.values(); + while(aItr < ai+alen && bItr < bi+blen) { + int aIdx = aix[aItr]; + int bIdx = bix[bItr]; + indexes[index] = aIdx; + values[index] = a[aItr] * b[bItr]; + index += aIdx == bIdx ? 1 : 0; + aItr += aIdx <= bIdx ? 1 : 0; + bItr += aIdx >= bIdx ? 1 : 0; + } + c.setSize(index); + return c; + } + public static void vectWrite(SparseRowVector a, double[] c, int ci, int len) { if( a == null ) return; int[] aix = a.indexes(); @@ -2248,7 +2268,7 @@ public static SparseRowVector vectDivWrite(int len, double bval, double[] a, int return c; } - public static SparseRowVector vectDivWrite(int len, double[] a, double[] b, int[] aix, int[] bix, int ai, int bi, int alen, int blen) { + public static SparseRowVector vectDivWriteB(int len, double[] a, double[] b, int[] aix, int[] bix, int ai, int bi, int alen, int blen) { SparseRowVector c = allocSparseVector(alen); int aIndex = ai; int bIndex = bi; @@ -2277,6 +2297,26 @@ public static SparseRowVector vectDivWrite(int len, double[] a, double[] b, int[ return c; } + public static SparseRowVector vectDivWrite(int len, double[] a, double[] b, int[] aix, int[] bix, int ai, int bi, int alen, int blen) { + SparseRowVector c = allocSparseVector(Math.min(alen, blen)); + int index = 0; + int aItr = ai; + int bItr = bi; + int[] indexes = c.indexes(); + double[] values = c.values(); + while(aItr < ai+alen && bItr < bi+blen) { + int aIdx = aix[aItr]; + int bIdx = bix[bItr]; + indexes[index] = aIdx; + values[index] = a[aItr] / b[bItr]; + index += aIdx == bIdx ? 1 : 0; + aItr += aIdx <= bIdx ? 1 : 0; + bItr += aIdx >= bIdx ? 1 : 0; + } + c.setSize(index); + return c; + } + public static SparseRowVector vectMinusWrite(int len, double[] a, double[] b, int[] aix, int[] bix, int ai, int bi, int alen, int blen) { SparseRowVector c = allocSparseVector(alen+blen); int aIndex = ai; From 1a52829a2a788e888bb509b4827152d25694448e Mon Sep 17 00:00:00 2001 From: Frxms Date: Mon, 23 Jun 2025 20:12:43 +0200 Subject: [PATCH 18/60] change int naming convention --- .../runtime/codegen/LibSpoofPrimitives.java | 534 +++++++++--------- 1 file changed, 267 insertions(+), 267 deletions(-) diff --git a/src/main/java/org/apache/sysds/runtime/codegen/LibSpoofPrimitives.java b/src/main/java/org/apache/sysds/runtime/codegen/LibSpoofPrimitives.java index 35fa4cd2a3a..061f0862fe7 100644 --- a/src/main/java/org/apache/sysds/runtime/codegen/LibSpoofPrimitives.java +++ b/src/main/java/org/apache/sysds/runtime/codegen/LibSpoofPrimitives.java @@ -2194,24 +2194,24 @@ public static SparseRowVector vectMultWrite(int len, double[] a, double bval, in public static SparseRowVector vectMultWriteB(int len, double[] a, double[] b, int[] aix, int[] bix, int ai, int bi, int alen, int blen) { SparseRowVector c = allocSparseVector(Math.min(alen, blen)); if( a == null || b == null ) return c; - int aIndex = ai; - int bIndex = bi; + int aItr = ai; + int bItr = bi; int index = 0; int[] indexes = c.indexes(); double[] values = c.values(); - while(aIndex < ai+alen && bIndex < bi+blen) { - int aIdx = aix[aIndex]; - int bIdx = bix[bIndex]; + while(aItr < ai+alen && bItr < bi+blen) { + int aIdx = aix[aItr]; + int bIdx = bix[bItr]; if(aIdx == bIdx) { indexes[index] = aIdx; - values[index] = a[aIndex] * b[bIndex]; - aIndex++; - bIndex++; + values[index] = a[aItr] * b[bItr]; + aItr++; + bItr++; index++; } else if(aIdx < bIdx) - aIndex++; + aItr++; else - bIndex++; + bItr++; } c.setSize(index); return c; @@ -2270,27 +2270,27 @@ public static SparseRowVector vectDivWrite(int len, double bval, double[] a, int public static SparseRowVector vectDivWriteB(int len, double[] a, double[] b, int[] aix, int[] bix, int ai, int bi, int alen, int blen) { SparseRowVector c = allocSparseVector(alen); - int aIndex = ai; - int bIndex = bi; + int aItr = ai; + int bItr = bi; int index = 0; int[] indexes = c.indexes(); double[] values = c.values(); - while(aIndex < ai+alen && bIndex < bi+blen) { - int aIdx = aix[aIndex]; - int bIdx = bix[bIndex]; + while(aItr < ai+alen && bItr < bi+blen) { + int aIdx = aix[aItr]; + int bIdx = bix[bItr]; if(aIdx == bIdx) { indexes[index] = aIdx; - values[index] = a[aIndex] / b[bIndex]; - aIndex++; - bIndex++; + values[index] = a[aItr] / b[bItr]; + aItr++; + bItr++; index++; } else if(aIdx < bIdx) { indexes[index] = aIdx; - values[index] = (a[aIndex]>0) ? Double.POSITIVE_INFINITY : Double.NEGATIVE_INFINITY; - aIndex++; + values[index] = (a[aItr]>0) ? Double.POSITIVE_INFINITY : Double.NEGATIVE_INFINITY; + aItr++; index++; } else { - bIndex++; + bItr++; } } c.setSize(index); @@ -2319,40 +2319,40 @@ public static SparseRowVector vectDivWrite(int len, double[] a, double[] b, int[ public static SparseRowVector vectMinusWrite(int len, double[] a, double[] b, int[] aix, int[] bix, int ai, int bi, int alen, int blen) { SparseRowVector c = allocSparseVector(alen+blen); - int aIndex = ai; - int bIndex = bi; + int aItr = ai; + int bItr = bi; int index = 0; int[] indexes = c.indexes(); double[] values = c.values(); - while(aIndex < ai+alen && bIndex < bi+blen) { - int aIdx = aix[aIndex]; - int bIdx = bix[bIndex]; + while(aItr < ai+alen && bItr < bi+blen) { + int aIdx = aix[aItr]; + int bIdx = bix[bItr]; if(aIdx == bIdx) { indexes[index] = aIdx; - values[index] = a[aIndex] - b[bIndex]; - aIndex++; - bIndex++; + values[index] = a[aItr] - b[bItr]; + aItr++; + bItr++; index++; } else if(aIdx < bIdx) { indexes[index] = aIdx; - values[index] = a[aIndex]; - aIndex++; + values[index] = a[aItr]; + aItr++; index++; } else { indexes[index] = bIdx; - values[index] = -b[bIndex]; - bIndex++; + values[index] = -b[bItr]; + bItr++; index++; } } - for (; aIndex < ai+alen; aIndex++) { - indexes[index] = aix[aIndex]; - values[index] = a[aIndex]; + for (; aItr < ai+alen; aItr++) { + indexes[index] = aix[aItr]; + values[index] = a[aItr]; index++; } - for (; bIndex < bi+blen; bIndex++) { - indexes[index] = bix[bIndex]; - values[index] = -b[bIndex]; + for (; bItr < bi+blen; bItr++) { + indexes[index] = bix[bItr]; + values[index] = -b[bItr]; index++; } c.setSize(index); @@ -2361,40 +2361,40 @@ public static SparseRowVector vectMinusWrite(int len, double[] a, double[] b, in public static SparseRowVector vectPlusWrite(int len, double[] a, double[] b, int[] aix, int[] bix, int ai, int bi, int alen, int blen) { SparseRowVector c = allocSparseVector(alen+blen); - int aIndex = ai; - int bIndex = bi; + int aItr = ai; + int bItr = bi; int index = 0; int[] indexes = c.indexes(); double[] values = c.values(); - while(aIndex < ai+alen && bIndex < bi+blen) { - int aIdx = aix[aIndex]; - int bIdx = bix[bIndex]; + while(aItr < ai+alen && bItr < bi+blen) { + int aIdx = aix[aItr]; + int bIdx = bix[bItr]; if(aIdx == bIdx) { indexes[index] = aIdx; - values[index] = a[aIndex] + b[bIndex]; - aIndex++; - bIndex++; + values[index] = a[aItr] + b[bItr]; + aItr++; + bItr++; index++; } else if(aIdx < bIdx) { indexes[index] = aIdx; - values[index] = a[aIndex]; - aIndex++; + values[index] = a[aItr]; + aItr++; index++; } else { indexes[index] = bIdx; - values[index] = b[bIndex]; - bIndex++; + values[index] = b[bItr]; + bItr++; index++; } } - for (; aIndex < ai+alen; aIndex++) { - indexes[index] = aix[aIndex]; - values[index] = a[aIndex]; + for (; aItr < ai+alen; aItr++) { + indexes[index] = aix[aItr]; + values[index] = a[aItr]; index++; } - for (; bIndex < bi+blen; bIndex++) { - indexes[index] = bix[bIndex]; - values[index] = b[bIndex]; + for (; bItr < bi+blen; bItr++) { + indexes[index] = bix[bItr]; + values[index] = b[bItr]; index++; } c.setSize(index); @@ -2407,12 +2407,12 @@ public static SparseRowVector vectXorWrite(int len, double[] a, double bval, int int[] indexes = c.indexes(); double[] values = c.values(); int index = 0; - int aIndex = 0; - while(aIndex < ai+alen && index < len) { + int aItr = 0; + while(aItr < ai+alen && index < len) { indexes[index] = index; - if(aix[aIndex] == index) { - values[index] = !(a[aIndex] != 0) ? 1 : 0; - aIndex++; + if(aix[aItr] == index) { + values[index] = !(a[aItr] != 0) ? 1 : 0; + aItr++; } else { values[index] = 1; } @@ -2443,40 +2443,40 @@ public static SparseRowVector vectXorWrite(int len, double bval, double[] a, int public static SparseRowVector vectXorWrite(int len, double[] a, double[] b, int[] aix, int[] bix, int ai, int bi, int alen, int blen) { SparseRowVector c = allocSparseVector(alen+blen); - int aIndex = ai; - int bIndex = bi; + int aItr = ai; + int bItr = bi; int index = 0; int[] indexes = c.indexes(); double[] values = c.values(); - while(aIndex < ai+alen && bIndex < bi+blen) { - int aIdx = aix[aIndex]; - int bIdx = bix[bIndex]; + while(aItr < ai+alen && bItr < bi+blen) { + int aIdx = aix[aItr]; + int bIdx = bix[bItr]; if(aIdx == bIdx) { indexes[index] = aIdx; - values[index] = ((a[aIndex] != 0) != (b[bIndex] != 0)) ? 1 : 0; - aIndex++; - bIndex++; + values[index] = ((a[aItr] != 0) != (b[bItr] != 0)) ? 1 : 0; + aItr++; + bItr++; index++; } else if(aIdx < bIdx) { indexes[index] = aIdx; - values[index] = (a[aIndex] != 0) ? 1 : 0; - aIndex++; + values[index] = (a[aItr] != 0) ? 1 : 0; + aItr++; index++; } else { indexes[index] = bIdx; - values[index] = (b[bIndex] != 0) ? 1 : 0; - bIndex++; + values[index] = (b[bItr] != 0) ? 1 : 0; + bItr++; index++; } } - for (; aIndex < ai+alen; aIndex++) { - indexes[index] = aix[aIndex]; - values[index] = (a[aIndex] != 0) ? 1 : 0; + for (; aItr < ai+alen; aItr++) { + indexes[index] = aix[aItr]; + values[index] = (a[aItr] != 0) ? 1 : 0; index++; } - for (; bIndex < bi+blen; bIndex++) { - indexes[index] = bix[bIndex]; - values[index] = (b[bIndex] != 0) ? 1 : 0; + for (; bItr < bi+blen; bItr++) { + indexes[index] = bix[bItr]; + values[index] = (b[bItr] != 0) ? 1 : 0; index++; } c.setSize(index); @@ -2489,12 +2489,12 @@ public static SparseRowVector vectPowWrite(int len, double[] a, double bval, int int[] indexes = c.indexes(); double[] values = c.values(); int index = 0; - int aIndex = 0; - while(aIndex < ai+alen && index < len) { + int aItr = 0; + while(aItr < ai+alen && index < len) { indexes[index] = index; - if(aix[aIndex] == index) { - values[index] = Math.pow(a[aIndex], bval) - 1; - aIndex++; + if(aix[aItr] == index) { + values[index] = Math.pow(a[aItr], bval) - 1; + aItr++; } else { values[index] = 1; } @@ -2525,12 +2525,12 @@ public static SparseRowVector vectMinWrite(int len, double[] a, double bval, int int[] indexes = c.indexes(); double[] values = c.values(); int index = 0; - int aIndex = 0; - while(aIndex < ai+alen && index < len) { + int aItr = 0; + while(aItr < ai+alen && index < len) { indexes[index] = index; - if(aix[aIndex] == index) { - values[index] = Math.min(a[aIndex], bval); - aIndex++; + if(aix[aItr] == index) { + values[index] = Math.min(a[aItr], bval); + aItr++; } else { values[index] = bval; } @@ -2561,40 +2561,40 @@ public static SparseRowVector vectMinWrite(int len, double bval, double[] a, int public static SparseRowVector vectMinWrite(int len, double[] a, double[] b, int[] aix, int[] bix, int ai, int bi, int alen, int blen) { SparseRowVector c = allocSparseVector(alen+blen); - int aIndex = ai; - int bIndex = bi; + int aItr = ai; + int bItr = bi; int index = 0; int[] indexes = c.indexes(); double[] values = c.values(); - while(aIndex < ai+alen && bIndex < bi+blen) { - int aIdx = aix[aIndex]; - int bIdx = bix[bIndex]; + while(aItr < ai+alen && bItr < bi+blen) { + int aIdx = aix[aItr]; + int bIdx = bix[bItr]; if(aIdx == bIdx) { indexes[index] = aIdx; - values[index] = Math.min(a[aIndex], b[bIndex]); - aIndex++; - bIndex++; + values[index] = Math.min(a[aItr], b[bItr]); + aItr++; + bItr++; index++; } else if(aIdx < bIdx) { indexes[index] = aIdx; - values[index] = Math.min(a[aIndex], 0); - aIndex++; + values[index] = Math.min(a[aItr], 0); + aItr++; index++; } else { indexes[index] = bIdx; - values[index] = Math.min(b[bIndex], 0); - bIndex++; + values[index] = Math.min(b[bItr], 0); + bItr++; index++; } } - for (; aIndex < ai+alen; aIndex++) { - indexes[index] = aix[aIndex]; - values[index] = Math.min(a[aIndex], 0); + for (; aItr < ai+alen; aItr++) { + indexes[index] = aix[aItr]; + values[index] = Math.min(a[aItr], 0); index++; } - for (; bIndex < bi+blen; bIndex++) { - indexes[index] = bix[bIndex]; - values[index] = Math.min(b[bIndex], 0); + for (; bItr < bi+blen; bItr++) { + indexes[index] = bix[bItr]; + values[index] = Math.min(b[bItr], 0); index++; } c.setSize(index); @@ -2607,12 +2607,12 @@ public static SparseRowVector vectMaxWrite(int len, double[] a, double bval, int int[] indexes = c.indexes(); double[] values = c.values(); int index = 0; - int aIndex = 0; - while(aIndex < ai+alen && index < len) { + int aItr = 0; + while(aItr < ai+alen && index < len) { indexes[index] = index; - if(aix[aIndex] == index) { - values[index] = Math.max(a[aIndex], bval); - aIndex++; + if(aix[aItr] == index) { + values[index] = Math.max(a[aItr], bval); + aItr++; } else { values[index] = bval; } @@ -2643,40 +2643,40 @@ public static SparseRowVector vectMaxWrite(int len, double bval, double[] a, int public static SparseRowVector vectMaxWrite(int len, double[] a, double[] b, int[] aix, int[] bix, int ai, int bi, int alen, int blen) { SparseRowVector c = allocSparseVector(alen+blen); - int aIndex = ai; - int bIndex = bi; + int aItr = ai; + int bItr = bi; int index = 0; int[] indexes = c.indexes(); double[] values = c.values(); - while(aIndex < ai+alen && bIndex < bi+blen) { - int aIdx = aix[aIndex]; - int bIdx = bix[bIndex]; + while(aItr < ai+alen && bItr < bi+blen) { + int aIdx = aix[aItr]; + int bIdx = bix[bItr]; if(aIdx == bIdx) { indexes[index] = aIdx; - values[index] = Math.max(a[aIndex], b[bIndex]); - aIndex++; - bIndex++; + values[index] = Math.max(a[aItr], b[bItr]); + aItr++; + bItr++; index++; } else if(aIdx < bIdx) { indexes[index] = aIdx; - values[index] = Math.max(a[aIndex], 0); - aIndex++; + values[index] = Math.max(a[aItr], 0); + aItr++; index++; } else { indexes[index] = bIdx; - values[index] = Math.max(b[bIndex], 0); - bIndex++; + values[index] = Math.max(b[bItr], 0); + bItr++; index++; } } - for (; aIndex < ai+alen; aIndex++) { - indexes[index] = aix[aIndex]; - values[index] = Math.max(a[aIndex], 0); + for (; aItr < ai+alen; aItr++) { + indexes[index] = aix[aItr]; + values[index] = Math.max(a[aItr], 0); index++; } - for (; bIndex < bi+blen; bIndex++) { - indexes[index] = bix[bIndex]; - values[index] = Math.max(b[bIndex], 0); + for (; bItr < bi+blen; bItr++) { + indexes[index] = bix[bItr]; + values[index] = Math.max(b[bItr], 0); index++; } c.setSize(index); @@ -2689,12 +2689,12 @@ public static SparseRowVector vectEqualWrite(int len, double[] a, double bval, i int[] indexes = c.indexes(); double[] values = c.values(); int index = 0; - int aIndex = 0; - while(aIndex < ai+alen && index < len) { + int aItr = 0; + while(aItr < ai+alen && index < len) { indexes[index] = index; - if(aix[aIndex] == index) { - values[index] = a[aIndex] == bval ? 1 : 0; - aIndex++; + if(aix[aItr] == index) { + values[index] = a[aItr] == bval ? 1 : 0; + aItr++; } else { values[index] = 1; } @@ -2726,28 +2726,28 @@ public static SparseRowVector vectEqualWrite(int len, double bval, double[] a, i //doesn't return SparseRowVector, but still uses two sparse vectors as inputs public static double[] vectEqualWrite(int len, double[] a, double[] b, int[] aix, int[] bix, int ai, int bi, int alen, int blen) { double[] c = allocVector(len, true, 1); - int aIndex = ai; - int bIndex = bi; - while(aIndex < ai+alen && bIndex < bi+blen) { - int aIdx = aix[aIndex]; - int bIdx = bix[bIndex]; + int aItr = ai; + int bItr = bi; + while(aItr < ai+alen && bItr < bi+blen) { + int aIdx = aix[aItr]; + int bIdx = bix[bItr]; if (aIdx == bIdx) { - c[aIdx] = (a[aIndex] == b[bIndex]) ? 1 : 0; - aIndex++; - bIndex++; + c[aIdx] = (a[aItr] == b[bItr]) ? 1 : 0; + aItr++; + bItr++; } else if(aIdx < bIdx) { //todo: this might be too unsafe - c[aIdx] = a[aIndex] == 0 ? 1 : 0; + c[aIdx] = a[aItr] == 0 ? 1 : 0; // c[aIdx] = 0; - aIndex++; + aItr++; } else { - c[bIdx] = b[bIndex] == 0 ? 1 : 0; + c[bIdx] = b[bItr] == 0 ? 1 : 0; // c[bIdx] = 0; - bIndex++; + bItr++; } } - for (; aIndex < ai+alen; aIndex++) c[aix[aIndex]] = 0; - for (; bIndex < bi+blen; bIndex++) c[bix[bIndex]] = 0; + for (; aItr < ai+alen; aItr++) c[aix[aItr]] = 0; + for (; bItr < bi+blen; bItr++) c[bix[bItr]] = 0; return c; } @@ -2757,12 +2757,12 @@ public static SparseRowVector vectNotequalWrite(int len, double[] a, double bval int[] indexes = c.indexes(); double[] values = c.values(); int index = 0; - int aIndex = 0; - while(aIndex < ai+alen && index < len) { + int aItr = 0; + while(aItr < ai+alen && index < len) { indexes[index] = index; - if(aix[aIndex] == index) { - values[index] = a[aIndex] != bval ? 1 : 0; - aIndex++; + if(aix[aItr] == index) { + values[index] = a[aItr] != bval ? 1 : 0; + aItr++; } else { values[index] = 1; } @@ -2793,40 +2793,40 @@ public static SparseRowVector vectNotequalWrite(int len, double bval, double[] a public static SparseRowVector vectNotequalWrite(int len, double[] a, double[] b, int[] aix, int[] bix, int ai, int bi, int alen, int blen) { SparseRowVector c = allocSparseVector(alen+blen); - int aIndex = ai; - int bIndex = bi; + int aItr = ai; + int bItr = bi; int index = 0; int[] indexes = c.indexes(); double[] values = c.values(); - while(aIndex < ai+alen && bIndex < bi+blen) { - int aIdx = aix[aIndex]; - int bIdx = bix[bIndex]; + while(aItr < ai+alen && bItr < bi+blen) { + int aIdx = aix[aItr]; + int bIdx = bix[bItr]; if(aIdx == bIdx) { indexes[index] = aIdx; - values[index] = (a[aIndex] != b[bIndex]) ? 1 : 0; - aIndex++; - bIndex++; + values[index] = (a[aItr] != b[bItr]) ? 1 : 0; + aItr++; + bItr++; index++; } else if(aIdx < bIdx) { indexes[index] = aIdx; - values[index] = a[aIndex] != 0 ? 1 : 0; - aIndex++; + values[index] = a[aItr] != 0 ? 1 : 0; + aItr++; index++; } else { indexes[index] = bIdx; - values[index] = b[bIndex] != 0 ? 1 : 0; - bIndex++; + values[index] = b[bItr] != 0 ? 1 : 0; + bItr++; index++; } } - for (; aIndex < ai+alen; aIndex++) { - indexes[index] = aix[aIndex]; - values[index] = a[aIndex] != 0 ? 1 : 0; + for (; aItr < ai+alen; aItr++) { + indexes[index] = aix[aItr]; + values[index] = a[aItr] != 0 ? 1 : 0; index++; } - for (; bIndex < bi+blen; bIndex++) { - indexes[index] = bix[bIndex]; - values[index] = b[bIndex] != 0 ? 1 : 0; + for (; bItr < bi+blen; bItr++) { + indexes[index] = bix[bItr]; + values[index] = b[bItr] != 0 ? 1 : 0; index++; } c.setSize(index); @@ -2839,12 +2839,12 @@ public static SparseRowVector vectLessWrite(int len, double[] a, double bval, in int[] indexes = c.indexes(); double[] values = c.values(); int index = 0; - int aIndex = 0; - while(aIndex < ai+alen && index < len) { + int aItr = 0; + while(aItr < ai+alen && index < len) { indexes[index] = index; - if(aix[aIndex] == index) { - values[index] = a[aIndex] < bval ? 1 : 0; - aIndex++; + if(aix[aItr] == index) { + values[index] = a[aItr] < bval ? 1 : 0; + aItr++; } else { values[index] = 1; } @@ -2875,40 +2875,40 @@ public static SparseRowVector vectLessWrite(int len, double bval, double[] a, in public static SparseRowVector vectLessWrite(int len, double[] a, double[] b, int[] aix, int[] bix, int ai, int bi, int alen, int blen) { SparseRowVector c = allocSparseVector(alen+blen); - int aIndex = ai; - int bIndex = bi; + int aItr = ai; + int bItr = bi; int index = 0; int[] indexes = c.indexes(); double[] values = c.values(); - while(aIndex < ai+alen && bIndex < bi+blen) { - int aIdx = aix[aIndex]; - int bIdx = bix[bIndex]; + while(aItr < ai+alen && bItr < bi+blen) { + int aIdx = aix[aItr]; + int bIdx = bix[bItr]; if(aIdx == bIdx) { indexes[index] = aIdx; - values[index] = (a[aIndex] < b[bIndex]) ? 1 : 0; - aIndex++; - bIndex++; + values[index] = (a[aItr] < b[bItr]) ? 1 : 0; + aItr++; + bItr++; index++; } else if(aIdx < bIdx) { indexes[index] = aIdx; - values[index] = a[aIndex] < 0 ? 1 : 0; - aIndex++; + values[index] = a[aItr] < 0 ? 1 : 0; + aItr++; index++; } else { indexes[index] = bIdx; - values[index] = 0 < b[bIndex] ? 1 : 0; - bIndex++; + values[index] = 0 < b[bItr] ? 1 : 0; + bItr++; index++; } } - for (; aIndex < ai+alen; aIndex++) { - indexes[index] = aix[aIndex]; - values[index] = a[aIndex] < 0? 1 : 0; + for (; aItr < ai+alen; aItr++) { + indexes[index] = aix[aItr]; + values[index] = a[aItr] < 0? 1 : 0; index++; } - for (; bIndex < bi+blen; bIndex++) { - indexes[index] = bix[bIndex]; - values[index] = 0 < b[bIndex] ? 1 : 0; + for (; bItr < bi+blen; bItr++) { + indexes[index] = bix[bItr]; + values[index] = 0 < b[bItr] ? 1 : 0; index++; } c.setSize(index); @@ -2921,12 +2921,12 @@ public static SparseRowVector vectLessequalWrite(int len, double[] a, double bva int[] indexes = c.indexes(); double[] values = c.values(); int index = 0; - int aIndex = 0; - while(aIndex < ai+alen && index < len) { + int aItr = 0; + while(aItr < ai+alen && index < len) { indexes[index] = index; - if(aix[aIndex] == index) { - values[index] = a[aIndex] <= bval ? 1 : 0; - aIndex++; + if(aix[aItr] == index) { + values[index] = a[aItr] <= bval ? 1 : 0; + aItr++; } else { values[index] = 1; } @@ -2958,25 +2958,25 @@ public static SparseRowVector vectLessequalWrite(int len, double bval, double[] //doesn't return SparseRowVector, but still uses two sparse vectors as inputs public static double[] vectLessequalWrite(int len, double[] a, double[] b, int[] aix, int[] bix, int ai, int bi, int alen, int blen) { double[] c = allocVector(len, true, 1); - int aIndex = ai; - int bIndex = bi; - while(aIndex < ai+alen && bIndex < bi+blen) { - int aIdx = aix[aIndex]; - int bIdx = bix[bIndex]; + int aItr = ai; + int bItr = bi; + while(aItr < ai+alen && bItr < bi+blen) { + int aIdx = aix[aItr]; + int bIdx = bix[bItr]; if(aIdx == bIdx) { - c[aIdx] = (a[aIndex] <= b[bIndex]) ? 1 : 0; - aIndex++; - bIndex++; + c[aIdx] = (a[aItr] <= b[bItr]) ? 1 : 0; + aItr++; + bItr++; } else if(aIdx < bIdx) { - c[aIdx] = (a[aIndex] <= 0) ? 1 : 0; - aIndex++; + c[aIdx] = (a[aItr] <= 0) ? 1 : 0; + aItr++; } else { - c[bIdx] = (0 <= b[bIndex]) ? 1 : 0; - bIndex++; + c[bIdx] = (0 <= b[bItr]) ? 1 : 0; + bItr++; } } - for(; aIndex < ai+alen; aIndex++) c[aix[aIndex]] = (a[aIndex] <= 0) ? 1 : 0; - for(; bIndex < bi+blen; bIndex++) c[bix[bIndex]] = (0 <= b[bIndex]) ? 1 : 0; + for(; aItr < ai+alen; aItr++) c[aix[aItr]] = (a[aItr] <= 0) ? 1 : 0; + for(; bItr < bi+blen; bItr++) c[bix[bItr]] = (0 <= b[bItr]) ? 1 : 0; return c; } @@ -2986,12 +2986,12 @@ public static SparseRowVector vectGreaterWrite(int len, double[] a, double bval, int[] indexes = c.indexes(); double[] values = c.values(); int index = 0; - int aIndex = 0; - while(aIndex < ai+alen && index < len) { + int aItr = 0; + while(aItr < ai+alen && index < len) { indexes[index] = index; - if(aix[aIndex] == index) { - values[index] = a[aIndex] > bval ? 1 : 0; - aIndex++; + if(aix[aItr] == index) { + values[index] = a[aItr] > bval ? 1 : 0; + aItr++; } else { values[index] = 1; } @@ -3022,40 +3022,40 @@ public static SparseRowVector vectGreaterWrite(int len, double bval, double[] a, public static SparseRowVector vectGreaterWrite(int len, double[] a, double[] b, int[] aix, int[] bix, int ai, int bi, int alen, int blen) { SparseRowVector c = allocSparseVector(alen+blen); - int aIndex = ai; - int bIndex = bi; + int aItr = ai; + int bItr = bi; int index = 0; int[] indexes = c.indexes(); double[] values = c.values(); - while(aIndex < ai+alen && bIndex < bi+blen) { - int aIdx = aix[aIndex]; - int bIdx = bix[bIndex]; + while(aItr < ai+alen && bItr < bi+blen) { + int aIdx = aix[aItr]; + int bIdx = bix[bItr]; if(aIdx == bIdx) { indexes[index] = aIdx; - values[index] = (a[aIndex] > b[bIndex]) ? 1 : 0; - aIndex++; - bIndex++; + values[index] = (a[aItr] > b[bItr]) ? 1 : 0; + aItr++; + bItr++; index++; } else if(aIdx < bIdx) { indexes[index] = aIdx; - values[index] = a[aIndex] > 0 ? 1 : 0; - aIndex++; + values[index] = a[aItr] > 0 ? 1 : 0; + aItr++; index++; } else { indexes[index] = bIdx; - values[index] = 0 > b[bIndex] ? 1 : 0; - bIndex++; + values[index] = 0 > b[bItr] ? 1 : 0; + bItr++; index++; } } - for (; aIndex < ai+alen; aIndex++) { - indexes[index] = aix[aIndex]; - values[index] = a[aIndex] > 0 ? 1 : 0; + for (; aItr < ai+alen; aItr++) { + indexes[index] = aix[aItr]; + values[index] = a[aItr] > 0 ? 1 : 0; index++; } - for (; bIndex < bi+blen; bIndex++) { - indexes[index] = bix[bIndex]; - values[index] = 0 > b[bIndex] ? 1 : 0; + for (; bItr < bi+blen; bItr++) { + indexes[index] = bix[bItr]; + values[index] = 0 > b[bItr] ? 1 : 0; index++; } c.setSize(index); @@ -3068,12 +3068,12 @@ public static SparseRowVector vectGreaterequalWrite(int len, double[] a, double int[] indexes = c.indexes(); double[] values = c.values(); int index = 0; - int aIndex = 0; - while(aIndex < ai+alen && index < len) { + int aItr = 0; + while(aItr < ai+alen && index < len) { indexes[index] = index; - if(aix[aIndex] == index) { - values[index] = a[aIndex] >= bval ? 1 : 0; - aIndex++; + if(aix[aItr] == index) { + values[index] = a[aItr] >= bval ? 1 : 0; + aItr++; } else { values[index] = 1; } @@ -3105,25 +3105,25 @@ public static SparseRowVector vectGreaterequalWrite(int len, double bval, double //doesn't return SparseRowVector, but still uses two sparse vectors as inputs public static double[] vectGreaterequalWrite(int len, double[] a, double[] b, int[] aix, int[] bix, int ai, int bi, int alen, int blen) { double[] c = allocVector(len, true, 1); - int aIndex = ai; - int bIndex = bi; - while(aIndex < ai+alen && bIndex < bi+blen) { - int aIdx = aix[aIndex]; - int bIdx = bix[bIndex]; + int aItr = ai; + int bItr = bi; + while(aItr < ai+alen && bItr < bi+blen) { + int aIdx = aix[aItr]; + int bIdx = bix[bItr]; if(aIdx == bIdx) { - c[aIdx] = (a[aIndex] >= b[bIndex]) ? 1 : 0; - aIndex++; - bIndex++; + c[aIdx] = (a[aItr] >= b[bItr]) ? 1 : 0; + aItr++; + bItr++; } else if(aIdx < bIdx) { - c[aIdx] = (a[aIndex] >= 0) ? 1 : 0; - aIndex++; + c[aIdx] = (a[aItr] >= 0) ? 1 : 0; + aItr++; } else { - c[bIdx] = (0 >= b[bIndex]) ? 1 : 0; - bIndex++; + c[bIdx] = (0 >= b[bItr]) ? 1 : 0; + bItr++; } } - for(; aIndex < ai+alen; aIndex++) c[aix[aIndex]] = (a[aIndex] >= 0) ? 1 : 0; - for(; bIndex < bi+blen; bIndex++) c[bix[bIndex]] = (0 >= b[bIndex]) ? 1 : 0; + for(; aItr < ai+alen; aItr++) c[aix[aItr]] = (a[aItr] >= 0) ? 1 : 0; + for(; bItr < bi+blen; bItr++) c[bix[bItr]] = (0 >= b[bItr]) ? 1 : 0; return c; } @@ -3146,24 +3146,24 @@ public static SparseRowVector vectBitwandWrite(int len, double bval, double[] a, public static SparseRowVector vectBitwandWrite(int len, double[] a, double[] b, int[] aix, int[] bix, int ai, int bi, int alen, int blen) { SparseRowVector c = allocSparseVector(alen); - int aIndex = ai; - int bIndex = bi; + int aItr = ai; + int bItr = bi; int index = 0; int[] indexes = c.indexes(); double[] values = c.values(); - while(aIndex < ai+alen && bIndex < bi+blen) { - int aIdx = aix[aIndex]; - int bIdx = bix[bIndex]; + while(aItr < ai+alen && bItr < bi+blen) { + int aIdx = aix[aItr]; + int bIdx = bix[bItr]; if(aIdx == bIdx) { indexes[index] = aIdx; - values[index] = bwAnd(a[aIndex], b[bIndex]); - aIndex++; - bIndex++; + values[index] = bwAnd(a[aItr], b[bItr]); + aItr++; + bItr++; index++; } else if(aIdx < bIdx) { - aIndex++; + aItr++; } else { - bIndex++; + bItr++; } } c.setSize(index); From 959d0762bf2759dbe526e57bbede14944755ed4a Mon Sep 17 00:00:00 2001 From: Frxms Date: Mon, 23 Jun 2025 23:47:03 +0200 Subject: [PATCH 19/60] cleanup code --- .../org/apache/sysds/runtime/codegen/LibSpoofPrimitives.java | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/main/java/org/apache/sysds/runtime/codegen/LibSpoofPrimitives.java b/src/main/java/org/apache/sysds/runtime/codegen/LibSpoofPrimitives.java index 061f0862fe7..1a41431499f 100644 --- a/src/main/java/org/apache/sysds/runtime/codegen/LibSpoofPrimitives.java +++ b/src/main/java/org/apache/sysds/runtime/codegen/LibSpoofPrimitives.java @@ -2736,13 +2736,10 @@ public static double[] vectEqualWrite(int len, double[] a, double[] b, int[] aix aItr++; bItr++; } else if(aIdx < bIdx) { - //todo: this might be too unsafe c[aIdx] = a[aItr] == 0 ? 1 : 0; - // c[aIdx] = 0; aItr++; } else { c[bIdx] = b[bItr] == 0 ? 1 : 0; - // c[bIdx] = 0; bItr++; } } From 207c375a34e018aceb0abbcaec6091d7b49f38a3 Mon Sep 17 00:00:00 2001 From: Frxms Date: Mon, 23 Jun 2025 23:52:00 +0200 Subject: [PATCH 20/60] cleanup code --- .../org/apache/sysds/hops/codegen/cplan/CNodeBinary.java | 5 ----- 1 file changed, 5 deletions(-) diff --git a/src/main/java/org/apache/sysds/hops/codegen/cplan/CNodeBinary.java b/src/main/java/org/apache/sysds/hops/codegen/cplan/CNodeBinary.java index e2bc0f47e9b..246e9f6cf5c 100644 --- a/src/main/java/org/apache/sysds/hops/codegen/cplan/CNodeBinary.java +++ b/src/main/java/org/apache/sysds/hops/codegen/cplan/CNodeBinary.java @@ -159,10 +159,6 @@ public String codegen(boolean sparse, GeneratorAPI api) { sb.append(_inputs.get(0).codegen(sparse, api)); sb.append(_inputs.get(1).codegen(sparse, api)); - /** - * todo: remember that only certain primitives will be called through this method, - * because the optimizer will choose which primitive functions should be calculated sparse and which not - */ if(DMLScript.SPARSE_INTERMEDIATE) { //generate binary operation (use sparse template, if data input) boolean lsparseLhs = sparse ? _inputs.get(0) instanceof CNodeData @@ -206,7 +202,6 @@ public String codegen(boolean sparse, GeneratorAPI api) { && TemplateUtils.isColVector(_inputs.get(j)))) && _type!=BinType.VECT_MATRIXMULT) ? varj + ".pos(rix)" : "0" : "0"); } - //todo: the following if else block could be simplified, because the first condition won't be true //replace length information (e.g., after matrix mult) if( _type == BinType.VECT_OUTERMULT_ADD || (_type == BinType.VECT_CBIND && vectorVector)) { for( int j=0; j<2; j++ ) From 193921a2ff32765483f724002e9fac9e30956272 Mon Sep 17 00:00:00 2001 From: Frxms Date: Mon, 30 Jun 2025 14:57:35 +0200 Subject: [PATCH 21/60] literal val parse --- .../sysds/hops/codegen/template/TemplateRow.java | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/src/main/java/org/apache/sysds/hops/codegen/template/TemplateRow.java b/src/main/java/org/apache/sysds/hops/codegen/template/TemplateRow.java index 64c4c2abace..def405db8b5 100644 --- a/src/main/java/org/apache/sysds/hops/codegen/template/TemplateRow.java +++ b/src/main/java/org/apache/sysds/hops/codegen/template/TemplateRow.java @@ -442,10 +442,14 @@ else if(hop instanceof BinaryOp) if( TemplateUtils.isColVector(cdata2) ) cdata2 = new CNodeUnary(cdata2, UnaryType.LOOKUP_R); String opName = ((BinaryOp)hop).getOp().name(); + Hop hopIn1 = hop.getInput(0); + Hop hopIn2 = hop.getInput(1); double sparsityEst = OptimizerUtils.getBinaryOpSparsity( - OptimizerUtils.getSparsity(hop.getInput(0)), - OptimizerUtils.getSparsity(hop.getInput(1)), OpOp2.valueOf(opName), true); - out = getVectorBinary(cdata1, cdata2, opName, sparsityEst); + OptimizerUtils.getSparsity(hopIn1), + OptimizerUtils.getSparsity(hopIn2), OpOp2.valueOf(opName), true); + double literalVal = hopIn1 instanceof LiteralOp ? ((LiteralOp) hopIn1).getDoubleValue() + : hopIn2 instanceof LiteralOp ? ((LiteralOp) hopIn2).getDoubleValue() : Double.NEGATIVE_INFINITY; + out = getVectorBinary(cdata1, cdata2, opName, sparsityEst, literalVal); if( cdata1 instanceof CNodeData && !inHops2.containsKey("X") && !(cdata1.getDataType()==DataType.SCALAR) ) { inHops2.put("X", hop.getInput().get(0)); @@ -575,13 +579,13 @@ private static CNodeBinary getVectorBinary(CNode cdata1, CNode cdata2, String na } } - private static CNodeBinary getVectorBinary(CNode cdata1, CNode cdata2, String name, double sparsity) { + private static CNodeBinary getVectorBinary(CNode cdata1, CNode cdata2, String name, double sparsity, double literalVal) { if( TemplateUtils.isMatrix(cdata1) && (TemplateUtils.isMatrix(cdata2) || TemplateUtils.isRowVector(cdata2)) ) { return new CNodeBinary(cdata1, cdata2, BinType.valueOf("VECT_"+name), sparsity); } else { - return new CNodeBinary(cdata1, cdata2, BinType.valueOf("VECT_"+name+"_SCALAR"), sparsity); + return new CNodeBinary(cdata1, cdata2, BinType.valueOf("VECT_"+name+"_SCALAR"), sparsity, literalVal); } } From 13db23f418c90b42050f31b1a31ccb51db409018 Mon Sep 17 00:00:00 2001 From: Frxms Date: Mon, 30 Jun 2025 14:57:39 +0200 Subject: [PATCH 22/60] literal val parse --- .../sysds/hops/codegen/cplan/CNodeBinary.java | 30 ++++++++++++++++--- 1 file changed, 26 insertions(+), 4 deletions(-) diff --git a/src/main/java/org/apache/sysds/hops/codegen/cplan/CNodeBinary.java b/src/main/java/org/apache/sysds/hops/codegen/cplan/CNodeBinary.java index 65d74827ee8..c95fd8c32c4 100644 --- a/src/main/java/org/apache/sysds/hops/codegen/cplan/CNodeBinary.java +++ b/src/main/java/org/apache/sysds/hops/codegen/cplan/CNodeBinary.java @@ -145,6 +145,7 @@ public CNodeBinary( CNode in1, CNode in2, BinType type ) { setOutputDims(); } + //constructor for sparse VECT-VECT functions public CNodeBinary( CNode in1, CNode in2, BinType type, double sparsityEst ) { //canonicalize commutative matrix-scalar operations //to increase reuse potential @@ -159,7 +160,25 @@ public CNodeBinary( CNode in1, CNode in2, BinType type, double sparsityEst ) { _inputs.add(in2); _type = type; setOutputDims(); - sparseTemplate = getTemplateTpe(sparsityEst); + sparseTemplate = getTemplateType(sparsityEst); + } + + //constructor for sparse SCALAR-VECT/VECT-SCALAR functions + public CNodeBinary( CNode in1, CNode in2, BinType type, double sparsityEst, double scalarVal ) { + //canonicalize commutative matrix-scalar operations + //to increase reuse potential + if( type.isCommutative() && in1 instanceof CNodeData + && in1.getDataType()==DataType.SCALAR ) { + CNode tmp = in1; + in1 = in2; + in2 = tmp; + } + + _inputs.add(in1); + _inputs.add(in2); + _type = type; + setOutputDims(); + sparseTemplate = getTemplateType(sparsityEst); } public BinType getType() { @@ -304,16 +323,19 @@ private CNode getIntermediateInputVector() { return null; } - private boolean getTemplateTpe(double sparsityEst) { + private boolean getTemplateType(double sparsityEst) { if(!DMLScript.SPARSE_INTERMEDIATE) { return false; } else { switch(_type) { case VECT_MULT: return sparsityEst < 0.08 ? true : false; case VECT_MULT_SCALAR: return sparsityEst < 0.15 ? true : false; - case VECT_LESS_SCALAR: return false; + case VECT_LESS_SCALAR: { + _inputs.get(0); + return false; + } case VECT_LESS: return sparsityEst < 0.035 ? true : false; - default: return true; + default: return sparsityEst < 0.3 ? true : false; } } } From b14986bf930b87c6d699b2cf5ff1d9140dc3e77f Mon Sep 17 00:00:00 2001 From: Frxms Date: Mon, 30 Jun 2025 18:26:26 +0200 Subject: [PATCH 23/60] implement BinaryOp and UnaryOp optimizer --- .../sysds/hops/codegen/cplan/CNodeBinary.java | 65 +++++++++++-------- .../sysds/hops/codegen/cplan/CNodeUnary.java | 34 +++++++++- .../hops/codegen/template/TemplateRow.java | 6 +- .../functions/codegen/rowAggPattern49.dml | 6 +- 4 files changed, 77 insertions(+), 34 deletions(-) diff --git a/src/main/java/org/apache/sysds/hops/codegen/cplan/CNodeBinary.java b/src/main/java/org/apache/sysds/hops/codegen/cplan/CNodeBinary.java index c95fd8c32c4..af10515ed9e 100644 --- a/src/main/java/org/apache/sysds/hops/codegen/cplan/CNodeBinary.java +++ b/src/main/java/org/apache/sysds/hops/codegen/cplan/CNodeBinary.java @@ -145,25 +145,6 @@ public CNodeBinary( CNode in1, CNode in2, BinType type ) { setOutputDims(); } - //constructor for sparse VECT-VECT functions - public CNodeBinary( CNode in1, CNode in2, BinType type, double sparsityEst ) { - //canonicalize commutative matrix-scalar operations - //to increase reuse potential - if( type.isCommutative() && in1 instanceof CNodeData - && in1.getDataType()==DataType.SCALAR ) { - CNode tmp = in1; - in1 = in2; - in2 = tmp; - } - - _inputs.add(in1); - _inputs.add(in2); - _type = type; - setOutputDims(); - sparseTemplate = getTemplateType(sparsityEst); - } - - //constructor for sparse SCALAR-VECT/VECT-SCALAR functions public CNodeBinary( CNode in1, CNode in2, BinType type, double sparsityEst, double scalarVal ) { //canonicalize commutative matrix-scalar operations //to increase reuse potential @@ -178,7 +159,7 @@ public CNodeBinary( CNode in1, CNode in2, BinType type, double sparsityEst, doub _inputs.add(in2); _type = type; setOutputDims(); - sparseTemplate = getTemplateType(sparsityEst); + sparseTemplate = getTemplateType(sparsityEst, scalarVal); } public BinType getType() { @@ -323,18 +304,46 @@ private CNode getIntermediateInputVector() { return null; } - private boolean getTemplateType(double sparsityEst) { - if(!DMLScript.SPARSE_INTERMEDIATE) { + private boolean getTemplateType(double sparsityEst, double scalarVal) { + if(!DMLScript.SPARSE_INTERMEDIATE) return false; - } else { + else { switch(_type) { - case VECT_MULT: return sparsityEst < 0.08 ? true : false; - case VECT_MULT_SCALAR: return sparsityEst < 0.15 ? true : false; + case VECT_MULT: return sparsityEst < 0.008; + case VECT_DIV: return sparsityEst < 0.04; + case VECT_LESS: return sparsityEst < 0.035; + case VECT_MINUS: + case VECT_PLUS: + case VECT_XOR: + case VECT_BITWAND: + case VECT_BIASADD: + case VECT_BIASMULT: + case VECT_MIN: + case VECT_MAX: + case VECT_NOTEQUAL: + case VECT_GREATER: + case VECT_EQUAL: + case VECT_LESSEQUAL: + case VECT_GREATEREQUAL: return sparsityEst < 0.3; + case VECT_MULT_SCALAR: return sparsityEst < 0.15; + case VECT_POW_SCALAR: + case VECT_DIV_SCALAR: + case VECT_XOR_SCALAR: + case VECT_MIN_SCALAR: + case VECT_MAX_SCALAR: + case VECT_EQUAL_SCALAR: + case VECT_NOTEQUAL_SCALAR: return sparsityEst < 0.3; case VECT_LESS_SCALAR: { - _inputs.get(0); - return false; + if(scalarVal != Double.NaN) { + return _inputs.get(0).getDataType().isScalar() ? scalarVal <= 0 && sparsityEst < 0.09 + : _inputs.get(0).getDataType().isScalar() && scalarVal > 0 && sparsityEst < 0.09; + } else + return false; } - case VECT_LESS: return sparsityEst < 0.035 ? true : false; + case VECT_LESSEQUAL_SCALAR: + case VECT_GREATER_SCALAR: + case VECT_GREATEREQUAL_SCALAR: + case VECT_BITWAND_SCALAR: return sparsityEst < 0.3; default: return sparsityEst < 0.3 ? true : false; } } diff --git a/src/main/java/org/apache/sysds/hops/codegen/cplan/CNodeUnary.java b/src/main/java/org/apache/sysds/hops/codegen/cplan/CNodeUnary.java index f8d5c3eeb4c..d92f08d2ba8 100644 --- a/src/main/java/org/apache/sysds/hops/codegen/cplan/CNodeUnary.java +++ b/src/main/java/org/apache/sysds/hops/codegen/cplan/CNodeUnary.java @@ -86,12 +86,20 @@ public boolean isNotSupportedBySpoofCUDA() { } private UnaryType _type; + private boolean sparseTemplate; public CNodeUnary( CNode in1, UnaryType type ) { _inputs.add(in1); _type = type; setOutputDims(); } + + public CNodeUnary( CNode in1, UnaryType type, double sparsity ) { + _inputs.add(in1); + _type = type; + setOutputDims(); + sparseTemplate = getTemplateType(sparsity); + } public UnaryType getType() { return _type; @@ -116,8 +124,8 @@ public String codegen(boolean sparse, GeneratorAPI api) { ((_inputs.get(0) instanceof CNodeData && _inputs.get(0).getVarname().startsWith("a") && !_inputs.get(0).isLiteral()) - || (DMLScript.SPARSE_INTERMEDIATE && _inputs.get(0).getVarname().startsWith("STMP"))); - String var = createVarname(DMLScript.SPARSE_INTERMEDIATE && lsparse && getOutputType()); + || (sparseTemplate && _inputs.get(0).getVarname().startsWith("STMP"))); + String var = createVarname(sparseTemplate && lsparse && getOutputType()); String tmp = getLanguageTemplateClass(this, api).getTemplate(_type, lsparse); tmp = tmp.replaceAll("%TMP%", var); @@ -134,6 +142,28 @@ public String codegen(boolean sparse, GeneratorAPI api) { return sb.toString(); } + public boolean getTemplateType(double sparsity) { + if(!DMLScript.SPARSE_INTERMEDIATE) + return false; + else { + switch(_type) { + case VECT_SQRT: + case VECT_ABS: + case VECT_ROUND: + case VECT_CEIL: + case VECT_FLOOR: + case VECT_SIN: + case VECT_TAN: + case VECT_ASIN: + case VECT_ATAN: + case VECT_SINH: + case VECT_TANH: + case VECT_SIGN: return sparsity < 0.3; + default: return false; + } + } + } + public boolean getOutputType() { switch(_type) { case VECT_SQRT: diff --git a/src/main/java/org/apache/sysds/hops/codegen/template/TemplateRow.java b/src/main/java/org/apache/sysds/hops/codegen/template/TemplateRow.java index def405db8b5..14fe0635bc9 100644 --- a/src/main/java/org/apache/sysds/hops/codegen/template/TemplateRow.java +++ b/src/main/java/org/apache/sysds/hops/codegen/template/TemplateRow.java @@ -404,7 +404,7 @@ else if(hop instanceof UnaryOp) { { cdata1 = TemplateUtils.wrapLookupIfNecessary(cdata1, hop.getInput().get(0)); String primitiveOpName = ((UnaryOp)hop).getOp().name(); - out = new CNodeUnary(cdata1, UnaryType.valueOf(primitiveOpName)); + out = new CNodeUnary(cdata1, UnaryType.valueOf(primitiveOpName), OptimizerUtils.getSparsity(hop)); } } else if(HopRewriteUtils.isBinary(hop, OpOp2.CBIND)) { @@ -448,7 +448,7 @@ else if(hop instanceof BinaryOp) OptimizerUtils.getSparsity(hopIn1), OptimizerUtils.getSparsity(hopIn2), OpOp2.valueOf(opName), true); double literalVal = hopIn1 instanceof LiteralOp ? ((LiteralOp) hopIn1).getDoubleValue() - : hopIn2 instanceof LiteralOp ? ((LiteralOp) hopIn2).getDoubleValue() : Double.NEGATIVE_INFINITY; + : hopIn2 instanceof LiteralOp ? ((LiteralOp) hopIn2).getDoubleValue() : Double.NaN; out = getVectorBinary(cdata1, cdata2, opName, sparsityEst, literalVal); if( cdata1 instanceof CNodeData && !inHops2.containsKey("X") && !(cdata1.getDataType()==DataType.SCALAR) ) { @@ -582,7 +582,7 @@ private static CNodeBinary getVectorBinary(CNode cdata1, CNode cdata2, String na private static CNodeBinary getVectorBinary(CNode cdata1, CNode cdata2, String name, double sparsity, double literalVal) { if( TemplateUtils.isMatrix(cdata1) && (TemplateUtils.isMatrix(cdata2) || TemplateUtils.isRowVector(cdata2)) ) { - return new CNodeBinary(cdata1, cdata2, BinType.valueOf("VECT_"+name), sparsity); + return new CNodeBinary(cdata1, cdata2, BinType.valueOf("VECT_"+name), sparsity, literalVal); } else { return new CNodeBinary(cdata1, cdata2, BinType.valueOf("VECT_"+name+"_SCALAR"), sparsity, literalVal); diff --git a/src/test/scripts/functions/codegen/rowAggPattern49.dml b/src/test/scripts/functions/codegen/rowAggPattern49.dml index 9d31565fba7..a8eda3d2a92 100644 --- a/src/test/scripts/functions/codegen/rowAggPattern49.dml +++ b/src/test/scripts/functions/codegen/rowAggPattern49.dml @@ -32,7 +32,11 @@ Z= cbind(J, W, J) Y = matrix(0, 10, 18) K = rbind(Z, Y, Y, Y, Y, Y, Y, Y, Y) -S = (X < rowSums(X*K)) +B = X < rowSums(X*K) +S = abs(21) * B +# S = B * rowSums(v) + +# S = (X < rowSums(X*K)) # S = X*(k>1)*X # S = X*rowSums(K*v)*X # S = (X*v)/rowSums(X*v) From f574ea6dd2415b91fad6bbadf3679742b623f97a Mon Sep 17 00:00:00 2001 From: Frxms Date: Tue, 8 Jul 2025 17:00:39 +0200 Subject: [PATCH 24/60] implement Unary Optimizer --- .../java/org/apache/sysds/hops/codegen/cplan/CNodeUnary.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/apache/sysds/hops/codegen/cplan/CNodeUnary.java b/src/main/java/org/apache/sysds/hops/codegen/cplan/CNodeUnary.java index d92f08d2ba8..e3e4b5f8c3f 100644 --- a/src/main/java/org/apache/sysds/hops/codegen/cplan/CNodeUnary.java +++ b/src/main/java/org/apache/sysds/hops/codegen/cplan/CNodeUnary.java @@ -158,7 +158,7 @@ public boolean getTemplateType(double sparsity) { case VECT_ATAN: case VECT_SINH: case VECT_TANH: - case VECT_SIGN: return sparsity < 0.3; + case VECT_SIGN: return sparsity <= 0.4; default: return false; } } From 3f484b34ce0af8ce6e978d7b6dd45d27855c1ca2 Mon Sep 17 00:00:00 2001 From: Frxms Date: Wed, 16 Jul 2025 18:31:18 +0200 Subject: [PATCH 25/60] minor optimizations with the sparse row checks and correct mistake in Unary.java --- .../org/apache/sysds/hops/codegen/cplan/CNode.java | 10 ++-------- .../apache/sysds/hops/codegen/cplan/CNodeBinary.java | 8 +++----- .../apache/sysds/hops/codegen/cplan/java/Unary.java | 2 +- 3 files changed, 6 insertions(+), 14 deletions(-) diff --git a/src/main/java/org/apache/sysds/hops/codegen/cplan/CNode.java b/src/main/java/org/apache/sysds/hops/codegen/cplan/CNode.java index f5998b73bf5..36ebd238aca 100644 --- a/src/main/java/org/apache/sysds/hops/codegen/cplan/CNode.java +++ b/src/main/java/org/apache/sysds/hops/codegen/cplan/CNode.java @@ -107,19 +107,13 @@ public String getVectorLength(GeneratorAPI api) { return "len"; if(getVarname().startsWith("b")) return getVarname() + ".clen"; + else if(getVarname().startsWith("STMP")) + return "len"; else if(_dataType == DataType.MATRIX) return getVarname() + ".length"; } return ""; } - - public String getVectorLength(GeneratorAPI api, boolean sparse) { - if(sparse && getVarname().startsWith("STMP")) { - return "len"; - } else { - return getVectorLength(api); - } - } public String getClassname() { return getVarname(); diff --git a/src/main/java/org/apache/sysds/hops/codegen/cplan/CNodeBinary.java b/src/main/java/org/apache/sysds/hops/codegen/cplan/CNodeBinary.java index 246e9f6cf5c..9be9e7b8f36 100644 --- a/src/main/java/org/apache/sysds/hops/codegen/cplan/CNodeBinary.java +++ b/src/main/java/org/apache/sysds/hops/codegen/cplan/CNodeBinary.java @@ -163,12 +163,10 @@ public String codegen(boolean sparse, GeneratorAPI api) { //generate binary operation (use sparse template, if data input) boolean lsparseLhs = sparse ? _inputs.get(0) instanceof CNodeData && _inputs.get(0).getVarname().startsWith("a") || - (_inputs.get(0).getVarname().startsWith("STMP")) - && _inputs.get(0).getDataType().isMatrix() : false; + _inputs.get(0).getVarname().startsWith("STMP") : false; boolean lsparseRhs = sparse ? _inputs.get(1) instanceof CNodeData && _inputs.get(1).getVarname().startsWith("a") || - (_inputs.get(1).getVarname().startsWith("STMP")) - && _inputs.get(1).getDataType().isMatrix() : false; + _inputs.get(1).getVarname().startsWith("STMP") : false; boolean scalarInput = _inputs.get(0).getDataType().isScalar(); boolean scalarVector = (_inputs.get(0).getDataType().isScalar() && _inputs.get(1).getDataType().isMatrix()); @@ -210,7 +208,7 @@ public String codegen(boolean sparse, GeneratorAPI api) { else { //general case CNode mInput = getIntermediateInputVector(); if( mInput != null ) - tmp = tmp.replace("%LEN%", mInput.getVectorLength(api, sparse)); + tmp = tmp.replace("%LEN%", mInput.getVectorLength(api)); } sb.append(tmp); diff --git a/src/main/java/org/apache/sysds/hops/codegen/cplan/java/Unary.java b/src/main/java/org/apache/sysds/hops/codegen/cplan/java/Unary.java index a1d21228982..68b8019e70a 100644 --- a/src/main/java/org/apache/sysds/hops/codegen/cplan/java/Unary.java +++ b/src/main/java/org/apache/sysds/hops/codegen/cplan/java/Unary.java @@ -54,7 +54,7 @@ public String getTemplate(UnaryType type, boolean sparse) { String vectName = type.getVectorPrimitiveName(); return sparse ? DMLScript.SPARSE_INTERMEDIATE ? " SparseRowVector %TMP% = LibSpoofPrimitives.vect"+vectName+"Write(len, %IN1v%, %IN1i%, %POS1%, alen);\n" : - " double[] %TMP% = LibSpoofPrimitives.vect"+vectName+"Write(%IN1%, %POS1%, %LEN%);\n" : + " double[] %TMP% = LibSpoofPrimitives.vect"+vectName+"Write%IN1v%, %IN1i%, %POS1%, alen, len);\n" : " double[] %TMP% = LibSpoofPrimitives.vect"+vectName+"Write(%IN1%, %POS1%, %LEN%);\n"; } case VECT_EXP: From 8926665bbec279459ee2ec49d8858ae819b5cb4c Mon Sep 17 00:00:00 2001 From: Frxms Date: Thu, 17 Jul 2025 16:40:50 +0200 Subject: [PATCH 26/60] add vectMult for scalar-vector --- .../sysds/runtime/codegen/LibSpoofPrimitives.java | 14 ++++++++++++-- .../codegen/CPlanVectorPrimitivesTest.java | 5 +++++ 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/apache/sysds/runtime/codegen/LibSpoofPrimitives.java b/src/main/java/org/apache/sysds/runtime/codegen/LibSpoofPrimitives.java index 1a41431499f..503eaf03b5b 100644 --- a/src/main/java/org/apache/sysds/runtime/codegen/LibSpoofPrimitives.java +++ b/src/main/java/org/apache/sysds/runtime/codegen/LibSpoofPrimitives.java @@ -2191,6 +2191,11 @@ public static SparseRowVector vectMultWrite(int len, double[] a, double bval, in return c; } + public static SparseRowVector vectMultWrite(int len, double bval, double[] a, int[] aix, int ai, int alen) { + return vectMultWrite(len, a, bval, aix, ai, alen); + } + + //version with branching public static SparseRowVector vectMultWriteB(int len, double[] a, double[] b, int[] aix, int[] bix, int ai, int bi, int alen, int blen) { SparseRowVector c = allocSparseVector(Math.min(alen, blen)); if( a == null || b == null ) return c; @@ -2217,6 +2222,7 @@ public static SparseRowVector vectMultWriteB(int len, double[] a, double[] b, in return c; } + //version without branching public static SparseRowVector vectMultWrite(int len, double[] a, double[] b, int[] aix, int[] bix, int ai, int bi, int alen, int blen) { SparseRowVector c = allocSparseVector(Math.min(alen, blen)); int index = 0; @@ -2240,8 +2246,9 @@ public static SparseRowVector vectMultWrite(int len, double[] a, double[] b, int public static void vectWrite(SparseRowVector a, double[] c, int ci, int len) { if( a == null ) return; int[] aix = a.indexes(); + double[] avals = a.values(); for(int j = 0; j < a.size(); j++) - c[ci+aix[j]] = a.get(aix[j]); + c[ci+aix[j]] = avals[j]; } public static SparseRowVector vectDivWrite(int len, double[] a, double bval, int[] aix, int ai, int alen) { @@ -2268,6 +2275,7 @@ public static SparseRowVector vectDivWrite(int len, double bval, double[] a, int return c; } + //version with branching public static SparseRowVector vectDivWriteB(int len, double[] a, double[] b, int[] aix, int[] bix, int ai, int bi, int alen, int blen) { SparseRowVector c = allocSparseVector(alen); int aItr = ai; @@ -2297,6 +2305,7 @@ public static SparseRowVector vectDivWriteB(int len, double[] a, double[] b, int return c; } + //version without branching public static SparseRowVector vectDivWrite(int len, double[] a, double[] b, int[] aix, int[] bix, int ai, int bi, int alen, int blen) { SparseRowVector c = allocSparseVector(Math.min(alen, blen)); int index = 0; @@ -2483,7 +2492,8 @@ public static SparseRowVector vectXorWrite(int len, double[] a, double[] b, int[ return c; } - public static SparseRowVector vectPowWrite(int len, double[] a, double bval, int[] aix, int ai, int alen) { + //version with branching + public static SparseRowVector vectPowWriteB(int len, double[] a, double bval, int[] aix, int ai, int alen) { if(bval == 0) { SparseRowVector c = allocSparseVector(len); int[] indexes = c.indexes(); diff --git a/src/test/java/org/apache/sysds/test/component/codegen/CPlanVectorPrimitivesTest.java b/src/test/java/org/apache/sysds/test/component/codegen/CPlanVectorPrimitivesTest.java index 9b66c11371d..bae22b1c384 100644 --- a/src/test/java/org/apache/sysds/test/component/codegen/CPlanVectorPrimitivesTest.java +++ b/src/test/java/org/apache/sysds/test/component/codegen/CPlanVectorPrimitivesTest.java @@ -788,6 +788,11 @@ public void testVectorScalarBitwAndSparseToSparse() { //scalar - vector + @Test + public void testScalarVectorMultSparseToSparse() { + testVectorBinarySparsePrimitive(BinType.VECT_MULT_SCALAR, InputType.SCALAR, InputType.VECTOR_SPARSE); + } + @Test public void testScalarVectorDivSparseToSparse() { testVectorBinarySparsePrimitive(BinType.VECT_DIV_SCALAR, InputType.SCALAR, InputType.VECTOR_SPARSE); From 207df6e8ec3f1963512c7037431d7efca2da9d12 Mon Sep 17 00:00:00 2001 From: Frxms Date: Thu, 17 Jul 2025 18:24:13 +0200 Subject: [PATCH 27/60] minor changes --- .../org/apache/sysds/hops/codegen/cplan/CNodeBinary.java | 7 ++----- .../org/apache/sysds/hops/codegen/cplan/CNodeUnary.java | 2 +- src/test/scripts/functions/codegen/rowAggPattern49.dml | 6 +++--- 3 files changed, 6 insertions(+), 9 deletions(-) diff --git a/src/main/java/org/apache/sysds/hops/codegen/cplan/CNodeBinary.java b/src/main/java/org/apache/sysds/hops/codegen/cplan/CNodeBinary.java index 409672b2cb2..388b49b4811 100644 --- a/src/main/java/org/apache/sysds/hops/codegen/cplan/CNodeBinary.java +++ b/src/main/java/org/apache/sysds/hops/codegen/cplan/CNodeBinary.java @@ -177,10 +177,7 @@ public String codegen(boolean sparse, GeneratorAPI api) { sb.append(_inputs.get(0).codegen(sparse, api)); sb.append(_inputs.get(1).codegen(sparse, api)); - /** - * todo: remember that only certain primitives will be called through this method, - * because the optimizer will choose which primitive functions should be calculated sparse and which not - */ + //todo: this if clause can be deleted if(sparseTemplate) { //generate binary operation (use sparse template, if data input) boolean lsparseLhs = sparse ? _inputs.get(0) instanceof CNodeData @@ -194,7 +191,7 @@ public String codegen(boolean sparse, GeneratorAPI api) { && _inputs.get(1).getDataType().isMatrix()); boolean vectorVector = _inputs.get(0).getDataType().isMatrix() && _inputs.get(1).getDataType().isMatrix(); - String var = createVarname(sparse && getOutputType(scalarVector, lsparseLhs, lsparseRhs)); + String var = createVarname(sparse && sparseTemplate && getOutputType(scalarVector, lsparseLhs, lsparseRhs)); String tmp = getLanguageTemplateClass(this, api) .getTemplate(_type, lsparseLhs, lsparseRhs, scalarVector, scalarInput, vectorVector, sparseTemplate); diff --git a/src/main/java/org/apache/sysds/hops/codegen/cplan/CNodeUnary.java b/src/main/java/org/apache/sysds/hops/codegen/cplan/CNodeUnary.java index e3e4b5f8c3f..560a3ff3435 100644 --- a/src/main/java/org/apache/sysds/hops/codegen/cplan/CNodeUnary.java +++ b/src/main/java/org/apache/sysds/hops/codegen/cplan/CNodeUnary.java @@ -124,7 +124,7 @@ public String codegen(boolean sparse, GeneratorAPI api) { ((_inputs.get(0) instanceof CNodeData && _inputs.get(0).getVarname().startsWith("a") && !_inputs.get(0).isLiteral()) - || (sparseTemplate && _inputs.get(0).getVarname().startsWith("STMP"))); + || _inputs.get(0).getVarname().startsWith("STMP")); String var = createVarname(sparseTemplate && lsparse && getOutputType()); String tmp = getLanguageTemplateClass(this, api).getTemplate(_type, lsparse); tmp = tmp.replaceAll("%TMP%", var); diff --git a/src/test/scripts/functions/codegen/rowAggPattern49.dml b/src/test/scripts/functions/codegen/rowAggPattern49.dml index a8eda3d2a92..4af201c7cc3 100644 --- a/src/test/scripts/functions/codegen/rowAggPattern49.dml +++ b/src/test/scripts/functions/codegen/rowAggPattern49.dml @@ -32,14 +32,14 @@ Z= cbind(J, W, J) Y = matrix(0, 10, 18) K = rbind(Z, Y, Y, Y, Y, Y, Y, Y, Y) -B = X < rowSums(X*K) -S = abs(21) * B +# B = X < rowSums(X*K) +# S = abs(21) * B # S = B * rowSums(v) # S = (X < rowSums(X*K)) # S = X*(k>1)*X # S = X*rowSums(K*v)*X -# S = (X*v)/rowSums(X*v) +S = (X*v)/rowSums(X*v) # S = abs((X*v)/rowSums(X*v)) # S = (X/v)+rowMeans(X-v) # S = (X*v)+rowSums(X*v) From e25a85176ee4e45d6188e687f0b138fcb5c146f2 Mon Sep 17 00:00:00 2001 From: Frxms Date: Thu, 17 Jul 2025 18:27:43 +0200 Subject: [PATCH 28/60] due to DMLScript.SPARSE_INTERMEDIATE getting checked inside getTemplateType() method, the other else clause can be deleted --- .../sysds/hops/codegen/cplan/CNodeBinary.java | 155 ++++++------------ 1 file changed, 50 insertions(+), 105 deletions(-) diff --git a/src/main/java/org/apache/sysds/hops/codegen/cplan/CNodeBinary.java b/src/main/java/org/apache/sysds/hops/codegen/cplan/CNodeBinary.java index 388b49b4811..d4e6baaf792 100644 --- a/src/main/java/org/apache/sysds/hops/codegen/cplan/CNodeBinary.java +++ b/src/main/java/org/apache/sysds/hops/codegen/cplan/CNodeBinary.java @@ -177,118 +177,63 @@ public String codegen(boolean sparse, GeneratorAPI api) { sb.append(_inputs.get(0).codegen(sparse, api)); sb.append(_inputs.get(1).codegen(sparse, api)); - //todo: this if clause can be deleted - if(sparseTemplate) { - //generate binary operation (use sparse template, if data input) - boolean lsparseLhs = sparse ? _inputs.get(0) instanceof CNodeData - && _inputs.get(0).getVarname().startsWith("a") || - _inputs.get(0).getVarname().startsWith("STMP") : false; - boolean lsparseRhs = sparse ? _inputs.get(1) instanceof CNodeData - && _inputs.get(1).getVarname().startsWith("a") || - _inputs.get(1).getVarname().startsWith("STMP") : false; - boolean scalarInput = _inputs.get(0).getDataType().isScalar(); - boolean scalarVector = (_inputs.get(0).getDataType().isScalar() - && _inputs.get(1).getDataType().isMatrix()); - boolean vectorVector = _inputs.get(0).getDataType().isMatrix() - && _inputs.get(1).getDataType().isMatrix(); - String var = createVarname(sparse && sparseTemplate && getOutputType(scalarVector, lsparseLhs, lsparseRhs)); - String tmp = getLanguageTemplateClass(this, api) - .getTemplate(_type, lsparseLhs, lsparseRhs, scalarVector, scalarInput, vectorVector, sparseTemplate); + //generate binary operation (use sparse template, if data input) + boolean lsparseLhs = sparse ? _inputs.get(0) instanceof CNodeData + && _inputs.get(0).getVarname().startsWith("a") || + _inputs.get(0).getVarname().startsWith("STMP") : false; + boolean lsparseRhs = sparse ? _inputs.get(1) instanceof CNodeData + && _inputs.get(1).getVarname().startsWith("a") || + _inputs.get(1).getVarname().startsWith("STMP") : false; + boolean scalarInput = _inputs.get(0).getDataType().isScalar(); + boolean scalarVector = (_inputs.get(0).getDataType().isScalar() + && _inputs.get(1).getDataType().isMatrix()); + boolean vectorVector = _inputs.get(0).getDataType().isMatrix() + && _inputs.get(1).getDataType().isMatrix(); + String var = createVarname(sparse && sparseTemplate && getOutputType(scalarVector, lsparseLhs, lsparseRhs)); + String tmp = getLanguageTemplateClass(this, api) + .getTemplate(_type, lsparseLhs, lsparseRhs, scalarVector, scalarInput, vectorVector, sparseTemplate); - tmp = tmp.replace("%TMP%", var); + tmp = tmp.replace("%TMP%", var); - //replace input references and start indexes - for( int j=0; j<2; j++ ) { - String varj = _inputs.get(j).getVarname(api); - //replace sparse and dense inputs - tmp = tmp.replace("%IN"+(j+1)+"v%", varj.startsWith("STMP") ? varj+".values()" : varj+"vals"); - tmp = tmp.replace("%IN"+(j+1)+"i%", varj.startsWith("STMP") ? varj+".indexes()" : varj+"ix"); - tmp = tmp.replace("%IN"+(j+1)+"%", - varj.startsWith("a") ? (api == GeneratorAPI.JAVA ? varj : - (_inputs.get(j).getDataType() == DataType.MATRIX ? varj + ".vals(0)" : varj)) : - varj.startsWith("b") ? (api == GeneratorAPI.JAVA ? varj + ".values(rix)" : - (_type == BinType.VECT_MATRIXMULT ? varj : varj + ".vals(0)")) : - _inputs.get(j).getDataType() == DataType.MATRIX ? (api == GeneratorAPI.JAVA ? varj : varj + ".vals(0)") : varj); + //replace input references and start indexes + for( int j=0; j<2; j++ ) { + String varj = _inputs.get(j).getVarname(api); + //replace sparse and dense inputs + tmp = tmp.replace("%IN"+(j+1)+"v%", varj.startsWith("STMP") ? varj+".values()" : varj+"vals"); + tmp = tmp.replace("%IN"+(j+1)+"i%", varj.startsWith("STMP") ? varj+".indexes()" : varj+"ix"); + tmp = tmp.replace("%IN"+(j+1)+"%", + varj.startsWith("a") ? (api == GeneratorAPI.JAVA ? varj : + (_inputs.get(j).getDataType() == DataType.MATRIX ? varj + ".vals(0)" : varj)) : + varj.startsWith("b") ? (api == GeneratorAPI.JAVA ? varj + ".values(rix)" : + (_type == BinType.VECT_MATRIXMULT ? varj : varj + ".vals(0)")) : + _inputs.get(j).getDataType() == DataType.MATRIX ? (api == GeneratorAPI.JAVA ? varj : varj + ".vals(0)") : varj); - tmp = tmp.replace("%LEN"+(j+1)+"%", varj.startsWith("STMP") ? varj+".size()" : j == 0 ? "alen" : "blen"); + tmp = tmp.replace("%LEN"+(j+1)+"%", varj.startsWith("STMP") ? varj+".size()" : j == 0 ? "alen" : "blen"); - //replace start position of main input - tmp = tmp.replace("%POS"+(j+1)+"%", (_inputs.get(j) instanceof CNodeData - && _inputs.get(j).getDataType().isMatrix()) ? (!varj.startsWith("b")) ? varj+"i" : - ((TemplateUtils.isMatrix(_inputs.get(j)) || (_type.isElementwise() - && TemplateUtils.isColVector(_inputs.get(j)))) && _type!=BinType.VECT_MATRIXMULT) ? - varj + ".pos(rix)" : "0" : "0"); - } - //replace length information (e.g., after matrix mult) - if( _type == BinType.VECT_OUTERMULT_ADD || (_type == BinType.VECT_CBIND && vectorVector)) { - for( int j=0; j<2; j++ ) - tmp = tmp.replace("%LEN"+(j+1)+"%", _inputs.get(j).getVectorLength(api)); - } - else { //general case - CNode mInput = getIntermediateInputVector(); - if( mInput != null ) - tmp = tmp.replace("%LEN%", mInput.getVectorLength(api)); - } - - sb.append(tmp); - - //mark as generated - _generated = true; - - return sb.toString(); - } else { - boolean lsparseLhs = - sparse && _inputs.get(0) instanceof CNodeData && _inputs.get(0).getVarname().startsWith("a"); - boolean lsparseRhs = - sparse && _inputs.get(1) instanceof CNodeData && _inputs.get(1).getVarname().startsWith("a"); - boolean scalarInput = _inputs.get(0).getDataType().isScalar(); - boolean scalarVector = (_inputs.get(0).getDataType().isScalar() && _inputs.get(1).getDataType().isMatrix()); - boolean vectorVector = _inputs.get(0).getDataType().isMatrix() && _inputs.get(1).getDataType().isMatrix(); - String var = createVarname(); - String tmp = getLanguageTemplateClass(this, api).getTemplate(_type, lsparseLhs, lsparseRhs, scalarVector, - scalarInput, vectorVector, false); - - tmp = tmp.replace("%TMP%", var); - - //replace input references and start indexes - for(int j = 0; j < 2; j++) { - String varj = _inputs.get(j).getVarname(api); - - //replace sparse and dense inputs - tmp = tmp.replace("%IN" + (j + 1) + "v%", varj + "vals"); - tmp = tmp.replace("%IN" + (j + 1) + "i%", varj + "ix"); - tmp = tmp.replace("%IN" + (j + 1) + "%", varj.startsWith("a") ? ( - api == GeneratorAPI.JAVA ? varj : (_inputs.get(j).getDataType() == DataType.MATRIX ? varj + - ".vals(0)" : varj)) : varj.startsWith("b") ? (api == GeneratorAPI.JAVA ? varj + - ".values(rix)" : (_type == BinType.VECT_MATRIXMULT ? varj : varj + ".vals(0)")) : - _inputs.get(j).getDataType() == DataType.MATRIX ? ( - api == GeneratorAPI.JAVA ? varj : varj + ".vals(0)") : varj); - - //replace start position of main input - tmp = tmp.replace("%POS" + (j + 1) + "%", (_inputs.get(j) instanceof CNodeData && - _inputs.get(j).getDataType().isMatrix()) ? (!varj.startsWith("b")) ? varj + "i" : ( - (TemplateUtils.isMatrix(_inputs.get(j)) || - (_type.isElementwise() && TemplateUtils.isColVector(_inputs.get(j)))) && - _type != BinType.VECT_MATRIXMULT) ? varj + ".pos(rix)" : "0" : "0"); - } - //replace length information (e.g., after matrix mult) - if(_type == BinType.VECT_OUTERMULT_ADD || (_type == BinType.VECT_CBIND && vectorVector)) { - for(int j = 0; j < 2; j++) - tmp = tmp.replace("%LEN" + (j + 1) + "%", _inputs.get(j).getVectorLength(api)); - } - else { //general case - CNode mInput = getIntermediateInputVector(); - if(mInput != null) - tmp = tmp.replace("%LEN%", mInput.getVectorLength(api)); - } + //replace start position of main input + tmp = tmp.replace("%POS"+(j+1)+"%", (_inputs.get(j) instanceof CNodeData + && _inputs.get(j).getDataType().isMatrix()) ? (!varj.startsWith("b")) ? varj+"i" : + ((TemplateUtils.isMatrix(_inputs.get(j)) || (_type.isElementwise() + && TemplateUtils.isColVector(_inputs.get(j)))) && _type!=BinType.VECT_MATRIXMULT) ? + varj + ".pos(rix)" : "0" : "0"); + } + //replace length information (e.g., after matrix mult) + if( _type == BinType.VECT_OUTERMULT_ADD || (_type == BinType.VECT_CBIND && vectorVector)) { + for( int j=0; j<2; j++ ) + tmp = tmp.replace("%LEN"+(j+1)+"%", _inputs.get(j).getVectorLength(api)); + } + else { //general case + CNode mInput = getIntermediateInputVector(); + if( mInput != null ) + tmp = tmp.replace("%LEN%", mInput.getVectorLength(api)); + } - sb.append(tmp); + sb.append(tmp); - //mark as generated - _generated = true; + //mark as generated + _generated = true; - return sb.toString(); - } + return sb.toString(); } private CNode getIntermediateInputVector() { From 4045b0bb21c2b251ea7608e6476542eab043c150 Mon Sep 17 00:00:00 2001 From: Frxms Date: Thu, 17 Jul 2025 18:44:44 +0200 Subject: [PATCH 29/60] make Ternary.java be able to process STMP sparse inputs --- .../org/apache/sysds/hops/codegen/cplan/CNodeTernary.java | 7 +++++-- .../org/apache/sysds/hops/codegen/cplan/java/Ternary.java | 2 +- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/src/main/java/org/apache/sysds/hops/codegen/cplan/CNodeTernary.java b/src/main/java/org/apache/sysds/hops/codegen/cplan/CNodeTernary.java index 5e811092836..c6ff9802b16 100644 --- a/src/main/java/org/apache/sysds/hops/codegen/cplan/CNodeTernary.java +++ b/src/main/java/org/apache/sysds/hops/codegen/cplan/CNodeTernary.java @@ -82,10 +82,13 @@ public String codegen(boolean sparse, GeneratorAPI api) { String varj = _inputs.get(j-1).getVarname(); //replace sparse and dense inputs tmp = tmp.replace("%IN"+j+"v%", - varj+(varj.startsWith("a")?"vals":"") ); + varj+(varj.startsWith("a")?"vals" : varj.startsWith("STMP") ? ".values()" :"") ); tmp = tmp.replace("%IN"+j+"i%", - varj+(varj.startsWith("a")?"ix":"") ); + varj+(varj.startsWith("a")?"ix": varj.startsWith("STMP") ? ".indexes()" :"") ); tmp = tmp.replace("%IN"+j+"%", varj ); + tmp = tmp.replace("%POS%", varj.startsWith("a") ? varj+"i" : varj.startsWith("STMP") ? "0" : ""); + tmp = tmp.replace("%LEN%", + varj.startsWith("a") ? "alen" : varj.startsWith("STMP") ? varj+".size()" : ""); } sb.append(tmp); diff --git a/src/main/java/org/apache/sysds/hops/codegen/cplan/java/Ternary.java b/src/main/java/org/apache/sysds/hops/codegen/cplan/java/Ternary.java index a86d51cca86..64d282bbb8d 100644 --- a/src/main/java/org/apache/sysds/hops/codegen/cplan/java/Ternary.java +++ b/src/main/java/org/apache/sysds/hops/codegen/cplan/java/Ternary.java @@ -51,7 +51,7 @@ public String getTemplate(CNodeTernary.TernaryType type, boolean sparse) { case LOOKUP_RC1: return sparse ? - " double %TMP% = getValue(%IN1v%, %IN1i%, ai, alen, %IN3%-1);\n" : + " double %TMP% = getValue(%IN1v%, %IN1i%, %POS%, %LEN%, %IN3%-1);\n" : " double %TMP% = getValue(%IN1%, %IN2%, rix, %IN3%-1);\n"; case LOOKUP_RVECT1: From 516906ccc9f88c2037b69cd01492b7907e473f38 Mon Sep 17 00:00:00 2001 From: Frxms Date: Fri, 18 Jul 2025 16:00:23 +0200 Subject: [PATCH 30/60] change test --- src/test/scripts/functions/codegen/rowAggPattern49.R | 4 ++-- src/test/scripts/functions/codegen/rowAggPattern49.dml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/test/scripts/functions/codegen/rowAggPattern49.R b/src/test/scripts/functions/codegen/rowAggPattern49.R index 7fc0c17aff0..06d5c63144b 100644 --- a/src/test/scripts/functions/codegen/rowAggPattern49.R +++ b/src/test/scripts/functions/codegen/rowAggPattern49.R @@ -37,9 +37,9 @@ Z= cbind(J, W, J) Y = matrix(0, 10, 18) K = rbind(Z, Y, Y, Y, Y, Y, Y, Y, Y) -S = (X < rowSums(X*K)) +# S = (X < rowSums(X*K)) # S = X*rowMins(K)*X -# S = X*rowSums(K*v)*X +S = X*rowSums(K*v)*X # S = (X*v)/rowSums(X*v) # S = abs((X*v)/rowSums(X*v)) # S = (X/v)+rowMeans(X-v) diff --git a/src/test/scripts/functions/codegen/rowAggPattern49.dml b/src/test/scripts/functions/codegen/rowAggPattern49.dml index 4af201c7cc3..c553c3e8745 100644 --- a/src/test/scripts/functions/codegen/rowAggPattern49.dml +++ b/src/test/scripts/functions/codegen/rowAggPattern49.dml @@ -38,8 +38,8 @@ K = rbind(Z, Y, Y, Y, Y, Y, Y, Y, Y) # S = (X < rowSums(X*K)) # S = X*(k>1)*X -# S = X*rowSums(K*v)*X -S = (X*v)/rowSums(X*v) +S = X*rowSums(K*v)*X +# S = (X*v)/rowSums(X*v) # S = abs((X*v)/rowSums(X*v)) # S = (X/v)+rowMeans(X-v) # S = (X*v)+rowSums(X*v) From 61715621dd77b0743f85f91e4fb014b27a5c3cc7 Mon Sep 17 00:00:00 2001 From: Frxms Date: Fri, 18 Jul 2025 16:38:34 +0200 Subject: [PATCH 31/60] debug pow primitive --- .../org/apache/sysds/runtime/codegen/LibSpoofPrimitives.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/main/java/org/apache/sysds/runtime/codegen/LibSpoofPrimitives.java b/src/main/java/org/apache/sysds/runtime/codegen/LibSpoofPrimitives.java index 503eaf03b5b..985fcdf0df8 100644 --- a/src/main/java/org/apache/sysds/runtime/codegen/LibSpoofPrimitives.java +++ b/src/main/java/org/apache/sysds/runtime/codegen/LibSpoofPrimitives.java @@ -2492,8 +2492,7 @@ public static SparseRowVector vectXorWrite(int len, double[] a, double[] b, int[ return c; } - //version with branching - public static SparseRowVector vectPowWriteB(int len, double[] a, double bval, int[] aix, int ai, int alen) { + public static SparseRowVector vectPowWrite(int len, double[] a, double bval, int[] aix, int ai, int alen) { if(bval == 0) { SparseRowVector c = allocSparseVector(len); int[] indexes = c.indexes(); From ea849a0c825105651ce259cc25a97ee127b78eae Mon Sep 17 00:00:00 2001 From: Frxms Date: Fri, 18 Jul 2025 18:38:11 +0200 Subject: [PATCH 32/60] change worst case flag --- .../org/apache/sysds/hops/codegen/template/TemplateRow.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/apache/sysds/hops/codegen/template/TemplateRow.java b/src/main/java/org/apache/sysds/hops/codegen/template/TemplateRow.java index 14fe0635bc9..05531c3d1e8 100644 --- a/src/main/java/org/apache/sysds/hops/codegen/template/TemplateRow.java +++ b/src/main/java/org/apache/sysds/hops/codegen/template/TemplateRow.java @@ -446,7 +446,7 @@ else if(hop instanceof BinaryOp) Hop hopIn2 = hop.getInput(1); double sparsityEst = OptimizerUtils.getBinaryOpSparsity( OptimizerUtils.getSparsity(hopIn1), - OptimizerUtils.getSparsity(hopIn2), OpOp2.valueOf(opName), true); + OptimizerUtils.getSparsity(hopIn2), OpOp2.valueOf(opName), false); double literalVal = hopIn1 instanceof LiteralOp ? ((LiteralOp) hopIn1).getDoubleValue() : hopIn2 instanceof LiteralOp ? ((LiteralOp) hopIn2).getDoubleValue() : Double.NaN; out = getVectorBinary(cdata1, cdata2, opName, sparsityEst, literalVal); From accf02d4e7d1cf6f5193b2878237c1efe63c823b Mon Sep 17 00:00:00 2001 From: Frxms Date: Sat, 19 Jul 2025 16:28:16 +0200 Subject: [PATCH 33/60] adjust SparseRowVector allocation --- .../org/apache/sysds/runtime/codegen/LibSpoofPrimitives.java | 4 ++-- .../java/org/apache/sysds/runtime/codegen/SpoofRowwise.java | 3 +-- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/src/main/java/org/apache/sysds/runtime/codegen/LibSpoofPrimitives.java b/src/main/java/org/apache/sysds/runtime/codegen/LibSpoofPrimitives.java index 985fcdf0df8..cd3c957a26d 100644 --- a/src/main/java/org/apache/sysds/runtime/codegen/LibSpoofPrimitives.java +++ b/src/main/java/org/apache/sysds/runtime/codegen/LibSpoofPrimitives.java @@ -3504,11 +3504,11 @@ public SparseVectorBuffer(int num, int len1, int len2) { _len2 = len2; } public SparseRowVector next(int len) { - if( _len1!=len && _len2!=len ) + if( _len1=_data.length) ? 0 : _pos+1; - } while( _data[_pos].values().length!=len ); + } while( _data[_pos].values().length inputs, ArrayList 0 ) if(inputs.get(0).isInSparseFormat() && DMLScript.SPARSE_INTERMEDIATE) { - //todo: look at wether n, n2 are correct for vector - vector calculations - LibSpoofPrimitives.setupSparseThreadLocalMemory(_reqVectMem, n, n2); + LibSpoofPrimitives.setupSparseThreadLocalMemory(_reqVectMem, n/2, n2); LibSpoofPrimitives.setupThreadLocalMemory(_reqVectMem, n, n2); } else { LibSpoofPrimitives.setupThreadLocalMemory(_reqVectMem, n, n2); From 7d3814423322b3359369c5c6dad348a5511ce222 Mon Sep 17 00:00:00 2001 From: Frxms Date: Sun, 20 Jul 2025 16:16:01 +0200 Subject: [PATCH 34/60] debug Unary class --- .../java/org/apache/sysds/hops/codegen/cplan/java/Unary.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/main/java/org/apache/sysds/hops/codegen/cplan/java/Unary.java b/src/main/java/org/apache/sysds/hops/codegen/cplan/java/Unary.java index 68b8019e70a..a5a35629108 100644 --- a/src/main/java/org/apache/sysds/hops/codegen/cplan/java/Unary.java +++ b/src/main/java/org/apache/sysds/hops/codegen/cplan/java/Unary.java @@ -33,6 +33,7 @@ public String getTemplate(UnaryType type, boolean sparse) { case ROW_MINS: case ROW_MAXS: case ROW_MEANS: + case ROW_VARS: case ROW_COUNTNNZS: { String vectName = StringUtils.capitalize(type.name().substring(4, type.name().length()-1).toLowerCase()); return sparse ? " double %TMP% = LibSpoofPrimitives.vect"+vectName+"(%IN1v%, %IN1i%, %POS1%, alen, len);\n": From c79ca0e68198423f26a6712d7a9508cb0f04e89f Mon Sep 17 00:00:00 2001 From: Frxms Date: Mon, 21 Jul 2025 16:02:46 +0200 Subject: [PATCH 35/60] debug Unary class - missing "(" --- .../java/org/apache/sysds/hops/codegen/cplan/java/Unary.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/apache/sysds/hops/codegen/cplan/java/Unary.java b/src/main/java/org/apache/sysds/hops/codegen/cplan/java/Unary.java index a5a35629108..8e5d28fb4a0 100644 --- a/src/main/java/org/apache/sysds/hops/codegen/cplan/java/Unary.java +++ b/src/main/java/org/apache/sysds/hops/codegen/cplan/java/Unary.java @@ -55,7 +55,7 @@ public String getTemplate(UnaryType type, boolean sparse) { String vectName = type.getVectorPrimitiveName(); return sparse ? DMLScript.SPARSE_INTERMEDIATE ? " SparseRowVector %TMP% = LibSpoofPrimitives.vect"+vectName+"Write(len, %IN1v%, %IN1i%, %POS1%, alen);\n" : - " double[] %TMP% = LibSpoofPrimitives.vect"+vectName+"Write%IN1v%, %IN1i%, %POS1%, alen, len);\n" : + " double[] %TMP% = LibSpoofPrimitives.vect"+vectName+"Write(%IN1v%, %IN1i%, %POS1%, alen, len);\n" : " double[] %TMP% = LibSpoofPrimitives.vect"+vectName+"Write(%IN1%, %POS1%, %LEN%);\n"; } case VECT_EXP: From 0e82e9cdf006453d09e9ec88fc8bc9c8332ffac0 Mon Sep 17 00:00:00 2001 From: Frxms Date: Thu, 24 Jul 2025 22:00:55 +0200 Subject: [PATCH 36/60] only one check is needed --- .../java/org/apache/sysds/hops/codegen/cplan/CNodeUnary.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/apache/sysds/hops/codegen/cplan/CNodeUnary.java b/src/main/java/org/apache/sysds/hops/codegen/cplan/CNodeUnary.java index f8d5c3eeb4c..9088d7c6e93 100644 --- a/src/main/java/org/apache/sysds/hops/codegen/cplan/CNodeUnary.java +++ b/src/main/java/org/apache/sysds/hops/codegen/cplan/CNodeUnary.java @@ -116,7 +116,7 @@ public String codegen(boolean sparse, GeneratorAPI api) { ((_inputs.get(0) instanceof CNodeData && _inputs.get(0).getVarname().startsWith("a") && !_inputs.get(0).isLiteral()) - || (DMLScript.SPARSE_INTERMEDIATE && _inputs.get(0).getVarname().startsWith("STMP"))); + || (_inputs.get(0).getVarname().startsWith("STMP"))); String var = createVarname(DMLScript.SPARSE_INTERMEDIATE && lsparse && getOutputType()); String tmp = getLanguageTemplateClass(this, api).getTemplate(_type, lsparse); tmp = tmp.replaceAll("%TMP%", var); From 07026111ac836733c0357aed27a7a41248055c63 Mon Sep 17 00:00:00 2001 From: Frxms Date: Fri, 25 Jul 2025 17:12:51 +0200 Subject: [PATCH 37/60] adjust len info --- .../java/org/apache/sysds/hops/codegen/cplan/CNodeBinary.java | 2 +- .../java/org/apache/sysds/hops/codegen/cplan/java/Binary.java | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/main/java/org/apache/sysds/hops/codegen/cplan/CNodeBinary.java b/src/main/java/org/apache/sysds/hops/codegen/cplan/CNodeBinary.java index 9be9e7b8f36..cf4e6d86b5a 100644 --- a/src/main/java/org/apache/sysds/hops/codegen/cplan/CNodeBinary.java +++ b/src/main/java/org/apache/sysds/hops/codegen/cplan/CNodeBinary.java @@ -191,7 +191,7 @@ public String codegen(boolean sparse, GeneratorAPI api) { (_type == BinType.VECT_MATRIXMULT ? varj : varj + ".vals(0)")) : _inputs.get(j).getDataType() == DataType.MATRIX ? (api == GeneratorAPI.JAVA ? varj : varj + ".vals(0)") : varj); - tmp = tmp.replace("%LEN"+(j+1)+"%", varj.startsWith("STMP") ? varj+".size()" : j == 0 ? "alen" : "blen"); + tmp = tmp.replace("%SLEN"+(j+1)+"%", varj.startsWith("STMP") ? varj+".size()" : j == 0 ? "alen" : "blen"); //replace start position of main input tmp = tmp.replace("%POS"+(j+1)+"%", (_inputs.get(j) instanceof CNodeData diff --git a/src/main/java/org/apache/sysds/hops/codegen/cplan/java/Binary.java b/src/main/java/org/apache/sysds/hops/codegen/cplan/java/Binary.java index 81b2517fe1f..d037ac15286 100644 --- a/src/main/java/org/apache/sysds/hops/codegen/cplan/java/Binary.java +++ b/src/main/java/org/apache/sysds/hops/codegen/cplan/java/Binary.java @@ -145,7 +145,7 @@ else if( !vectorVector ) case VECT_GREATER:{ String vectName = type.getVectorPrimitiveName(); if(DMLScript.SPARSE_INTERMEDIATE && sparseLhs && sparseRhs) { - return " SparseRowVector %TMP% = LibSpoofPrimitives.vect"+vectName+"Write(%LEN%, %IN1v%, %IN2v%, %IN1i%, %IN2i%, %POS1%, %POS2%, %LEN1%, %LEN2%);\n"; + return " SparseRowVector %TMP% = LibSpoofPrimitives.vect"+vectName+"Write(%LEN%, %IN1v%, %IN2v%, %IN1i%, %IN2i%, %POS1%, %POS2%, %SLEN1%, %SLEN2%);\n"; } else { return sparseLhs ? " double[] %TMP% = LibSpoofPrimitives.vect"+vectName+"Write(%IN1v%, %IN2%, %IN1i%, %POS1%, %POS2%, alen, %LEN%);\n" : @@ -159,7 +159,7 @@ else if( !vectorVector ) case VECT_GREATEREQUAL: { String vectName = type.getVectorPrimitiveName(); if(DMLScript.SPARSE_INTERMEDIATE && sparseLhs && sparseRhs) { - return " double[] %TMP% = LibSpoofPrimitives.vect"+vectName+"Write(%LEN%, %IN1v%, %IN2v%, %IN1i%, %IN2i%, %POS1%, %POS2%, %LEN1%, %LEN2%);\n"; + return " double[] %TMP% = LibSpoofPrimitives.vect"+vectName+"Write(%LEN%, %IN1v%, %IN2v%, %IN1i%, %IN2i%, %POS1%, %POS2%, %SLEN1%, %SLEN2%);\n"; } else { return sparseLhs ? " double[] %TMP% = LibSpoofPrimitives.vect"+vectName+"Write(%IN1v%, %IN2%, %IN1i%, %POS1%, %POS2%, alen, %LEN%);\n" : From 3ba64c2dd757d2925e563bcc2f302ff17eb15bc6 Mon Sep 17 00:00:00 2001 From: Frxms Date: Fri, 25 Jul 2025 17:23:50 +0200 Subject: [PATCH 38/60] adjust alloc test to new alloc implementation --- .../sysds/test/component/codegen/SparseVectorAllocTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/java/org/apache/sysds/test/component/codegen/SparseVectorAllocTest.java b/src/test/java/org/apache/sysds/test/component/codegen/SparseVectorAllocTest.java index c6a2b10f031..c03dd9adb1b 100644 --- a/src/test/java/org/apache/sysds/test/component/codegen/SparseVectorAllocTest.java +++ b/src/test/java/org/apache/sysds/test/component/codegen/SparseVectorAllocTest.java @@ -59,7 +59,7 @@ public void testBasicSparseVectorAllocation(int numVectors, int len, int expLen) LibSpoofPrimitives.setupSparseThreadLocalMemory(numVectors, len, -1); SparseRowVector sparseVec = LibSpoofPrimitives.allocSparseVector(expLen); - Assert.assertEquals("Vector capacity should be initialized correctly", expLen, sparseVec.capacity()); + Assert.assertTrue("Vector capacity should be initialized correctly", expLen <= sparseVec.capacity()); Assert.assertEquals("Vector size should be initialized with 0", 0, sparseVec.size()); LibSpoofPrimitives.cleanupSparseThreadLocalMemory(); From cc1be656c63e93e0640f80e18907e76a5082ef07 Mon Sep 17 00:00:00 2001 From: Frxms Date: Fri, 25 Jul 2025 17:34:56 +0200 Subject: [PATCH 39/60] debug %SLEN% implementation --- .../org/apache/sysds/hops/codegen/cplan/java/Binary.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/main/java/org/apache/sysds/hops/codegen/cplan/java/Binary.java b/src/main/java/org/apache/sysds/hops/codegen/cplan/java/Binary.java index d037ac15286..a86259c0e74 100644 --- a/src/main/java/org/apache/sysds/hops/codegen/cplan/java/Binary.java +++ b/src/main/java/org/apache/sysds/hops/codegen/cplan/java/Binary.java @@ -76,7 +76,7 @@ public String getTemplate(BinType type, boolean sparseLhs, boolean sparseRhs, return sparseRhs ? " double[] %TMP% = LibSpoofPrimitives.vect"+vectName+"Write(%IN1%, %IN2v%, %IN2i%, %POS2%, alen, %LEN%);\n" : " double[] %TMP% = LibSpoofPrimitives.vect"+vectName+"Write(%IN1%, %IN2%, %POS2%, %LEN%);\n"; else if(DMLScript.SPARSE_INTERMEDIATE) { - return sparseLhs ? " SparseRowVector %TMP% = LibSpoofPrimitives.vect"+vectName+"Write(%LEN%, %IN1v%, %IN2%, %IN1i%, %POS1%, %LEN1%);\n" : + return sparseLhs ? " SparseRowVector %TMP% = LibSpoofPrimitives.vect"+vectName+"Write(%LEN%, %IN1v%, %IN2%, %IN1i%, %POS1%, %SLEN1%);\n" : " double[] %TMP% = LibSpoofPrimitives.vect"+vectName+"Write(%IN1%, %IN2%, %POS1%, %LEN%);\n"; } else { return sparseLhs ? " double[] %TMP% = LibSpoofPrimitives.vect"+vectName+"Write(%IN1v%, %IN2%, %IN1i%, %POS1%, alen, %LEN%);\n" : @@ -97,11 +97,11 @@ else if(DMLScript.SPARSE_INTERMEDIATE) { String vectName = type.getVectorPrimitiveName(); if(scalarVector) { if(sparseRhs) - return DMLScript.SPARSE_INTERMEDIATE ? " SparseRowVector %TMP% = LibSpoofPrimitives.vect"+vectName+"Write(%LEN%, %IN1%, %IN2v%, %IN2i%, %POS2%, %LEN1%);\n" : + return DMLScript.SPARSE_INTERMEDIATE ? " SparseRowVector %TMP% = LibSpoofPrimitives.vect"+vectName+"Write(%LEN%, %IN1%, %IN2v%, %IN2i%, %POS2%, %SLEN1%);\n" : " double[] %TMP% = LibSpoofPrimitives.vect"+vectName+"Write(%IN1%, %IN2v%, %IN2i%, %POS2%, alen, %LEN%);\n"; } else { if(sparseLhs) - return DMLScript.SPARSE_INTERMEDIATE ? " SparseRowVector %TMP% = LibSpoofPrimitives.vect"+vectName+"Write(%LEN%, %IN1v%, %IN2%, %IN1i%, %POS1%, %LEN1%);\n" : + return DMLScript.SPARSE_INTERMEDIATE ? " SparseRowVector %TMP% = LibSpoofPrimitives.vect"+vectName+"Write(%LEN%, %IN1v%, %IN2%, %IN1i%, %POS1%, %SLEN1%);\n" : " double[] %TMP% = LibSpoofPrimitives.vect"+vectName+"Write(%IN1v%, %IN2%, %IN1i%, %POS1%, alen, %LEN%);\n"; } return " double[] %TMP% = LibSpoofPrimitives.vect"+vectName+"Write(%IN1%, %IN2%, %POS1%, %LEN%);\n"; From abdfac7a9d4810f9dfc3c6500e14227f633f4443 Mon Sep 17 00:00:00 2001 From: Frxms Date: Fri, 25 Jul 2025 18:29:22 +0200 Subject: [PATCH 40/60] implement Unary optimizer --- .../java/org/apache/sysds/hops/codegen/cplan/CNodeUnary.java | 2 +- .../org/apache/sysds/hops/codegen/cplan/CodeTemplate.java | 4 ++++ .../java/org/apache/sysds/hops/codegen/cplan/java/Unary.java | 4 ++-- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/src/main/java/org/apache/sysds/hops/codegen/cplan/CNodeUnary.java b/src/main/java/org/apache/sysds/hops/codegen/cplan/CNodeUnary.java index 0e3308bb040..0231bfa9ce5 100644 --- a/src/main/java/org/apache/sysds/hops/codegen/cplan/CNodeUnary.java +++ b/src/main/java/org/apache/sysds/hops/codegen/cplan/CNodeUnary.java @@ -126,7 +126,7 @@ public String codegen(boolean sparse, GeneratorAPI api) { && !_inputs.get(0).isLiteral()) || _inputs.get(0).getVarname().startsWith("STMP")); String var = createVarname(sparseTemplate && lsparse && getOutputType()); - String tmp = getLanguageTemplateClass(this, api).getTemplate(_type, lsparse); + String tmp = getLanguageTemplateClass(this, api).getTemplate(_type, lsparse, sparseTemplate); tmp = tmp.replaceAll("%TMP%", var); //replace sparse and dense inputs diff --git a/src/main/java/org/apache/sysds/hops/codegen/cplan/CodeTemplate.java b/src/main/java/org/apache/sysds/hops/codegen/cplan/CodeTemplate.java index 477ed765724..1ca8b4a3c1d 100644 --- a/src/main/java/org/apache/sysds/hops/codegen/cplan/CodeTemplate.java +++ b/src/main/java/org/apache/sysds/hops/codegen/cplan/CodeTemplate.java @@ -52,6 +52,10 @@ public String getTemplate(CNodeTernary.TernaryType type, boolean sparse) { public String getTemplate(CNodeUnary.UnaryType type, boolean sparse) { throw new RuntimeException("Calling wrong getTemplate method on " + getClass().getCanonicalName()); } + + public String getTemplate(CNodeUnary.UnaryType type, boolean sparse, boolean sparseTemplate) { + throw new RuntimeException("Calling wrong getTemplate method on " + getClass().getCanonicalName()); + } public static String getTemplate(String templateFileName) { try { diff --git a/src/main/java/org/apache/sysds/hops/codegen/cplan/java/Unary.java b/src/main/java/org/apache/sysds/hops/codegen/cplan/java/Unary.java index 8e5d28fb4a0..21c4ea26266 100644 --- a/src/main/java/org/apache/sysds/hops/codegen/cplan/java/Unary.java +++ b/src/main/java/org/apache/sysds/hops/codegen/cplan/java/Unary.java @@ -26,7 +26,7 @@ public class Unary extends CodeTemplate { @Override - public String getTemplate(UnaryType type, boolean sparse) { + public String getTemplate(UnaryType type, boolean sparse, boolean sparseTemplate) { switch( type ) { case ROW_SUMS: case ROW_SUMSQS: @@ -53,7 +53,7 @@ public String getTemplate(UnaryType type, boolean sparse) { case VECT_TANH: case VECT_SIGN:{ String vectName = type.getVectorPrimitiveName(); - return sparse ? DMLScript.SPARSE_INTERMEDIATE ? + return sparse ? sparseTemplate ? " SparseRowVector %TMP% = LibSpoofPrimitives.vect"+vectName+"Write(len, %IN1v%, %IN1i%, %POS1%, alen);\n" : " double[] %TMP% = LibSpoofPrimitives.vect"+vectName+"Write(%IN1v%, %IN1i%, %POS1%, alen, len);\n" : " double[] %TMP% = LibSpoofPrimitives.vect"+vectName+"Write(%IN1%, %POS1%, %LEN%);\n"; From e2323f89ab3a625e67ef0c50db4599a5eac92bcb Mon Sep 17 00:00:00 2001 From: Frxms Date: Fri, 25 Jul 2025 18:45:24 +0200 Subject: [PATCH 41/60] adapt new LEN feature for sparse rows --- .../java/org/apache/sysds/hops/codegen/cplan/java/Binary.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/apache/sysds/hops/codegen/cplan/java/Binary.java b/src/main/java/org/apache/sysds/hops/codegen/cplan/java/Binary.java index b45ddd5ac36..14b3c83ae0b 100644 --- a/src/main/java/org/apache/sysds/hops/codegen/cplan/java/Binary.java +++ b/src/main/java/org/apache/sysds/hops/codegen/cplan/java/Binary.java @@ -76,7 +76,7 @@ public String getTemplate(BinType type, boolean sparseLhs, boolean sparseRhs, return sparseRhs ? " double[] %TMP% = LibSpoofPrimitives.vect"+vectName+"Write(%IN1%, %IN2v%, %IN2i%, %POS2%, alen, %LEN%);\n" : " double[] %TMP% = LibSpoofPrimitives.vect"+vectName+"Write(%IN1%, %IN2%, %POS2%, %LEN%);\n"; else if(sparseTemplate) { - return sparseLhs ? " SparseRowVector %TMP% = LibSpoofPrimitives.vect"+vectName+"Write(%LEN%, %IN1v%, %IN2%, %IN1i%, %POS1%, %LEN1%);\n" : + return sparseLhs ? " SparseRowVector %TMP% = LibSpoofPrimitives.vect"+vectName+"Write(%LEN%, %IN1v%, %IN2%, %IN1i%, %POS1%, %SLEN1%);\n" : " double[] %TMP% = LibSpoofPrimitives.vect"+vectName+"Write(%IN1%, %IN2%, %POS1%, %LEN%);\n"; } else { return sparseLhs ? " double[] %TMP% = LibSpoofPrimitives.vect"+vectName+"Write(%IN1v%, %IN2%, %IN1i%, %POS1%, alen, %LEN%);\n" : From 8ce70d83642fe319a73f5df2a9594007f11e835d Mon Sep 17 00:00:00 2001 From: Frxms Date: Fri, 25 Jul 2025 18:52:23 +0200 Subject: [PATCH 42/60] implement optimizer for unary --- .../org/apache/sysds/hops/codegen/template/TemplateRow.java | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/main/java/org/apache/sysds/hops/codegen/template/TemplateRow.java b/src/main/java/org/apache/sysds/hops/codegen/template/TemplateRow.java index 05531c3d1e8..7fbdef03142 100644 --- a/src/main/java/org/apache/sysds/hops/codegen/template/TemplateRow.java +++ b/src/main/java/org/apache/sysds/hops/codegen/template/TemplateRow.java @@ -392,7 +392,7 @@ else if(hop instanceof UnaryOp) { { if( HopRewriteUtils.isUnary(hop, SUPPORTED_VECT_UNARY) ) { String opname = "VECT_"+((UnaryOp)hop).getOp().name(); - out = new CNodeUnary(cdata1, UnaryType.valueOf(opname)); + out = new CNodeUnary(cdata1, UnaryType.valueOf(opname), hop.getInput(0).getSparsity()); if( cdata1 instanceof CNodeData && !inHops2.containsKey("X") ) inHops2.put("X", hop.getInput().get(0)); } @@ -445,8 +445,7 @@ else if(hop instanceof BinaryOp) Hop hopIn1 = hop.getInput(0); Hop hopIn2 = hop.getInput(1); double sparsityEst = OptimizerUtils.getBinaryOpSparsity( - OptimizerUtils.getSparsity(hopIn1), - OptimizerUtils.getSparsity(hopIn2), OpOp2.valueOf(opName), false); + hopIn1.getSparsity(), hopIn2.getSparsity(), OpOp2.valueOf(opName), false); double literalVal = hopIn1 instanceof LiteralOp ? ((LiteralOp) hopIn1).getDoubleValue() : hopIn2 instanceof LiteralOp ? ((LiteralOp) hopIn2).getDoubleValue() : Double.NaN; out = getVectorBinary(cdata1, cdata2, opName, sparsityEst, literalVal); From 46d744936b70b342b7cf5b3e814f637325aa31c8 Mon Sep 17 00:00:00 2001 From: Frxms Date: Fri, 25 Jul 2025 20:36:38 +0200 Subject: [PATCH 43/60] implement new sparse intermediates for vectWrite --- .../java/org/apache/sysds/hops/codegen/cplan/CNodeNary.java | 5 ++++- .../java/org/apache/sysds/hops/codegen/cplan/CNodeRow.java | 4 ++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/src/main/java/org/apache/sysds/hops/codegen/cplan/CNodeNary.java b/src/main/java/org/apache/sysds/hops/codegen/cplan/CNodeNary.java index dcf18ec6569..3288e8f45ba 100644 --- a/src/main/java/org/apache/sysds/hops/codegen/cplan/CNodeNary.java +++ b/src/main/java/org/apache/sysds/hops/codegen/cplan/CNodeNary.java @@ -60,7 +60,10 @@ public String getTemplate(boolean sparseGen, long len, ArrayList inputs, sb.append( sparseInput ? " LibSpoofPrimitives.vectWrite("+varj+"vals, %TMP%, " +varj+"ix, "+pos+", "+off+", "+input._cols+");\n" : - " LibSpoofPrimitives.vectWrite("+(varj.startsWith("b")?varj+".values(rix)":varj) + varj.startsWith("STMP") ? + " LibSpoofPrimitives.vectWrite("+input._cols+", "+varj+".values(), %TMP%, " + +varj+".indexes(), "+pos+", "+off+", "+varj+".size());\n" : + " LibSpoofPrimitives.vectWrite("+(varj.startsWith("b")?varj+".values(rix)":varj) +", %TMP%, "+pos+", "+off+", "+input._cols+");\n"); off += input._cols; } diff --git a/src/main/java/org/apache/sysds/hops/codegen/cplan/CNodeRow.java b/src/main/java/org/apache/sysds/hops/codegen/cplan/CNodeRow.java index 93e3585c378..c0d06b4bcbc 100644 --- a/src/main/java/org/apache/sysds/hops/codegen/cplan/CNodeRow.java +++ b/src/main/java/org/apache/sysds/hops/codegen/cplan/CNodeRow.java @@ -163,8 +163,8 @@ private String getOutputStatement(String varName) { case NO_AGG_B1: case NO_AGG_CONST: if(api == GeneratorAPI.JAVA) - return TEMPLATE_NOAGG_OUT.replace("%IN%", varName).replace("%LEN%", - _output.getVarname().startsWith("STMP") ? _output.getVarname()+".size()" : _output.getVarname()+".length"); + return TEMPLATE_NOAGG_OUT.replace("%IN%", varName.startsWith("STMP")?varName+".values(), "+varName+".indexes()":varName).replace("%LEN%", + varName.startsWith("STMP") ? varName+".size()" : _output.getVarname()+".length"); else return TEMPLATE_NOAGG_CONST_OUT_CUDA.replace("%IN%", varName + ".vals(0)").replaceAll("%LEN%", _output.getVarname()+".length"); case FULL_AGG: From 497e11129968d910f2ec54e8a4eeb5962015104c Mon Sep 17 00:00:00 2001 From: Frxms Date: Fri, 25 Jul 2025 20:37:45 +0200 Subject: [PATCH 44/60] implement new vectWrite that use sparse intermediates --- .../sysds/runtime/codegen/LibSpoofPrimitives.java | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/src/main/java/org/apache/sysds/runtime/codegen/LibSpoofPrimitives.java b/src/main/java/org/apache/sysds/runtime/codegen/LibSpoofPrimitives.java index cd3c957a26d..e600cc81e41 100644 --- a/src/main/java/org/apache/sysds/runtime/codegen/LibSpoofPrimitives.java +++ b/src/main/java/org/apache/sysds/runtime/codegen/LibSpoofPrimitives.java @@ -2243,12 +2243,16 @@ public static SparseRowVector vectMultWrite(int len, double[] a, double[] b, int return c; } - public static void vectWrite(SparseRowVector a, double[] c, int ci, int len) { + public static void vectWrite(double[] a, int[] aix, double[] c, int ci, int len) { if( a == null ) return; - int[] aix = a.indexes(); - double[] avals = a.values(); - for(int j = 0; j < a.size(); j++) - c[ci+aix[j]] = avals[j]; + for(int j = 0; j < len; j++) + c[ci+aix[j]] = a[j]; + } + + public static void vectWrite(int len, double[] a, double[] c, int[] aix, int ai, int ci, int alen) { + if( a == null ) return; + for(int j = 0; j < ai+alen; j++) + c[ci+aix[j]] = a[j]; } public static SparseRowVector vectDivWrite(int len, double[] a, double bval, int[] aix, int ai, int alen) { From 87dcaeebecfb197a7e4e4cccfdc8ba936c83f538 Mon Sep 17 00:00:00 2001 From: Frxms Date: Fri, 25 Jul 2025 22:11:56 +0200 Subject: [PATCH 45/60] adjust method signature --- .../java/org/apache/sysds/hops/codegen/cplan/CNodeNary.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/apache/sysds/hops/codegen/cplan/CNodeNary.java b/src/main/java/org/apache/sysds/hops/codegen/cplan/CNodeNary.java index 3288e8f45ba..35c351546da 100644 --- a/src/main/java/org/apache/sysds/hops/codegen/cplan/CNodeNary.java +++ b/src/main/java/org/apache/sysds/hops/codegen/cplan/CNodeNary.java @@ -61,7 +61,7 @@ public String getTemplate(boolean sparseGen, long len, ArrayList inputs, " LibSpoofPrimitives.vectWrite("+varj+"vals, %TMP%, " +varj+"ix, "+pos+", "+off+", "+input._cols+");\n" : varj.startsWith("STMP") ? - " LibSpoofPrimitives.vectWrite("+input._cols+", "+varj+".values(), %TMP%, " + " LibSpoofPrimitives.vectWrite("+varj+".values(), %TMP%, " +varj+".indexes(), "+pos+", "+off+", "+varj+".size());\n" : " LibSpoofPrimitives.vectWrite("+(varj.startsWith("b")?varj+".values(rix)":varj) +", %TMP%, "+pos+", "+off+", "+input._cols+");\n"); From 9051ff62b12b466a7364c22bd6ea4e6596b52c0a Mon Sep 17 00:00:00 2001 From: Frxms Date: Fri, 25 Jul 2025 22:12:20 +0200 Subject: [PATCH 46/60] adjust method signature --- .../org/apache/sysds/runtime/codegen/LibSpoofPrimitives.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/apache/sysds/runtime/codegen/LibSpoofPrimitives.java b/src/main/java/org/apache/sysds/runtime/codegen/LibSpoofPrimitives.java index e600cc81e41..d9181802e9e 100644 --- a/src/main/java/org/apache/sysds/runtime/codegen/LibSpoofPrimitives.java +++ b/src/main/java/org/apache/sysds/runtime/codegen/LibSpoofPrimitives.java @@ -2249,7 +2249,7 @@ public static void vectWrite(double[] a, int[] aix, double[] c, int ci, int len) c[ci+aix[j]] = a[j]; } - public static void vectWrite(int len, double[] a, double[] c, int[] aix, int ai, int ci, int alen) { + public static void vectWrite(double[] a, double[] c, int[] aix, int ai, int ci, int alen) { if( a == null ) return; for(int j = 0; j < ai+alen; j++) c[ci+aix[j]] = a[j]; From 64e79847361ca1649f6261dedc6480a5490be4d9 Mon Sep 17 00:00:00 2001 From: Frxms Date: Fri, 25 Jul 2025 22:27:56 +0200 Subject: [PATCH 47/60] debug div method --- .../org/apache/sysds/runtime/codegen/LibSpoofPrimitives.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/apache/sysds/runtime/codegen/LibSpoofPrimitives.java b/src/main/java/org/apache/sysds/runtime/codegen/LibSpoofPrimitives.java index d9181802e9e..879ae931292 100644 --- a/src/main/java/org/apache/sysds/runtime/codegen/LibSpoofPrimitives.java +++ b/src/main/java/org/apache/sysds/runtime/codegen/LibSpoofPrimitives.java @@ -2259,7 +2259,7 @@ public static SparseRowVector vectDivWrite(int len, double[] a, double bval, int SparseRowVector c = allocSparseVector(alen); int[] indexes = c.indexes(); double[] values = c.values(); - for( int j = ai; j < ai+alen; j++ ) { + for( int j = 0; j < ai+alen; j++ ) { indexes[j] = aix[j]; values[j] = a[j] / bval; } From 8eff70105e7669f01f6e3296fb11954e95113d0c Mon Sep 17 00:00:00 2001 From: Frxms Date: Fri, 25 Jul 2025 23:33:52 +0200 Subject: [PATCH 48/60] debug binary class --- .../java/org/apache/sysds/hops/codegen/cplan/java/Binary.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/apache/sysds/hops/codegen/cplan/java/Binary.java b/src/main/java/org/apache/sysds/hops/codegen/cplan/java/Binary.java index 14b3c83ae0b..301830c5887 100644 --- a/src/main/java/org/apache/sysds/hops/codegen/cplan/java/Binary.java +++ b/src/main/java/org/apache/sysds/hops/codegen/cplan/java/Binary.java @@ -145,7 +145,7 @@ else if( !vectorVector ) case VECT_GREATER:{ String vectName = type.getVectorPrimitiveName(); if(sparseTemplate && sparseLhs && sparseRhs) { - return " SparseRowVector %TMP% = LibSpoofPrimitives.vect"+vectName+"Write(%LEN%, %IN1v%, %IN2v%, %IN1i%, %IN2i%, %POS1%, %POS2%, %LEN1%, %LEN2%);\n"; + return " SparseRowVector %TMP% = LibSpoofPrimitives.vect"+vectName+"Write(%LEN%, %IN1v%, %IN2v%, %IN1i%, %IN2i%, %POS1%, %POS2%, %SLEN1%, %SLEN2%);\n"; } else { return sparseLhs ? " double[] %TMP% = LibSpoofPrimitives.vect"+vectName+"Write(%IN1v%, %IN2%, %IN1i%, %POS1%, %POS2%, alen, %LEN%);\n" : @@ -159,7 +159,7 @@ else if( !vectorVector ) case VECT_GREATEREQUAL: { String vectName = type.getVectorPrimitiveName(); if(sparseTemplate && sparseLhs && sparseRhs) { - return " double[] %TMP% = LibSpoofPrimitives.vect"+vectName+"Write(%LEN%, %IN1v%, %IN2v%, %IN1i%, %IN2i%, %POS1%, %POS2%, %LEN1%, %LEN2%);\n"; + return " double[] %TMP% = LibSpoofPrimitives.vect"+vectName+"Write(%LEN%, %IN1v%, %IN2v%, %IN1i%, %IN2i%, %POS1%, %POS2%, %SLEN1%, %SLEN2%);\n"; } else { return sparseLhs ? " double[] %TMP% = LibSpoofPrimitives.vect"+vectName+"Write(%IN1v%, %IN2%, %IN1i%, %POS1%, %POS2%, alen, %LEN%);\n" : From 834b6739fdd903caffeafdaa755f53f60ac8b88b Mon Sep 17 00:00:00 2001 From: Frxms Date: Wed, 13 Aug 2025 15:03:03 +0200 Subject: [PATCH 49/60] delete branching with tail for loops --- .../runtime/codegen/LibSpoofPrimitives.java | 39 ++++++++++++++++++- 1 file changed, 38 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/apache/sysds/runtime/codegen/LibSpoofPrimitives.java b/src/main/java/org/apache/sysds/runtime/codegen/LibSpoofPrimitives.java index 879ae931292..05ec57fbbfd 100644 --- a/src/main/java/org/apache/sysds/runtime/codegen/LibSpoofPrimitives.java +++ b/src/main/java/org/apache/sysds/runtime/codegen/LibSpoofPrimitives.java @@ -2330,7 +2330,7 @@ public static SparseRowVector vectDivWrite(int len, double[] a, double[] b, int[ return c; } - public static SparseRowVector vectMinusWrite(int len, double[] a, double[] b, int[] aix, int[] bix, int ai, int bi, int alen, int blen) { + public static SparseRowVector vectMinusWriteB(int len, double[] a, double[] b, int[] aix, int[] bix, int ai, int bi, int alen, int blen) { SparseRowVector c = allocSparseVector(alen+blen); int aItr = ai; int bItr = bi; @@ -2372,6 +2372,43 @@ public static SparseRowVector vectMinusWrite(int len, double[] a, double[] b, in return c; } + public static SparseRowVector vectMinusWrite(int len, double[] a, double[] b, int[] aix, int[] bix, int ai, int bi, int alen, int blen) { + SparseRowVector c = allocSparseVector(alen+blen); + int aEnd = ai+alen; + int bEnd = bi+blen; + int aItr = ai; + int bItr = bi; + int index = 0; + int[] indexes = c.indexes(); + double[] values = c.values(); + while (aItr < aEnd && bItr < bEnd) { + int aIdx = aix[aItr]; + int bIdx = bix[bItr]; + + int useA = (aIdx <= bIdx) ? 1 : 0; + int useB = (aIdx >= bIdx) ? 1 : 0; + + indexes[index] = (useA == 1) ? aIdx : bIdx; + double av = (useA == 1) ? a[aItr] : 0.0; + double bv = (useB == 1) ? b[bItr] : 0.0; + values[index] = av - bv; + + aItr += useA; + bItr += useB; + index++; + } + for (; aItr < aEnd; aItr++, index++) { + indexes[index] = aix[aItr]; + values[index] = a[aItr]; + } + for (; bItr < bEnd; bItr++, index++) { + indexes[index] = bix[bItr]; + values[index] = -b[bItr]; + } + c.setSize(index); + return c; + } + public static SparseRowVector vectPlusWrite(int len, double[] a, double[] b, int[] aix, int[] bix, int ai, int bi, int alen, int blen) { SparseRowVector c = allocSparseVector(alen+blen); int aItr = ai; From b9191c7bee14862a6c9a12391629a5d7510d26a5 Mon Sep 17 00:00:00 2001 From: Frxms Date: Wed, 13 Aug 2025 15:07:04 +0200 Subject: [PATCH 50/60] j = 0 needs to be changed to j = ai --- .../runtime/codegen/LibSpoofPrimitives.java | 52 +++++++++---------- 1 file changed, 26 insertions(+), 26 deletions(-) diff --git a/src/main/java/org/apache/sysds/runtime/codegen/LibSpoofPrimitives.java b/src/main/java/org/apache/sysds/runtime/codegen/LibSpoofPrimitives.java index 05ec57fbbfd..8320d585b98 100644 --- a/src/main/java/org/apache/sysds/runtime/codegen/LibSpoofPrimitives.java +++ b/src/main/java/org/apache/sysds/runtime/codegen/LibSpoofPrimitives.java @@ -2183,7 +2183,7 @@ public static SparseRowVector vectMultWrite(int len, double[] a, double bval, in if( a == null ) return c; int[] indexes = c.indexes(); double[] values = c.values(); - for(int j = 0; j < ai+alen; j++) { + for(int j = ai; j < ai+alen; j++) { indexes[j] = aix[j]; values[j] = a[j]*bval; } @@ -2251,7 +2251,7 @@ public static void vectWrite(double[] a, int[] aix, double[] c, int ci, int len) public static void vectWrite(double[] a, double[] c, int[] aix, int ai, int ci, int alen) { if( a == null ) return; - for(int j = 0; j < ai+alen; j++) + for(int j = ai; j < ai+alen; j++) c[ci+aix[j]] = a[j]; } @@ -2259,7 +2259,7 @@ public static SparseRowVector vectDivWrite(int len, double[] a, double bval, int SparseRowVector c = allocSparseVector(alen); int[] indexes = c.indexes(); double[] values = c.values(); - for( int j = 0; j < ai+alen; j++ ) { + for( int j = ai; j < ai+alen; j++ ) { indexes[j] = aix[j]; values[j] = a[j] / bval; } @@ -2271,7 +2271,7 @@ public static SparseRowVector vectDivWrite(int len, double bval, double[] a, int SparseRowVector c = allocSparseVector(alen); int[] indexes = c.indexes(); double[] values = c.values(); - for(int j = 0; j < ai+alen; j++) { + for(int j = ai; j < ai+alen; j++) { indexes[j] = aix[j]; values[j] = bval / a[j]; } @@ -2478,7 +2478,7 @@ public static SparseRowVector vectXorWrite(int len, double[] a, double bval, int SparseRowVector c = allocSparseVector(alen); int[] indexes = c.indexes(); double[] values = c.values(); - for(int j = 0; j < ai+alen; j++) { + for(int j = ai; j < ai+alen; j++) { indexes[j] = aix[j]; values[j] = (a[j] != 0) ? 1 : 0; } @@ -2560,7 +2560,7 @@ public static SparseRowVector vectPowWrite(int len, double[] a, double bval, int SparseRowVector c = allocSparseVector(alen); int[] indexes = c.indexes(); double[] values = c.values(); - for(int j = 0; j < ai+alen; j++) { + for(int j = ai; j < ai+alen; j++) { indexes[j] = aix[j]; values[j] = Math.pow(a[j], bval); } @@ -2596,7 +2596,7 @@ public static SparseRowVector vectMinWrite(int len, double[] a, double bval, int SparseRowVector c = allocSparseVector(alen); int[] indexes = c.indexes(); double[] values = c.values(); - for(int j = 0; j < ai+alen; j++) { + for(int j = ai; j < ai+alen; j++) { indexes[j] = aix[j]; values[j] = Math.min(a[j], bval); } @@ -2678,7 +2678,7 @@ public static SparseRowVector vectMaxWrite(int len, double[] a, double bval, int SparseRowVector c = allocSparseVector(alen); int[] indexes = c.indexes(); double[] values = c.values(); - for(int j = 0; j < ai+alen; j++) { + for(int j = ai; j < ai+alen; j++) { indexes[j] = aix[j]; values[j] = Math.max(a[j], bval); } @@ -2760,7 +2760,7 @@ public static SparseRowVector vectEqualWrite(int len, double[] a, double bval, i SparseRowVector c = allocSparseVector(alen); int[] indexes = c.indexes(); double[] values = c.values(); - for(int j = 0; j < ai+alen; j++) { + for(int j = ai; j < ai+alen; j++) { indexes[j] = aix[j]; values[j] = a[j] == bval ? 1 : 0; } @@ -2825,7 +2825,7 @@ public static SparseRowVector vectNotequalWrite(int len, double[] a, double bval SparseRowVector c = allocSparseVector(alen); int[] indexes = c.indexes(); double[] values = c.values(); - for(int j = 0; j < ai+alen; j++) { + for(int j = ai; j < ai+alen; j++) { indexes[j] = aix[j]; values[j] = a[j] != bval ? 1 : 0; } @@ -2907,7 +2907,7 @@ public static SparseRowVector vectLessWrite(int len, double[] a, double bval, in SparseRowVector c = allocSparseVector(alen); int[] indexes = c.indexes(); double[] values = c.values(); - for(int j = 0; j < ai+alen; j++) { + for(int j = ai; j < ai+alen; j++) { indexes[j] = aix[j]; values[j] = a[j] < bval ? 1 : 0; } @@ -2989,7 +2989,7 @@ public static SparseRowVector vectLessequalWrite(int len, double[] a, double bva SparseRowVector c = allocSparseVector(alen); int[] indexes = c.indexes(); double[] values = c.values(); - for(int j = 0; j < ai+alen; j++) { + for(int j = ai; j < ai+alen; j++) { indexes[j] = aix[j]; values[j] = a[j] <= bval ? 1 : 0; } @@ -3054,7 +3054,7 @@ public static SparseRowVector vectGreaterWrite(int len, double[] a, double bval, SparseRowVector c = allocSparseVector(alen); int[] indexes = c.indexes(); double[] values = c.values(); - for(int j = 0; j < ai+alen; j++) { + for(int j = ai; j < ai+alen; j++) { indexes[j] = aix[j]; values[j] = a[j] > bval ? 1 : 0; } @@ -3136,7 +3136,7 @@ public static SparseRowVector vectGreaterequalWrite(int len, double[] a, double SparseRowVector c = allocSparseVector(alen); int[] indexes = c.indexes(); double[] values = c.values(); - for(int j = 0; j < ai+alen; j++) { + for(int j = ai; j < ai+alen; j++) { indexes[j] = aix[j]; values[j] = a[j] >= bval ? 1 : 0; } @@ -3221,7 +3221,7 @@ public static SparseRowVector vectSqrtWrite(int len, double[] a, int[] aix, int SparseRowVector c = allocSparseVector(alen); int[] indexes = c.indexes(); double[] values = c.values(); - for(int j = 0; j < ai+alen; j++) { + for(int j = ai; j < ai+alen; j++) { indexes[j] = aix[j]; values[j] = Math.sqrt(a[j]); } @@ -3233,7 +3233,7 @@ public static SparseRowVector vectAbsWrite(int len, double[] a, int[] aix, int a SparseRowVector c = allocSparseVector(alen); int[] indexes = c.indexes(); double[] values = c.values(); - for(int j = 0; j < ai+alen; j++) { + for(int j = ai; j < ai+alen; j++) { indexes[j] = aix[j]; values[j] = Math.abs(a[j]); } @@ -3245,7 +3245,7 @@ public static SparseRowVector vectRoundWrite(int len, double[] a, int[] aix, int SparseRowVector c = allocSparseVector(alen); int[] indexes = c.indexes(); double[] values = c.values(); - for(int j = 0; j < ai+alen; j++) { + for(int j = ai; j < ai+alen; j++) { indexes[j] = aix[j]; values[j] = Math.round(a[j]); } @@ -3257,7 +3257,7 @@ public static SparseRowVector vectCeilWrite(int len, double[] a, int[] aix, int SparseRowVector c = allocSparseVector(alen); int[] indexes = c.indexes(); double[] values = c.values(); - for(int j = 0; j < ai+alen; j++) { + for(int j = ai; j < ai+alen; j++) { indexes[j] = aix[j]; values[j] = Math.ceil(a[j]); } @@ -3269,7 +3269,7 @@ public static SparseRowVector vectFloorWrite(int len, double[] a, int[] aix, int SparseRowVector c = allocSparseVector(alen); int[] indexes = c.indexes(); double[] values = c.values(); - for(int j = 0; j < ai+alen; j++) { + for(int j = ai; j < ai+alen; j++) { indexes[j] = aix[j]; values[j] = Math.floor(a[j]); } @@ -3281,7 +3281,7 @@ public static SparseRowVector vectSinWrite(int len, double[] a, int[] aix, int a SparseRowVector c = allocSparseVector(alen); int[] indexes = c.indexes(); double[] values = c.values(); - for(int j = 0; j < ai+alen; j++) { + for(int j = ai; j < ai+alen; j++) { indexes[j] = aix[j]; values[j] = Math.sin(a[j]); } @@ -3293,7 +3293,7 @@ public static SparseRowVector vectTanWrite(int len, double[] a, int[] aix, int a SparseRowVector c = allocSparseVector(alen); int[] indexes = c.indexes(); double[] values = c.values(); - for(int j = 0; j < ai+alen; j++) { + for(int j = ai; j < ai+alen; j++) { indexes[j] = aix[j]; values[j] = Math.tan(a[j]); } @@ -3305,7 +3305,7 @@ public static SparseRowVector vectAsinWrite(int len, double[] a, int[] aix, int SparseRowVector c = allocSparseVector(alen); int[] indexes = c.indexes(); double[] values = c.values(); - for(int j = 0; j < ai+alen; j++) { + for(int j = ai; j < ai+alen; j++) { indexes[j] = aix[j]; values[j] = Math.asin(a[j]); } @@ -3317,7 +3317,7 @@ public static SparseRowVector vectAtanWrite(int len, double[] a, int[] aix, int SparseRowVector c = allocSparseVector(alen); int[] indexes = c.indexes(); double[] values = c.values(); - for(int j = 0; j < ai+alen; j++) { + for(int j = ai; j < ai+alen; j++) { indexes[j] = aix[j]; values[j] = Math.atan(a[j]); } @@ -3329,7 +3329,7 @@ public static SparseRowVector vectSinhWrite(int len, double[] a, int[] aix, int SparseRowVector c = allocSparseVector(alen); int[] indexes = c.indexes(); double[] values = c.values(); - for(int j = 0; j < ai+alen; j++) { + for(int j = ai; j < ai+alen; j++) { indexes[j] = aix[j]; values[j] = Math.sinh(a[j]); } @@ -3341,7 +3341,7 @@ public static SparseRowVector vectTanhWrite(int len, double[] a, int[] aix, int SparseRowVector c = allocSparseVector(alen); int[] indexes = c.indexes(); double[] values = c.values(); - for(int j = 0; j < ai+alen; j++) { + for(int j = ai; j < ai+alen; j++) { indexes[j] = aix[j]; values[j] = Math.tanh(a[j]); } @@ -3353,7 +3353,7 @@ public static SparseRowVector vectSignWrite(int len, double[] a, int[] aix, int SparseRowVector c = allocSparseVector(alen); int[] indexes = c.indexes(); double[] values = c.values(); - for(int j = 0; j < ai+alen; j++) { + for(int j = ai; j < ai+alen; j++) { indexes[j] = aix[j]; values[j] = Math.signum(a[j]); } From 577fbe42e3f3088a174af47df7a0ae9553359481 Mon Sep 17 00:00:00 2001 From: Frxms Date: Wed, 13 Aug 2025 19:36:30 +0200 Subject: [PATCH 51/60] adjust optimizer values. Changed optimizer to be less harsh, as the benefit doesn't come from one operation being sparse, but consecutive sparse operations. Change templateRow to check the sparsity of the input in CNodeUnary constructor call --- .../sysds/hops/codegen/cplan/CNodeBinary.java | 54 ++++++++++++------- .../hops/codegen/template/TemplateRow.java | 2 +- 2 files changed, 37 insertions(+), 19 deletions(-) diff --git a/src/main/java/org/apache/sysds/hops/codegen/cplan/CNodeBinary.java b/src/main/java/org/apache/sysds/hops/codegen/cplan/CNodeBinary.java index 9d031dfb262..72fab314c00 100644 --- a/src/main/java/org/apache/sysds/hops/codegen/cplan/CNodeBinary.java +++ b/src/main/java/org/apache/sysds/hops/codegen/cplan/CNodeBinary.java @@ -248,9 +248,9 @@ private boolean getTemplateType(double sparsityEst, double scalarVal) { return false; else { switch(_type) { - case VECT_MULT: return sparsityEst < 0.008; - case VECT_DIV: return sparsityEst < 0.04; - case VECT_LESS: return sparsityEst < 0.035; + case VECT_MULT: + case VECT_DIV: + case VECT_LESS: case VECT_MINUS: case VECT_PLUS: case VECT_XOR: @@ -260,30 +260,48 @@ private boolean getTemplateType(double sparsityEst, double scalarVal) { case VECT_MIN: case VECT_MAX: case VECT_NOTEQUAL: - case VECT_GREATER: + case VECT_GREATER: return sparsityEst < 0.1; case VECT_EQUAL: case VECT_LESSEQUAL: - case VECT_GREATEREQUAL: return sparsityEst < 0.3; - case VECT_MULT_SCALAR: return sparsityEst < 0.15; - case VECT_POW_SCALAR: + case VECT_GREATEREQUAL: return sparsityEst < 0.5; + case VECT_MULT_SCALAR: return sparsityEst < 0.2; case VECT_DIV_SCALAR: case VECT_XOR_SCALAR: - case VECT_MIN_SCALAR: - case VECT_MAX_SCALAR: - case VECT_EQUAL_SCALAR: - case VECT_NOTEQUAL_SCALAR: return sparsityEst < 0.3; - case VECT_LESS_SCALAR: { + case VECT_BITWAND_SCALAR: return sparsityEst < 0.3; + case VECT_GREATER_SCALAR: + case VECT_GREATEREQUAL_SCALAR: + case VECT_MIN_SCALAR: { if(scalarVal != Double.NaN) { - return _inputs.get(0).getDataType().isScalar() ? scalarVal <= 0 && sparsityEst < 0.09 - : _inputs.get(0).getDataType().isScalar() && scalarVal > 0 && sparsityEst < 0.09; + return _inputs.get(1).getDataType().isScalar() ? scalarVal >= 0 && sparsityEst < 0.2 + : _inputs.get(0).getDataType().isScalar() && scalarVal < 0 && sparsityEst < 0.2; } else return false; } + case VECT_LESS_SCALAR: case VECT_LESSEQUAL_SCALAR: - case VECT_GREATER_SCALAR: - case VECT_GREATEREQUAL_SCALAR: - case VECT_BITWAND_SCALAR: return sparsityEst < 0.3; - default: return sparsityEst < 0.3 ? true : false; + case VECT_MAX_SCALAR: { + if(scalarVal != Double.NaN) { + return _inputs.get(1).getDataType().isScalar() ? scalarVal <= 0 && sparsityEst < 0.2 + : _inputs.get(0).getDataType().isScalar() && scalarVal > 0 && sparsityEst < 0.2; + } else + return false; + } + case VECT_POW_SCALAR: + case VECT_EQUAL_SCALAR:{ + if(scalarVal != Double.NaN) { + return _inputs.get(1).getDataType().isScalar() ? scalarVal != 0 && sparsityEst < 0.2 + : _inputs.get(0).getDataType().isScalar() && scalarVal != 0 && sparsityEst < 0.2; + } else + return false; + } + case VECT_NOTEQUAL_SCALAR:{ + if(scalarVal != Double.NaN) { + return _inputs.get(1).getDataType().isScalar() ? scalarVal == 0 && sparsityEst < 0.2 + : _inputs.get(0).getDataType().isScalar() && scalarVal == 0 && sparsityEst < 0.2; + } else + return false; + } + default: return sparsityEst < 0.3; } } } diff --git a/src/main/java/org/apache/sysds/hops/codegen/template/TemplateRow.java b/src/main/java/org/apache/sysds/hops/codegen/template/TemplateRow.java index 7fbdef03142..8efb0245e9f 100644 --- a/src/main/java/org/apache/sysds/hops/codegen/template/TemplateRow.java +++ b/src/main/java/org/apache/sysds/hops/codegen/template/TemplateRow.java @@ -404,7 +404,7 @@ else if(hop instanceof UnaryOp) { { cdata1 = TemplateUtils.wrapLookupIfNecessary(cdata1, hop.getInput().get(0)); String primitiveOpName = ((UnaryOp)hop).getOp().name(); - out = new CNodeUnary(cdata1, UnaryType.valueOf(primitiveOpName), OptimizerUtils.getSparsity(hop)); + out = new CNodeUnary(cdata1, UnaryType.valueOf(primitiveOpName), hop.getInput(0).getSparsity()); } } else if(HopRewriteUtils.isBinary(hop, OpOp2.CBIND)) { From d5c4b5fd40de776c20d1eae1be5ace5bc170f762 Mon Sep 17 00:00:00 2001 From: Frxms Date: Thu, 14 Aug 2025 01:02:46 +0200 Subject: [PATCH 52/60] update vect-vect primitives with branchless versions --- .../runtime/codegen/LibSpoofPrimitives.java | 341 ++++++++---------- 1 file changed, 149 insertions(+), 192 deletions(-) diff --git a/src/main/java/org/apache/sysds/runtime/codegen/LibSpoofPrimitives.java b/src/main/java/org/apache/sysds/runtime/codegen/LibSpoofPrimitives.java index 8320d585b98..29695b214c0 100644 --- a/src/main/java/org/apache/sysds/runtime/codegen/LibSpoofPrimitives.java +++ b/src/main/java/org/apache/sysds/runtime/codegen/LibSpoofPrimitives.java @@ -2384,15 +2384,13 @@ public static SparseRowVector vectMinusWrite(int len, double[] a, double[] b, in while (aItr < aEnd && bItr < bEnd) { int aIdx = aix[aItr]; int bIdx = bix[bItr]; - int useA = (aIdx <= bIdx) ? 1 : 0; int useB = (aIdx >= bIdx) ? 1 : 0; - indexes[index] = (useA == 1) ? aIdx : bIdx; double av = (useA == 1) ? a[aItr] : 0.0; double bv = (useB == 1) ? b[bItr] : 0.0; - values[index] = av - bv; + values[index] = av - bv; aItr += useA; bItr += useB; index++; @@ -2411,38 +2409,33 @@ public static SparseRowVector vectMinusWrite(int len, double[] a, double[] b, in public static SparseRowVector vectPlusWrite(int len, double[] a, double[] b, int[] aix, int[] bix, int ai, int bi, int alen, int blen) { SparseRowVector c = allocSparseVector(alen+blen); + int aEnd = ai+alen; + int bEnd = bi+blen; int aItr = ai; int bItr = bi; int index = 0; int[] indexes = c.indexes(); double[] values = c.values(); - while(aItr < ai+alen && bItr < bi+blen) { + while (aItr < aEnd && bItr < bEnd) { int aIdx = aix[aItr]; int bIdx = bix[bItr]; - if(aIdx == bIdx) { - indexes[index] = aIdx; - values[index] = a[aItr] + b[bItr]; - aItr++; - bItr++; - index++; - } else if(aIdx < bIdx) { - indexes[index] = aIdx; - values[index] = a[aItr]; - aItr++; - index++; - } else { - indexes[index] = bIdx; - values[index] = b[bItr]; - bItr++; - index++; - } + int useA = (aIdx <= bIdx) ? 1 : 0; + int useB = (aIdx >= bIdx) ? 1 : 0; + indexes[index] = (useA == 1) ? aIdx : bIdx; + double av = (useA == 1) ? a[aItr] : 0.0; + double bv = (useB == 1) ? b[bItr] : 0.0; + + values[index] = av + bv; + aItr += useA; + bItr += useB; + index++; } - for (; aItr < ai+alen; aItr++) { + for (; aItr < aEnd; aItr++) { indexes[index] = aix[aItr]; values[index] = a[aItr]; index++; } - for (; bItr < bi+blen; bItr++) { + for (; bItr < bEnd; bItr++) { indexes[index] = bix[bItr]; values[index] = b[bItr]; index++; @@ -2493,38 +2486,33 @@ public static SparseRowVector vectXorWrite(int len, double bval, double[] a, int public static SparseRowVector vectXorWrite(int len, double[] a, double[] b, int[] aix, int[] bix, int ai, int bi, int alen, int blen) { SparseRowVector c = allocSparseVector(alen+blen); + int aEnd = ai+alen; + int bEnd = bi+blen; int aItr = ai; int bItr = bi; int index = 0; int[] indexes = c.indexes(); double[] values = c.values(); - while(aItr < ai+alen && bItr < bi+blen) { + while (aItr < aEnd && bItr < bEnd) { int aIdx = aix[aItr]; int bIdx = bix[bItr]; - if(aIdx == bIdx) { - indexes[index] = aIdx; - values[index] = ((a[aItr] != 0) != (b[bItr] != 0)) ? 1 : 0; - aItr++; - bItr++; - index++; - } else if(aIdx < bIdx) { - indexes[index] = aIdx; - values[index] = (a[aItr] != 0) ? 1 : 0; - aItr++; - index++; - } else { - indexes[index] = bIdx; - values[index] = (b[bItr] != 0) ? 1 : 0; - bItr++; - index++; - } + int useA = (aIdx <= bIdx) ? 1 : 0; + int useB = (aIdx >= bIdx) ? 1 : 0; + indexes[index] = (useA == 1) ? aIdx : bIdx; + double av = (useA == 1) ? a[aItr] : 0.0; + double bv = (useB == 1) ? b[bItr] : 0.0; + + values[index] = ((av != 0) != (bv != 0)) ? 1 : 0; + aItr += useA; + bItr += useB; + index++; } - for (; aItr < ai+alen; aItr++) { + for (; aItr < aEnd; aItr++) { indexes[index] = aix[aItr]; values[index] = (a[aItr] != 0) ? 1 : 0; index++; } - for (; bItr < bi+blen; bItr++) { + for (; bItr < bEnd; bItr++) { indexes[index] = bix[bItr]; values[index] = (b[bItr] != 0) ? 1 : 0; index++; @@ -2611,38 +2599,33 @@ public static SparseRowVector vectMinWrite(int len, double bval, double[] a, int public static SparseRowVector vectMinWrite(int len, double[] a, double[] b, int[] aix, int[] bix, int ai, int bi, int alen, int blen) { SparseRowVector c = allocSparseVector(alen+blen); + int aEnd = ai+alen; + int bEnd = bi+blen; int aItr = ai; int bItr = bi; int index = 0; int[] indexes = c.indexes(); double[] values = c.values(); - while(aItr < ai+alen && bItr < bi+blen) { + while (aItr < aEnd && bItr < bEnd) { int aIdx = aix[aItr]; int bIdx = bix[bItr]; - if(aIdx == bIdx) { - indexes[index] = aIdx; - values[index] = Math.min(a[aItr], b[bItr]); - aItr++; - bItr++; - index++; - } else if(aIdx < bIdx) { - indexes[index] = aIdx; - values[index] = Math.min(a[aItr], 0); - aItr++; - index++; - } else { - indexes[index] = bIdx; - values[index] = Math.min(b[bItr], 0); - bItr++; - index++; - } + int useA = (aIdx <= bIdx) ? 1 : 0; + int useB = (aIdx >= bIdx) ? 1 : 0; + indexes[index] = (useA == 1) ? aIdx : bIdx; + double av = (useA == 1) ? a[aItr] : 0.0; + double bv = (useB == 1) ? b[bItr] : 0.0; + + values[index] = Math.min(av, bv); + aItr += useA; + bItr += useB; + index++; } - for (; aItr < ai+alen; aItr++) { + for (; aItr < aEnd; aItr++) { indexes[index] = aix[aItr]; values[index] = Math.min(a[aItr], 0); index++; } - for (; bItr < bi+blen; bItr++) { + for (; bItr < bEnd; bItr++) { indexes[index] = bix[bItr]; values[index] = Math.min(b[bItr], 0); index++; @@ -2693,38 +2676,33 @@ public static SparseRowVector vectMaxWrite(int len, double bval, double[] a, int public static SparseRowVector vectMaxWrite(int len, double[] a, double[] b, int[] aix, int[] bix, int ai, int bi, int alen, int blen) { SparseRowVector c = allocSparseVector(alen+blen); + int aEnd = ai+alen; + int bEnd = bi+blen; int aItr = ai; int bItr = bi; int index = 0; int[] indexes = c.indexes(); double[] values = c.values(); - while(aItr < ai+alen && bItr < bi+blen) { + while (aItr < aEnd && bItr < bEnd) { int aIdx = aix[aItr]; int bIdx = bix[bItr]; - if(aIdx == bIdx) { - indexes[index] = aIdx; - values[index] = Math.max(a[aItr], b[bItr]); - aItr++; - bItr++; - index++; - } else if(aIdx < bIdx) { - indexes[index] = aIdx; - values[index] = Math.max(a[aItr], 0); - aItr++; - index++; - } else { - indexes[index] = bIdx; - values[index] = Math.max(b[bItr], 0); - bItr++; - index++; - } + int useA = (aIdx <= bIdx) ? 1 : 0; + int useB = (aIdx >= bIdx) ? 1 : 0; + indexes[index] = (useA == 1) ? aIdx : bIdx; + double av = (useA == 1) ? a[aItr] : 0.0; + double bv = (useB == 1) ? b[bItr] : 0.0; + + values[index] = Math.max(av, bv); + aItr += useA; + bItr += useB; + index++; } - for (; aItr < ai+alen; aItr++) { + for (; aItr < aEnd; aItr++) { indexes[index] = aix[aItr]; values[index] = Math.max(a[aItr], 0); index++; } - for (; bItr < bi+blen; bItr++) { + for (; bItr < bEnd; bItr++) { indexes[index] = bix[bItr]; values[index] = Math.max(b[bItr], 0); index++; @@ -2776,25 +2754,25 @@ public static SparseRowVector vectEqualWrite(int len, double bval, double[] a, i //doesn't return SparseRowVector, but still uses two sparse vectors as inputs public static double[] vectEqualWrite(int len, double[] a, double[] b, int[] aix, int[] bix, int ai, int bi, int alen, int blen) { double[] c = allocVector(len, true, 1); + int aEnd = ai+alen; + int bEnd = bi+blen; int aItr = ai; int bItr = bi; - while(aItr < ai+alen && bItr < bi+blen) { + while(aItr < aEnd && bItr < bEnd) { int aIdx = aix[aItr]; int bIdx = bix[bItr]; - if (aIdx == bIdx) { - c[aIdx] = (a[aItr] == b[bItr]) ? 1 : 0; - aItr++; - bItr++; - } else if(aIdx < bIdx) { - c[aIdx] = a[aItr] == 0 ? 1 : 0; - aItr++; - } else { - c[bIdx] = b[bItr] == 0 ? 1 : 0; - bItr++; - } + int useA = (aIdx <= bIdx) ? 1 : 0; + int useB = (aIdx >= bIdx) ? 1 : 0; + int index = (useA == 1) ? aIdx : bIdx; + double av = (useA == 1) ? a[aItr] : 0.0; + double bv = (useB == 1) ? b[bItr] : 0.0; + + c[index] = av == bv ? 1 : 0; + aItr += useA; + bItr += useB; } - for (; aItr < ai+alen; aItr++) c[aix[aItr]] = 0; - for (; bItr < bi+blen; bItr++) c[bix[bItr]] = 0; + for (; aItr < aEnd; aItr++) c[aix[aItr]] = 0; + for (; bItr < bEnd; bItr++) c[bix[bItr]] = 0; return c; } @@ -2840,38 +2818,33 @@ public static SparseRowVector vectNotequalWrite(int len, double bval, double[] a public static SparseRowVector vectNotequalWrite(int len, double[] a, double[] b, int[] aix, int[] bix, int ai, int bi, int alen, int blen) { SparseRowVector c = allocSparseVector(alen+blen); + int aEnd = ai+alen; + int bEnd = bi+blen; int aItr = ai; int bItr = bi; int index = 0; int[] indexes = c.indexes(); double[] values = c.values(); - while(aItr < ai+alen && bItr < bi+blen) { + while (aItr < aEnd && bItr < bEnd) { int aIdx = aix[aItr]; int bIdx = bix[bItr]; - if(aIdx == bIdx) { - indexes[index] = aIdx; - values[index] = (a[aItr] != b[bItr]) ? 1 : 0; - aItr++; - bItr++; - index++; - } else if(aIdx < bIdx) { - indexes[index] = aIdx; - values[index] = a[aItr] != 0 ? 1 : 0; - aItr++; - index++; - } else { - indexes[index] = bIdx; - values[index] = b[bItr] != 0 ? 1 : 0; - bItr++; - index++; - } + int useA = (aIdx <= bIdx) ? 1 : 0; + int useB = (aIdx >= bIdx) ? 1 : 0; + indexes[index] = (useA == 1) ? aIdx : bIdx; + double av = (useA == 1) ? a[aItr] : 0.0; + double bv = (useB == 1) ? b[bItr] : 0.0; + + values[index] = (av != bv) ? 1 : 0; + aItr += useA; + bItr += useB; + index++; } - for (; aItr < ai+alen; aItr++) { + for (; aItr < aEnd; aItr++) { indexes[index] = aix[aItr]; values[index] = a[aItr] != 0 ? 1 : 0; index++; } - for (; bItr < bi+blen; bItr++) { + for (; bItr < bEnd; bItr++) { indexes[index] = bix[bItr]; values[index] = b[bItr] != 0 ? 1 : 0; index++; @@ -2922,38 +2895,33 @@ public static SparseRowVector vectLessWrite(int len, double bval, double[] a, in public static SparseRowVector vectLessWrite(int len, double[] a, double[] b, int[] aix, int[] bix, int ai, int bi, int alen, int blen) { SparseRowVector c = allocSparseVector(alen+blen); + int aEnd = ai+alen; + int bEnd = bi+blen; int aItr = ai; int bItr = bi; int index = 0; int[] indexes = c.indexes(); double[] values = c.values(); - while(aItr < ai+alen && bItr < bi+blen) { + while (aItr < aEnd && bItr < bEnd) { int aIdx = aix[aItr]; int bIdx = bix[bItr]; - if(aIdx == bIdx) { - indexes[index] = aIdx; - values[index] = (a[aItr] < b[bItr]) ? 1 : 0; - aItr++; - bItr++; - index++; - } else if(aIdx < bIdx) { - indexes[index] = aIdx; - values[index] = a[aItr] < 0 ? 1 : 0; - aItr++; - index++; - } else { - indexes[index] = bIdx; - values[index] = 0 < b[bItr] ? 1 : 0; - bItr++; - index++; - } + int useA = (aIdx <= bIdx) ? 1 : 0; + int useB = (aIdx >= bIdx) ? 1 : 0; + indexes[index] = (useA == 1) ? aIdx : bIdx; + double av = (useA == 1) ? a[aItr] : 0.0; + double bv = (useB == 1) ? b[bItr] : 0.0; + + values[index] = av < bv ? 1 : 0; + aItr += useA; + bItr += useB; + index++; } - for (; aItr < ai+alen; aItr++) { + for (; aItr < aEnd; aItr++) { indexes[index] = aix[aItr]; values[index] = a[aItr] < 0? 1 : 0; index++; } - for (; bItr < bi+blen; bItr++) { + for (; bItr < bEnd; bItr++) { indexes[index] = bix[bItr]; values[index] = 0 < b[bItr] ? 1 : 0; index++; @@ -3005,22 +2973,22 @@ public static SparseRowVector vectLessequalWrite(int len, double bval, double[] //doesn't return SparseRowVector, but still uses two sparse vectors as inputs public static double[] vectLessequalWrite(int len, double[] a, double[] b, int[] aix, int[] bix, int ai, int bi, int alen, int blen) { double[] c = allocVector(len, true, 1); + int aEnd = ai+alen; + int bEnd = bi+blen; int aItr = ai; int bItr = bi; - while(aItr < ai+alen && bItr < bi+blen) { + while(aItr < aEnd && bItr < bEnd) { int aIdx = aix[aItr]; int bIdx = bix[bItr]; - if(aIdx == bIdx) { - c[aIdx] = (a[aItr] <= b[bItr]) ? 1 : 0; - aItr++; - bItr++; - } else if(aIdx < bIdx) { - c[aIdx] = (a[aItr] <= 0) ? 1 : 0; - aItr++; - } else { - c[bIdx] = (0 <= b[bItr]) ? 1 : 0; - bItr++; - } + int useA = (aIdx <= bIdx) ? 1 : 0; + int useB = (aIdx >= bIdx) ? 1 : 0; + int index = (useA == 1) ? aIdx : bIdx; + double av = (useA == 1) ? a[aItr] : 0.0; + double bv = (useB == 1) ? b[bItr] : 0.0; + + c[index] = av <= bv ? 1 : 0; + aItr += useA; + bItr += useB; } for(; aItr < ai+alen; aItr++) c[aix[aItr]] = (a[aItr] <= 0) ? 1 : 0; for(; bItr < bi+blen; bItr++) c[bix[bItr]] = (0 <= b[bItr]) ? 1 : 0; @@ -3069,31 +3037,26 @@ public static SparseRowVector vectGreaterWrite(int len, double bval, double[] a, public static SparseRowVector vectGreaterWrite(int len, double[] a, double[] b, int[] aix, int[] bix, int ai, int bi, int alen, int blen) { SparseRowVector c = allocSparseVector(alen+blen); + int aEnd = ai+alen; + int bEnd = bi+blen; int aItr = ai; int bItr = bi; int index = 0; int[] indexes = c.indexes(); double[] values = c.values(); - while(aItr < ai+alen && bItr < bi+blen) { + while (aItr < aEnd && bItr < bEnd) { int aIdx = aix[aItr]; int bIdx = bix[bItr]; - if(aIdx == bIdx) { - indexes[index] = aIdx; - values[index] = (a[aItr] > b[bItr]) ? 1 : 0; - aItr++; - bItr++; - index++; - } else if(aIdx < bIdx) { - indexes[index] = aIdx; - values[index] = a[aItr] > 0 ? 1 : 0; - aItr++; - index++; - } else { - indexes[index] = bIdx; - values[index] = 0 > b[bItr] ? 1 : 0; - bItr++; - index++; - } + int useA = (aIdx <= bIdx) ? 1 : 0; + int useB = (aIdx >= bIdx) ? 1 : 0; + indexes[index] = (useA == 1) ? aIdx : bIdx; + double av = (useA == 1) ? a[aItr] : 0.0; + double bv = (useB == 1) ? b[bItr] : 0.0; + + values[index] = av >bv ? 1 : 0; + aItr += useA; + bItr += useB; + index++; } for (; aItr < ai+alen; aItr++) { indexes[index] = aix[aItr]; @@ -3152,22 +3115,22 @@ public static SparseRowVector vectGreaterequalWrite(int len, double bval, double //doesn't return SparseRowVector, but still uses two sparse vectors as inputs public static double[] vectGreaterequalWrite(int len, double[] a, double[] b, int[] aix, int[] bix, int ai, int bi, int alen, int blen) { double[] c = allocVector(len, true, 1); + int aEnd = ai+alen; + int bEnd = bi+blen; int aItr = ai; int bItr = bi; - while(aItr < ai+alen && bItr < bi+blen) { + while(aItr < aEnd && bItr < bEnd) { int aIdx = aix[aItr]; int bIdx = bix[bItr]; - if(aIdx == bIdx) { - c[aIdx] = (a[aItr] >= b[bItr]) ? 1 : 0; - aItr++; - bItr++; - } else if(aIdx < bIdx) { - c[aIdx] = (a[aItr] >= 0) ? 1 : 0; - aItr++; - } else { - c[bIdx] = (0 >= b[bItr]) ? 1 : 0; - bItr++; - } + int useA = (aIdx <= bIdx) ? 1 : 0; + int useB = (aIdx >= bIdx) ? 1 : 0; + int index = (useA == 1) ? aIdx : bIdx; + double av = (useA == 1) ? a[aItr] : 0.0; + double bv = (useB == 1) ? b[bItr] : 0.0; + + c[index] = av >= bv ? 1 : 0; + aItr += useA; + bItr += useB; } for(; aItr < ai+alen; aItr++) c[aix[aItr]] = (a[aItr] >= 0) ? 1 : 0; for(; bItr < bi+blen; bItr++) c[bix[bItr]] = (0 >= b[bItr]) ? 1 : 0; @@ -3192,26 +3155,20 @@ public static SparseRowVector vectBitwandWrite(int len, double bval, double[] a, } public static SparseRowVector vectBitwandWrite(int len, double[] a, double[] b, int[] aix, int[] bix, int ai, int bi, int alen, int blen) { - SparseRowVector c = allocSparseVector(alen); + SparseRowVector c = allocSparseVector(Math.min(alen, blen)); + int index = 0; int aItr = ai; int bItr = bi; - int index = 0; int[] indexes = c.indexes(); double[] values = c.values(); while(aItr < ai+alen && bItr < bi+blen) { int aIdx = aix[aItr]; int bIdx = bix[bItr]; - if(aIdx == bIdx) { - indexes[index] = aIdx; - values[index] = bwAnd(a[aItr], b[bItr]); - aItr++; - bItr++; - index++; - } else if(aIdx < bIdx) { - aItr++; - } else { - bItr++; - } + indexes[index] = aIdx; + values[index] = bwAnd(a[aItr], b[bItr]); + index += aIdx == bIdx ? 1 : 0; + aItr += aIdx <= bIdx ? 1 : 0; + bItr += aIdx >= bIdx ? 1 : 0; } c.setSize(index); return c; From e3b3eba61c8ffb27deaf32d9e96446f3fb4ca03e Mon Sep 17 00:00:00 2001 From: Frxms Date: Thu, 14 Aug 2025 01:21:19 +0200 Subject: [PATCH 53/60] correct Optimizer functionality --- .../sysds/hops/codegen/cplan/CNodeBinary.java | 36 +++++++++++++++---- 1 file changed, 30 insertions(+), 6 deletions(-) diff --git a/src/main/java/org/apache/sysds/hops/codegen/cplan/CNodeBinary.java b/src/main/java/org/apache/sysds/hops/codegen/cplan/CNodeBinary.java index 72fab314c00..f5d918789c9 100644 --- a/src/main/java/org/apache/sysds/hops/codegen/cplan/CNodeBinary.java +++ b/src/main/java/org/apache/sysds/hops/codegen/cplan/CNodeBinary.java @@ -268,24 +268,48 @@ private boolean getTemplateType(double sparsityEst, double scalarVal) { case VECT_DIV_SCALAR: case VECT_XOR_SCALAR: case VECT_BITWAND_SCALAR: return sparsityEst < 0.3; - case VECT_GREATER_SCALAR: - case VECT_GREATEREQUAL_SCALAR: - case VECT_MIN_SCALAR: { + case VECT_GREATER_SCALAR: { if(scalarVal != Double.NaN) { return _inputs.get(1).getDataType().isScalar() ? scalarVal >= 0 && sparsityEst < 0.2 : _inputs.get(0).getDataType().isScalar() && scalarVal < 0 && sparsityEst < 0.2; } else return false; } - case VECT_LESS_SCALAR: - case VECT_LESSEQUAL_SCALAR: - case VECT_MAX_SCALAR: { + case VECT_GREATEREQUAL_SCALAR: { + if(scalarVal != Double.NaN) { + return _inputs.get(1).getDataType().isScalar() ? scalarVal > 0 && sparsityEst < 0.2 + : _inputs.get(0).getDataType().isScalar() && scalarVal <= 0 && sparsityEst < 0.2; + } else + return false; + } + case VECT_MIN_SCALAR: { + if(scalarVal != Double.NaN) { + return _inputs.get(1).getDataType().isScalar() ? scalarVal >= 0 && sparsityEst < 0.2 + : _inputs.get(0).getDataType().isScalar() && scalarVal >= 0 && sparsityEst < 0.2; + } else + return false; + } + case VECT_LESS_SCALAR: { if(scalarVal != Double.NaN) { return _inputs.get(1).getDataType().isScalar() ? scalarVal <= 0 && sparsityEst < 0.2 : _inputs.get(0).getDataType().isScalar() && scalarVal > 0 && sparsityEst < 0.2; } else return false; } + case VECT_LESSEQUAL_SCALAR: { + if(scalarVal != Double.NaN) { + return _inputs.get(1).getDataType().isScalar() ? scalarVal < 0 && sparsityEst < 0.2 + : _inputs.get(0).getDataType().isScalar() && scalarVal >= 0 && sparsityEst < 0.2; + } else + return false; + } + case VECT_MAX_SCALAR: { + if(scalarVal != Double.NaN) { + return _inputs.get(1).getDataType().isScalar() ? scalarVal <= 0 && sparsityEst < 0.2 + : _inputs.get(0).getDataType().isScalar() && scalarVal <= 0 && sparsityEst < 0.2; + } else + return false; + } case VECT_POW_SCALAR: case VECT_EQUAL_SCALAR:{ if(scalarVal != Double.NaN) { From 7b5ccc5c1adc932560a91170d003362e16d51b07 Mon Sep 17 00:00:00 2001 From: Frxms Date: Thu, 14 Aug 2025 01:23:20 +0200 Subject: [PATCH 54/60] debug vectMultWrite --- .../java/org/apache/sysds/hops/codegen/cplan/CNodeBinary.java | 2 +- .../java/org/apache/sysds/hops/codegen/cplan/java/Binary.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/apache/sysds/hops/codegen/cplan/CNodeBinary.java b/src/main/java/org/apache/sysds/hops/codegen/cplan/CNodeBinary.java index cf4e6d86b5a..adadb41d2e6 100644 --- a/src/main/java/org/apache/sysds/hops/codegen/cplan/CNodeBinary.java +++ b/src/main/java/org/apache/sysds/hops/codegen/cplan/CNodeBinary.java @@ -281,8 +281,8 @@ private CNode getIntermediateInputVector() { public boolean getOutputType(boolean scalarVector, boolean lsparseLhs, boolean lsparseRhs) { switch(_type) { - case VECT_MULT_SCALAR: case VECT_POW_SCALAR: return !scalarVector && lsparseLhs; + case VECT_MULT_SCALAR: case VECT_DIV_SCALAR: case VECT_XOR_SCALAR: case VECT_MIN_SCALAR: diff --git a/src/main/java/org/apache/sysds/hops/codegen/cplan/java/Binary.java b/src/main/java/org/apache/sysds/hops/codegen/cplan/java/Binary.java index a86259c0e74..8bce035604c 100644 --- a/src/main/java/org/apache/sysds/hops/codegen/cplan/java/Binary.java +++ b/src/main/java/org/apache/sysds/hops/codegen/cplan/java/Binary.java @@ -69,7 +69,6 @@ public String getTemplate(BinType type, boolean sparseLhs, boolean sparseRhs, } //vector-scalar operations - case VECT_MULT_SCALAR: case VECT_POW_SCALAR: { String vectName = type.getVectorPrimitiveName(); if( scalarVector ) @@ -83,6 +82,7 @@ else if(DMLScript.SPARSE_INTERMEDIATE) { " double[] %TMP% = LibSpoofPrimitives.vect"+vectName+"Write(%IN1%, %IN2%, %POS1%, %LEN%);\n"; } } + case VECT_MULT_SCALAR: case VECT_DIV_SCALAR: case VECT_XOR_SCALAR: case VECT_MIN_SCALAR: From e50fa723867bbeb22ee955855400f222f8c03865 Mon Sep 17 00:00:00 2001 From: Frxms Date: Thu, 14 Aug 2025 13:39:52 +0200 Subject: [PATCH 55/60] minor value changes --- .../org/apache/sysds/hops/codegen/cplan/CNodeBinary.java | 6 +++--- .../org/apache/sysds/hops/codegen/cplan/CNodeUnary.java | 2 +- src/test/scripts/functions/codegen/rowAggPattern49.R | 4 ++-- src/test/scripts/functions/codegen/rowAggPattern49.dml | 6 +++--- 4 files changed, 9 insertions(+), 9 deletions(-) diff --git a/src/main/java/org/apache/sysds/hops/codegen/cplan/CNodeBinary.java b/src/main/java/org/apache/sysds/hops/codegen/cplan/CNodeBinary.java index c184734167f..37539ba4ecf 100644 --- a/src/main/java/org/apache/sysds/hops/codegen/cplan/CNodeBinary.java +++ b/src/main/java/org/apache/sysds/hops/codegen/cplan/CNodeBinary.java @@ -260,11 +260,11 @@ private boolean getTemplateType(double sparsityEst, double scalarVal) { case VECT_MIN: case VECT_MAX: case VECT_NOTEQUAL: - case VECT_GREATER: return sparsityEst < 0.1; + case VECT_GREATER: case VECT_EQUAL: case VECT_LESSEQUAL: - case VECT_GREATEREQUAL: return sparsityEst < 0.5; - case VECT_MULT_SCALAR: return sparsityEst < 0.2; + case VECT_GREATEREQUAL: return sparsityEst < 0.1; + case VECT_MULT_SCALAR: case VECT_DIV_SCALAR: case VECT_XOR_SCALAR: case VECT_BITWAND_SCALAR: return sparsityEst < 0.3; diff --git a/src/main/java/org/apache/sysds/hops/codegen/cplan/CNodeUnary.java b/src/main/java/org/apache/sysds/hops/codegen/cplan/CNodeUnary.java index 0231bfa9ce5..ffbe0087f1d 100644 --- a/src/main/java/org/apache/sysds/hops/codegen/cplan/CNodeUnary.java +++ b/src/main/java/org/apache/sysds/hops/codegen/cplan/CNodeUnary.java @@ -158,7 +158,7 @@ public boolean getTemplateType(double sparsity) { case VECT_ATAN: case VECT_SINH: case VECT_TANH: - case VECT_SIGN: return sparsity <= 0.4; + case VECT_SIGN: return sparsity <= 0.3; default: return false; } } diff --git a/src/test/scripts/functions/codegen/rowAggPattern49.R b/src/test/scripts/functions/codegen/rowAggPattern49.R index 06d5c63144b..18cd419dc21 100644 --- a/src/test/scripts/functions/codegen/rowAggPattern49.R +++ b/src/test/scripts/functions/codegen/rowAggPattern49.R @@ -39,8 +39,8 @@ K = rbind(Z, Y, Y, Y, Y, Y, Y, Y, Y) # S = (X < rowSums(X*K)) # S = X*rowMins(K)*X -S = X*rowSums(K*v)*X -# S = (X*v)/rowSums(X*v) +# S = X*rowSums(K*v)*X +S = (X*v)/rowSums(X*v) # S = abs((X*v)/rowSums(X*v)) # S = (X/v)+rowMeans(X-v) # S = (X*v)+rowSums(X*v) diff --git a/src/test/scripts/functions/codegen/rowAggPattern49.dml b/src/test/scripts/functions/codegen/rowAggPattern49.dml index c553c3e8745..60e3b271d32 100644 --- a/src/test/scripts/functions/codegen/rowAggPattern49.dml +++ b/src/test/scripts/functions/codegen/rowAggPattern49.dml @@ -31,15 +31,15 @@ J = matrix(0, 1, 8) Z= cbind(J, W, J) Y = matrix(0, 10, 18) K = rbind(Z, Y, Y, Y, Y, Y, Y, Y, Y) +while(FALSE) { } # B = X < rowSums(X*K) # S = abs(21) * B # S = B * rowSums(v) - # S = (X < rowSums(X*K)) # S = X*(k>1)*X -S = X*rowSums(K*v)*X -# S = (X*v)/rowSums(X*v) +# S = X*rowSums(K*v)*X +S = (X*v)/rowSums(X*v) # S = abs((X*v)/rowSums(X*v)) # S = (X/v)+rowMeans(X-v) # S = (X*v)+rowSums(X*v) From c086be76416fbc30a08a46cb1858feda9e9c4c44 Mon Sep 17 00:00:00 2001 From: Frxms Date: Thu, 14 Aug 2025 20:03:00 +0200 Subject: [PATCH 56/60] create new test class --- .../runtime/codegen/LibSpoofPrimitives.java | 88 +++++++++---------- 1 file changed, 44 insertions(+), 44 deletions(-) diff --git a/src/main/java/org/apache/sysds/runtime/codegen/LibSpoofPrimitives.java b/src/main/java/org/apache/sysds/runtime/codegen/LibSpoofPrimitives.java index 29695b214c0..1535fb71bd2 100644 --- a/src/main/java/org/apache/sysds/runtime/codegen/LibSpoofPrimitives.java +++ b/src/main/java/org/apache/sysds/runtime/codegen/LibSpoofPrimitives.java @@ -2183,9 +2183,9 @@ public static SparseRowVector vectMultWrite(int len, double[] a, double bval, in if( a == null ) return c; int[] indexes = c.indexes(); double[] values = c.values(); - for(int j = ai; j < ai+alen; j++) { - indexes[j] = aix[j]; - values[j] = a[j]*bval; + for(int j = 0; j < alen; j++) { + indexes[j] = aix[ai+j]; + values[j] = a[ai+j]*bval; } c.setSize(alen); return c; @@ -2251,17 +2251,17 @@ public static void vectWrite(double[] a, int[] aix, double[] c, int ci, int len) public static void vectWrite(double[] a, double[] c, int[] aix, int ai, int ci, int alen) { if( a == null ) return; - for(int j = ai; j < ai+alen; j++) - c[ci+aix[j]] = a[j]; + for(int j = 0; j < alen; j++) + c[ci+aix[ai+j]] = a[ai+j]; } public static SparseRowVector vectDivWrite(int len, double[] a, double bval, int[] aix, int ai, int alen) { SparseRowVector c = allocSparseVector(alen); int[] indexes = c.indexes(); double[] values = c.values(); - for( int j = ai; j < ai+alen; j++ ) { - indexes[j] = aix[j]; - values[j] = a[j] / bval; + for( int j = 0; j < alen; j++ ) { + indexes[j] = aix[ai+j]; + values[j] = a[ai+j] / bval; } c.setSize(alen); return c; @@ -2271,9 +2271,9 @@ public static SparseRowVector vectDivWrite(int len, double bval, double[] a, int SparseRowVector c = allocSparseVector(alen); int[] indexes = c.indexes(); double[] values = c.values(); - for(int j = ai; j < ai+alen; j++) { - indexes[j] = aix[j]; - values[j] = bval / a[j]; + for(int j = 0; j < alen; j++) { + indexes[j] = aix[ai+j]; + values[j] = bval / a[ai+j]; } c.setSize(alen); return c; @@ -2471,9 +2471,9 @@ public static SparseRowVector vectXorWrite(int len, double[] a, double bval, int SparseRowVector c = allocSparseVector(alen); int[] indexes = c.indexes(); double[] values = c.values(); - for(int j = ai; j < ai+alen; j++) { - indexes[j] = aix[j]; - values[j] = (a[j] != 0) ? 1 : 0; + for(int j = 0; j < alen; j++) { + indexes[j] = aix[ai+j]; + values[j] = (a[ai+j] != 0) ? 1 : 0; } c.setSize(alen); return c; @@ -2548,9 +2548,9 @@ public static SparseRowVector vectPowWrite(int len, double[] a, double bval, int SparseRowVector c = allocSparseVector(alen); int[] indexes = c.indexes(); double[] values = c.values(); - for(int j = ai; j < ai+alen; j++) { - indexes[j] = aix[j]; - values[j] = Math.pow(a[j], bval); + for(int j = 0; j < alen; j++) { + indexes[j] = aix[ai+j]; + values[j] = Math.pow(a[ai+j], bval); } c.setSize(alen); return c; @@ -2584,9 +2584,9 @@ public static SparseRowVector vectMinWrite(int len, double[] a, double bval, int SparseRowVector c = allocSparseVector(alen); int[] indexes = c.indexes(); double[] values = c.values(); - for(int j = ai; j < ai+alen; j++) { - indexes[j] = aix[j]; - values[j] = Math.min(a[j], bval); + for(int j = 0; j < alen; j++) { + indexes[j] = aix[ai+j]; + values[j] = Math.min(a[ai+j], bval); } c.setSize(alen); return c; @@ -2661,9 +2661,9 @@ public static SparseRowVector vectMaxWrite(int len, double[] a, double bval, int SparseRowVector c = allocSparseVector(alen); int[] indexes = c.indexes(); double[] values = c.values(); - for(int j = ai; j < ai+alen; j++) { - indexes[j] = aix[j]; - values[j] = Math.max(a[j], bval); + for(int j = 0; j < alen; j++) { + indexes[j] = aix[ai+j]; + values[j] = Math.max(a[ai+j], bval); } c.setSize(alen); return c; @@ -2738,9 +2738,9 @@ public static SparseRowVector vectEqualWrite(int len, double[] a, double bval, i SparseRowVector c = allocSparseVector(alen); int[] indexes = c.indexes(); double[] values = c.values(); - for(int j = ai; j < ai+alen; j++) { - indexes[j] = aix[j]; - values[j] = a[j] == bval ? 1 : 0; + for(int j = 0; j < alen; j++) { + indexes[j] = aix[ai+j]; + values[j] = a[ai+j] == bval ? 1 : 0; } c.setSize(alen); return c; @@ -2803,9 +2803,9 @@ public static SparseRowVector vectNotequalWrite(int len, double[] a, double bval SparseRowVector c = allocSparseVector(alen); int[] indexes = c.indexes(); double[] values = c.values(); - for(int j = ai; j < ai+alen; j++) { - indexes[j] = aix[j]; - values[j] = a[j] != bval ? 1 : 0; + for(int j = 0; j < alen; j++) { + indexes[j] = aix[ai+j]; + values[j] = a[ai+j] != bval ? 1 : 0; } c.setSize(alen); return c; @@ -2880,9 +2880,9 @@ public static SparseRowVector vectLessWrite(int len, double[] a, double bval, in SparseRowVector c = allocSparseVector(alen); int[] indexes = c.indexes(); double[] values = c.values(); - for(int j = ai; j < ai+alen; j++) { - indexes[j] = aix[j]; - values[j] = a[j] < bval ? 1 : 0; + for(int j = 0; j < alen; j++) { + indexes[j] = aix[ai+j]; + values[j] = a[ai+j] < bval ? 1 : 0; } c.setSize(alen); return c; @@ -2957,9 +2957,9 @@ public static SparseRowVector vectLessequalWrite(int len, double[] a, double bva SparseRowVector c = allocSparseVector(alen); int[] indexes = c.indexes(); double[] values = c.values(); - for(int j = ai; j < ai+alen; j++) { - indexes[j] = aix[j]; - values[j] = a[j] <= bval ? 1 : 0; + for(int j = 0; j < alen; j++) { + indexes[j] = aix[ai+j]; + values[j] = a[ai+j] <= bval ? 1 : 0; } c.setSize(alen); return c; @@ -3022,9 +3022,9 @@ public static SparseRowVector vectGreaterWrite(int len, double[] a, double bval, SparseRowVector c = allocSparseVector(alen); int[] indexes = c.indexes(); double[] values = c.values(); - for(int j = ai; j < ai+alen; j++) { - indexes[j] = aix[j]; - values[j] = a[j] > bval ? 1 : 0; + for(int j = 0; j < alen; j++) { + indexes[j] = aix[ai+j]; + values[j] = a[ai+j] > bval ? 1 : 0; } c.setSize(alen); return c; @@ -3099,9 +3099,9 @@ public static SparseRowVector vectGreaterequalWrite(int len, double[] a, double SparseRowVector c = allocSparseVector(alen); int[] indexes = c.indexes(); double[] values = c.values(); - for(int j = ai; j < ai+alen; j++) { - indexes[j] = aix[j]; - values[j] = a[j] >= bval ? 1 : 0; + for(int j = 0; j < alen; j++) { + indexes[j] = aix[ai+j]; + values[j] = a[ai+j] >= bval ? 1 : 0; } c.setSize(alen); return c; @@ -3142,9 +3142,9 @@ public static SparseRowVector vectBitwandWrite(int len, double[] a, double bval, int[] indexes = c.indexes(); double[] values = c.values(); int bval1 = (int) bval; - for( int j = ai; j < ai+alen; j++ ) { - indexes[j] = aix[j]; - values[j] = bwAnd(a[j], bval1); + for( int j = 0; j < alen; j++ ) { + indexes[j] = aix[ai+j]; + values[j] = bwAnd(a[ai+j], bval1); } c.setSize(alen); return c; From e5943a38f6bb725dd7b40256c1359738b872d8db Mon Sep 17 00:00:00 2001 From: Frxms Date: Tue, 19 Aug 2025 23:17:48 +0200 Subject: [PATCH 57/60] debug setupSparseThreadLocalMem --- .../runtime/codegen/LibSpoofPrimitives.java | 29 ++--------------- .../sysds/runtime/codegen/SpoofRowwise.java | 32 +++++++++++++++---- 2 files changed, 28 insertions(+), 33 deletions(-) diff --git a/src/main/java/org/apache/sysds/runtime/codegen/LibSpoofPrimitives.java b/src/main/java/org/apache/sysds/runtime/codegen/LibSpoofPrimitives.java index 1535fb71bd2..f253b5d3983 100644 --- a/src/main/java/org/apache/sysds/runtime/codegen/LibSpoofPrimitives.java +++ b/src/main/java/org/apache/sysds/runtime/codegen/LibSpoofPrimitives.java @@ -2195,7 +2195,7 @@ public static SparseRowVector vectMultWrite(int len, double bval, double[] a, in return vectMultWrite(len, a, bval, aix, ai, alen); } - //version with branching + //old version with branching (not used) public static SparseRowVector vectMultWriteB(int len, double[] a, double[] b, int[] aix, int[] bix, int ai, int bi, int alen, int blen) { SparseRowVector c = allocSparseVector(Math.min(alen, blen)); if( a == null || b == null ) return c; @@ -2279,7 +2279,7 @@ public static SparseRowVector vectDivWrite(int len, double bval, double[] a, int return c; } - //version with branching + //old version with branching (not used) public static SparseRowVector vectDivWriteB(int len, double[] a, double[] b, int[] aix, int[] bix, int ai, int bi, int alen, int blen) { SparseRowVector c = allocSparseVector(alen); int aItr = ai; @@ -3318,31 +3318,6 @@ public static SparseRowVector vectSignWrite(int len, double[] a, int[] aix, int return c; } - //todo MatrixMult, pow2 and mult2 drafts -// public static SparseRowVector vectMatrixMult(int len, double[] a, double[] b, int[] aix, int[] bix, int ai, int bi, int alen, int blen) { -// //note: assumption b is already transposed for efficient dot products -// int m2clen = b.length / len; -// SparseRowVector c = allocSparseVector(m2clen); -// for(int i = 0; i < m2clen; i++) { -// c.set(bix[i], LibMatrixMult.dotProduct(a, aix, ai, alen, b, bix, bi, blen)); -// } -// return c; -// } -// -// public static SparseRowVector vectPow2Write(int len, double[] a, int[] aix, int ai, int alen) { -// SparseRowVector c = allocSparseVector(len); -// for(int j = 0; j < ai+alen; j++) -// c.set(aix[j], a[j] * a[j]); -// return c; -// } -// -// public static SparseRowVector vectMult2Write(int len, double[] a, int[] aix, int ai, int alen) { -// SparseRowVector c = allocSparseVector(len); -// for(int j = 0; j < ai+alen; j++) -// c.set(aix[j], a[j] + a[j]); -// return c; -// } - //complex builtin functions that are not directly generated //(included here in order to reduce the number of imports) diff --git a/src/main/java/org/apache/sysds/runtime/codegen/SpoofRowwise.java b/src/main/java/org/apache/sysds/runtime/codegen/SpoofRowwise.java index eabbb390069..e48d44f33fb 100644 --- a/src/main/java/org/apache/sysds/runtime/codegen/SpoofRowwise.java +++ b/src/main/java/org/apache/sysds/runtime/codegen/SpoofRowwise.java @@ -189,7 +189,7 @@ public MatrixBlock execute(ArrayList inputs, ArrayList 0 ) if(inputs.get(0).isInSparseFormat() && DMLScript.SPARSE_INTERMEDIATE) { - LibSpoofPrimitives.setupSparseThreadLocalMemory(_reqVectMem, n/2, n2); + LibSpoofPrimitives.setupSparseThreadLocalMemory(_reqVectMem, n, n2); LibSpoofPrimitives.setupThreadLocalMemory(_reqVectMem, n, n2); } else { LibSpoofPrimitives.setupThreadLocalMemory(_reqVectMem, n, n2); @@ -442,7 +442,12 @@ public DenseBlock call() { //allocate vector intermediates and partial output if( _reqVectMem > 0 ) - LibSpoofPrimitives.setupThreadLocalMemory(_reqVectMem, _clen, _clen2); + if(_a.isInSparseFormat() && DMLScript.SPARSE_INTERMEDIATE) { + LibSpoofPrimitives.setupSparseThreadLocalMemory(_reqVectMem, _clen, _clen2); + LibSpoofPrimitives.setupThreadLocalMemory(_reqVectMem, _clen, _clen2); + } else { + LibSpoofPrimitives.setupThreadLocalMemory(_reqVectMem, _clen, _clen2); + } DenseBlock c = DenseBlockFactory.createDenseBlock(1, _outLen); if( !_a.isInSparseFormat() ) @@ -451,7 +456,12 @@ public DenseBlock call() { executeSparse(_a.getSparseBlock(), _b, _scalars, c, _clen, _rl, _ru, 0); if( _reqVectMem > 0 ) - LibSpoofPrimitives.cleanupThreadLocalMemory(); + if(_a.isInSparseFormat() && DMLScript.SPARSE_INTERMEDIATE) { + LibSpoofPrimitives.cleanupSparseThreadLocalMemory(); + LibSpoofPrimitives.cleanupThreadLocalMemory(); + } else { + LibSpoofPrimitives.cleanupThreadLocalMemory(); + } return c; } } @@ -485,15 +495,25 @@ protected ParExecTask( MatrixBlock a, SideInput[] b, MatrixBlock c, double[] sca public Long call() { //allocate vector intermediates if( _reqVectMem > 0 ) - LibSpoofPrimitives.setupThreadLocalMemory(_reqVectMem, _clen, _clen2); + if(_a.isInSparseFormat() && DMLScript.SPARSE_INTERMEDIATE) { + LibSpoofPrimitives.setupSparseThreadLocalMemory(_reqVectMem, _clen, _clen2); + LibSpoofPrimitives.setupThreadLocalMemory(_reqVectMem, _clen, _clen2); + } else { + LibSpoofPrimitives.setupThreadLocalMemory(_reqVectMem, _clen, _clen2); + } if( !_a.isInSparseFormat() ) executeDense(_a.getDenseBlock(), _b, _scalars, _c.getDenseBlock(), _clen, _rl, _ru, 0); else executeSparse(_a.getSparseBlock(), _b, _scalars, _c.getDenseBlock(), _clen, _rl, _ru, 0); - + if( _reqVectMem > 0 ) - LibSpoofPrimitives.cleanupThreadLocalMemory(); + if(_a.isInSparseFormat() && DMLScript.SPARSE_INTERMEDIATE) { + LibSpoofPrimitives.cleanupSparseThreadLocalMemory(); + LibSpoofPrimitives.cleanupThreadLocalMemory(); + } else { + LibSpoofPrimitives.cleanupThreadLocalMemory(); + } //maintain nnz for row partition return _c.recomputeNonZeros(_rl, _ru-1, 0, _c.getNumColumns()-1); From c761d8fc2d30d3c68ee819580f56989a5bcb6ab3 Mon Sep 17 00:00:00 2001 From: Frxms Date: Tue, 19 Aug 2025 23:19:54 +0200 Subject: [PATCH 58/60] debug alen/blen replacement on vector-vector primitives --- .../java/org/apache/sysds/hops/codegen/cplan/CNodeBinary.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/apache/sysds/hops/codegen/cplan/CNodeBinary.java b/src/main/java/org/apache/sysds/hops/codegen/cplan/CNodeBinary.java index adadb41d2e6..b71aa8945fa 100644 --- a/src/main/java/org/apache/sysds/hops/codegen/cplan/CNodeBinary.java +++ b/src/main/java/org/apache/sysds/hops/codegen/cplan/CNodeBinary.java @@ -191,7 +191,7 @@ public String codegen(boolean sparse, GeneratorAPI api) { (_type == BinType.VECT_MATRIXMULT ? varj : varj + ".vals(0)")) : _inputs.get(j).getDataType() == DataType.MATRIX ? (api == GeneratorAPI.JAVA ? varj : varj + ".vals(0)") : varj); - tmp = tmp.replace("%SLEN"+(j+1)+"%", varj.startsWith("STMP") ? varj+".size()" : j == 0 ? "alen" : "blen"); + tmp = tmp.replace("%SLEN"+(j+1)+"%", varj.startsWith("STMP") ? varj+".size()" : varj.startsWith("a") ? "alen" : "blen"); //replace start position of main input tmp = tmp.replace("%POS"+(j+1)+"%", (_inputs.get(j) instanceof CNodeData From 304774bf1aedcbfc4de502f96c39bd4b2b0154ce Mon Sep 17 00:00:00 2001 From: Frxms Date: Tue, 19 Aug 2025 23:52:00 +0200 Subject: [PATCH 59/60] debug unary primitives --- .../runtime/codegen/LibSpoofPrimitives.java | 72 +++++++++---------- 1 file changed, 36 insertions(+), 36 deletions(-) diff --git a/src/main/java/org/apache/sysds/runtime/codegen/LibSpoofPrimitives.java b/src/main/java/org/apache/sysds/runtime/codegen/LibSpoofPrimitives.java index f253b5d3983..bc6ba198951 100644 --- a/src/main/java/org/apache/sysds/runtime/codegen/LibSpoofPrimitives.java +++ b/src/main/java/org/apache/sysds/runtime/codegen/LibSpoofPrimitives.java @@ -3178,9 +3178,9 @@ public static SparseRowVector vectSqrtWrite(int len, double[] a, int[] aix, int SparseRowVector c = allocSparseVector(alen); int[] indexes = c.indexes(); double[] values = c.values(); - for(int j = ai; j < ai+alen; j++) { - indexes[j] = aix[j]; - values[j] = Math.sqrt(a[j]); + for(int j = 0; j < alen; j++) { + indexes[j] = aix[ai+j]; + values[j] = Math.sqrt(a[ai+j]); } c.setSize(alen); return c; @@ -3190,9 +3190,9 @@ public static SparseRowVector vectAbsWrite(int len, double[] a, int[] aix, int a SparseRowVector c = allocSparseVector(alen); int[] indexes = c.indexes(); double[] values = c.values(); - for(int j = ai; j < ai+alen; j++) { - indexes[j] = aix[j]; - values[j] = Math.abs(a[j]); + for(int j = 0; j < alen; j++) { + indexes[j] = aix[ai+j]; + values[j] = Math.abs(a[ai+j]); } c.setSize(alen); return c; @@ -3202,9 +3202,9 @@ public static SparseRowVector vectRoundWrite(int len, double[] a, int[] aix, int SparseRowVector c = allocSparseVector(alen); int[] indexes = c.indexes(); double[] values = c.values(); - for(int j = ai; j < ai+alen; j++) { - indexes[j] = aix[j]; - values[j] = Math.round(a[j]); + for(int j = 0; j < alen; j++) { + indexes[j] = aix[ai+j]; + values[j] = Math.round(a[ai+j]); } c.setSize(alen); return c; @@ -3214,9 +3214,9 @@ public static SparseRowVector vectCeilWrite(int len, double[] a, int[] aix, int SparseRowVector c = allocSparseVector(alen); int[] indexes = c.indexes(); double[] values = c.values(); - for(int j = ai; j < ai+alen; j++) { - indexes[j] = aix[j]; - values[j] = Math.ceil(a[j]); + for(int j = 0; j < alen; j++) { + indexes[j] = aix[ai+j]; + values[j] = Math.ceil(a[ai+j]); } c.setSize(alen); return c; @@ -3226,9 +3226,9 @@ public static SparseRowVector vectFloorWrite(int len, double[] a, int[] aix, int SparseRowVector c = allocSparseVector(alen); int[] indexes = c.indexes(); double[] values = c.values(); - for(int j = ai; j < ai+alen; j++) { - indexes[j] = aix[j]; - values[j] = Math.floor(a[j]); + for(int j = 0; j < alen; j++) { + indexes[j] = aix[ai+j]; + values[j] = Math.floor(a[ai+j]); } c.setSize(alen); return c; @@ -3238,9 +3238,9 @@ public static SparseRowVector vectSinWrite(int len, double[] a, int[] aix, int a SparseRowVector c = allocSparseVector(alen); int[] indexes = c.indexes(); double[] values = c.values(); - for(int j = ai; j < ai+alen; j++) { - indexes[j] = aix[j]; - values[j] = Math.sin(a[j]); + for(int j = 0; j < alen; j++) { + indexes[j] = aix[ai+j]; + values[j] = Math.sin(a[ai+j]); } c.setSize(alen); return c; @@ -3250,9 +3250,9 @@ public static SparseRowVector vectTanWrite(int len, double[] a, int[] aix, int a SparseRowVector c = allocSparseVector(alen); int[] indexes = c.indexes(); double[] values = c.values(); - for(int j = ai; j < ai+alen; j++) { - indexes[j] = aix[j]; - values[j] = Math.tan(a[j]); + for(int j = 0; j < alen; j++) { + indexes[j] = aix[ai+j]; + values[j] = Math.tan(a[ai+j]); } c.setSize(alen); return c; @@ -3262,9 +3262,9 @@ public static SparseRowVector vectAsinWrite(int len, double[] a, int[] aix, int SparseRowVector c = allocSparseVector(alen); int[] indexes = c.indexes(); double[] values = c.values(); - for(int j = ai; j < ai+alen; j++) { - indexes[j] = aix[j]; - values[j] = Math.asin(a[j]); + for(int j = 0; j < alen; j++) { + indexes[j] = aix[ai+j]; + values[j] = Math.asin(a[ai+j]); } c.setSize(alen); return c; @@ -3274,9 +3274,9 @@ public static SparseRowVector vectAtanWrite(int len, double[] a, int[] aix, int SparseRowVector c = allocSparseVector(alen); int[] indexes = c.indexes(); double[] values = c.values(); - for(int j = ai; j < ai+alen; j++) { - indexes[j] = aix[j]; - values[j] = Math.atan(a[j]); + for(int j = 0; j < alen; j++) { + indexes[j] = aix[ai+j]; + values[j] = Math.atan(a[ai+j]); } c.setSize(alen); return c; @@ -3286,9 +3286,9 @@ public static SparseRowVector vectSinhWrite(int len, double[] a, int[] aix, int SparseRowVector c = allocSparseVector(alen); int[] indexes = c.indexes(); double[] values = c.values(); - for(int j = ai; j < ai+alen; j++) { - indexes[j] = aix[j]; - values[j] = Math.sinh(a[j]); + for(int j = 0; j < alen; j++) { + indexes[j] = aix[ai+j]; + values[j] = Math.sinh(a[ai+j]); } c.setSize(alen); return c; @@ -3298,9 +3298,9 @@ public static SparseRowVector vectTanhWrite(int len, double[] a, int[] aix, int SparseRowVector c = allocSparseVector(alen); int[] indexes = c.indexes(); double[] values = c.values(); - for(int j = ai; j < ai+alen; j++) { - indexes[j] = aix[j]; - values[j] = Math.tanh(a[j]); + for(int j = 0; j < alen; j++) { + indexes[j] = aix[ai+j]; + values[j] = Math.tanh(a[ai+j]); } c.setSize(alen); return c; @@ -3310,9 +3310,9 @@ public static SparseRowVector vectSignWrite(int len, double[] a, int[] aix, int SparseRowVector c = allocSparseVector(alen); int[] indexes = c.indexes(); double[] values = c.values(); - for(int j = ai; j < ai+alen; j++) { - indexes[j] = aix[j]; - values[j] = Math.signum(a[j]); + for(int j = 0; j < alen; j++) { + indexes[j] = aix[ai+j]; + values[j] = Math.signum(a[ai+j]); } c.setSize(alen); return c; From 63aac6bace991543fe7f71ad138ef7b82af5a99a Mon Sep 17 00:00:00 2001 From: Frxms Date: Wed, 20 Aug 2025 01:06:15 +0200 Subject: [PATCH 60/60] add test case --- .../functions/codegen/RowAggTmplTest.java | 8 +++- .../functions/codegen/rowAggPattern49.R | 4 +- .../functions/codegen/rowAggPattern49.dml | 4 +- .../functions/codegen/rowAggPattern50.R | 43 +++++++++++++++++++ .../functions/codegen/rowAggPattern50.dml | 40 +++++++++++++++++ 5 files changed, 93 insertions(+), 6 deletions(-) create mode 100644 src/test/scripts/functions/codegen/rowAggPattern50.R create mode 100644 src/test/scripts/functions/codegen/rowAggPattern50.dml diff --git a/src/test/java/org/apache/sysds/test/functions/codegen/RowAggTmplTest.java b/src/test/java/org/apache/sysds/test/functions/codegen/RowAggTmplTest.java index 99798988746..0c2387a4c0f 100644 --- a/src/test/java/org/apache/sysds/test/functions/codegen/RowAggTmplTest.java +++ b/src/test/java/org/apache/sysds/test/functions/codegen/RowAggTmplTest.java @@ -89,7 +89,8 @@ public class RowAggTmplTest extends AutomatedTestBase private static final String TEST_NAME46 = TEST_NAME+"46"; //conv2d(X - mean(X), F1) + conv2d(X - mean(X), F2); private static final String TEST_NAME47 = TEST_NAME+"47"; //sum(X + rowVars(X)) private static final String TEST_NAME48 = TEST_NAME+"48"; //sum(rowVars(X)) - private static final String TEST_NAME49 = TEST_NAME+"49"; + private static final String TEST_NAME49 = TEST_NAME+"49"; //X*rowSums(K*v)*X + private static final String TEST_NAME50 = TEST_NAME+"50"; //(abs(A)*B)+(B*v) private static final String TEST_DIR = "functions/codegen/"; private static final String TEST_CLASS_DIR = TEST_DIR + RowAggTmplTest.class.getSimpleName() + "/"; @@ -101,7 +102,7 @@ public class RowAggTmplTest extends AutomatedTestBase @Override public void setUp() { TestUtils.clearAssertionInformation(); - for(int i=1; i<=49; i++) + for(int i=1; i<=50; i++) addTestConfiguration( TEST_NAME+i, new TestConfiguration(TEST_CLASS_DIR, TEST_NAME+i, new String[] { String.valueOf(i) }) ); } @@ -831,6 +832,9 @@ public void testCodegenRowAgg48SP() { @Test public void testCodegenRowAgg49CP() {testCodegenIntegration( TEST_NAME49, false, ExecType.CP );} + @Test + public void testCodegenRowAgg50CP() {testCodegenIntegration( TEST_NAME50, false, ExecType.CP );} + private void testCodegenIntegration( String testname, boolean rewrites, ExecType instType ) { boolean oldFlag = OptimizerUtils.ALLOW_ALGEBRAIC_SIMPLIFICATION; diff --git a/src/test/scripts/functions/codegen/rowAggPattern49.R b/src/test/scripts/functions/codegen/rowAggPattern49.R index 18cd419dc21..06d5c63144b 100644 --- a/src/test/scripts/functions/codegen/rowAggPattern49.R +++ b/src/test/scripts/functions/codegen/rowAggPattern49.R @@ -39,8 +39,8 @@ K = rbind(Z, Y, Y, Y, Y, Y, Y, Y, Y) # S = (X < rowSums(X*K)) # S = X*rowMins(K)*X -# S = X*rowSums(K*v)*X -S = (X*v)/rowSums(X*v) +S = X*rowSums(K*v)*X +# S = (X*v)/rowSums(X*v) # S = abs((X*v)/rowSums(X*v)) # S = (X/v)+rowMeans(X-v) # S = (X*v)+rowSums(X*v) diff --git a/src/test/scripts/functions/codegen/rowAggPattern49.dml b/src/test/scripts/functions/codegen/rowAggPattern49.dml index 60e3b271d32..8c0d1db2251 100644 --- a/src/test/scripts/functions/codegen/rowAggPattern49.dml +++ b/src/test/scripts/functions/codegen/rowAggPattern49.dml @@ -38,8 +38,8 @@ while(FALSE) { } # S = B * rowSums(v) # S = (X < rowSums(X*K)) # S = X*(k>1)*X -# S = X*rowSums(K*v)*X -S = (X*v)/rowSums(X*v) +S = X*rowSums(K*v)*X +# S = (X*v)/rowSums(X*v) # S = abs((X*v)/rowSums(X*v)) # S = (X/v)+rowMeans(X-v) # S = (X*v)+rowSums(X*v) diff --git a/src/test/scripts/functions/codegen/rowAggPattern50.R b/src/test/scripts/functions/codegen/rowAggPattern50.R new file mode 100644 index 00000000000..af2c3301a75 --- /dev/null +++ b/src/test/scripts/functions/codegen/rowAggPattern50.R @@ -0,0 +1,43 @@ +#------------------------------------------------------------- +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +#------------------------------------------------------------- + +args<-commandArgs(TRUE) +options(digits=22) +library("Matrix") +# library("matrixStats") + +W = matrix(seq(28,29), 1, 2) +J = matrix(0, 1, 8) +Z = cbind(J, W, J) +Y = matrix(0, 10, 18) +X = rbind(Z, Y, Y, Y, Y, Y, Y, Y, Y) +v = seq(1,81) +v1 = seq(20, 37) +W = matrix(seq(13,14), 1, 2) +J = matrix(0, 1, 8) +Z= cbind(J, W, J) +Y = matrix(0, 10, 18) +K = rbind(Z, Y, Y, Y, Y, Y, Y, Y, Y) + +S = (abs(X)*K)+rowSums(X*v) + + +writeMM(as(S, "CsparseMatrix"), paste(args[2], "S", sep="")); diff --git a/src/test/scripts/functions/codegen/rowAggPattern50.dml b/src/test/scripts/functions/codegen/rowAggPattern50.dml new file mode 100644 index 00000000000..64efe460f9b --- /dev/null +++ b/src/test/scripts/functions/codegen/rowAggPattern50.dml @@ -0,0 +1,40 @@ +#------------------------------------------------------------- +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +#------------------------------------------------------------- + +W = matrix(seq(28,29), 1, 2) +J = matrix(0, 1, 8) +Z= cbind(J, W, J) +Y = matrix(0, 10, 18) +X = rbind(Z, Y, Y, Y, Y, Y, Y, Y, Y) +v = seq(1,81) +v1 = seq(20, 37) +W = matrix(seq(13,14), 1, 2) +J = matrix(0, 1, 8) +Z= cbind(J, W, J) +Y = matrix(0, 10, 18) +K = rbind(Z, Y, Y, Y, Y, Y, Y, Y, Y) +while(FALSE) { } + + +S = (abs(X)*K)+rowSums(X*v) + + +write(S,$1)