From e2bbb0b88b7cfb78d0120350412cf976ea97ff45 Mon Sep 17 00:00:00 2001 From: Selin Kahvecioglu Date: Thu, 15 Jun 2023 13:51:00 +0200 Subject: [PATCH 1/8] Testing mybranch --- test | 1 + 1 file changed, 1 insertion(+) create mode 100644 test diff --git a/test b/test new file mode 100644 index 00000000000..06c812cbb33 --- /dev/null +++ b/test @@ -0,0 +1 @@ +“a test file” From 73a13acf753d7af8cf97f1a157334c3bec3d5b7a Mon Sep 17 00:00:00 2001 From: Selin Kahvecioglu Date: Thu, 10 Aug 2023 22:27:38 +0200 Subject: [PATCH 2/8] imgcutoutlin draft finalized, test created --- scripts/builtin/img_cutout_linearized.dml | 70 +++++++++++++++++++ .../org/apache/sysds/common/Builtins.java | 1 + .../part1/BuiltinImgCutoutLinTest.java | 49 +++++++++++++ test | 1 - 4 files changed, 120 insertions(+), 1 deletion(-) create mode 100644 scripts/builtin/img_cutout_linearized.dml create mode 100644 src/test/java/org/apache/sysds/test/functions/builtin/part1/BuiltinImgCutoutLinTest.java delete mode 100644 test diff --git a/scripts/builtin/img_cutout_linearized.dml b/scripts/builtin/img_cutout_linearized.dml new file mode 100644 index 00000000000..7c0e1e63bb7 --- /dev/null +++ b/scripts/builtin/img_cutout_linearized.dml @@ -0,0 +1,70 @@ +#------------------------------------------------------------- +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +#------------------------------------------------------------- + +# Image Cutout function replaces a rectangular section of an image with a constant value. +# +# INPUT: +# --------------------------------------------------------------------------------------------- +# img_in Input image as 2D matrix with top left corner at [1, 1] +# x Column index of the top left corner of the rectangle (starting at 1) +# y Row index of the top left corner of the rectangle (starting at 1) +# width Width of the rectangle (must be positive) +# height Height of the rectangle (must be positive) +# fill_value The value to set for the rectangle +# s_cols Width of a single image +# s_rows Height of a single image +# --------------------------------------------------------------------------------------------- +# +# OUTPUT: +# ------------------------------------------------------------------------------------------ +# img_out Output images as linearized 2D matrix with top left corner at [1, 1] +# ------------------------------------------------------------------------------------------ + + +m_img_cutout_linearized = function(Matrix[Double] img_in, Integer x, Integer y, Integer width, Integer height, Double fill_value, Integer s_rows, Integer s_cols) return (Matrix[Double] img_out) { + # size is s_cols : s_rows + rows = nrow(img_in) # number of images + cols = ncol(img_in) # s_cols * s_rows + + if (width < 1 | height < 1) { + print("Invalid width or height. Returning input") + img_out = img_in + } else { + end_x = x + width - 1 + end_y = (y + height)*s_cols #the last row's start + + start_x = max(1, x) + start_y = max(1, y*s_cols) + end_x = min(cols, end_x) + end_y = min(rows, end_y) + + img_out = img_in + + # Iterate through each row of the rectangular region + for (i in start_y: end_y){ + + start_idx = i * s_cols + start_x + end_idx = i * s_cols + end_x + + img_out[, start_idx:end_idx] = matrix(fill_value, rows=rows, cols=(end_x-start_x+1)) + } + } +} diff --git a/src/main/java/org/apache/sysds/common/Builtins.java b/src/main/java/org/apache/sysds/common/Builtins.java index 883e57aa22f..f59b90b7332 100644 --- a/src/main/java/org/apache/sysds/common/Builtins.java +++ b/src/main/java/org/apache/sysds/common/Builtins.java @@ -162,6 +162,7 @@ public enum Builtins { IMG_ROTATE("img_rotate", true), IMG_SHEAR("img_shear", true), IMG_CUTOUT("img_cutout", true), + IMG_CUTOUT_LINEARIZED("img_cutout_linearized", true), IMG_SAMPLE_PAIRING("img_sample_pairing", true), IMG_INVERT("img_invert", true), IMG_POSTERIZE("img_posterize", true), diff --git a/src/test/java/org/apache/sysds/test/functions/builtin/part1/BuiltinImgCutoutLinTest.java b/src/test/java/org/apache/sysds/test/functions/builtin/part1/BuiltinImgCutoutLinTest.java new file mode 100644 index 00000000000..a4e6f016201 --- /dev/null +++ b/src/test/java/org/apache/sysds/test/functions/builtin/part1/BuiltinImgCutoutLinTest.java @@ -0,0 +1,49 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.sysds.test.functions.builtin.part1; + +import org.junit.Test; +import org.apache.sysds.common.Types.ExecMode; +import org.apache.sysds.common.Types.ExecType; +import org.apache.sysds.runtime.matrix.data.MatrixValue; +import org.apache.sysds.test.AutomatedTestBase; +import org.apache.sysds.test.TestConfiguration; +import org.apache.sysds.test.TestUtils; + +import java.util.HashMap; + +public class BuiltinImgCutoutLinTest extends AutomatedTestBase +{ + + private final static String TEST_NAME = "img_cutout_linearized"; + private final static String TEST_DIR = "functions/builtin/"; + + // TODO: Get more info on Rscripts + + + @Override + public void setUp() { + // TODO Auto-generated method stub + throw new UnsupportedOperationException("Unimplemented method 'setUp'"); + } + + + + +} \ No newline at end of file diff --git a/test b/test deleted file mode 100644 index 06c812cbb33..00000000000 --- a/test +++ /dev/null @@ -1 +0,0 @@ -“a test file” From 2fbee138d751d33e18452132e15be02dc6b311c2 Mon Sep 17 00:00:00 2001 From: Selin Kahvecioglu Date: Thu, 31 Aug 2023 16:32:18 +0200 Subject: [PATCH 3/8] read issue persists --- bin/systemds | 8 +- scripts/builtin/img_cutout_linearized.dml | 22 +-- .../part1/BuiltinImgCutoutLinTest.java | 49 ------ .../pipelines/BuiltinImageCutoutLinTest.java | 146 ++++++++++++++++++ .../builtin/image_cutout_linearized.dml | 36 +++++ 5 files changed, 199 insertions(+), 62 deletions(-) delete mode 100644 src/test/java/org/apache/sysds/test/functions/builtin/part1/BuiltinImgCutoutLinTest.java create mode 100644 src/test/java/org/apache/sysds/test/functions/pipelines/BuiltinImageCutoutLinTest.java create mode 100644 src/test/scripts/functions/builtin/image_cutout_linearized.dml diff --git a/bin/systemds b/bin/systemds index efa47c1abcb..76df8d39584 100755 --- a/bin/systemds +++ b/bin/systemds @@ -60,7 +60,7 @@ if [[ -z $SYSTEMDS_ROOT ]] ; then print_out "SYSTEMDS_ROOT not set defaulting to current dir $(pwd)" else # construct a relative path - SYSTEMDS_ROOT=$(realpath --relative-to=. ${SYSTEMDS_ROOT}) + SYSTEMDS_ROOT=$(grealpath --relative-to=. ${SYSTEMDS_ROOT}) fi; # when using find, look in the directories in this order @@ -375,11 +375,11 @@ fi # find absolute path to hadoop home in SYSTEMDS_ROOT if [ -z "$HADOOP_HOME" ]; then - HADOOP_HOME=$(realpath "$(find "$SYSTEMDS_ROOT" -iname hadoop | tail -n 1 )") + HADOOP_HOME=$(grealpath "$(find "$SYSTEMDS_ROOT" -iname hadoop | tail -n 1 )") export HADOOP_HOME fi # add hadoop home to path and lib path for loading hadoop jni -HADOOP_REL=$(realpath --relative-to=. "$HADOOP_HOME") +HADOOP_REL=$(grealpath --relative-to=. "$HADOOP_HOME") # default directory separator unix style DIR_SEP=/ @@ -396,7 +396,7 @@ fi JARNAME=$(basename "$SYSTEMDS_JAR_FILE") # relative path to jar file -SYSTEMDS_JAR_FILE=$(realpath --relative-to=. "$(dirname "$SYSTEMDS_JAR_FILE")")${DIR_SEP}${JARNAME} +SYSTEMDS_JAR_FILE=$(grealpath --relative-to=. "$(dirname "$SYSTEMDS_JAR_FILE")")${DIR_SEP}${JARNAME} NATIVE_LIBS="$SYSTEMDS_ROOT${DIR_SEP}target${DIR_SEP}classes${DIR_SEP}lib" export PATH=${HADOOP_REL}${DIR_SEP}bin${PATH_SEP}${PATH}${PATH_SEP}$NATIVE_LIBS diff --git a/scripts/builtin/img_cutout_linearized.dml b/scripts/builtin/img_cutout_linearized.dml index 7c0e1e63bb7..1d3d9c1af67 100644 --- a/scripts/builtin/img_cutout_linearized.dml +++ b/scripts/builtin/img_cutout_linearized.dml @@ -23,7 +23,7 @@ # # INPUT: # --------------------------------------------------------------------------------------------- -# img_in Input image as 2D matrix with top left corner at [1, 1] +# img_in Input images as linearized 2D matrix with top left corner at [1, 1] # x Column index of the top left corner of the rectangle (starting at 1) # y Row index of the top left corner of the rectangle (starting at 1) # width Width of the rectangle (must be positive) @@ -39,7 +39,7 @@ # ------------------------------------------------------------------------------------------ -m_img_cutout_linearized = function(Matrix[Double] img_in, Integer x, Integer y, Integer width, Integer height, Double fill_value, Integer s_rows, Integer s_cols) return (Matrix[Double] img_out) { +m_img_cutout_linearized = function(Matrix[Double] img_in, Integer x, Integer y, Integer width, Integer height, Double fill_value, Integer s_cols, Integer s_rows) return (Matrix[Double] img_out) { # size is s_cols : s_rows rows = nrow(img_in) # number of images cols = ncol(img_in) # s_cols * s_rows @@ -48,23 +48,27 @@ m_img_cutout_linearized = function(Matrix[Double] img_in, Integer x, Integer y, print("Invalid width or height. Returning input") img_out = img_in } else { - end_x = x + width - 1 - end_y = (y + height)*s_cols #the last row's start start_x = max(1, x) - start_y = max(1, y*s_cols) - end_x = min(cols, end_x) - end_y = min(rows, end_y) + start_y = max(1, y) + + end_x = start_x + width - 1 + end_x = min(s_cols, end_x) + + end_y = start_y + height - 1 + end_y = min(s_rows, end_y) img_out = img_in # Iterate through each row of the rectangular region for (i in start_y: end_y){ - start_idx = i * s_cols + start_x - end_idx = i * s_cols + end_x + start_idx = (i-1) * s_cols + start_x + end_idx = (i-1) * s_cols + end_x img_out[, start_idx:end_idx] = matrix(fill_value, rows=rows, cols=(end_x-start_x+1)) } } + } + diff --git a/src/test/java/org/apache/sysds/test/functions/builtin/part1/BuiltinImgCutoutLinTest.java b/src/test/java/org/apache/sysds/test/functions/builtin/part1/BuiltinImgCutoutLinTest.java deleted file mode 100644 index a4e6f016201..00000000000 --- a/src/test/java/org/apache/sysds/test/functions/builtin/part1/BuiltinImgCutoutLinTest.java +++ /dev/null @@ -1,49 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.sysds.test.functions.builtin.part1; - -import org.junit.Test; -import org.apache.sysds.common.Types.ExecMode; -import org.apache.sysds.common.Types.ExecType; -import org.apache.sysds.runtime.matrix.data.MatrixValue; -import org.apache.sysds.test.AutomatedTestBase; -import org.apache.sysds.test.TestConfiguration; -import org.apache.sysds.test.TestUtils; - -import java.util.HashMap; - -public class BuiltinImgCutoutLinTest extends AutomatedTestBase -{ - - private final static String TEST_NAME = "img_cutout_linearized"; - private final static String TEST_DIR = "functions/builtin/"; - - // TODO: Get more info on Rscripts - - - @Override - public void setUp() { - // TODO Auto-generated method stub - throw new UnsupportedOperationException("Unimplemented method 'setUp'"); - } - - - - -} \ No newline at end of file diff --git a/src/test/java/org/apache/sysds/test/functions/pipelines/BuiltinImageCutoutLinTest.java b/src/test/java/org/apache/sysds/test/functions/pipelines/BuiltinImageCutoutLinTest.java new file mode 100644 index 00000000000..d1b30f02152 --- /dev/null +++ b/src/test/java/org/apache/sysds/test/functions/pipelines/BuiltinImageCutoutLinTest.java @@ -0,0 +1,146 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +// like BuiltinImageCutoutTest.java but this time input and output are matrices with each row representing a linearized image + +package org.apache.sysds.test.functions.pipelines; + +import org.apache.sysds.common.Types.ExecMode; +import org.apache.sysds.common.Types.ExecType; +import org.apache.sysds.runtime.matrix.data.MatrixValue; +import org.apache.sysds.test.AutomatedTestBase; +import org.apache.sysds.test.TestConfiguration; +import org.apache.sysds.test.TestUtils; +import org.junit.Test; + +import java.util.HashMap; +import java.util.Random; + +public class BuiltinImageCutoutLinTest extends AutomatedTestBase { + private final static String TEST_NAME = "image_cutout_linearized"; + private final static String TEST_DIR = "functions/builtin/"; + private final static String TEST_CLASS_DIR = TEST_DIR + BuiltinImageCutoutLinTest.class.getSimpleName() + "/"; + + private final static double eps = 1e-10; + private final static double spSparse = 0.1; + private final static double spDense = 0.9; + private final static Random random = new Random(); + + @Override + public void setUp() { + addTestConfiguration(TEST_NAME, new TestConfiguration(TEST_CLASS_DIR, TEST_NAME, new String[] {"B"})); + } + + @Test + public void testImageTranslateMatrixDenseCP() { runImageCutoutLinTest(false, ExecType.CP); } + + @Test + public void testImageTranslateMatrixSparseCP() { + runImageCutoutLinTest(true, ExecType.CP); + } + + @Test + public void testImageTranslateMatrixDenseSP() { + runImageCutoutLinTest(false, ExecType.SPARK); + } + + @Test + public void testImageTranslateMatrixSparseSP() { + runImageCutoutLinTest(false, ExecType.SPARK); + } + + private void runImageCutoutLinTest(boolean sparse, ExecType instType) { + ExecMode platformOld = setExecMode(instType); + disableOutAndExpectedDeletion(); + + setOutputBuffering(true); +/* int s_cols = random.nextInt(100) + 1; + int s_rows = random.nextInt(100) + 1; + int x = random.nextInt(s_cols); + int y = random.nextInt(s_rows); + int width = random.nextInt(s_cols - x) + 1; + int height = random.nextInt(s_rows - y) + 1; + + int n_imgs = random.nextInt(100) + 1; + int fill_color = random.nextInt(256); + */ + + + // constant variables for this command: +/* systemds ./src/test/scripts/functions/builtin/image_cutout_linearized.dml -nvargs \ +in_file=./target/testTemp/functions/builtin/BuiltinImageCutoutLinTestin/in/A.mtx \ +out_file=./target/testTemp/functions/builtin/BuiltinImageCutoutLinTestin/out/B.mtx \ +width=3358 \ +height=41 \ +x=12 \ +y=63 \ +w=11 \ +h=4 \ +fill_color=57 \ +s_cols=46 \ +s_rows=73 */ + + int s_cols = 46; + int s_rows = 73; + int x = 12; + int y = 63; + int width = 11; + int height = 4; + + int n_imgs = 41; + int fill_color = 57; + + + + try { + loadTestConfiguration(getTestConfiguration(TEST_NAME)); + double sparsity = sparse ? spSparse : spDense; + + String HOME = SCRIPT_DIR + TEST_DIR; + fullDMLScriptName= HOME + TEST_NAME + ".dml"; + programArgs = new String[] {"-nvargs", "in_file=" + input("A"), "out_file=" + output("B"), "width=" + s_cols*s_rows, + "height=" + n_imgs, "x=" + (x+1), "y=" + (y+1), "w=" + width, "h=" + height, "fill_color=" + fill_color, "s_cols=" + s_cols, "s_rows" + s_rows}; + + //generate actual dataset + double[][] A = getRandomMatrix(n_imgs, s_cols*s_rows, 0, 255, sparsity, 7); + writeInputMatrixWithMTD("A", A, true); + + double[][] ref = new double[n_imgs][s_cols*s_rows]; + for (int i = 0; i < n_imgs; i++) { + for (int j = 0; j < s_cols*s_rows; j++) { + ref[i][j] = A[i][j]; + if (y <= Math.floor(j/s_cols) && Math.floor(j/s_cols) < y + height && x <= j%s_cols && j%s_cols < x + width) { + ref[i][j] = fill_color; + } + } + } + + runTest(true, false, null, -1); + + HashMap dmlfile = readDMLMatrixFromOutputDir("B"); + double[][] dml_res = TestUtils.convertHashMapToDoubleArray(dmlfile, n_imgs, s_cols*s_rows); + TestUtils.compareMatrices(ref, dml_res, eps, "Java vs. DML"); + + } + finally { + rtplatform = platformOld; + } + } +} + diff --git a/src/test/scripts/functions/builtin/image_cutout_linearized.dml b/src/test/scripts/functions/builtin/image_cutout_linearized.dml new file mode 100644 index 00000000000..36326cf5540 --- /dev/null +++ b/src/test/scripts/functions/builtin/image_cutout_linearized.dml @@ -0,0 +1,36 @@ +#------------------------------------------------------------- +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +#------------------------------------------------------------- + +input = read($in_file) +width = ifdef($width, 512) +height = ifdef($height, 512) +x = ifdef($x, 0) +y = ifdef($y, 0) +w = ifdef($w, width) +h = ifdef($h, height) +s_cols = ifdef($s_cols, 512) +s_rows = ifdef($s_rows, 512) +fill_value = ifdef($fill_color, 0) + +input = matrix(input, rows=width, cols=height) + +res = img_cutout_linearized(input, x, y, w, h, fill_value, s_cols, s_rows) +write(res, $out_file) \ No newline at end of file From 87009fb88c732b6bfbdcbe3a63d5ce8cfefc4e18 Mon Sep 17 00:00:00 2001 From: Selin Kahvecioglu Date: Thu, 31 Aug 2023 16:37:07 +0200 Subject: [PATCH 4/8] test --- .../pipelines/BuiltinImageCutoutLinTest.java | 30 ++++--------------- 1 file changed, 5 insertions(+), 25 deletions(-) diff --git a/src/test/java/org/apache/sysds/test/functions/pipelines/BuiltinImageCutoutLinTest.java b/src/test/java/org/apache/sysds/test/functions/pipelines/BuiltinImageCutoutLinTest.java index d1b30f02152..ffd73d04686 100644 --- a/src/test/java/org/apache/sysds/test/functions/pipelines/BuiltinImageCutoutLinTest.java +++ b/src/test/java/org/apache/sysds/test/functions/pipelines/BuiltinImageCutoutLinTest.java @@ -70,7 +70,7 @@ private void runImageCutoutLinTest(boolean sparse, ExecType instType) { disableOutAndExpectedDeletion(); setOutputBuffering(true); -/* int s_cols = random.nextInt(100) + 1; + int s_cols = random.nextInt(100) + 1; int s_rows = random.nextInt(100) + 1; int x = random.nextInt(s_cols); int y = random.nextInt(s_rows); @@ -79,32 +79,12 @@ private void runImageCutoutLinTest(boolean sparse, ExecType instType) { int n_imgs = random.nextInt(100) + 1; int fill_color = random.nextInt(256); - */ + + + + - // constant variables for this command: -/* systemds ./src/test/scripts/functions/builtin/image_cutout_linearized.dml -nvargs \ -in_file=./target/testTemp/functions/builtin/BuiltinImageCutoutLinTestin/in/A.mtx \ -out_file=./target/testTemp/functions/builtin/BuiltinImageCutoutLinTestin/out/B.mtx \ -width=3358 \ -height=41 \ -x=12 \ -y=63 \ -w=11 \ -h=4 \ -fill_color=57 \ -s_cols=46 \ -s_rows=73 */ - - int s_cols = 46; - int s_rows = 73; - int x = 12; - int y = 63; - int width = 11; - int height = 4; - - int n_imgs = 41; - int fill_color = 57; From 6bb53d83c5339aa88908d12a327291c62c8f5d00 Mon Sep 17 00:00:00 2001 From: Selin Kahvecioglu Date: Thu, 31 Aug 2023 21:18:23 +0200 Subject: [PATCH 5/8] write issue persists --- .../part1}/BuiltinImageCutoutLinTest.java | 13 +++---------- .../functions/builtin/image_cutout_linearized.dml | 4 ++-- 2 files changed, 5 insertions(+), 12 deletions(-) rename src/test/java/org/apache/sysds/test/functions/{pipelines => builtin/part1}/BuiltinImageCutoutLinTest.java (98%) diff --git a/src/test/java/org/apache/sysds/test/functions/pipelines/BuiltinImageCutoutLinTest.java b/src/test/java/org/apache/sysds/test/functions/builtin/part1/BuiltinImageCutoutLinTest.java similarity index 98% rename from src/test/java/org/apache/sysds/test/functions/pipelines/BuiltinImageCutoutLinTest.java rename to src/test/java/org/apache/sysds/test/functions/builtin/part1/BuiltinImageCutoutLinTest.java index ffd73d04686..ff4b2961bf4 100644 --- a/src/test/java/org/apache/sysds/test/functions/pipelines/BuiltinImageCutoutLinTest.java +++ b/src/test/java/org/apache/sysds/test/functions/builtin/part1/BuiltinImageCutoutLinTest.java @@ -70,23 +70,16 @@ private void runImageCutoutLinTest(boolean sparse, ExecType instType) { disableOutAndExpectedDeletion(); setOutputBuffering(true); + int s_cols = random.nextInt(100) + 1; int s_rows = random.nextInt(100) + 1; int x = random.nextInt(s_cols); int y = random.nextInt(s_rows); int width = random.nextInt(s_cols - x) + 1; int height = random.nextInt(s_rows - y) + 1; - int n_imgs = random.nextInt(100) + 1; - int fill_color = random.nextInt(256); - - - - - - - - + int fill_color = random.nextInt(256); + try { loadTestConfiguration(getTestConfiguration(TEST_NAME)); diff --git a/src/test/scripts/functions/builtin/image_cutout_linearized.dml b/src/test/scripts/functions/builtin/image_cutout_linearized.dml index 36326cf5540..d81a9f37e5d 100644 --- a/src/test/scripts/functions/builtin/image_cutout_linearized.dml +++ b/src/test/scripts/functions/builtin/image_cutout_linearized.dml @@ -30,7 +30,7 @@ s_cols = ifdef($s_cols, 512) s_rows = ifdef($s_rows, 512) fill_value = ifdef($fill_color, 0) -input = matrix(input, rows=width, cols=height) +input = matrix(input, rows=height, cols=width) res = img_cutout_linearized(input, x, y, w, h, fill_value, s_cols, s_rows) -write(res, $out_file) \ No newline at end of file +write(res, $out_file) From bb01f9bca657b63045a89a4131c75949f05a6483 Mon Sep 17 00:00:00 2001 From: Selin Kahvecioglu Date: Tue, 12 Sep 2023 20:05:36 +0300 Subject: [PATCH 6/8] java part is done --- .../part1/BuiltinImageCutoutLinTest.java | 60 ++++++++++++------- 1 file changed, 37 insertions(+), 23 deletions(-) diff --git a/src/test/java/org/apache/sysds/test/functions/builtin/part1/BuiltinImageCutoutLinTest.java b/src/test/java/org/apache/sysds/test/functions/builtin/part1/BuiltinImageCutoutLinTest.java index ff4b2961bf4..ac5760268b3 100644 --- a/src/test/java/org/apache/sysds/test/functions/builtin/part1/BuiltinImageCutoutLinTest.java +++ b/src/test/java/org/apache/sysds/test/functions/builtin/part1/BuiltinImageCutoutLinTest.java @@ -17,9 +17,7 @@ * under the License. */ -// like BuiltinImageCutoutTest.java but this time input and output are matrices with each row representing a linearized image - -package org.apache.sysds.test.functions.pipelines; +package org.apache.sysds.test.functions.builtin.part1; import org.apache.sysds.common.Types.ExecMode; import org.apache.sysds.common.Types.ExecType; @@ -31,6 +29,9 @@ import java.util.HashMap; import java.util.Random; +// if A should be generated one row at a time +//import java.util.stream.Stream; +//import java.util.stream.DoubleStream; public class BuiltinImageCutoutLinTest extends AutomatedTestBase { private final static String TEST_NAME = "image_cutout_linearized"; @@ -44,11 +45,13 @@ public class BuiltinImageCutoutLinTest extends AutomatedTestBase { @Override public void setUp() { - addTestConfiguration(TEST_NAME, new TestConfiguration(TEST_CLASS_DIR, TEST_NAME, new String[] {"B"})); + addTestConfiguration(TEST_NAME, new TestConfiguration(TEST_CLASS_DIR, TEST_NAME, new String[] { "B" })); } @Test - public void testImageTranslateMatrixDenseCP() { runImageCutoutLinTest(false, ExecType.CP); } + public void testImageTranslateMatrixDenseCP() { + runImageCutoutLinTest(false, ExecType.CP); + } @Test public void testImageTranslateMatrixSparseCP() { @@ -67,38 +70,49 @@ public void testImageTranslateMatrixSparseSP() { private void runImageCutoutLinTest(boolean sparse, ExecType instType) { ExecMode platformOld = setExecMode(instType); - disableOutAndExpectedDeletion(); + disableOutAndExpectedDeletion(); setOutputBuffering(true); - - int s_cols = random.nextInt(100) + 1; + int s_rows = random.nextInt(100) + 1; + int s_cols = random.nextInt(100) + 1; int x = random.nextInt(s_cols); int y = random.nextInt(s_rows); int width = random.nextInt(s_cols - x) + 1; int height = random.nextInt(s_rows - y) + 1; + int fill_color = random.nextInt(256); int n_imgs = random.nextInt(100) + 1; - int fill_color = random.nextInt(256); - - + try { loadTestConfiguration(getTestConfiguration(TEST_NAME)); double sparsity = sparse ? spSparse : spDense; String HOME = SCRIPT_DIR + TEST_DIR; - fullDMLScriptName= HOME + TEST_NAME + ".dml"; - programArgs = new String[] {"-nvargs", "in_file=" + input("A"), "out_file=" + output("B"), "width=" + s_cols*s_rows, - "height=" + n_imgs, "x=" + (x+1), "y=" + (y+1), "w=" + width, "h=" + height, "fill_color=" + fill_color, "s_cols=" + s_cols, "s_rows" + s_rows}; + fullDMLScriptName = HOME + TEST_NAME + ".dml"; + programArgs = new String[] { "-nvargs", "in_file=" + input("A"), "out_file=" + output("B"), + "width=" + (s_cols * s_rows), + "height=" + n_imgs, "x=" + (x + 1), "y=" + (y + 1), "w=" + width, "h=" + height, + "fill_color=" + fill_color, "s_cols=" + s_cols, "s_rows=" + s_rows }; + + // overall sparsity of the dataset or a single image/row? + double[][] A = getRandomMatrix(n_imgs, s_cols * s_rows, 0, 255, sparsity, 7); + /* + * double[][] A = new double[n_imgs][s_cols*s_rows]; + * for (int i = 0; i < n_imgs; i++) { + * double[][] matrix = getRandomMatrix(s_cols, s_rows, 0, 255, sparsity, 7); + * double[] row = Stream.of(matrix).flatMapToDouble(DoubleStream::of).toArray(); + * A[i] = row; + * } + */ - //generate actual dataset - double[][] A = getRandomMatrix(n_imgs, s_cols*s_rows, 0, 255, sparsity, 7); writeInputMatrixWithMTD("A", A, true); - double[][] ref = new double[n_imgs][s_cols*s_rows]; + double[][] ref = new double[n_imgs][s_cols * s_rows]; for (int i = 0; i < n_imgs; i++) { - for (int j = 0; j < s_cols*s_rows; j++) { + for (int j = 0; j < s_cols * s_rows; j++) { ref[i][j] = A[i][j]; - if (y <= Math.floor(j/s_cols) && Math.floor(j/s_cols) < y + height && x <= j%s_cols && j%s_cols < x + width) { + if (y <= (int) Math.floor(j / s_cols) && (int) Math.floor(j / s_cols) < y + height + && x <= (j % s_cols) && (j % s_cols) < x + width) { ref[i][j] = fill_color; } } @@ -107,13 +121,13 @@ private void runImageCutoutLinTest(boolean sparse, ExecType instType) { runTest(true, false, null, -1); HashMap dmlfile = readDMLMatrixFromOutputDir("B"); - double[][] dml_res = TestUtils.convertHashMapToDoubleArray(dmlfile, n_imgs, s_cols*s_rows); + double[][] dml_res = TestUtils.convertHashMapToDoubleArray(dmlfile, n_imgs, (s_cols * s_rows)); + + writeInputMatrixWithMTD("ref", ref, true); TestUtils.compareMatrices(ref, dml_res, eps, "Java vs. DML"); - } - finally { + } finally { rtplatform = platformOld; } } } - From 76d9eb14a54bdd6a972917b44e8ea0f9244d5b71 Mon Sep 17 00:00:00 2001 From: Selin Kahvecioglu Date: Thu, 14 Sep 2023 16:22:15 +0200 Subject: [PATCH 7/8] adjustments made according to the comments --- bin/systemds | 8 +-- scripts/builtin/img_cutout_linearized.dml | 26 ++++---- .../part1/BuiltinImageCutoutLinTest.java | 62 +++++++++++-------- 3 files changed, 52 insertions(+), 44 deletions(-) diff --git a/bin/systemds b/bin/systemds index 76df8d39584..efa47c1abcb 100755 --- a/bin/systemds +++ b/bin/systemds @@ -60,7 +60,7 @@ if [[ -z $SYSTEMDS_ROOT ]] ; then print_out "SYSTEMDS_ROOT not set defaulting to current dir $(pwd)" else # construct a relative path - SYSTEMDS_ROOT=$(grealpath --relative-to=. ${SYSTEMDS_ROOT}) + SYSTEMDS_ROOT=$(realpath --relative-to=. ${SYSTEMDS_ROOT}) fi; # when using find, look in the directories in this order @@ -375,11 +375,11 @@ fi # find absolute path to hadoop home in SYSTEMDS_ROOT if [ -z "$HADOOP_HOME" ]; then - HADOOP_HOME=$(grealpath "$(find "$SYSTEMDS_ROOT" -iname hadoop | tail -n 1 )") + HADOOP_HOME=$(realpath "$(find "$SYSTEMDS_ROOT" -iname hadoop | tail -n 1 )") export HADOOP_HOME fi # add hadoop home to path and lib path for loading hadoop jni -HADOOP_REL=$(grealpath --relative-to=. "$HADOOP_HOME") +HADOOP_REL=$(realpath --relative-to=. "$HADOOP_HOME") # default directory separator unix style DIR_SEP=/ @@ -396,7 +396,7 @@ fi JARNAME=$(basename "$SYSTEMDS_JAR_FILE") # relative path to jar file -SYSTEMDS_JAR_FILE=$(grealpath --relative-to=. "$(dirname "$SYSTEMDS_JAR_FILE")")${DIR_SEP}${JARNAME} +SYSTEMDS_JAR_FILE=$(realpath --relative-to=. "$(dirname "$SYSTEMDS_JAR_FILE")")${DIR_SEP}${JARNAME} NATIVE_LIBS="$SYSTEMDS_ROOT${DIR_SEP}target${DIR_SEP}classes${DIR_SEP}lib" export PATH=${HADOOP_REL}${DIR_SEP}bin${PATH_SEP}${PATH}${PATH_SEP}$NATIVE_LIBS diff --git a/scripts/builtin/img_cutout_linearized.dml b/scripts/builtin/img_cutout_linearized.dml index 1d3d9c1af67..f3e09fa40ba 100644 --- a/scripts/builtin/img_cutout_linearized.dml +++ b/scripts/builtin/img_cutout_linearized.dml @@ -28,21 +28,20 @@ # y Row index of the top left corner of the rectangle (starting at 1) # width Width of the rectangle (must be positive) # height Height of the rectangle (must be positive) -# fill_value The value to set for the rectangle -# s_cols Width of a single image -# s_rows Height of a single image +# fill_value The value to set for the rectangle +# s_cols Width of a single image +# s_rows Height of a single image # --------------------------------------------------------------------------------------------- # # OUTPUT: # ------------------------------------------------------------------------------------------ -# img_out Output images as linearized 2D matrix with top left corner at [1, 1] +# img_out Output images as linearized 2D matrix with top left corner at [1, 1] # ------------------------------------------------------------------------------------------ - -m_img_cutout_linearized = function(Matrix[Double] img_in, Integer x, Integer y, Integer width, Integer height, Double fill_value, Integer s_cols, Integer s_rows) return (Matrix[Double] img_out) { - # size is s_cols : s_rows - rows = nrow(img_in) # number of images - cols = ncol(img_in) # s_cols * s_rows +m_img_cutout_linearized = function(Matrix[Double] img_in, Integer x, Integer y, Integer width, Integer height, + Double fill_value, Integer s_cols, Integer s_rows) return (Matrix[Double] img_out) { + rows = nrow(img_in) + cols = ncol(img_in) if (width < 1 | height < 1) { print("Invalid width or height. Returning input") @@ -61,14 +60,11 @@ m_img_cutout_linearized = function(Matrix[Double] img_in, Integer x, Integer y, img_out = img_in # Iterate through each row of the rectangular region - for (i in start_y: end_y){ - + parfor (i in start_y: end_y, check=0){ start_idx = (i-1) * s_cols + start_x end_idx = (i-1) * s_cols + end_x img_out[, start_idx:end_idx] = matrix(fill_value, rows=rows, cols=(end_x-start_x+1)) - } + } } - -} - +} \ No newline at end of file diff --git a/src/test/java/org/apache/sysds/test/functions/builtin/part1/BuiltinImageCutoutLinTest.java b/src/test/java/org/apache/sysds/test/functions/builtin/part1/BuiltinImageCutoutLinTest.java index ac5760268b3..73adc746930 100644 --- a/src/test/java/org/apache/sysds/test/functions/builtin/part1/BuiltinImageCutoutLinTest.java +++ b/src/test/java/org/apache/sysds/test/functions/builtin/part1/BuiltinImageCutoutLinTest.java @@ -26,12 +26,15 @@ import org.apache.sysds.test.TestConfiguration; import org.apache.sysds.test.TestUtils; import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; import java.util.HashMap; -import java.util.Random; -// if A should be generated one row at a time -//import java.util.stream.Stream; -//import java.util.stream.DoubleStream; +import java.util.Arrays; +import java.util.Collection; + +@RunWith(Parameterized.class) +@net.jcip.annotations.NotThreadSafe public class BuiltinImageCutoutLinTest extends AutomatedTestBase { private final static String TEST_NAME = "image_cutout_linearized"; @@ -41,7 +44,36 @@ public class BuiltinImageCutoutLinTest extends AutomatedTestBase { private final static double eps = 1e-10; private final static double spSparse = 0.1; private final static double spDense = 0.9; - private final static Random random = new Random(); + + @Parameterized.Parameter(0) + public int s_rows; + @Parameterized.Parameter(1) + public int s_cols; + @Parameterized.Parameter(2) + public int x; + @Parameterized.Parameter(3) + public int y; + @Parameterized.Parameter(4) + public int width; + @Parameterized.Parameter(5) + public int height; + @Parameterized.Parameter(6) + public int fill_color; + @Parameterized.Parameter(7) + public int n_imgs; + + @Parameterized.Parameters + public static Collection data() { + return Arrays.asList(new Object[][] { + { 12, 12, 7, 5, 6, 2, 0, 512 }, + { 13, 11, 10, 7, 2, 3, 32, 175 }, + { 32, 32, 2, 11, 1, 60, 64, 4 }, + { 64, 64, 50, 17, 10, 109, 96, 5 }, + { 64, 61, 33, 20, 30, 10, 128, 32 }, + { 128, 128, 2, 3, 2, 9, 192, 5 }, + { 123, 128, 83, 70, 50, 2, 225, 12 }, + }); + } @Override public void setUp() { @@ -74,15 +106,6 @@ private void runImageCutoutLinTest(boolean sparse, ExecType instType) { setOutputBuffering(true); - int s_rows = random.nextInt(100) + 1; - int s_cols = random.nextInt(100) + 1; - int x = random.nextInt(s_cols); - int y = random.nextInt(s_rows); - int width = random.nextInt(s_cols - x) + 1; - int height = random.nextInt(s_rows - y) + 1; - int fill_color = random.nextInt(256); - int n_imgs = random.nextInt(100) + 1; - try { loadTestConfiguration(getTestConfiguration(TEST_NAME)); double sparsity = sparse ? spSparse : spDense; @@ -94,17 +117,7 @@ private void runImageCutoutLinTest(boolean sparse, ExecType instType) { "height=" + n_imgs, "x=" + (x + 1), "y=" + (y + 1), "w=" + width, "h=" + height, "fill_color=" + fill_color, "s_cols=" + s_cols, "s_rows=" + s_rows }; - // overall sparsity of the dataset or a single image/row? double[][] A = getRandomMatrix(n_imgs, s_cols * s_rows, 0, 255, sparsity, 7); - /* - * double[][] A = new double[n_imgs][s_cols*s_rows]; - * for (int i = 0; i < n_imgs; i++) { - * double[][] matrix = getRandomMatrix(s_cols, s_rows, 0, 255, sparsity, 7); - * double[] row = Stream.of(matrix).flatMapToDouble(DoubleStream::of).toArray(); - * A[i] = row; - * } - */ - writeInputMatrixWithMTD("A", A, true); double[][] ref = new double[n_imgs][s_cols * s_rows]; @@ -123,7 +136,6 @@ private void runImageCutoutLinTest(boolean sparse, ExecType instType) { HashMap dmlfile = readDMLMatrixFromOutputDir("B"); double[][] dml_res = TestUtils.convertHashMapToDoubleArray(dmlfile, n_imgs, (s_cols * s_rows)); - writeInputMatrixWithMTD("ref", ref, true); TestUtils.compareMatrices(ref, dml_res, eps, "Java vs. DML"); } finally { From ce81f802d9ff8d3c5ca25971608c9f5e24c52919 Mon Sep 17 00:00:00 2001 From: slnkahveci <76944633+slnkahveci@users.noreply.github.com> Date: Thu, 28 Sep 2023 19:50:38 +0200 Subject: [PATCH 8/8] suggested parfor performs badly, reversed back to for --- scripts/builtin/img_cutout_linearized.dml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/builtin/img_cutout_linearized.dml b/scripts/builtin/img_cutout_linearized.dml index f3e09fa40ba..cb923e31ba4 100644 --- a/scripts/builtin/img_cutout_linearized.dml +++ b/scripts/builtin/img_cutout_linearized.dml @@ -60,11 +60,11 @@ m_img_cutout_linearized = function(Matrix[Double] img_in, Integer x, Integer y, img_out = img_in # Iterate through each row of the rectangular region - parfor (i in start_y: end_y, check=0){ + for (i in start_y: end_y){ start_idx = (i-1) * s_cols + start_x end_idx = (i-1) * s_cols + end_x img_out[, start_idx:end_idx] = matrix(fill_value, rows=rows, cols=(end_x-start_x+1)) } } -} \ No newline at end of file +}