[SYSTEMDS-3330] Documentation of builtin functions (main)

.github/workflows/python.yml

@@ -53,7 +53,7 @@ jobs:
         os: [ubuntu-24.04]
         java: ['17']
         javadist: ['adopt']
-        test_mode: [env, noenv, federated]
+        test_mode: [env, noenv, federated, scuro]
 
     name: ${{ matrix.os }} Java ${{ matrix.java }} ${{ matrix.javadist }} Python ${{ matrix.python-version }}/ ${{ matrix.test_mode}}
     steps:
@@ -150,4 +150,10 @@ jobs:
         cd src/main/python
         ./tests/federated/runFedTest.sh
 
+    - name: Run Scuro Python Tests 
+      if: ${{ matrix.test_mode == 'scuro' }}
+      run: |
+        cd src/main/python
+        python -m unittest discover -s tests/scuro -p 'test_*.py'
+
 

.github/workflows/pythonFormatting.yml

@@ -52,5 +52,5 @@ jobs:
 
       - name: Run Black Check
         run: |
-          black --check --exclude operator/algorithm \
+          black --check --exclude '(operator/algorithm/|auto_tests/)' \
           src/main/python/systemds src/main/python/tests

scripts/builtin/dist.dml

@@ -21,6 +21,21 @@
 
 # Returns Euclidean distance matrix (distances between N n-dimensional points)
 #
+# .. code-block:: python
+#
+#   >>> import numpy as np
+#   >>> from systemds.context import SystemDSContext
+#   >>> from systemds.operator.algorithm import dist
+#   >>> 
+#   >>> with SystemDSContext() as sds:
+#   ...     X = sds.from_numpy(np.array([[0], [3], [4]]))
+#   ...     out = dist(X).compute()
+#   ...     print(out)
+#   [[0. 3. 4.]
+#    [3. 0. 1.]
+#    [4. 1. 0.]]
+#
+#
 # INPUT:
 # --------------------------------------------------------------------------------
 # X       Matrix to calculate the distance inside

scripts/builtin/img_brightness.dml

@@ -21,9 +21,26 @@
 
 # The img_brightness-function is an image data augmentation function. It changes the brightness of the image.
 #
+# .. code-block:: python
+#
+#   >>> import numpy as np
+#   >>> from systemds.context import SystemDSContext
+#   >>> from systemds.operator.algorithm import img_brightness
+#   >>> 
+#   >>> with SystemDSContext() as sds:
+#   ...     img = sds.from_numpy(
+#   ...         np.array([[ 50, 100,
+#   ...                     150, 200 ]], dtype=np.float32)
+#   ...     )
+#   ...     result_img = img_brightness(img, 30.0, 150).compute()
+#   ...     print(result_img.reshape(2, 2))
+#   [[ 80. 130.]
+#    [150. 150.]]
+#
+#
 # INPUT:
 # -----------------------------------------------------------------------------------------
-# img_in       Input matrix/image
+# img_in       Input image as 2D matrix with top left corner at [1, 1]
 # value        The amount of brightness to be changed for the image
 # channel_max  Maximum value of the brightness of the image
 # -----------------------------------------------------------------------------------------

scripts/builtin/img_brightness_linearized.dml

@@ -21,9 +21,26 @@
 
 # The img_brightness_linearized-function is an image data augmentation function. It changes the brightness of one or multiple images.
 #
+# .. code-block:: python
+#
+#   >>> import numpy as np
+#   >>> from systemds.context import SystemDSContext
+#   >>> from systemds.operator.algorithm import img_brightness_linearized
+#   >>> 
+#   >>> with SystemDSContext() as sds:
+#   ...     img = sds.from_numpy(
+#   ...         np.array([[ 50, 100,
+#   ...                     150, 200 ]], dtype=np.float32)
+#   ...     )
+#   ...     result_img = img_brightness_linearized(img, 30.0, 255).compute()
+#   ...     print(result_img.reshape(2, 2))
+#   [[ 80. 130.]
+#    [180. 230.]]
+#
+#
 # INPUT:
 # -----------------------------------------------------------------------------------------
-# img_in       Input matrix/image (can represent multiple images every row of the matrix represents a linearized image)
+# img_in       Input images as linearized 2D matrix with top left corner at [1, 1] (every row represents a linearized matrix/image)
 # value        The amount of brightness to be changed for the image
 # channel_max  Maximum value of the brightness of the image
 # -----------------------------------------------------------------------------------------

scripts/builtin/img_crop.dml

@@ -21,9 +21,26 @@
 
 # The img_crop-function is an image data augmentation function. It cuts out a subregion of an image.
 #
+# .. code-block:: python
+#
+#   >>> import numpy as np
+#   >>> from systemds.context import SystemDSContext
+#   >>> from systemds.operator.algorithm import img_crop
+#   >>> 
+#   >>> with SystemDSContext() as sds:
+#   ...     img = sds.from_numpy(
+#   ...         np.array([[ 50., 100., 150.],
+#   ...                   [150., 200., 250.],
+#   ...                   [250., 200., 200.]], dtype=np.float32)
+#   ...     )
+#   ...     result_img = img_crop(img, 1, 1, 1, 1).compute()
+#   ...     print(result_img)
+#   [[200.]]
+#
+#
 # INPUT:
 # ----------------------------------------------------------------------------------------
-# img_in    Input matrix/image
+# img_in    Input image as 2D matrix with top left corner at [1, 1]
 # w         The width of the subregion required
 # h         The height of the subregion required
 # x_offset  The horizontal coordinate in the image to begin the crop operation

scripts/builtin/img_crop_linearized.dml

@@ -21,9 +21,26 @@
 
 # The img_crop_linearized cuts out a rectangular section of multiple linearized images.
 #
+# .. code-block:: python
+#
+#   >>> import numpy as np
+#   >>> from systemds.context import SystemDSContext
+#   >>> from systemds.operator.algorithm import img_crop_linearized
+#   >>> 
+#   >>> with SystemDSContext() as sds:
+#   ...     img = sds.from_numpy(
+#   ...         np.array([[ 50., 100., 150.,
+#   ...                     150., 200., 250.,
+#   ...                     250., 200., 200. ]], dtype=np.float32)
+#   ...     )
+#   ...     result_img = img_crop_linearized(img, 1, 1, 1, 1, 3, 3).compute()
+#   ...     print(result_img)
+#   [[200.]]
+#
+#
 # INPUT:
 # ----------------------------------------------------------------------------------------
-# img_in     Linearized input images as 2D matrix
+# img_in     Input images as linearized 2D matrix with top left corner at [1, 1] (every row represents a linearized matrix/image)
 # w          The width of the subregion required
 # h          The height of the subregion required
 # x_offset   The horizontal offset for the center of the crop region

scripts/builtin/img_cutout.dml

@@ -21,6 +21,26 @@
 
 # Image Cutout function replaces a rectangular section of an image with a constant value.
 #
+#
+# .. code-block:: python
+#
+#   >>> import numpy as np
+#   >>> from systemds.context import SystemDSContext
+#   >>> from systemds.operator.algorithm import img_cutout
+#   >>> 
+#   >>> with SystemDSContext() as sds:
+#   ...     img = sds.from_numpy(
+#   ...         np.array([[ 50., 100., 150.],
+#   ...                   [150., 200., 250.],
+#   ...                   [250., 200., 200.]], dtype=np.float32)
+#   ...     )
+#   ...     result_img = img_cutout(img, 2, 2, 1, 1, 49.).compute()
+#   ...     print(result_img)
+#   [[ 50. 100. 150.]
+#    [150.  49. 250.]
+#    [250. 200. 200.]]
+#
+#
 # INPUT:
 # ---------------------------------------------------------------------------------------------
 # img_in      Input image as 2D matrix with top left corner at [1, 1]

scripts/builtin/img_cutout_linearized.dml

@@ -21,14 +21,33 @@
 
 # Image Cutout function replaces a rectangular section of an image with a constant value.
 #
+# .. code-block:: python
+#
+#   >>> import numpy as np
+#   >>> from systemds.context import SystemDSContext
+#   >>> from systemds.operator.algorithm import img_cutout_linearized
+#   >>> 
+#   >>> with SystemDSContext() as sds:
+#   ...     img = sds.from_numpy(
+#   ...         np.array([[ 50., 100., 150.,
+#   ...                     150., 200., 250.,
+#   ...                     250., 200., 200. ]], dtype=np.float32)
+#   ...     )
+#   ...     result_img = img_cutout_linearized(img, 2, 2, 1, 1, 25.0, 3, 3).compute()
+#   ...     print(result_img.reshape(3, 3))
+#   [[ 50. 100. 150.]
+#    [150.  25. 250.]
+#    [250. 200. 200.]]
+#
+#
 # INPUT:
 # ---------------------------------------------------------------------------------------------
-# img_in      Input images as linearized 2D matrix with top left corner at [1, 1]
+# img_in      Input images as linearized 2D matrix with top left corner at [1, 1] (every row represents a linearized matrix/image)
 # x           Column index of the top left corner of the rectangle (starting at 1)
 # y           Row index of the top left corner of the rectangle (starting at 1)
 # width       Width of the rectangle (must be positive)
 # height      Height of the rectangle (must be positive)
-# fill_value   The value to set for the rectangle
+# fill_value  The value to set for the rectangle
 # s_cols      Width of a single image
 # s_rows      Height of a single image
 # ---------------------------------------------------------------------------------------------

scripts/builtin/img_invert.dml

@@ -21,9 +21,28 @@
 
 # This is an image data augmentation function. It inverts an image.
 #
+# .. code-block:: python
+#
+#   >>> import numpy as np
+#   >>> from systemds.context import SystemDSContext
+#   >>> from systemds.operator.algorithm import img_invert
+#   >>> 
+#   >>> with SystemDSContext() as sds:
+#   ...     img = sds.from_numpy(
+#   ...         np.array([[ 10., 20., 30.],
+#   ...                   [ 40., 50., 60.],
+#   ...                   [ 70., 80., 90.]], dtype=np.float32)
+#   ...     )
+#   ...     result_img = img_invert(img, 210.).compute()
+#   ...     print(result_img)
+#   [[200. 190. 180.]
+#    [170. 160. 150.]
+#    [140. 130. 120.]]
+#
+#
 # INPUT:
 # ---------------------------------------------------------------------------------------------
-# img_in     Input image
+# img_in     Input image as 2D matrix with top left corner at [1, 1]
 # max_value  The maximum value pixels can have
 # ---------------------------------------------------------------------------------------------
 #

scripts/builtin/img_invert_linearized.dml

@@ -21,9 +21,28 @@
 
 # This is an image data augmentation function. It inverts an image.It can handle one or multiple images 
 #
+# .. code-block:: python
+#
+#   >>> import numpy as np
+#   >>> from systemds.context import SystemDSContext
+#   >>> from systemds.operator.algorithm import img_invert_linearized
+#   >>> 
+#   >>> with SystemDSContext() as sds:
+#   ...     img = sds.from_numpy(
+#   ...         np.array([[ 10., 20., 30.,
+#   ...                     40., 50., 60.,
+#   ...                     70., 80., 90. ]], dtype=np.float32)
+#   ...     )
+#   ...     result_img = img_invert_linearized(img, 200.).compute()
+#   ...     print(result_img.reshape(3, 3))
+#   [[190. 180. 170.]
+#    [160. 150. 140.]
+#    [130. 120. 110.]]
+#
+#
 # INPUT:
 # ---------------------------------------------------------------------------------------------
-# img_in     Input matrix/image (every row of the matrix represents a linearized image)
+# img_in     Input images as linearized 2D matrix with top left corner at [1, 1] (every row represents a linearized matrix/image)
 # max_value  The maximum value pixels can have
 # ---------------------------------------------------------------------------------------------
 #

scripts/builtin/img_mirror.dml

@@ -22,10 +22,29 @@
 # This function is an image data augmentation function.
 # It flips an image on the X (horizontal) or Y (vertical) axis.
 #
+# .. code-block:: python
+#
+#   >>> import numpy as np
+#   >>> from systemds.context import SystemDSContext
+#   >>> from systemds.operator.algorithm import img_mirror
+#   >>> 
+#   >>> with SystemDSContext() as sds:
+#   ...     img = sds.from_numpy(
+#   ...         np.array([[ 10., 20., 30.],
+#   ...                   [ 40., 50., 60.],
+#   ...                   [ 70., 80., 90.]], dtype=np.float32)
+#   ...     )
+#   ...     result_img = img_mirror(img, False).compute()
+#   ...     print(result_img)
+#   [[30. 20. 10.]
+#    [60. 50. 40.]
+#    [90. 80. 70.]]
+#
+#
 # INPUT:
 # ---------------------------------------------------------------------------------------------
-# img_in     Input matrix/image
-# max_value  The maximum value pixels can have
+# img_in            Input image as 2D matrix with top left corner at [1, 1]
+# horizontal_axis   Flip either in X or Y axis
 # ---------------------------------------------------------------------------------------------
 #
 # OUTPUT:

scripts/builtin/img_mirror_linearized.dml

@@ -22,9 +22,55 @@
 # This function has  the same functionality with img_mirror but it handles multiple images at
 # the same time. Each row of the input and output matrix represents a linearized image/matrix
 # It flips an image on the X (horizontal) or Y (vertical) axis.
+#
+# .. code-block:: python
+#
+#   >>> import numpy as np
+#   >>> from systemds.context import SystemDSContext
+#   >>> from systemds.operator.algorithm import img_mirror_linearized
+#   >>> 
+#   >>> with SystemDSContext() as sds:
+#   ...     img = sds.from_numpy(
+#   ...         np.array([[ 10., 20., 30.,
+#   ...                     40., 50., 60.,
+#   ...                     70., 80., 90. ]], dtype=np.float32)
+#   ...     )
+#   ...     result_img = img_mirror_linearized(img, True, 3, 3).compute()
+#   ...     print(result_img.reshape(3, 3))
+#   [[70. 80. 90.]
+#    [40. 50. 60.]
+#    [10. 20. 30.]]
+#
+#
+# .. code-block:: python
+#
+#   >>> import numpy as np
+#   >>> from systemds.context import SystemDSContext
+#   >>> from systemds.operator.algorithm import img_mirror_linearized
+#   >>> 
+#   >>> with SystemDSContext() as sds:
+#   ...     imgs = sds.from_numpy(
+#   ...         np.array([[ 10., 20., 30.,
+#   ...                     40., 50., 60.,
+#   ...                     70., 80., 90. ],
+#   ...                   [ 70., 80., 90.,
+#   ...                     40., 50., 60.,
+#   ...                     10., 20., 30. ]], dtype=np.float32)
+#   ...     )
+#   ...     result_imgs = img_mirror_linearized(imgs, True, 3, 3).compute()
+#   ...     print(result_imgs[0].reshape(3, 3))
+#   ...     print(result_imgs[1].reshape(3, 3))
+#   [[70. 80. 90.]
+#    [40. 50. 60.]
+#    [10. 20. 30.]]
+#   [[10. 20. 30.]
+#    [40. 50. 60.]
+#    [70. 80. 90.]]
+#
+#
 # INPUT:
 # -----------------------------------------------------------------------------------------
-# img_matrix           Input matrix/image (every row represents a linearized matrix/image)
+# img_matrix           Input images as linearized 2D matrix with top left corner at [1, 1] (every row represents a linearized matrix/image)
 # horizontal_axis      flip either in X or Y axis
 # original_rows        number of rows in the original 2-D images
 # original_cols        number of cols in the original 2-D images