From 1b4fe90456dd57a6b0f45310c49555347191ba2b Mon Sep 17 00:00:00 2001 From: anuunchin <88698977+anuunchin@users.noreply.github.com> Date: Sun, 13 Jul 2025 21:10:55 +0200 Subject: [PATCH 1/4] Fixes in generator.py, builtin function fixes --- scripts/builtin/ampute.dml | 1 - scripts/builtin/confusionMatrix.dml | 2 +- scripts/builtin/cooccurrenceMatrix.dml | 13 ++- scripts/builtin/decisionTree.dml | 4 +- scripts/builtin/differenceStatistics.dml | 5 ++ scripts/builtin/glove.dml | 29 ++++++- scripts/builtin/imputeByKNN.dml | 25 ++---- scripts/builtin/quantizeByCluster.dml | 5 +- scripts/builtin/randomForest.dml | 7 +- scripts/builtin/shapExplainer.dml | 1 + scripts/builtin/topk_cleaning.dml | 40 ++++++++- src/main/python/docs/README.md | 2 +- src/main/python/docs/requires-docs.txt | 3 +- src/main/python/generator/dml_parser.py | 16 ++-- .../systemds/operator/algorithm/__init__.py | 36 ++++++++ .../operator/algorithm/builtin/ampute.py | 10 +++ .../algorithm/builtin/apply_pipeline.py | 1 - .../algorithm/builtin/confusionMatrix.py | 2 +- .../algorithm/builtin/cooccurrenceMatrix.py | 58 +++++++++++++ .../algorithm/builtin/correctTypos.py | 1 - .../algorithm/builtin/correctTyposApply.py | 1 - .../algorithm/builtin/decisionTree.py | 4 +- .../algorithm/builtin/denialConstraints.py | 1 - .../algorithm/builtin/differenceStatistics.py | 5 ++ .../operator/algorithm/builtin/dmv.py | 1 - .../operator/algorithm/builtin/ema.py | 1 - .../algorithm/builtin/executePipeline.py | 38 +++++++-- .../algorithm/builtin/fit_pipeline.py | 1 - .../algorithm/builtin/fixInvalidLengths.py | 1 - .../builtin/fixInvalidLengthsApply.py | 1 - .../operator/algorithm/builtin/frameSort.py | 1 - .../operator/algorithm/builtin/glove.py | 67 +++++++++++++++ .../operator/algorithm/builtin/imputeByKNN.py | 19 ++++- .../operator/algorithm/builtin/mdedup.py | 1 - .../algorithm/builtin/quantizeByCluster.py | 83 +++++++++++++++++++ .../algorithm/builtin/randomForest.py | 7 +- .../algorithm/builtin/shapExplainer.py | 78 +++++++++++++++++ .../algorithm/builtin/topk_cleaning.py | 34 +++++++- .../operator/algorithm/builtin/wer.py | 48 +++++++++++ 39 files changed, 578 insertions(+), 75 deletions(-) create mode 100644 src/main/python/systemds/operator/algorithm/builtin/cooccurrenceMatrix.py create mode 100644 src/main/python/systemds/operator/algorithm/builtin/glove.py create mode 100644 src/main/python/systemds/operator/algorithm/builtin/quantizeByCluster.py create mode 100644 src/main/python/systemds/operator/algorithm/builtin/shapExplainer.py create mode 100644 src/main/python/systemds/operator/algorithm/builtin/wer.py diff --git a/scripts/builtin/ampute.dml b/scripts/builtin/ampute.dml index 0da6af20fdb..ffc645b5b35 100644 --- a/scripts/builtin/ampute.dml +++ b/scripts/builtin/ampute.dml @@ -30,7 +30,6 @@ # mech a string [either "MAR", "MNAR", or "MCAR"] specifying the missingness mechanism. Chosen "MAR" and "MNAR" settings will be overridden if a non-default weight matrix is specified # weights a weight matrix [shape: k-by-m], containing weights that will be used to calculate the weighted sum scores. Will be overridden if mech == "MCAR" # seed a manually defined seed for reproducible RNG - # ------------------------------------------------------------------------------------- # # OUTPUT: diff --git a/scripts/builtin/confusionMatrix.dml b/scripts/builtin/confusionMatrix.dml index 3ac70fb3f87..b21088f2cfa 100644 --- a/scripts/builtin/confusionMatrix.dml +++ b/scripts/builtin/confusionMatrix.dml @@ -23,7 +23,7 @@ # and actual labels. We return both the counts and relative frequency # (normalized by sum of true labels) # -# .. code-block:: +# .. code-block:: text # # True Labels # 1 2 diff --git a/scripts/builtin/cooccurrenceMatrix.dml b/scripts/builtin/cooccurrenceMatrix.dml index 86b8b9ca169..590f4ba1e00 100644 --- a/scripts/builtin/cooccurrenceMatrix.dml +++ b/scripts/builtin/cooccurrenceMatrix.dml @@ -18,22 +18,21 @@ # under the License. # #------------------------------------------------------------- -# -# The implementation is based on + +# Cleans and processes text data by removing punctuation, converting it to lowercase, and reformatting. +# Adds an index column to the result. The implementation is based on # https://github.com/stanfordnlp/GloVe/blob/master/src/cooccur.c # -#------------------------------------------------------------- - -## Cleans and processes text data by removing punctuation, converting it to lowercase, and reformatting. -## Adds an index column to the result. # INPUT: # ------------------------------------------------------------------------------ # S (Frame[Unknown]): 1D input data frame containing text data. # ------------------------------------------------------------------------------ +# # OUTPUT: # ------------------------------------------------------------------------------ # result (Frame[Unknown]): Processed text data with an index column. # ------------------------------------------------------------------------------ + processText = function(Frame[Unknown] S) return (Frame[Unknown] result){ print("processText"); tmpStr = map(S[,1], "x -> x.replaceAll(\"[.]\", \"\")"); @@ -172,4 +171,4 @@ f_cooccurrenceMatrix = function( [wordPosition, docID] = getWordPosition(processedResult, maxTokens); [recodedWordPosition, tableSize, column] = getRecodedMatrix(wordPosition); coocMatrix = createCoocMatrix(cbind(docID, recodedWordPosition), tableSize, distanceWeighting, symmetric, windowSize); -} +} \ No newline at end of file diff --git a/scripts/builtin/decisionTree.dml b/scripts/builtin/decisionTree.dml index 69bf12af90c..94c292d8554 100644 --- a/scripts/builtin/decisionTree.dml +++ b/scripts/builtin/decisionTree.dml @@ -30,9 +30,9 @@ # and the following trees, M would look as follows: # # (L1) |d<5| -# / \ +# / \\ # (L2) P1:2 |a<7| -# / \ +# / \\ # (L3) P2:2 P3:1 # # --> M := diff --git a/scripts/builtin/differenceStatistics.dml b/scripts/builtin/differenceStatistics.dml index 0e9019f0963..30f207091e4 100644 --- a/scripts/builtin/differenceStatistics.dml +++ b/scripts/builtin/differenceStatistics.dml @@ -28,6 +28,11 @@ # X First Matrix to compare # Y Second Matrix to compare # -------------------------------------------------------------------------------- +# +# OUTPUT: +# ------------------------------------------------------------------------------------- +# stats. Difference statistics +# ------------------------------------------------------------------------------------- m_differenceStatistics = function(Matrix[Double] X, Matrix[Double] Y) { diff --git a/scripts/builtin/glove.dml b/scripts/builtin/glove.dml index fc5ee9bafb3..1fe99968605 100644 --- a/scripts/builtin/glove.dml +++ b/scripts/builtin/glove.dml @@ -18,6 +18,31 @@ # under the License. #------------------------------------------------------------- +# Computes the vector embeddings for words in a large text corpus. +# +# INPUT: +# -------------------------------------------------------------------------------- +# input 1DInput corpus in CSV format. +# seed Random seed for reproducibility. +# vector_size Dimensionality of word vectors, V. +# eta Learning rate for optimization, recommended value: 0.05. +# alpha Weighting function parameter, recommended value: 0.75. +# x_max Maximum co-occurrence value as per the GloVe paper: 100. +# tol Tolerance value to avoid overfitting, recommended value: 1e-4. +# iterations Total number of training iterations. +# print_loss_it Interval (in iterations) for printing the loss. +# maxTokens Maximum number of tokens per text entry. +# windowSize Context window size. +# distanceWeighting Whether to apply distance-based weighting. +# symmetric Determines if the matrix is symmetric (TRUE) or asymmetric (FALSE). +# ------------------------------------------------------------------------------ +# +# OUTPUT: +# ------------------------------------------------------------------------------ +# G The word indices and their word vectors, of shape (N, V). Each represented as a vector, of shape (1,V) +# ------------------------------------------------------------------------------ + + init = function(matrix[double] cooc_matrix, double x_max, double alpha) return(matrix[double] weights, matrix[double] log_cooc_matrix){ E = 2.718281828; @@ -119,7 +144,7 @@ gloveWithCoocMatrix = function(matrix[double] cooc_matrix, frame[Unknown] cooc_i G = cbind(cooc_index[,2], as.frame(G)); } -glove = function( +f_glove = function( Frame[Unknown] input, int seed, int vector_size, double alpha, double eta, @@ -159,4 +184,4 @@ glove = function( [cooc_matrix, cooc_index] = cooccurrenceMatrix(input, maxTokens, windowSize, distanceWeighting, symmetric); G = gloveWithCoocMatrix(cooc_matrix, cooc_index, seed, vector_size, alpha, eta, x_max, tol, iterations, print_loss_it); -} +} \ No newline at end of file diff --git a/scripts/builtin/imputeByKNN.dml b/scripts/builtin/imputeByKNN.dml index 13136ff2c9a..edd8e7727d2 100644 --- a/scripts/builtin/imputeByKNN.dml +++ b/scripts/builtin/imputeByKNN.dml @@ -25,23 +25,16 @@ # the missing values by column means. Currently, only the column with the most # missing values is actually imputed. # -# ------------------------------------------------------------------------------ # INPUT: # ------------------------------------------------------------------------------ -# X Matrix with missing values, which are represented as NaNs -# method Method used for imputing missing values with different performance -# and accuracy tradeoffs: -# 'dist' (default): Compute all-pairs distances and impute the -# missing values by closest. O(N^2 * #features) -# 'dist_missing': Compute distances between data and records with -# missing values. O(N*M * #features), assuming -# that the number of records with MV is M< M := # [[1, 7, 3, 3, 2, 4, 0, 2, 0, 1, 0, 1, 0, 2], (1st tree) # [4, 5, 1, 7, 0, 2, 0, 2, 0, 1, 0, 0, 0, 0]] (2nd tree) diff --git a/scripts/builtin/shapExplainer.dml b/scripts/builtin/shapExplainer.dml index b78a5dbcefb..39d365bf013 100644 --- a/scripts/builtin/shapExplainer.dml +++ b/scripts/builtin/shapExplainer.dml @@ -51,6 +51,7 @@ # S Matrix holding the shapley values along the cols, one row per instance. # expected Double holding the average prediction of all instances. # ----------------------------------------------------------------------------- + s_shapExplainer = function(String model_function, list[unknown] model_args, Matrix[Double] x_instances, Matrix[Double] X_bg, Integer n_permutations = 10, Integer n_samples = 100, Integer remove_non_var=0, Matrix[Double] partitions=as.matrix(-1), Integer seed = -1, Integer verbose = 0) diff --git a/scripts/builtin/topk_cleaning.dml b/scripts/builtin/topk_cleaning.dml index 6f946c7729c..d54c3f4e1bd 100644 --- a/scripts/builtin/topk_cleaning.dml +++ b/scripts/builtin/topk_cleaning.dml @@ -19,8 +19,44 @@ # #------------------------------------------------------------- -# This function cleans top-K item (where K is given as input)for a given list of users. +# This function cleans top-K item (where K is given as input) for a given list of users. # metaData[3, ncol(X)] : metaData[1] stores mask, metaData[2] stores schema, metaData[3] stores FD mask +# +# INPUT: +# ------------------------------------------------------------------------------ +# dataTrain Training set +# dataTest Test set ignored when cv is set to True +# metaData 3×n frame with schema, categorical mask, and FD mask for dataTrain +# primitives Library of primitive cleaning operators +# parameters Hyperparameter search space that matches the primitives +# refSol Reference solution +# evaluationFunc Name of a SystemDS DML function that scores a pipeline +# evalFunHp Hyperparameter matrix for the above evaluation function +# topK Number of best pipelines to return +# resource_val Maximum resource R for the Bandit search +# max_iter Maximum iterations while enumerating logical pipelines +# lq Lower quantile used by utils::doErrorSample when triggered +# uq Upper quantile used by utils::doErrorSample when triggered +# sample Fraction of rows to subsample from dataTrain +# expectedIncrease Minimum improvement over dirtyScore that a candidate must deliver +# seed Seed number +# cv TRUE means k-fold CV, FALSE means hold-out split +# cvk Number of folds if cv = TRUE +# isLastLabel TRUE if the last column is the label +# rowCount Row-count threshold above which doErrorSample may replace uniform sampling +# correctTypos Run spelling correction in the string preprocessing step +# enablePruning Enable pruning inside the Bandit phase +# ------------------------------------------------------------------------------ +# +# OUTPUT: +#------------------------------------------------------------------------------- +# topKPipelines K cleaned-data pipelines +# topKHyperParams Hyperparameter matrix with rows aligning with topKPipelines +# topKScores Evaluation scores with rows aligning with topKPipelines +# dirtyScore Baseline score on the unclean data +# evalFunHps Updated evaluation function hyperparameters +# evalFunHp Frame of “apply” functions for deploying each of the top-K pipelines +#------------------------------------------------------------------------------- source("scripts/pipelines/scripts/utils.dml") as utils; source("scripts/pipelines/scripts/enumerateLogical.dml") as lg; @@ -32,7 +68,7 @@ f_topk_cleaning = function(Frame[Unknown] dataTrain, Frame[Unknown] dataTest = a Boolean isLastLabel = TRUE, Integer rowCount = 3700, Boolean correctTypos=FALSE, Boolean enablePruning = FALSE) return (Frame[Unknown] topKPipelines, Matrix[Double] topKHyperParams, Matrix[Double] topKScores, - Double dirtyScore, Matrix[Double] evalFunHp, Frame[Unknown] applyFunc) + Double dirtyScore, Matrix[Double] evalFunHp, Frame[Unknown] evalFunHp) { t1 = time(); print("TopK-Cleaning:"); diff --git a/src/main/python/docs/README.md b/src/main/python/docs/README.md index 61bdd24a3e9..e5bc5c59583 100644 --- a/src/main/python/docs/README.md +++ b/src/main/python/docs/README.md @@ -39,4 +39,4 @@ and then run `make html`: make html ``` -The docs will then be created at: `/src/main/python/build`in HTML will be placed in the `./_build` directory. +The docs will then be created at: `/src/main/python/docs/build/html/`. \ No newline at end of file diff --git a/src/main/python/docs/requires-docs.txt b/src/main/python/docs/requires-docs.txt index 9305d9320fa..1022b652401 100644 --- a/src/main/python/docs/requires-docs.txt +++ b/src/main/python/docs/requires-docs.txt @@ -24,4 +24,5 @@ sphinx_rtd_theme numpy py4j scipy -requests \ No newline at end of file +requests +pandas \ No newline at end of file diff --git a/src/main/python/generator/dml_parser.py b/src/main/python/generator/dml_parser.py index 2abffb021f6..8e835e96a12 100644 --- a/src/main/python/generator/dml_parser.py +++ b/src/main/python/generator/dml_parser.py @@ -28,7 +28,7 @@ class FunctionParser(object): header_input_pattern = r"^[ \t\n]*[#]+[ \t\n]*input[ \t\n\w:;.,#]*[\s#\-]*[#]+[\w\s\d:,.()\" \t\n\-]*[\s#\-]*$" header_output_pattern = r"[\s#\-]*[#]+[ \t]*(return|output)[ \t\w:;.,#]*[\s#\-]*[#]+[\w\s\d:,.()\" \t\-]*[\s#\-]*$" - function_pattern = r"^[ms]_[\w]+[ \t\n]*=[ \t\n]+function[^#{]*" + function_pattern = r"^[fms]_[\w]+[ \t\n]*=[ \t\n]+function[^#{]*" # parameter_pattern = r"^m_[\w]+[\s]+=[\s]+function[\s]*\([\s]*(?=return)[\s]*\)[\s]*return[\s]*\([\s]*([\w\[\]\s,\d=.\-_]*)[\s]*\)[\s]*" header_parameter_pattern = r"[\s#\-]*[#]+[ \t]*([\w|-]+)[\s]+([\w]+)[\s]+([\w,\d.\"\-]+)[\s]+([\w|\W]+)" divider_pattern = r"[\s#\-]*" @@ -57,15 +57,13 @@ def parse_function(self, path: str): """ file_name = os.path.basename(path) function_name, extension = os.path.splitext(file_name) - # try: - function_definition = self.find_function_definition(path) - # pattern = re.compile( - # self.__class__.parameter_pattern, flags=re.I | re.M) - # match = pattern.match(function_definition) - - # if match: + try: + function_definition = self.find_function_definition(path) + except AttributeError: + print(f"[INFO] Skipping '{function_name}': does not match function name pattern. It is likely an internal function.") + return - func_split = function_definition.split("function")[1].split("return") + func_split = function_definition.split("function", 1)[1].split("return") param_str = self.extract_param_str(func_split[0]) retval_str = None diff --git a/src/main/python/systemds/operator/algorithm/__init__.py b/src/main/python/systemds/operator/algorithm/__init__.py index bd611ee6cc6..c198a7138e9 100644 --- a/src/main/python/systemds/operator/algorithm/__init__.py +++ b/src/main/python/systemds/operator/algorithm/__init__.py @@ -31,6 +31,7 @@ from .builtin.alsPredict import alsPredict from .builtin.alsTopkPredict import alsTopkPredict from .builtin.ampute import ampute +from .builtin.apply_pipeline import apply_pipeline from .builtin.arima import arima from .builtin.auc import auc from .builtin.autoencoder_2layer import autoencoder_2layer @@ -38,7 +39,10 @@ from .builtin.bivar import bivar from .builtin.components import components from .builtin.confusionMatrix import confusionMatrix +from .builtin.cooccurrenceMatrix import cooccurrenceMatrix from .builtin.cor import cor +from .builtin.correctTypos import correctTypos +from .builtin.correctTyposApply import correctTyposApply from .builtin.cov import cov from .builtin.cox import cox from .builtin.cspline import cspline @@ -50,15 +54,22 @@ from .builtin.decisionTree import decisionTree from .builtin.decisionTreePredict import decisionTreePredict from .builtin.deepWalk import deepWalk +from .builtin.denialConstraints import denialConstraints from .builtin.differenceStatistics import differenceStatistics from .builtin.discoverFD import discoverFD from .builtin.dist import dist +from .builtin.dmv import dmv +from .builtin.ema import ema from .builtin.executePipeline import executePipeline from .builtin.f1Score import f1Score from .builtin.fdr import fdr from .builtin.ffPredict import ffPredict from .builtin.ffTrain import ffTrain +from .builtin.fit_pipeline import fit_pipeline +from .builtin.fixInvalidLengths import fixInvalidLengths +from .builtin.fixInvalidLengthsApply import fixInvalidLengthsApply from .builtin.flattenQuantile import flattenQuantile +from .builtin.frameSort import frameSort from .builtin.frequencyEncode import frequencyEncode from .builtin.frequencyEncodeApply import frequencyEncodeApply from .builtin.garch import garch @@ -66,6 +77,7 @@ from .builtin.getAccuracy import getAccuracy from .builtin.glm import glm from .builtin.glmPredict import glmPredict +from .builtin.glove import glove from .builtin.gmm import gmm from .builtin.gmmPredict import gmmPredict from .builtin.gnmf import gnmf @@ -97,6 +109,7 @@ from .builtin.impurityMeasures import impurityMeasures from .builtin.imputeByFD import imputeByFD from .builtin.imputeByFDApply import imputeByFDApply +from .builtin.imputeByKNN import imputeByKNN from .builtin.imputeByMean import imputeByMean from .builtin.imputeByMeanApply import imputeByMeanApply from .builtin.imputeByMedian import imputeByMedian @@ -126,6 +139,7 @@ from .builtin.mape import mape from .builtin.matrixProfile import matrixProfile from .builtin.mcc import mcc +from .builtin.mdedup import mdedup from .builtin.mice import mice from .builtin.miceApply import miceApply from .builtin.mse import mse @@ -153,6 +167,7 @@ from .builtin.pnmf import pnmf from .builtin.ppca import ppca from .builtin.psnr import psnr +from .builtin.quantizeByCluster import quantizeByCluster from .builtin.raGroupby import raGroupby from .builtin.raJoin import raJoin from .builtin.raSelection import raSelection @@ -165,6 +180,7 @@ from .builtin.selectByVarThresh import selectByVarThresh from .builtin.ses import ses from .builtin.setdiff import setdiff +from .builtin.shapExplainer import shapExplainer from .builtin.sherlock import sherlock from .builtin.sherlockPredict import sherlockPredict from .builtin.shortestPath import shortestPath @@ -189,10 +205,12 @@ from .builtin.tSNE import tSNE from .builtin.toOneHot import toOneHot from .builtin.tomeklink import tomeklink +from .builtin.topk_cleaning import topk_cleaning from .builtin.underSampling import underSampling from .builtin.union import union from .builtin.univar import univar from .builtin.vectorToCsv import vectorToCsv +from .builtin.wer import wer from .builtin.winsorize import winsorize from .builtin.winsorizeApply import winsorizeApply from .builtin.xdummy1 import xdummy1 @@ -211,6 +229,7 @@ 'alsPredict', 'alsTopkPredict', 'ampute', + 'apply_pipeline', 'arima', 'auc', 'autoencoder_2layer', @@ -218,7 +237,10 @@ 'bivar', 'components', 'confusionMatrix', + 'cooccurrenceMatrix', 'cor', + 'correctTypos', + 'correctTyposApply', 'cov', 'cox', 'cspline', @@ -230,15 +252,22 @@ 'decisionTree', 'decisionTreePredict', 'deepWalk', + 'denialConstraints', 'differenceStatistics', 'discoverFD', 'dist', + 'dmv', + 'ema', 'executePipeline', 'f1Score', 'fdr', 'ffPredict', 'ffTrain', + 'fit_pipeline', + 'fixInvalidLengths', + 'fixInvalidLengthsApply', 'flattenQuantile', + 'frameSort', 'frequencyEncode', 'frequencyEncodeApply', 'garch', @@ -246,6 +275,7 @@ 'getAccuracy', 'glm', 'glmPredict', + 'glove', 'gmm', 'gmmPredict', 'gnmf', @@ -277,6 +307,7 @@ 'impurityMeasures', 'imputeByFD', 'imputeByFDApply', + 'imputeByKNN', 'imputeByMean', 'imputeByMeanApply', 'imputeByMedian', @@ -306,6 +337,7 @@ 'mape', 'matrixProfile', 'mcc', + 'mdedup', 'mice', 'miceApply', 'mse', @@ -333,6 +365,7 @@ 'pnmf', 'ppca', 'psnr', + 'quantizeByCluster', 'raGroupby', 'raJoin', 'raSelection', @@ -345,6 +378,7 @@ 'selectByVarThresh', 'ses', 'setdiff', + 'shapExplainer', 'sherlock', 'sherlockPredict', 'shortestPath', @@ -369,10 +403,12 @@ 'tSNE', 'toOneHot', 'tomeklink', + 'topk_cleaning', 'underSampling', 'union', 'univar', 'vectorToCsv', + 'wer', 'winsorize', 'winsorizeApply', 'xdummy1', diff --git a/src/main/python/systemds/operator/algorithm/builtin/ampute.py b/src/main/python/systemds/operator/algorithm/builtin/ampute.py index d323000710e..fb3a82a380f 100644 --- a/src/main/python/systemds/operator/algorithm/builtin/ampute.py +++ b/src/main/python/systemds/operator/algorithm/builtin/ampute.py @@ -33,6 +33,16 @@ def ampute(X: Matrix, """ This function injects missing values into a multivariate a given dataset, similarly to the ampute() method in R's MICE package. + + + :param X: a multivariate numeric dataset [shape: n-by-m] + :param prop: a number in the (0, 1] range specifying the proportion of amputed rows across the entire dataset + :param patterns: a pattern matrix of 0's and 1's [shape: k-by-m] where each row corresponds to a pattern. 0 indicates that a variable should have missing values and 1 indicating that a variable should remain complete + :param freq: a vector [length: k] containing the relative frequency with which each pattern in the patterns matrix should occur + :param mech: a string [either "MAR", "MNAR", or "MCAR"] specifying the missingness mechanism. Chosen "MAR" and "MNAR" settings will be overridden if a non-default weight matrix is specified + :param weights: a weight matrix [shape: k-by-m], containing weights that will be used to calculate the weighted sum scores. Will be overridden if mech == "MCAR" + :param seed: a manually defined seed for reproducible RNG + :return: amputed output dataset """ params_dict = {'X': X} diff --git a/src/main/python/systemds/operator/algorithm/builtin/apply_pipeline.py b/src/main/python/systemds/operator/algorithm/builtin/apply_pipeline.py index be1100b4127..63ffc3f66b3 100644 --- a/src/main/python/systemds/operator/algorithm/builtin/apply_pipeline.py +++ b/src/main/python/systemds/operator/algorithm/builtin/apply_pipeline.py @@ -25,7 +25,6 @@ from typing import Dict, Iterable from systemds.operator import OperationNode, Matrix, Frame, List, MultiReturn, Scalar -from systemds.script_building.dag import OutputType from systemds.utils.consts import VALID_INPUT_TYPES diff --git a/src/main/python/systemds/operator/algorithm/builtin/confusionMatrix.py b/src/main/python/systemds/operator/algorithm/builtin/confusionMatrix.py index 81c549b5982..66a01780b0e 100644 --- a/src/main/python/systemds/operator/algorithm/builtin/confusionMatrix.py +++ b/src/main/python/systemds/operator/algorithm/builtin/confusionMatrix.py @@ -35,7 +35,7 @@ def confusionMatrix(P: Matrix, and actual labels. We return both the counts and relative frequency (normalized by sum of true labels) - .. code-block:: + .. code-block:: text True Labels 1 2 diff --git a/src/main/python/systemds/operator/algorithm/builtin/cooccurrenceMatrix.py b/src/main/python/systemds/operator/algorithm/builtin/cooccurrenceMatrix.py new file mode 100644 index 00000000000..6df77d3e7dd --- /dev/null +++ b/src/main/python/systemds/operator/algorithm/builtin/cooccurrenceMatrix.py @@ -0,0 +1,58 @@ +# ------------------------------------------------------------- +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# ------------------------------------------------------------- + +# Autogenerated By : src/main/python/generator/generator.py +# Autogenerated From : scripts/builtin/cooccurrenceMatrix.dml + +from typing import Dict, Iterable + +from systemds.operator import OperationNode, Matrix, Frame, List, MultiReturn, Scalar +from systemds.utils.consts import VALID_INPUT_TYPES + + +def cooccurrenceMatrix(input: Frame, + maxTokens: int, + windowSize: int, + distanceWeighting: bool, + symmetric: bool): + """ + Cleans and processes text data by removing punctuation, converting it to lowercase, and reformatting. + Adds an index column to the result. The implementation is based on + https://github.com/stanfordnlp/GloVe/blob/master/src/cooccur.c + + + + :param S: (Frame[Unknown]): 1D input data frame containing text data. + :return: (Frame[Unknown]): Processed text data with an index column. + """ + + params_dict = {'input': input, 'maxTokens': maxTokens, 'windowSize': windowSize, 'distanceWeighting': distanceWeighting, 'symmetric': symmetric} + + vX_0 = Matrix(input.sds_context, '') + vX_1 = Frame(input.sds_context, '') + output_nodes = [vX_0, vX_1, ] + + op = MultiReturn(input.sds_context, 'cooccurrenceMatrix', output_nodes, named_input_nodes=params_dict) + + vX_0._unnamed_input_nodes = [op] + vX_1._unnamed_input_nodes = [op] + + return op diff --git a/src/main/python/systemds/operator/algorithm/builtin/correctTypos.py b/src/main/python/systemds/operator/algorithm/builtin/correctTypos.py index 321a1949f58..64354a9bc3e 100644 --- a/src/main/python/systemds/operator/algorithm/builtin/correctTypos.py +++ b/src/main/python/systemds/operator/algorithm/builtin/correctTypos.py @@ -25,7 +25,6 @@ from typing import Dict, Iterable from systemds.operator import OperationNode, Matrix, Frame, List, MultiReturn, Scalar -from systemds.script_building.dag import OutputType from systemds.utils.consts import VALID_INPUT_TYPES diff --git a/src/main/python/systemds/operator/algorithm/builtin/correctTyposApply.py b/src/main/python/systemds/operator/algorithm/builtin/correctTyposApply.py index 0a2c61a6f40..5da8769509c 100644 --- a/src/main/python/systemds/operator/algorithm/builtin/correctTyposApply.py +++ b/src/main/python/systemds/operator/algorithm/builtin/correctTyposApply.py @@ -25,7 +25,6 @@ from typing import Dict, Iterable from systemds.operator import OperationNode, Matrix, Frame, List, MultiReturn, Scalar -from systemds.script_building.dag import OutputType from systemds.utils.consts import VALID_INPUT_TYPES diff --git a/src/main/python/systemds/operator/algorithm/builtin/decisionTree.py b/src/main/python/systemds/operator/algorithm/builtin/decisionTree.py index a1a751d0aad..3fe565b8c7e 100644 --- a/src/main/python/systemds/operator/algorithm/builtin/decisionTree.py +++ b/src/main/python/systemds/operator/algorithm/builtin/decisionTree.py @@ -44,9 +44,9 @@ def decisionTree(X: Matrix, and the following trees, M would look as follows: (L1) |d<5| - / \ + / \\ (L2) P1:2 |a<7| - / \ + / \\ (L3) P2:2 P3:1 --> M := diff --git a/src/main/python/systemds/operator/algorithm/builtin/denialConstraints.py b/src/main/python/systemds/operator/algorithm/builtin/denialConstraints.py index 347502b848e..5cdec212965 100644 --- a/src/main/python/systemds/operator/algorithm/builtin/denialConstraints.py +++ b/src/main/python/systemds/operator/algorithm/builtin/denialConstraints.py @@ -25,7 +25,6 @@ from typing import Dict, Iterable from systemds.operator import OperationNode, Matrix, Frame, List, MultiReturn, Scalar -from systemds.script_building.dag import OutputType from systemds.utils.consts import VALID_INPUT_TYPES diff --git a/src/main/python/systemds/operator/algorithm/builtin/differenceStatistics.py b/src/main/python/systemds/operator/algorithm/builtin/differenceStatistics.py index dfe2218a424..b6597bb6e4b 100644 --- a/src/main/python/systemds/operator/algorithm/builtin/differenceStatistics.py +++ b/src/main/python/systemds/operator/algorithm/builtin/differenceStatistics.py @@ -35,6 +35,11 @@ def differenceStatistics(X: Matrix, they are different. This can be used for instance in comparison of lossy compression techniques, that reduce the fidelity of the data. + + + :param X: First Matrix to compare + :param Y: Second Matrix to compare + :return: Difference statistics """ params_dict = {'X': X, 'Y': Y} diff --git a/src/main/python/systemds/operator/algorithm/builtin/dmv.py b/src/main/python/systemds/operator/algorithm/builtin/dmv.py index deaf3ea8a6b..2955e505e13 100644 --- a/src/main/python/systemds/operator/algorithm/builtin/dmv.py +++ b/src/main/python/systemds/operator/algorithm/builtin/dmv.py @@ -25,7 +25,6 @@ from typing import Dict, Iterable from systemds.operator import OperationNode, Matrix, Frame, List, MultiReturn, Scalar -from systemds.script_building.dag import OutputType from systemds.utils.consts import VALID_INPUT_TYPES diff --git a/src/main/python/systemds/operator/algorithm/builtin/ema.py b/src/main/python/systemds/operator/algorithm/builtin/ema.py index 4e0ccca6bbb..90f9a852d76 100644 --- a/src/main/python/systemds/operator/algorithm/builtin/ema.py +++ b/src/main/python/systemds/operator/algorithm/builtin/ema.py @@ -25,7 +25,6 @@ from typing import Dict, Iterable from systemds.operator import OperationNode, Matrix, Frame, List, MultiReturn, Scalar -from systemds.script_building.dag import OutputType from systemds.utils.consts import VALID_INPUT_TYPES diff --git a/src/main/python/systemds/operator/algorithm/builtin/executePipeline.py b/src/main/python/systemds/operator/algorithm/builtin/executePipeline.py index 1fffb46f100..66750fc0711 100644 --- a/src/main/python/systemds/operator/algorithm/builtin/executePipeline.py +++ b/src/main/python/systemds/operator/algorithm/builtin/executePipeline.py @@ -28,7 +28,18 @@ from systemds.utils.consts import VALID_INPUT_TYPES -def executePipeline(X: Matrix): +def executePipeline(pipeline: Frame, + Xtrain: Matrix, + Ytrain: Matrix, + Xtest: Matrix, + Ytest: Matrix, + metaList: List, + hyperParameters: Matrix, + flagsCount: int, + verbose: bool, + startInd: int, + endInd: int, + **kwargs: Dict[str, VALID_INPUT_TYPES]): """ This function execute pipeline. @@ -56,17 +67,30 @@ def executePipeline(X: Matrix): :return: --- """ - params_dict = {'X': X} + params_dict = {'pipeline': pipeline, 'Xtrain': Xtrain, 'Ytrain': Ytrain, 'Xtest': Xtest, 'Ytest': Ytest, 'metaList': metaList, 'hyperParameters': hyperParameters, 'flagsCount': flagsCount, 'verbose': verbose, 'startInd': startInd, 'endInd': endInd} + params_dict.update(kwargs) - vX_0 = Matrix(X.sds_context, '') - vX_1 = Matrix(X.sds_context, '') - vX_2 = Matrix(X.sds_context, '') - output_nodes = [vX_0, vX_1, vX_2, ] + vX_0 = Matrix(pipeline.sds_context, '') + vX_1 = Matrix(pipeline.sds_context, '') + vX_2 = Matrix(pipeline.sds_context, '') + vX_3 = Matrix(pipeline.sds_context, '') + vX_4 = Scalar(pipeline.sds_context, '') + vX_5 = Matrix(pipeline.sds_context, '') + vX_6 = Matrix(pipeline.sds_context, '') + vX_7 = Scalar(pipeline.sds_context, '') + vX_8 = List(pipeline.sds_context, '') + output_nodes = [vX_0, vX_1, vX_2, vX_3, vX_4, vX_5, vX_6, vX_7, vX_8, ] - op = MultiReturn(X.sds_context, 'executePipeline', output_nodes, named_input_nodes=params_dict) + op = MultiReturn(pipeline.sds_context, 'executePipeline', output_nodes, named_input_nodes=params_dict) vX_0._unnamed_input_nodes = [op] vX_1._unnamed_input_nodes = [op] vX_2._unnamed_input_nodes = [op] + vX_3._unnamed_input_nodes = [op] + vX_4._unnamed_input_nodes = [op] + vX_5._unnamed_input_nodes = [op] + vX_6._unnamed_input_nodes = [op] + vX_7._unnamed_input_nodes = [op] + vX_8._unnamed_input_nodes = [op] return op diff --git a/src/main/python/systemds/operator/algorithm/builtin/fit_pipeline.py b/src/main/python/systemds/operator/algorithm/builtin/fit_pipeline.py index 5de40c745f8..48363035d8b 100644 --- a/src/main/python/systemds/operator/algorithm/builtin/fit_pipeline.py +++ b/src/main/python/systemds/operator/algorithm/builtin/fit_pipeline.py @@ -25,7 +25,6 @@ from typing import Dict, Iterable from systemds.operator import OperationNode, Matrix, Frame, List, MultiReturn, Scalar -from systemds.script_building.dag import OutputType from systemds.utils.consts import VALID_INPUT_TYPES diff --git a/src/main/python/systemds/operator/algorithm/builtin/fixInvalidLengths.py b/src/main/python/systemds/operator/algorithm/builtin/fixInvalidLengths.py index b635f31b298..cc0e83a51e4 100644 --- a/src/main/python/systemds/operator/algorithm/builtin/fixInvalidLengths.py +++ b/src/main/python/systemds/operator/algorithm/builtin/fixInvalidLengths.py @@ -25,7 +25,6 @@ from typing import Dict, Iterable from systemds.operator import OperationNode, Matrix, Frame, List, MultiReturn, Scalar -from systemds.script_building.dag import OutputType from systemds.utils.consts import VALID_INPUT_TYPES diff --git a/src/main/python/systemds/operator/algorithm/builtin/fixInvalidLengthsApply.py b/src/main/python/systemds/operator/algorithm/builtin/fixInvalidLengthsApply.py index cc8fe68aacc..ed2572368d3 100644 --- a/src/main/python/systemds/operator/algorithm/builtin/fixInvalidLengthsApply.py +++ b/src/main/python/systemds/operator/algorithm/builtin/fixInvalidLengthsApply.py @@ -25,7 +25,6 @@ from typing import Dict, Iterable from systemds.operator import OperationNode, Matrix, Frame, List, MultiReturn, Scalar -from systemds.script_building.dag import OutputType from systemds.utils.consts import VALID_INPUT_TYPES diff --git a/src/main/python/systemds/operator/algorithm/builtin/frameSort.py b/src/main/python/systemds/operator/algorithm/builtin/frameSort.py index 0bfc7f3afec..2575baefe4b 100644 --- a/src/main/python/systemds/operator/algorithm/builtin/frameSort.py +++ b/src/main/python/systemds/operator/algorithm/builtin/frameSort.py @@ -25,7 +25,6 @@ from typing import Dict, Iterable from systemds.operator import OperationNode, Matrix, Frame, List, MultiReturn, Scalar -from systemds.script_building.dag import OutputType from systemds.utils.consts import VALID_INPUT_TYPES diff --git a/src/main/python/systemds/operator/algorithm/builtin/glove.py b/src/main/python/systemds/operator/algorithm/builtin/glove.py new file mode 100644 index 00000000000..cbf9c421c40 --- /dev/null +++ b/src/main/python/systemds/operator/algorithm/builtin/glove.py @@ -0,0 +1,67 @@ +# ------------------------------------------------------------- +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# ------------------------------------------------------------- + +# Autogenerated By : src/main/python/generator/generator.py +# Autogenerated From : scripts/builtin/glove.dml + +from typing import Dict, Iterable + +from systemds.operator import OperationNode, Matrix, Frame, List, MultiReturn, Scalar +from systemds.utils.consts import VALID_INPUT_TYPES + + +def glove(input: Frame, + seed: int, + vector_size: int, + alpha: float, + eta: float, + x_max: float, + tol: float, + iterations: int, + print_loss_it: int, + maxTokens: int, + windowSize: int, + distanceWeighting: bool, + symmetric: bool): + """ + + + + :param input: 1DInput corpus in CSV format. + :param seed: Random seed for reproducibility. + :param vector_size: Dimensionality of word vectors, V. + :param eta: Learning rate for optimization, recommended value: 0.05. + :param alpha: Weighting function parameter, recommended value: 0.75. + :param x_max: Maximum co-occurrence value as per the GloVe paper: 100. + :param tol: Tolerance value to avoid overfitting, recommended value: 1e-4. + :param iterations: Total number of training iterations. + :param print_loss_it: Interval (in iterations) for printing the loss. + :param maxTokens: Maximum number of tokens per text entry. + :param windowSize: Context window size. + :param distanceWeighting: Whether to apply distance-based weighting. + :param symmetric: Determines if the matrix is symmetric (TRUE) or asymmetric (FALSE). + :return: The word indices and their word vectors, of shape (N, V). Each represented as a vector, of shape (1,V) + """ + + params_dict = {'input': input, 'seed': seed, 'vector_size': vector_size, 'alpha': alpha, 'eta': eta, 'x_max': x_max, 'tol': tol, 'iterations': iterations, 'print_loss_it': print_loss_it, 'maxTokens': maxTokens, 'windowSize': windowSize, 'distanceWeighting': distanceWeighting, 'symmetric': symmetric} + return Matrix(input.sds_context, + 'glove', + named_input_nodes=params_dict) diff --git a/src/main/python/systemds/operator/algorithm/builtin/imputeByKNN.py b/src/main/python/systemds/operator/algorithm/builtin/imputeByKNN.py index fcc096180b9..f04aa098514 100644 --- a/src/main/python/systemds/operator/algorithm/builtin/imputeByKNN.py +++ b/src/main/python/systemds/operator/algorithm/builtin/imputeByKNN.py @@ -25,13 +25,30 @@ from typing import Dict, Iterable from systemds.operator import OperationNode, Matrix, Frame, List, MultiReturn, Scalar -from systemds.script_building.dag import OutputType from systemds.utils.consts import VALID_INPUT_TYPES def imputeByKNN(X: Matrix, **kwargs: Dict[str, VALID_INPUT_TYPES]): + """ + Imputes missing values, indicated by NaNs, using KNN-based methods + (k-nearest neighbors by euclidean distance). In order to avoid NaNs in + distance computation and meaningful nearest neighbor search, we initialize + the missing values by column means. Currently, only the column with the most + missing values is actually imputed. + + + :param X: Matrix with missing values, which are represented as NaNs + :param method: Method used for imputing missing values with different performance and accuracy tradeoffs:\n + - 'dist' (default): Compute all-pairs distances and impute the missing values by closest. O(N^2 * #features) + - 'dist_missing': Compute distances between data and records with missing values. O(N*M * #features), assuming that the number of records with MV is M< M := [[1, 7, 3, 3, 2, 4, 0, 2, 0, 1, 0, 1, 0, 2], (1st tree) [4, 5, 1, 7, 0, 2, 0, 2, 0, 1, 0, 0, 0, 0]] (2nd tree) diff --git a/src/main/python/systemds/operator/algorithm/builtin/shapExplainer.py b/src/main/python/systemds/operator/algorithm/builtin/shapExplainer.py new file mode 100644 index 00000000000..42a0afb6e69 --- /dev/null +++ b/src/main/python/systemds/operator/algorithm/builtin/shapExplainer.py @@ -0,0 +1,78 @@ +# ------------------------------------------------------------- +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# ------------------------------------------------------------- + +# Autogenerated By : src/main/python/generator/generator.py +# Autogenerated From : scripts/builtin/shapExplainer.dml + +from typing import Dict, Iterable + +from systemds.operator import OperationNode, Matrix, Frame, List, MultiReturn, Scalar +from systemds.utils.consts import VALID_INPUT_TYPES + + +def shapExplainer(model_function: str, + model_args: List, + x_instances: Matrix, + X_bg: Matrix, + **kwargs: Dict[str, VALID_INPUT_TYPES]): + """ + Computes shapley values for multiple instances in parallel using antithetic permutation sampling. + The resulting matrix phis holds the shapley values for each feature in the column given by the index of the feature in the sample. + + This method first creates two large matrices for masks and masked background data for all permutations and + then runs in paralell on all instances in x. + While the prepared matrices can become very large (2 * #features * #permuations * #n_samples * #features), + the preparation of a row for the model call breaks down to a single element-wise multiplication of this mask with the row and + an addition to the masked background data, since masks can be reused for each instance. + + + + :param model_function: The function of the model to be evaluated as a String. This function has to take a matrix of samples + and return a vector of predictions. + It might be usefull to wrap the model into a function the takes and returns the desired shapes and + use this wrapper here. + :param model_args: Arguments in order for the model, if desired. This will be prepended by the created instances-matrix. + :param x_instances: Multiple instances as rows for which to compute the shapley values. + :param X_bg: The background dataset from which to pull the random samples to perform Monte Carlo integration. + :param n_permutations: The number of permutaions. Defaults to 10. Theoretical 1 should already be enough for models with up + to second order interaction effects. + :param n_samples: Number of samples from X_bg used for marginalization. + :param remove_non_var: EXPERIMENTAL: If set, for every instance the varaince of each feature is checked against this feature in the + background data. If it does not change, we do not run any model cals for it. + :param seed: A seed, in case the sampling has to be deterministic. + :param verbose: A boolean to enable logging of each step of the function. + :return: Matrix holding the shapley values along the cols, one row per instance. + :return: Double holding the average prediction of all instances. + """ + + params_dict = {'model_function': model_function, 'model_args': model_args, 'x_instances': x_instances, 'X_bg': X_bg} + params_dict.update(kwargs) + + vX_0 = Matrix(model_function.sds_context, '') + vX_1 = Scalar(model_function.sds_context, '') + output_nodes = [vX_0, vX_1, ] + + op = MultiReturn(model_function.sds_context, 'shapExplainer', output_nodes, named_input_nodes=params_dict) + + vX_0._unnamed_input_nodes = [op] + vX_1._unnamed_input_nodes = [op] + + return op diff --git a/src/main/python/systemds/operator/algorithm/builtin/topk_cleaning.py b/src/main/python/systemds/operator/algorithm/builtin/topk_cleaning.py index 16a20d20e08..270a6d7b166 100644 --- a/src/main/python/systemds/operator/algorithm/builtin/topk_cleaning.py +++ b/src/main/python/systemds/operator/algorithm/builtin/topk_cleaning.py @@ -25,7 +25,6 @@ from typing import Dict, Iterable from systemds.operator import OperationNode, Matrix, Frame, List, MultiReturn, Scalar -from systemds.script_building.dag import OutputType from systemds.utils.consts import VALID_INPUT_TYPES @@ -36,8 +35,39 @@ def topk_cleaning(dataTrain: Frame, evalFunHp: Matrix, **kwargs: Dict[str, VALID_INPUT_TYPES]): """ - This function cleans top-K item (where K is given as input)for a given list of users. + This function cleans top-K item (where K is given as input) for a given list of users. metaData[3, ncol(X)] : metaData[1] stores mask, metaData[2] stores schema, metaData[3] stores FD mask + + + + :param dataTrain: Training set + :param dataTest: Test set ignored when cv is set to True + :param metaData: 3×n frame with schema, categorical mask, and FD mask for dataTrain + :param primitives: Library of primitive cleaning operators + :param parameters: Hyperparameter search space that matches the primitives + :param refSol: Reference solution + :param evaluationFunc: Name of a SystemDS DML function that scores a pipeline + :param evalFunHp: Hyperparameter matrix for the above evaluation function + :param topK: Number of best pipelines to return + :param resource_val: Maximum resource R for the Bandit search + :param max_iter: Maximum iterations while enumerating logical pipelines + :param lq: Lower quantile used by utils::doErrorSample when triggered + :param uq: Upper quantile used by utils::doErrorSample when triggered + :param sample: Fraction of rows to subsample from dataTrain + :param expectedIncrease: Minimum improvement over dirtyScore that a candidate must deliver + :param seed: Seed number + :param cv: TRUE means k-fold CV, FALSE means hold-out split + :param cvk: Number of folds if cv = TRUE + :param isLastLabel: TRUE if the last column is the label + :param rowCount: Row-count threshold above which doErrorSample may replace uniform sampling + :param correctTypos: Run spelling correction in the string preprocessing step + :param enablePruning: Enable pruning inside the Bandit phase + :return: K cleaned-data pipelines + :return: Hyperparameter matrix with rows aligning with topKPipelines + :return: Evaluation scores with rows aligning with topKPipelines + :return: Baseline score on the unclean data + :return: Updated evaluation function hyperparameters + :return: Frame of “apply” functions for deploying each of the top-K pipelines """ params_dict = {'dataTrain': dataTrain, 'primitives': primitives, 'parameters': parameters, 'evaluationFunc': evaluationFunc, 'evalFunHp': evalFunHp} diff --git a/src/main/python/systemds/operator/algorithm/builtin/wer.py b/src/main/python/systemds/operator/algorithm/builtin/wer.py new file mode 100644 index 00000000000..99d278461cf --- /dev/null +++ b/src/main/python/systemds/operator/algorithm/builtin/wer.py @@ -0,0 +1,48 @@ +# ------------------------------------------------------------- +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# ------------------------------------------------------------- + +# Autogenerated By : src/main/python/generator/generator.py +# Autogenerated From : scripts/builtin/wer.dml + +from typing import Dict, Iterable + +from systemds.operator import OperationNode, Matrix, Frame, List, MultiReturn, Scalar +from systemds.utils.consts import VALID_INPUT_TYPES + + +def wer(R: Frame, + H: Frame): + """ + This built-in function computes the word error rate (WER) + defined as wer = (numSubst + numDel + numIns) / length(r) + + + + :param R: Input frame of reference strings, shape: [N x 1] + :param H: Input frame of hypothesis strings, shape: [N x 1] + :return: Output matrix of word error rate per pair of strings, + shape: [N x 1], where W[i,1] = wer(R[i,1], H[i,1]) + """ + + params_dict = {'R': R, 'H': H} + return Matrix(R.sds_context, + 'wer', + named_input_nodes=params_dict) From 4e40df50674a62677d2fc82343a252b1bea29515 Mon Sep 17 00:00:00 2001 From: anuunchin <88698977+anuunchin@users.noreply.github.com> Date: Sun, 13 Jul 2025 21:40:58 +0200 Subject: [PATCH 2/4] Test and rst file generator funcs, html structure adjusted --- .../docs/source/api/operator/algorithms.rst | 7 +- .../source/api/operator/algorithms/WoE.rst | 4 ++ .../api/operator/algorithms/WoEApply.rst | 4 ++ .../api/operator/algorithms/abstain.rst | 4 ++ .../source/api/operator/algorithms/adasyn.rst | 4 ++ .../source/api/operator/algorithms/als.rst | 4 ++ .../source/api/operator/algorithms/alsCG.rst | 4 ++ .../source/api/operator/algorithms/alsDS.rst | 4 ++ .../api/operator/algorithms/alsPredict.rst | 4 ++ .../operator/algorithms/alsTopkPredict.rst | 4 ++ .../source/api/operator/algorithms/ampute.rst | 4 ++ .../operator/algorithms/apply_pipeline.rst | 4 ++ .../source/api/operator/algorithms/arima.rst | 4 ++ .../source/api/operator/algorithms/auc.rst | 4 ++ .../algorithms/autoencoder_2layer.rst | 4 ++ .../source/api/operator/algorithms/bandit.rst | 4 ++ .../source/api/operator/algorithms/bivar.rst | 4 ++ .../api/operator/algorithms/components.rst | 4 ++ .../operator/algorithms/confusionMatrix.rst | 4 ++ .../algorithms/cooccurrenceMatrix.rst | 4 ++ .../source/api/operator/algorithms/cor.rst | 4 ++ .../api/operator/algorithms/correctTypos.rst | 4 ++ .../operator/algorithms/correctTyposApply.rst | 4 ++ .../source/api/operator/algorithms/cox.rst | 4 ++ .../api/operator/algorithms/cspline.rst | 4 ++ .../api/operator/algorithms/csplineCG.rst | 4 ++ .../api/operator/algorithms/csplineDS.rst | 4 ++ .../source/api/operator/algorithms/cvlm.rst | 4 ++ .../source/api/operator/algorithms/dbscan.rst | 4 ++ .../api/operator/algorithms/dbscanApply.rst | 4 ++ .../api/operator/algorithms/decisionTree.rst | 4 ++ .../algorithms/decisionTreePredict.rst | 4 ++ .../api/operator/algorithms/deepWalk.rst | 4 ++ .../operator/algorithms/denialConstraints.rst | 4 ++ .../algorithms/differenceStatistics.rst | 4 ++ .../api/operator/algorithms/discoverFD.rst | 4 ++ .../source/api/operator/algorithms/dist.rst | 4 ++ .../source/api/operator/algorithms/dmv.rst | 4 ++ .../source/api/operator/algorithms/ema.rst | 4 ++ .../operator/algorithms/executePipeline.rst | 4 ++ .../api/operator/algorithms/f1Score.rst | 4 ++ .../source/api/operator/algorithms/fdr.rst | 4 ++ .../api/operator/algorithms/ffPredict.rst | 4 ++ .../api/operator/algorithms/ffTrain.rst | 4 ++ .../api/operator/algorithms/fit_pipeline.rst | 4 ++ .../operator/algorithms/fixInvalidLengths.rst | 4 ++ .../algorithms/fixInvalidLengthsApply.rst | 4 ++ .../operator/algorithms/flattenQuantile.rst | 4 ++ .../api/operator/algorithms/frameSort.rst | 4 ++ .../operator/algorithms/frequencyEncode.rst | 4 ++ .../algorithms/frequencyEncodeApply.rst | 4 ++ .../source/api/operator/algorithms/garch.rst | 4 ++ .../algorithms/gaussianClassifier.rst | 4 ++ .../api/operator/algorithms/getAccuracy.rst | 4 ++ .../source/api/operator/algorithms/glm.rst | 4 ++ .../api/operator/algorithms/glmPredict.rst | 4 ++ .../source/api/operator/algorithms/glove.rst | 4 ++ .../source/api/operator/algorithms/gmm.rst | 4 ++ .../api/operator/algorithms/gmmPredict.rst | 4 ++ .../source/api/operator/algorithms/gnmf.rst | 4 ++ .../api/operator/algorithms/gridSearch.rst | 4 ++ .../algorithms/hospitalResidencyMatch.rst | 4 ++ .../api/operator/algorithms/hyperband.rst | 4 ++ .../operator/algorithms/img_brightness.rst | 4 ++ .../algorithms/img_brightness_linearized.rst | 4 ++ .../api/operator/algorithms/img_crop.rst | 4 ++ .../algorithms/img_crop_linearized.rst | 4 ++ .../api/operator/algorithms/img_cutout.rst | 4 ++ .../algorithms/img_cutout_linearized.rst | 4 ++ .../api/operator/algorithms/img_invert.rst | 4 ++ .../algorithms/img_invert_linearized.rst | 4 ++ .../api/operator/algorithms/img_mirror.rst | 4 ++ .../algorithms/img_mirror_linearized.rst | 4 ++ .../api/operator/algorithms/img_posterize.rst | 4 ++ .../algorithms/img_posterize_linearized.rst | 4 ++ .../api/operator/algorithms/img_rotate.rst | 4 ++ .../algorithms/img_rotate_linearized.rst | 4 ++ .../algorithms/img_sample_pairing.rst | 4 ++ .../img_sample_pairing_linearized.rst | 4 ++ .../api/operator/algorithms/img_shear.rst | 4 ++ .../algorithms/img_shear_linearized.rst | 4 ++ .../api/operator/algorithms/img_transform.rst | 4 ++ .../algorithms/img_transform_linearized.rst | 4 ++ .../api/operator/algorithms/img_translate.rst | 4 ++ .../algorithms/img_translate_linearized.rst | 4 ++ .../operator/algorithms/impurityMeasures.rst | 4 ++ .../api/operator/algorithms/imputeByFD.rst | 4 ++ .../operator/algorithms/imputeByFDApply.rst | 4 ++ .../api/operator/algorithms/imputeByKNN.rst | 4 ++ .../api/operator/algorithms/imputeByMean.rst | 4 ++ .../operator/algorithms/imputeByMeanApply.rst | 4 ++ .../operator/algorithms/imputeByMedian.rst | 4 ++ .../algorithms/imputeByMedianApply.rst | 4 ++ .../api/operator/algorithms/imputeByMode.rst | 4 ++ .../operator/algorithms/imputeByModeApply.rst | 4 ++ .../api/operator/algorithms/incSliceLine.rst | 4 ++ .../api/operator/algorithms/intersect.rst | 4 ++ .../source/api/operator/algorithms/km.rst | 4 ++ .../source/api/operator/algorithms/kmeans.rst | 4 ++ .../api/operator/algorithms/kmeansPredict.rst | 4 ++ .../source/api/operator/algorithms/knn.rst | 4 ++ .../api/operator/algorithms/knnGraph.rst | 4 ++ .../source/api/operator/algorithms/knnbf.rst | 4 ++ .../source/api/operator/algorithms/l2svm.rst | 4 ++ .../api/operator/algorithms/l2svmPredict.rst | 4 ++ .../source/api/operator/algorithms/lasso.rst | 4 ++ .../api/operator/algorithms/lenetPredict.rst | 4 ++ .../api/operator/algorithms/lenetTrain.rst | 4 ++ .../source/api/operator/algorithms/lm.rst | 4 ++ .../source/api/operator/algorithms/lmCG.rst | 4 ++ .../source/api/operator/algorithms/lmDS.rst | 4 ++ .../api/operator/algorithms/lmPredict.rst | 4 ++ .../operator/algorithms/lmPredictStats.rst | 4 ++ .../api/operator/algorithms/logSumExp.rst | 4 ++ .../source/api/operator/algorithms/mae.rst | 4 ++ .../source/api/operator/algorithms/mape.rst | 4 ++ .../api/operator/algorithms/matrixProfile.rst | 4 ++ .../source/api/operator/algorithms/mcc.rst | 4 ++ .../source/api/operator/algorithms/mdedup.rst | 4 ++ .../source/api/operator/algorithms/mice.rst | 4 ++ .../api/operator/algorithms/miceApply.rst | 4 ++ .../source/api/operator/algorithms/mse.rst | 4 ++ .../source/api/operator/algorithms/msmape.rst | 4 ++ .../source/api/operator/algorithms/msvm.rst | 4 ++ .../api/operator/algorithms/msvmPredict.rst | 4 ++ .../api/operator/algorithms/multiLogReg.rst | 4 ++ .../algorithms/multiLogRegPredict.rst | 4 ++ .../api/operator/algorithms/na_locf.rst | 4 ++ .../api/operator/algorithms/naiveBayes.rst | 4 ++ .../operator/algorithms/naiveBayesPredict.rst | 4 ++ .../api/operator/algorithms/normalize.rst | 4 ++ .../operator/algorithms/normalizeApply.rst | 4 ++ .../source/api/operator/algorithms/nrmse.rst | 4 ++ .../api/operator/algorithms/outlier.rst | 4 ++ .../operator/algorithms/outlierByArima.rst | 4 ++ .../api/operator/algorithms/outlierByIQR.rst | 4 ++ .../operator/algorithms/outlierByIQRApply.rst | 4 ++ .../api/operator/algorithms/outlierBySd.rst | 4 ++ .../operator/algorithms/outlierBySdApply.rst | 4 ++ .../api/operator/algorithms/pageRank.rst | 4 ++ .../source/api/operator/algorithms/pca.rst | 4 ++ .../api/operator/algorithms/pcaInverse.rst | 4 ++ .../api/operator/algorithms/pcaTransform.rst | 4 ++ .../source/api/operator/algorithms/pnmf.rst | 4 ++ .../source/api/operator/algorithms/ppca.rst | 4 ++ .../source/api/operator/algorithms/psnr.rst | 4 ++ .../operator/algorithms/quantizeByCluster.rst | 4 ++ .../api/operator/algorithms/raGroupby.rst | 4 ++ .../source/api/operator/algorithms/raJoin.rst | 4 ++ .../api/operator/algorithms/raSelection.rst | 4 ++ .../api/operator/algorithms/randomForest.rst | 4 ++ .../algorithms/randomForestPredict.rst | 4 ++ .../source/api/operator/algorithms/rmse.rst | 4 ++ .../source/api/operator/algorithms/scale.rst | 4 ++ .../api/operator/algorithms/scaleApply.rst | 4 ++ .../api/operator/algorithms/scaleMinMax.rst | 4 ++ .../operator/algorithms/selectByVarThresh.rst | 4 ++ .../source/api/operator/algorithms/ses.rst | 4 ++ .../api/operator/algorithms/setdiff.rst | 4 ++ .../api/operator/algorithms/shapExplainer.rst | 4 ++ .../api/operator/algorithms/sherlock.rst | 4 ++ .../operator/algorithms/sherlockPredict.rst | 4 ++ .../api/operator/algorithms/shortestPath.rst | 4 ++ .../api/operator/algorithms/sigmoid.rst | 4 ++ .../api/operator/algorithms/skewness.rst | 4 ++ .../api/operator/algorithms/sliceLine.rst | 4 ++ .../operator/algorithms/sliceLineDebug.rst | 4 ++ .../operator/algorithms/sliceLineExtract.rst | 4 ++ .../api/operator/algorithms/slicefinder.rst | 4 ++ .../source/api/operator/algorithms/smape.rst | 4 ++ .../source/api/operator/algorithms/smote.rst | 4 ++ .../api/operator/algorithms/softmax.rst | 4 ++ .../source/api/operator/algorithms/split.rst | 4 ++ .../api/operator/algorithms/splitBalanced.rst | 4 ++ .../api/operator/algorithms/sqrtMatrix.rst | 4 ++ .../operator/algorithms/stableMarriage.rst | 4 ++ .../api/operator/algorithms/statsNA.rst | 4 ++ .../source/api/operator/algorithms/steplm.rst | 4 ++ .../api/operator/algorithms/stratstats.rst | 4 ++ .../algorithms/symmetricDifference.rst | 4 ++ .../source/api/operator/algorithms/tSNE.rst | 4 ++ .../api/operator/algorithms/toOneHot.rst | 4 ++ .../api/operator/algorithms/tomeklink.rst | 4 ++ .../api/operator/algorithms/topk_cleaning.rst | 4 ++ .../api/operator/algorithms/underSampling.rst | 4 ++ .../source/api/operator/algorithms/union.rst | 4 ++ .../source/api/operator/algorithms/univar.rst | 4 ++ .../api/operator/algorithms/vectorToCsv.rst | 4 ++ .../source/api/operator/algorithms/wer.rst | 4 ++ .../api/operator/algorithms/winsorize.rst | 4 ++ .../operator/algorithms/winsorizeApply.rst | 4 ++ .../api/operator/algorithms/xdummy1.rst | 4 ++ .../api/operator/algorithms/xdummy2.rst | 4 ++ .../api/operator/algorithms/xgboost.rst | 4 ++ .../xgboostPredictClassification.rst | 4 ++ .../algorithms/xgboostPredictRegression.rst | 4 ++ src/main/python/generator/dml_parser.py | 24 ++++++- src/main/python/generator/generator.py | 68 ++++++++++++++++++- 198 files changed, 874 insertions(+), 5 deletions(-) create mode 100644 src/main/python/docs/source/api/operator/algorithms/WoE.rst create mode 100644 src/main/python/docs/source/api/operator/algorithms/WoEApply.rst create mode 100644 src/main/python/docs/source/api/operator/algorithms/abstain.rst create mode 100644 src/main/python/docs/source/api/operator/algorithms/adasyn.rst create mode 100644 src/main/python/docs/source/api/operator/algorithms/als.rst create mode 100644 src/main/python/docs/source/api/operator/algorithms/alsCG.rst create mode 100644 src/main/python/docs/source/api/operator/algorithms/alsDS.rst create mode 100644 src/main/python/docs/source/api/operator/algorithms/alsPredict.rst create mode 100644 src/main/python/docs/source/api/operator/algorithms/alsTopkPredict.rst create mode 100644 src/main/python/docs/source/api/operator/algorithms/ampute.rst create mode 100644 src/main/python/docs/source/api/operator/algorithms/apply_pipeline.rst create mode 100644 src/main/python/docs/source/api/operator/algorithms/arima.rst create mode 100644 src/main/python/docs/source/api/operator/algorithms/auc.rst create mode 100644 src/main/python/docs/source/api/operator/algorithms/autoencoder_2layer.rst create mode 100644 src/main/python/docs/source/api/operator/algorithms/bandit.rst create mode 100644 src/main/python/docs/source/api/operator/algorithms/bivar.rst create mode 100644 src/main/python/docs/source/api/operator/algorithms/components.rst create mode 100644 src/main/python/docs/source/api/operator/algorithms/confusionMatrix.rst create mode 100644 src/main/python/docs/source/api/operator/algorithms/cooccurrenceMatrix.rst create mode 100644 src/main/python/docs/source/api/operator/algorithms/cor.rst create mode 100644 src/main/python/docs/source/api/operator/algorithms/correctTypos.rst create mode 100644 src/main/python/docs/source/api/operator/algorithms/correctTyposApply.rst create mode 100644 src/main/python/docs/source/api/operator/algorithms/cox.rst create mode 100644 src/main/python/docs/source/api/operator/algorithms/cspline.rst create mode 100644 src/main/python/docs/source/api/operator/algorithms/csplineCG.rst create mode 100644 src/main/python/docs/source/api/operator/algorithms/csplineDS.rst create mode 100644 src/main/python/docs/source/api/operator/algorithms/cvlm.rst create mode 100644 src/main/python/docs/source/api/operator/algorithms/dbscan.rst create mode 100644 src/main/python/docs/source/api/operator/algorithms/dbscanApply.rst create mode 100644 src/main/python/docs/source/api/operator/algorithms/decisionTree.rst create mode 100644 src/main/python/docs/source/api/operator/algorithms/decisionTreePredict.rst create mode 100644 src/main/python/docs/source/api/operator/algorithms/deepWalk.rst create mode 100644 src/main/python/docs/source/api/operator/algorithms/denialConstraints.rst create mode 100644 src/main/python/docs/source/api/operator/algorithms/differenceStatistics.rst create mode 100644 src/main/python/docs/source/api/operator/algorithms/discoverFD.rst create mode 100644 src/main/python/docs/source/api/operator/algorithms/dist.rst create mode 100644 src/main/python/docs/source/api/operator/algorithms/dmv.rst create mode 100644 src/main/python/docs/source/api/operator/algorithms/ema.rst create mode 100644 src/main/python/docs/source/api/operator/algorithms/executePipeline.rst create mode 100644 src/main/python/docs/source/api/operator/algorithms/f1Score.rst create mode 100644 src/main/python/docs/source/api/operator/algorithms/fdr.rst create mode 100644 src/main/python/docs/source/api/operator/algorithms/ffPredict.rst create mode 100644 src/main/python/docs/source/api/operator/algorithms/ffTrain.rst create mode 100644 src/main/python/docs/source/api/operator/algorithms/fit_pipeline.rst create mode 100644 src/main/python/docs/source/api/operator/algorithms/fixInvalidLengths.rst create mode 100644 src/main/python/docs/source/api/operator/algorithms/fixInvalidLengthsApply.rst create mode 100644 src/main/python/docs/source/api/operator/algorithms/flattenQuantile.rst create mode 100644 src/main/python/docs/source/api/operator/algorithms/frameSort.rst create mode 100644 src/main/python/docs/source/api/operator/algorithms/frequencyEncode.rst create mode 100644 src/main/python/docs/source/api/operator/algorithms/frequencyEncodeApply.rst create mode 100644 src/main/python/docs/source/api/operator/algorithms/garch.rst create mode 100644 src/main/python/docs/source/api/operator/algorithms/gaussianClassifier.rst create mode 100644 src/main/python/docs/source/api/operator/algorithms/getAccuracy.rst create mode 100644 src/main/python/docs/source/api/operator/algorithms/glm.rst create mode 100644 src/main/python/docs/source/api/operator/algorithms/glmPredict.rst create mode 100644 src/main/python/docs/source/api/operator/algorithms/glove.rst create mode 100644 src/main/python/docs/source/api/operator/algorithms/gmm.rst create mode 100644 src/main/python/docs/source/api/operator/algorithms/gmmPredict.rst create mode 100644 src/main/python/docs/source/api/operator/algorithms/gnmf.rst create mode 100644 src/main/python/docs/source/api/operator/algorithms/gridSearch.rst create mode 100644 src/main/python/docs/source/api/operator/algorithms/hospitalResidencyMatch.rst create mode 100644 src/main/python/docs/source/api/operator/algorithms/hyperband.rst create mode 100644 src/main/python/docs/source/api/operator/algorithms/img_brightness.rst create mode 100644 src/main/python/docs/source/api/operator/algorithms/img_brightness_linearized.rst create mode 100644 src/main/python/docs/source/api/operator/algorithms/img_crop.rst create mode 100644 src/main/python/docs/source/api/operator/algorithms/img_crop_linearized.rst create mode 100644 src/main/python/docs/source/api/operator/algorithms/img_cutout.rst create mode 100644 src/main/python/docs/source/api/operator/algorithms/img_cutout_linearized.rst create mode 100644 src/main/python/docs/source/api/operator/algorithms/img_invert.rst create mode 100644 src/main/python/docs/source/api/operator/algorithms/img_invert_linearized.rst create mode 100644 src/main/python/docs/source/api/operator/algorithms/img_mirror.rst create mode 100644 src/main/python/docs/source/api/operator/algorithms/img_mirror_linearized.rst create mode 100644 src/main/python/docs/source/api/operator/algorithms/img_posterize.rst create mode 100644 src/main/python/docs/source/api/operator/algorithms/img_posterize_linearized.rst create mode 100644 src/main/python/docs/source/api/operator/algorithms/img_rotate.rst create mode 100644 src/main/python/docs/source/api/operator/algorithms/img_rotate_linearized.rst create mode 100644 src/main/python/docs/source/api/operator/algorithms/img_sample_pairing.rst create mode 100644 src/main/python/docs/source/api/operator/algorithms/img_sample_pairing_linearized.rst create mode 100644 src/main/python/docs/source/api/operator/algorithms/img_shear.rst create mode 100644 src/main/python/docs/source/api/operator/algorithms/img_shear_linearized.rst create mode 100644 src/main/python/docs/source/api/operator/algorithms/img_transform.rst create mode 100644 src/main/python/docs/source/api/operator/algorithms/img_transform_linearized.rst create mode 100644 src/main/python/docs/source/api/operator/algorithms/img_translate.rst create mode 100644 src/main/python/docs/source/api/operator/algorithms/img_translate_linearized.rst create mode 100644 src/main/python/docs/source/api/operator/algorithms/impurityMeasures.rst create mode 100644 src/main/python/docs/source/api/operator/algorithms/imputeByFD.rst create mode 100644 src/main/python/docs/source/api/operator/algorithms/imputeByFDApply.rst create mode 100644 src/main/python/docs/source/api/operator/algorithms/imputeByKNN.rst create mode 100644 src/main/python/docs/source/api/operator/algorithms/imputeByMean.rst create mode 100644 src/main/python/docs/source/api/operator/algorithms/imputeByMeanApply.rst create mode 100644 src/main/python/docs/source/api/operator/algorithms/imputeByMedian.rst create mode 100644 src/main/python/docs/source/api/operator/algorithms/imputeByMedianApply.rst create mode 100644 src/main/python/docs/source/api/operator/algorithms/imputeByMode.rst create mode 100644 src/main/python/docs/source/api/operator/algorithms/imputeByModeApply.rst create mode 100644 src/main/python/docs/source/api/operator/algorithms/incSliceLine.rst create mode 100644 src/main/python/docs/source/api/operator/algorithms/intersect.rst create mode 100644 src/main/python/docs/source/api/operator/algorithms/km.rst create mode 100644 src/main/python/docs/source/api/operator/algorithms/kmeans.rst create mode 100644 src/main/python/docs/source/api/operator/algorithms/kmeansPredict.rst create mode 100644 src/main/python/docs/source/api/operator/algorithms/knn.rst create mode 100644 src/main/python/docs/source/api/operator/algorithms/knnGraph.rst create mode 100644 src/main/python/docs/source/api/operator/algorithms/knnbf.rst create mode 100644 src/main/python/docs/source/api/operator/algorithms/l2svm.rst create mode 100644 src/main/python/docs/source/api/operator/algorithms/l2svmPredict.rst create mode 100644 src/main/python/docs/source/api/operator/algorithms/lasso.rst create mode 100644 src/main/python/docs/source/api/operator/algorithms/lenetPredict.rst create mode 100644 src/main/python/docs/source/api/operator/algorithms/lenetTrain.rst create mode 100644 src/main/python/docs/source/api/operator/algorithms/lm.rst create mode 100644 src/main/python/docs/source/api/operator/algorithms/lmCG.rst create mode 100644 src/main/python/docs/source/api/operator/algorithms/lmDS.rst create mode 100644 src/main/python/docs/source/api/operator/algorithms/lmPredict.rst create mode 100644 src/main/python/docs/source/api/operator/algorithms/lmPredictStats.rst create mode 100644 src/main/python/docs/source/api/operator/algorithms/logSumExp.rst create mode 100644 src/main/python/docs/source/api/operator/algorithms/mae.rst create mode 100644 src/main/python/docs/source/api/operator/algorithms/mape.rst create mode 100644 src/main/python/docs/source/api/operator/algorithms/matrixProfile.rst create mode 100644 src/main/python/docs/source/api/operator/algorithms/mcc.rst create mode 100644 src/main/python/docs/source/api/operator/algorithms/mdedup.rst create mode 100644 src/main/python/docs/source/api/operator/algorithms/mice.rst create mode 100644 src/main/python/docs/source/api/operator/algorithms/miceApply.rst create mode 100644 src/main/python/docs/source/api/operator/algorithms/mse.rst create mode 100644 src/main/python/docs/source/api/operator/algorithms/msmape.rst create mode 100644 src/main/python/docs/source/api/operator/algorithms/msvm.rst create mode 100644 src/main/python/docs/source/api/operator/algorithms/msvmPredict.rst create mode 100644 src/main/python/docs/source/api/operator/algorithms/multiLogReg.rst create mode 100644 src/main/python/docs/source/api/operator/algorithms/multiLogRegPredict.rst create mode 100644 src/main/python/docs/source/api/operator/algorithms/na_locf.rst create mode 100644 src/main/python/docs/source/api/operator/algorithms/naiveBayes.rst create mode 100644 src/main/python/docs/source/api/operator/algorithms/naiveBayesPredict.rst create mode 100644 src/main/python/docs/source/api/operator/algorithms/normalize.rst create mode 100644 src/main/python/docs/source/api/operator/algorithms/normalizeApply.rst create mode 100644 src/main/python/docs/source/api/operator/algorithms/nrmse.rst create mode 100644 src/main/python/docs/source/api/operator/algorithms/outlier.rst create mode 100644 src/main/python/docs/source/api/operator/algorithms/outlierByArima.rst create mode 100644 src/main/python/docs/source/api/operator/algorithms/outlierByIQR.rst create mode 100644 src/main/python/docs/source/api/operator/algorithms/outlierByIQRApply.rst create mode 100644 src/main/python/docs/source/api/operator/algorithms/outlierBySd.rst create mode 100644 src/main/python/docs/source/api/operator/algorithms/outlierBySdApply.rst create mode 100644 src/main/python/docs/source/api/operator/algorithms/pageRank.rst create mode 100644 src/main/python/docs/source/api/operator/algorithms/pca.rst create mode 100644 src/main/python/docs/source/api/operator/algorithms/pcaInverse.rst create mode 100644 src/main/python/docs/source/api/operator/algorithms/pcaTransform.rst create mode 100644 src/main/python/docs/source/api/operator/algorithms/pnmf.rst create mode 100644 src/main/python/docs/source/api/operator/algorithms/ppca.rst create mode 100644 src/main/python/docs/source/api/operator/algorithms/psnr.rst create mode 100644 src/main/python/docs/source/api/operator/algorithms/quantizeByCluster.rst create mode 100644 src/main/python/docs/source/api/operator/algorithms/raGroupby.rst create mode 100644 src/main/python/docs/source/api/operator/algorithms/raJoin.rst create mode 100644 src/main/python/docs/source/api/operator/algorithms/raSelection.rst create mode 100644 src/main/python/docs/source/api/operator/algorithms/randomForest.rst create mode 100644 src/main/python/docs/source/api/operator/algorithms/randomForestPredict.rst create mode 100644 src/main/python/docs/source/api/operator/algorithms/rmse.rst create mode 100644 src/main/python/docs/source/api/operator/algorithms/scale.rst create mode 100644 src/main/python/docs/source/api/operator/algorithms/scaleApply.rst create mode 100644 src/main/python/docs/source/api/operator/algorithms/scaleMinMax.rst create mode 100644 src/main/python/docs/source/api/operator/algorithms/selectByVarThresh.rst create mode 100644 src/main/python/docs/source/api/operator/algorithms/ses.rst create mode 100644 src/main/python/docs/source/api/operator/algorithms/setdiff.rst create mode 100644 src/main/python/docs/source/api/operator/algorithms/shapExplainer.rst create mode 100644 src/main/python/docs/source/api/operator/algorithms/sherlock.rst create mode 100644 src/main/python/docs/source/api/operator/algorithms/sherlockPredict.rst create mode 100644 src/main/python/docs/source/api/operator/algorithms/shortestPath.rst create mode 100644 src/main/python/docs/source/api/operator/algorithms/sigmoid.rst create mode 100644 src/main/python/docs/source/api/operator/algorithms/skewness.rst create mode 100644 src/main/python/docs/source/api/operator/algorithms/sliceLine.rst create mode 100644 src/main/python/docs/source/api/operator/algorithms/sliceLineDebug.rst create mode 100644 src/main/python/docs/source/api/operator/algorithms/sliceLineExtract.rst create mode 100644 src/main/python/docs/source/api/operator/algorithms/slicefinder.rst create mode 100644 src/main/python/docs/source/api/operator/algorithms/smape.rst create mode 100644 src/main/python/docs/source/api/operator/algorithms/smote.rst create mode 100644 src/main/python/docs/source/api/operator/algorithms/softmax.rst create mode 100644 src/main/python/docs/source/api/operator/algorithms/split.rst create mode 100644 src/main/python/docs/source/api/operator/algorithms/splitBalanced.rst create mode 100644 src/main/python/docs/source/api/operator/algorithms/sqrtMatrix.rst create mode 100644 src/main/python/docs/source/api/operator/algorithms/stableMarriage.rst create mode 100644 src/main/python/docs/source/api/operator/algorithms/statsNA.rst create mode 100644 src/main/python/docs/source/api/operator/algorithms/steplm.rst create mode 100644 src/main/python/docs/source/api/operator/algorithms/stratstats.rst create mode 100644 src/main/python/docs/source/api/operator/algorithms/symmetricDifference.rst create mode 100644 src/main/python/docs/source/api/operator/algorithms/tSNE.rst create mode 100644 src/main/python/docs/source/api/operator/algorithms/toOneHot.rst create mode 100644 src/main/python/docs/source/api/operator/algorithms/tomeklink.rst create mode 100644 src/main/python/docs/source/api/operator/algorithms/topk_cleaning.rst create mode 100644 src/main/python/docs/source/api/operator/algorithms/underSampling.rst create mode 100644 src/main/python/docs/source/api/operator/algorithms/union.rst create mode 100644 src/main/python/docs/source/api/operator/algorithms/univar.rst create mode 100644 src/main/python/docs/source/api/operator/algorithms/vectorToCsv.rst create mode 100644 src/main/python/docs/source/api/operator/algorithms/wer.rst create mode 100644 src/main/python/docs/source/api/operator/algorithms/winsorize.rst create mode 100644 src/main/python/docs/source/api/operator/algorithms/winsorizeApply.rst create mode 100644 src/main/python/docs/source/api/operator/algorithms/xdummy1.rst create mode 100644 src/main/python/docs/source/api/operator/algorithms/xdummy2.rst create mode 100644 src/main/python/docs/source/api/operator/algorithms/xgboost.rst create mode 100644 src/main/python/docs/source/api/operator/algorithms/xgboostPredictClassification.rst create mode 100644 src/main/python/docs/source/api/operator/algorithms/xgboostPredictRegression.rst diff --git a/src/main/python/docs/source/api/operator/algorithms.rst b/src/main/python/docs/source/api/operator/algorithms.rst index 1ea5de4a435..27163896437 100644 --- a/src/main/python/docs/source/api/operator/algorithms.rst +++ b/src/main/python/docs/source/api/operator/algorithms.rst @@ -66,4 +66,9 @@ The output should be similar to [ 0.37957689]] .. automodule:: systemds.operator.algorithm - :members: \ No newline at end of file + +.. toctree:: + :maxdepth: 1 + :glob: + + algorithms/* \ No newline at end of file diff --git a/src/main/python/docs/source/api/operator/algorithms/WoE.rst b/src/main/python/docs/source/api/operator/algorithms/WoE.rst new file mode 100644 index 00000000000..9712759986c --- /dev/null +++ b/src/main/python/docs/source/api/operator/algorithms/WoE.rst @@ -0,0 +1,4 @@ +WoE +=== + +.. autofunction:: systemds.operator.algorithm.WoE \ No newline at end of file diff --git a/src/main/python/docs/source/api/operator/algorithms/WoEApply.rst b/src/main/python/docs/source/api/operator/algorithms/WoEApply.rst new file mode 100644 index 00000000000..1b612b82bbc --- /dev/null +++ b/src/main/python/docs/source/api/operator/algorithms/WoEApply.rst @@ -0,0 +1,4 @@ +WoEApply +======== + +.. autofunction:: systemds.operator.algorithm.WoEApply \ No newline at end of file diff --git a/src/main/python/docs/source/api/operator/algorithms/abstain.rst b/src/main/python/docs/source/api/operator/algorithms/abstain.rst new file mode 100644 index 00000000000..dcef8791593 --- /dev/null +++ b/src/main/python/docs/source/api/operator/algorithms/abstain.rst @@ -0,0 +1,4 @@ +abstain +======= + +.. autofunction:: systemds.operator.algorithm.abstain \ No newline at end of file diff --git a/src/main/python/docs/source/api/operator/algorithms/adasyn.rst b/src/main/python/docs/source/api/operator/algorithms/adasyn.rst new file mode 100644 index 00000000000..28762393754 --- /dev/null +++ b/src/main/python/docs/source/api/operator/algorithms/adasyn.rst @@ -0,0 +1,4 @@ +adasyn +====== + +.. autofunction:: systemds.operator.algorithm.adasyn \ No newline at end of file diff --git a/src/main/python/docs/source/api/operator/algorithms/als.rst b/src/main/python/docs/source/api/operator/algorithms/als.rst new file mode 100644 index 00000000000..67ff3426dcd --- /dev/null +++ b/src/main/python/docs/source/api/operator/algorithms/als.rst @@ -0,0 +1,4 @@ +als +=== + +.. autofunction:: systemds.operator.algorithm.als \ No newline at end of file diff --git a/src/main/python/docs/source/api/operator/algorithms/alsCG.rst b/src/main/python/docs/source/api/operator/algorithms/alsCG.rst new file mode 100644 index 00000000000..11407a8cd5c --- /dev/null +++ b/src/main/python/docs/source/api/operator/algorithms/alsCG.rst @@ -0,0 +1,4 @@ +alsCG +===== + +.. autofunction:: systemds.operator.algorithm.alsCG \ No newline at end of file diff --git a/src/main/python/docs/source/api/operator/algorithms/alsDS.rst b/src/main/python/docs/source/api/operator/algorithms/alsDS.rst new file mode 100644 index 00000000000..cffde102cef --- /dev/null +++ b/src/main/python/docs/source/api/operator/algorithms/alsDS.rst @@ -0,0 +1,4 @@ +alsDS +===== + +.. autofunction:: systemds.operator.algorithm.alsDS \ No newline at end of file diff --git a/src/main/python/docs/source/api/operator/algorithms/alsPredict.rst b/src/main/python/docs/source/api/operator/algorithms/alsPredict.rst new file mode 100644 index 00000000000..872eeff8d4f --- /dev/null +++ b/src/main/python/docs/source/api/operator/algorithms/alsPredict.rst @@ -0,0 +1,4 @@ +alsPredict +========== + +.. autofunction:: systemds.operator.algorithm.alsPredict \ No newline at end of file diff --git a/src/main/python/docs/source/api/operator/algorithms/alsTopkPredict.rst b/src/main/python/docs/source/api/operator/algorithms/alsTopkPredict.rst new file mode 100644 index 00000000000..f68c5c4e3fc --- /dev/null +++ b/src/main/python/docs/source/api/operator/algorithms/alsTopkPredict.rst @@ -0,0 +1,4 @@ +alsTopkPredict +============== + +.. autofunction:: systemds.operator.algorithm.alsTopkPredict \ No newline at end of file diff --git a/src/main/python/docs/source/api/operator/algorithms/ampute.rst b/src/main/python/docs/source/api/operator/algorithms/ampute.rst new file mode 100644 index 00000000000..f8b7783ef67 --- /dev/null +++ b/src/main/python/docs/source/api/operator/algorithms/ampute.rst @@ -0,0 +1,4 @@ +ampute +====== + +.. autofunction:: systemds.operator.algorithm.ampute \ No newline at end of file diff --git a/src/main/python/docs/source/api/operator/algorithms/apply_pipeline.rst b/src/main/python/docs/source/api/operator/algorithms/apply_pipeline.rst new file mode 100644 index 00000000000..1af55455dad --- /dev/null +++ b/src/main/python/docs/source/api/operator/algorithms/apply_pipeline.rst @@ -0,0 +1,4 @@ +apply_pipeline +============== + +.. autofunction:: systemds.operator.algorithm.apply_pipeline \ No newline at end of file diff --git a/src/main/python/docs/source/api/operator/algorithms/arima.rst b/src/main/python/docs/source/api/operator/algorithms/arima.rst new file mode 100644 index 00000000000..af74a3f85fd --- /dev/null +++ b/src/main/python/docs/source/api/operator/algorithms/arima.rst @@ -0,0 +1,4 @@ +arima +===== + +.. autofunction:: systemds.operator.algorithm.arima \ No newline at end of file diff --git a/src/main/python/docs/source/api/operator/algorithms/auc.rst b/src/main/python/docs/source/api/operator/algorithms/auc.rst new file mode 100644 index 00000000000..06457bdb4a5 --- /dev/null +++ b/src/main/python/docs/source/api/operator/algorithms/auc.rst @@ -0,0 +1,4 @@ +auc +=== + +.. autofunction:: systemds.operator.algorithm.auc \ No newline at end of file diff --git a/src/main/python/docs/source/api/operator/algorithms/autoencoder_2layer.rst b/src/main/python/docs/source/api/operator/algorithms/autoencoder_2layer.rst new file mode 100644 index 00000000000..fee877e3ab7 --- /dev/null +++ b/src/main/python/docs/source/api/operator/algorithms/autoencoder_2layer.rst @@ -0,0 +1,4 @@ +autoencoder_2layer +================== + +.. autofunction:: systemds.operator.algorithm.autoencoder_2layer \ No newline at end of file diff --git a/src/main/python/docs/source/api/operator/algorithms/bandit.rst b/src/main/python/docs/source/api/operator/algorithms/bandit.rst new file mode 100644 index 00000000000..6884f821ce3 --- /dev/null +++ b/src/main/python/docs/source/api/operator/algorithms/bandit.rst @@ -0,0 +1,4 @@ +bandit +====== + +.. autofunction:: systemds.operator.algorithm.bandit \ No newline at end of file diff --git a/src/main/python/docs/source/api/operator/algorithms/bivar.rst b/src/main/python/docs/source/api/operator/algorithms/bivar.rst new file mode 100644 index 00000000000..4def7aeac4f --- /dev/null +++ b/src/main/python/docs/source/api/operator/algorithms/bivar.rst @@ -0,0 +1,4 @@ +bivar +===== + +.. autofunction:: systemds.operator.algorithm.bivar \ No newline at end of file diff --git a/src/main/python/docs/source/api/operator/algorithms/components.rst b/src/main/python/docs/source/api/operator/algorithms/components.rst new file mode 100644 index 00000000000..e42ca98cf68 --- /dev/null +++ b/src/main/python/docs/source/api/operator/algorithms/components.rst @@ -0,0 +1,4 @@ +components +========== + +.. autofunction:: systemds.operator.algorithm.components \ No newline at end of file diff --git a/src/main/python/docs/source/api/operator/algorithms/confusionMatrix.rst b/src/main/python/docs/source/api/operator/algorithms/confusionMatrix.rst new file mode 100644 index 00000000000..90189523894 --- /dev/null +++ b/src/main/python/docs/source/api/operator/algorithms/confusionMatrix.rst @@ -0,0 +1,4 @@ +confusionMatrix +=============== + +.. autofunction:: systemds.operator.algorithm.confusionMatrix \ No newline at end of file diff --git a/src/main/python/docs/source/api/operator/algorithms/cooccurrenceMatrix.rst b/src/main/python/docs/source/api/operator/algorithms/cooccurrenceMatrix.rst new file mode 100644 index 00000000000..d768c2f775b --- /dev/null +++ b/src/main/python/docs/source/api/operator/algorithms/cooccurrenceMatrix.rst @@ -0,0 +1,4 @@ +cooccurrenceMatrix +================== + +.. autofunction:: systemds.operator.algorithm.cooccurrenceMatrix \ No newline at end of file diff --git a/src/main/python/docs/source/api/operator/algorithms/cor.rst b/src/main/python/docs/source/api/operator/algorithms/cor.rst new file mode 100644 index 00000000000..34484d65ce3 --- /dev/null +++ b/src/main/python/docs/source/api/operator/algorithms/cor.rst @@ -0,0 +1,4 @@ +cor +=== + +.. autofunction:: systemds.operator.algorithm.cor \ No newline at end of file diff --git a/src/main/python/docs/source/api/operator/algorithms/correctTypos.rst b/src/main/python/docs/source/api/operator/algorithms/correctTypos.rst new file mode 100644 index 00000000000..38a5b7695cb --- /dev/null +++ b/src/main/python/docs/source/api/operator/algorithms/correctTypos.rst @@ -0,0 +1,4 @@ +correctTypos +============ + +.. autofunction:: systemds.operator.algorithm.correctTypos \ No newline at end of file diff --git a/src/main/python/docs/source/api/operator/algorithms/correctTyposApply.rst b/src/main/python/docs/source/api/operator/algorithms/correctTyposApply.rst new file mode 100644 index 00000000000..a7f3a77a312 --- /dev/null +++ b/src/main/python/docs/source/api/operator/algorithms/correctTyposApply.rst @@ -0,0 +1,4 @@ +correctTyposApply +================= + +.. autofunction:: systemds.operator.algorithm.correctTyposApply \ No newline at end of file diff --git a/src/main/python/docs/source/api/operator/algorithms/cox.rst b/src/main/python/docs/source/api/operator/algorithms/cox.rst new file mode 100644 index 00000000000..a8cbdf4248e --- /dev/null +++ b/src/main/python/docs/source/api/operator/algorithms/cox.rst @@ -0,0 +1,4 @@ +cox +=== + +.. autofunction:: systemds.operator.algorithm.cox \ No newline at end of file diff --git a/src/main/python/docs/source/api/operator/algorithms/cspline.rst b/src/main/python/docs/source/api/operator/algorithms/cspline.rst new file mode 100644 index 00000000000..00861c0b63b --- /dev/null +++ b/src/main/python/docs/source/api/operator/algorithms/cspline.rst @@ -0,0 +1,4 @@ +cspline +======= + +.. autofunction:: systemds.operator.algorithm.cspline \ No newline at end of file diff --git a/src/main/python/docs/source/api/operator/algorithms/csplineCG.rst b/src/main/python/docs/source/api/operator/algorithms/csplineCG.rst new file mode 100644 index 00000000000..27785cfd48d --- /dev/null +++ b/src/main/python/docs/source/api/operator/algorithms/csplineCG.rst @@ -0,0 +1,4 @@ +csplineCG +========= + +.. autofunction:: systemds.operator.algorithm.csplineCG \ No newline at end of file diff --git a/src/main/python/docs/source/api/operator/algorithms/csplineDS.rst b/src/main/python/docs/source/api/operator/algorithms/csplineDS.rst new file mode 100644 index 00000000000..01e5a9719f0 --- /dev/null +++ b/src/main/python/docs/source/api/operator/algorithms/csplineDS.rst @@ -0,0 +1,4 @@ +csplineDS +========= + +.. autofunction:: systemds.operator.algorithm.csplineDS \ No newline at end of file diff --git a/src/main/python/docs/source/api/operator/algorithms/cvlm.rst b/src/main/python/docs/source/api/operator/algorithms/cvlm.rst new file mode 100644 index 00000000000..e11cf61abac --- /dev/null +++ b/src/main/python/docs/source/api/operator/algorithms/cvlm.rst @@ -0,0 +1,4 @@ +cvlm +==== + +.. autofunction:: systemds.operator.algorithm.cvlm \ No newline at end of file diff --git a/src/main/python/docs/source/api/operator/algorithms/dbscan.rst b/src/main/python/docs/source/api/operator/algorithms/dbscan.rst new file mode 100644 index 00000000000..5a72b28bfbf --- /dev/null +++ b/src/main/python/docs/source/api/operator/algorithms/dbscan.rst @@ -0,0 +1,4 @@ +dbscan +====== + +.. autofunction:: systemds.operator.algorithm.dbscan \ No newline at end of file diff --git a/src/main/python/docs/source/api/operator/algorithms/dbscanApply.rst b/src/main/python/docs/source/api/operator/algorithms/dbscanApply.rst new file mode 100644 index 00000000000..57b52e92ae1 --- /dev/null +++ b/src/main/python/docs/source/api/operator/algorithms/dbscanApply.rst @@ -0,0 +1,4 @@ +dbscanApply +=========== + +.. autofunction:: systemds.operator.algorithm.dbscanApply \ No newline at end of file diff --git a/src/main/python/docs/source/api/operator/algorithms/decisionTree.rst b/src/main/python/docs/source/api/operator/algorithms/decisionTree.rst new file mode 100644 index 00000000000..39b871ddb9f --- /dev/null +++ b/src/main/python/docs/source/api/operator/algorithms/decisionTree.rst @@ -0,0 +1,4 @@ +decisionTree +============ + +.. autofunction:: systemds.operator.algorithm.decisionTree \ No newline at end of file diff --git a/src/main/python/docs/source/api/operator/algorithms/decisionTreePredict.rst b/src/main/python/docs/source/api/operator/algorithms/decisionTreePredict.rst new file mode 100644 index 00000000000..c2cadd0df33 --- /dev/null +++ b/src/main/python/docs/source/api/operator/algorithms/decisionTreePredict.rst @@ -0,0 +1,4 @@ +decisionTreePredict +=================== + +.. autofunction:: systemds.operator.algorithm.decisionTreePredict \ No newline at end of file diff --git a/src/main/python/docs/source/api/operator/algorithms/deepWalk.rst b/src/main/python/docs/source/api/operator/algorithms/deepWalk.rst new file mode 100644 index 00000000000..48a60442bdf --- /dev/null +++ b/src/main/python/docs/source/api/operator/algorithms/deepWalk.rst @@ -0,0 +1,4 @@ +deepWalk +======== + +.. autofunction:: systemds.operator.algorithm.deepWalk \ No newline at end of file diff --git a/src/main/python/docs/source/api/operator/algorithms/denialConstraints.rst b/src/main/python/docs/source/api/operator/algorithms/denialConstraints.rst new file mode 100644 index 00000000000..6f9580b5ed7 --- /dev/null +++ b/src/main/python/docs/source/api/operator/algorithms/denialConstraints.rst @@ -0,0 +1,4 @@ +denialConstraints +================= + +.. autofunction:: systemds.operator.algorithm.denialConstraints \ No newline at end of file diff --git a/src/main/python/docs/source/api/operator/algorithms/differenceStatistics.rst b/src/main/python/docs/source/api/operator/algorithms/differenceStatistics.rst new file mode 100644 index 00000000000..bd5eff6c47d --- /dev/null +++ b/src/main/python/docs/source/api/operator/algorithms/differenceStatistics.rst @@ -0,0 +1,4 @@ +differenceStatistics +==================== + +.. autofunction:: systemds.operator.algorithm.differenceStatistics \ No newline at end of file diff --git a/src/main/python/docs/source/api/operator/algorithms/discoverFD.rst b/src/main/python/docs/source/api/operator/algorithms/discoverFD.rst new file mode 100644 index 00000000000..acb39f63b0c --- /dev/null +++ b/src/main/python/docs/source/api/operator/algorithms/discoverFD.rst @@ -0,0 +1,4 @@ +discoverFD +========== + +.. autofunction:: systemds.operator.algorithm.discoverFD \ No newline at end of file diff --git a/src/main/python/docs/source/api/operator/algorithms/dist.rst b/src/main/python/docs/source/api/operator/algorithms/dist.rst new file mode 100644 index 00000000000..048d11444ef --- /dev/null +++ b/src/main/python/docs/source/api/operator/algorithms/dist.rst @@ -0,0 +1,4 @@ +dist +==== + +.. autofunction:: systemds.operator.algorithm.dist \ No newline at end of file diff --git a/src/main/python/docs/source/api/operator/algorithms/dmv.rst b/src/main/python/docs/source/api/operator/algorithms/dmv.rst new file mode 100644 index 00000000000..c01e47ea639 --- /dev/null +++ b/src/main/python/docs/source/api/operator/algorithms/dmv.rst @@ -0,0 +1,4 @@ +dmv +=== + +.. autofunction:: systemds.operator.algorithm.dmv \ No newline at end of file diff --git a/src/main/python/docs/source/api/operator/algorithms/ema.rst b/src/main/python/docs/source/api/operator/algorithms/ema.rst new file mode 100644 index 00000000000..1fab50bc65a --- /dev/null +++ b/src/main/python/docs/source/api/operator/algorithms/ema.rst @@ -0,0 +1,4 @@ +ema +=== + +.. autofunction:: systemds.operator.algorithm.ema \ No newline at end of file diff --git a/src/main/python/docs/source/api/operator/algorithms/executePipeline.rst b/src/main/python/docs/source/api/operator/algorithms/executePipeline.rst new file mode 100644 index 00000000000..e06fc6e96b0 --- /dev/null +++ b/src/main/python/docs/source/api/operator/algorithms/executePipeline.rst @@ -0,0 +1,4 @@ +executePipeline +=============== + +.. autofunction:: systemds.operator.algorithm.executePipeline \ No newline at end of file diff --git a/src/main/python/docs/source/api/operator/algorithms/f1Score.rst b/src/main/python/docs/source/api/operator/algorithms/f1Score.rst new file mode 100644 index 00000000000..fe436fc41bb --- /dev/null +++ b/src/main/python/docs/source/api/operator/algorithms/f1Score.rst @@ -0,0 +1,4 @@ +f1Score +======= + +.. autofunction:: systemds.operator.algorithm.f1Score \ No newline at end of file diff --git a/src/main/python/docs/source/api/operator/algorithms/fdr.rst b/src/main/python/docs/source/api/operator/algorithms/fdr.rst new file mode 100644 index 00000000000..a776cfa3050 --- /dev/null +++ b/src/main/python/docs/source/api/operator/algorithms/fdr.rst @@ -0,0 +1,4 @@ +fdr +=== + +.. autofunction:: systemds.operator.algorithm.fdr \ No newline at end of file diff --git a/src/main/python/docs/source/api/operator/algorithms/ffPredict.rst b/src/main/python/docs/source/api/operator/algorithms/ffPredict.rst new file mode 100644 index 00000000000..9772e15959d --- /dev/null +++ b/src/main/python/docs/source/api/operator/algorithms/ffPredict.rst @@ -0,0 +1,4 @@ +ffPredict +========= + +.. autofunction:: systemds.operator.algorithm.ffPredict \ No newline at end of file diff --git a/src/main/python/docs/source/api/operator/algorithms/ffTrain.rst b/src/main/python/docs/source/api/operator/algorithms/ffTrain.rst new file mode 100644 index 00000000000..8e2b14ddfed --- /dev/null +++ b/src/main/python/docs/source/api/operator/algorithms/ffTrain.rst @@ -0,0 +1,4 @@ +ffTrain +======= + +.. autofunction:: systemds.operator.algorithm.ffTrain \ No newline at end of file diff --git a/src/main/python/docs/source/api/operator/algorithms/fit_pipeline.rst b/src/main/python/docs/source/api/operator/algorithms/fit_pipeline.rst new file mode 100644 index 00000000000..d3d32e880cd --- /dev/null +++ b/src/main/python/docs/source/api/operator/algorithms/fit_pipeline.rst @@ -0,0 +1,4 @@ +fit_pipeline +============ + +.. autofunction:: systemds.operator.algorithm.fit_pipeline \ No newline at end of file diff --git a/src/main/python/docs/source/api/operator/algorithms/fixInvalidLengths.rst b/src/main/python/docs/source/api/operator/algorithms/fixInvalidLengths.rst new file mode 100644 index 00000000000..31b0cd12335 --- /dev/null +++ b/src/main/python/docs/source/api/operator/algorithms/fixInvalidLengths.rst @@ -0,0 +1,4 @@ +fixInvalidLengths +================= + +.. autofunction:: systemds.operator.algorithm.fixInvalidLengths \ No newline at end of file diff --git a/src/main/python/docs/source/api/operator/algorithms/fixInvalidLengthsApply.rst b/src/main/python/docs/source/api/operator/algorithms/fixInvalidLengthsApply.rst new file mode 100644 index 00000000000..95e5802fb38 --- /dev/null +++ b/src/main/python/docs/source/api/operator/algorithms/fixInvalidLengthsApply.rst @@ -0,0 +1,4 @@ +fixInvalidLengthsApply +====================== + +.. autofunction:: systemds.operator.algorithm.fixInvalidLengthsApply \ No newline at end of file diff --git a/src/main/python/docs/source/api/operator/algorithms/flattenQuantile.rst b/src/main/python/docs/source/api/operator/algorithms/flattenQuantile.rst new file mode 100644 index 00000000000..e6fffa71402 --- /dev/null +++ b/src/main/python/docs/source/api/operator/algorithms/flattenQuantile.rst @@ -0,0 +1,4 @@ +flattenQuantile +=============== + +.. autofunction:: systemds.operator.algorithm.flattenQuantile \ No newline at end of file diff --git a/src/main/python/docs/source/api/operator/algorithms/frameSort.rst b/src/main/python/docs/source/api/operator/algorithms/frameSort.rst new file mode 100644 index 00000000000..87ee40c0351 --- /dev/null +++ b/src/main/python/docs/source/api/operator/algorithms/frameSort.rst @@ -0,0 +1,4 @@ +frameSort +========= + +.. autofunction:: systemds.operator.algorithm.frameSort \ No newline at end of file diff --git a/src/main/python/docs/source/api/operator/algorithms/frequencyEncode.rst b/src/main/python/docs/source/api/operator/algorithms/frequencyEncode.rst new file mode 100644 index 00000000000..e54e934d0cb --- /dev/null +++ b/src/main/python/docs/source/api/operator/algorithms/frequencyEncode.rst @@ -0,0 +1,4 @@ +frequencyEncode +=============== + +.. autofunction:: systemds.operator.algorithm.frequencyEncode \ No newline at end of file diff --git a/src/main/python/docs/source/api/operator/algorithms/frequencyEncodeApply.rst b/src/main/python/docs/source/api/operator/algorithms/frequencyEncodeApply.rst new file mode 100644 index 00000000000..db4d94e92b4 --- /dev/null +++ b/src/main/python/docs/source/api/operator/algorithms/frequencyEncodeApply.rst @@ -0,0 +1,4 @@ +frequencyEncodeApply +==================== + +.. autofunction:: systemds.operator.algorithm.frequencyEncodeApply \ No newline at end of file diff --git a/src/main/python/docs/source/api/operator/algorithms/garch.rst b/src/main/python/docs/source/api/operator/algorithms/garch.rst new file mode 100644 index 00000000000..0c605ab8a35 --- /dev/null +++ b/src/main/python/docs/source/api/operator/algorithms/garch.rst @@ -0,0 +1,4 @@ +garch +===== + +.. autofunction:: systemds.operator.algorithm.garch \ No newline at end of file diff --git a/src/main/python/docs/source/api/operator/algorithms/gaussianClassifier.rst b/src/main/python/docs/source/api/operator/algorithms/gaussianClassifier.rst new file mode 100644 index 00000000000..e202cefae8c --- /dev/null +++ b/src/main/python/docs/source/api/operator/algorithms/gaussianClassifier.rst @@ -0,0 +1,4 @@ +gaussianClassifier +================== + +.. autofunction:: systemds.operator.algorithm.gaussianClassifier \ No newline at end of file diff --git a/src/main/python/docs/source/api/operator/algorithms/getAccuracy.rst b/src/main/python/docs/source/api/operator/algorithms/getAccuracy.rst new file mode 100644 index 00000000000..da9b573d3db --- /dev/null +++ b/src/main/python/docs/source/api/operator/algorithms/getAccuracy.rst @@ -0,0 +1,4 @@ +getAccuracy +=========== + +.. autofunction:: systemds.operator.algorithm.getAccuracy \ No newline at end of file diff --git a/src/main/python/docs/source/api/operator/algorithms/glm.rst b/src/main/python/docs/source/api/operator/algorithms/glm.rst new file mode 100644 index 00000000000..44a40c2a06e --- /dev/null +++ b/src/main/python/docs/source/api/operator/algorithms/glm.rst @@ -0,0 +1,4 @@ +glm +=== + +.. autofunction:: systemds.operator.algorithm.glm \ No newline at end of file diff --git a/src/main/python/docs/source/api/operator/algorithms/glmPredict.rst b/src/main/python/docs/source/api/operator/algorithms/glmPredict.rst new file mode 100644 index 00000000000..556ac2bbc4a --- /dev/null +++ b/src/main/python/docs/source/api/operator/algorithms/glmPredict.rst @@ -0,0 +1,4 @@ +glmPredict +========== + +.. autofunction:: systemds.operator.algorithm.glmPredict \ No newline at end of file diff --git a/src/main/python/docs/source/api/operator/algorithms/glove.rst b/src/main/python/docs/source/api/operator/algorithms/glove.rst new file mode 100644 index 00000000000..e5788753789 --- /dev/null +++ b/src/main/python/docs/source/api/operator/algorithms/glove.rst @@ -0,0 +1,4 @@ +glove +===== + +.. autofunction:: systemds.operator.algorithm.glove \ No newline at end of file diff --git a/src/main/python/docs/source/api/operator/algorithms/gmm.rst b/src/main/python/docs/source/api/operator/algorithms/gmm.rst new file mode 100644 index 00000000000..081f197ffa1 --- /dev/null +++ b/src/main/python/docs/source/api/operator/algorithms/gmm.rst @@ -0,0 +1,4 @@ +gmm +=== + +.. autofunction:: systemds.operator.algorithm.gmm \ No newline at end of file diff --git a/src/main/python/docs/source/api/operator/algorithms/gmmPredict.rst b/src/main/python/docs/source/api/operator/algorithms/gmmPredict.rst new file mode 100644 index 00000000000..9dd34ee83b5 --- /dev/null +++ b/src/main/python/docs/source/api/operator/algorithms/gmmPredict.rst @@ -0,0 +1,4 @@ +gmmPredict +========== + +.. autofunction:: systemds.operator.algorithm.gmmPredict \ No newline at end of file diff --git a/src/main/python/docs/source/api/operator/algorithms/gnmf.rst b/src/main/python/docs/source/api/operator/algorithms/gnmf.rst new file mode 100644 index 00000000000..2a7295d9da5 --- /dev/null +++ b/src/main/python/docs/source/api/operator/algorithms/gnmf.rst @@ -0,0 +1,4 @@ +gnmf +==== + +.. autofunction:: systemds.operator.algorithm.gnmf \ No newline at end of file diff --git a/src/main/python/docs/source/api/operator/algorithms/gridSearch.rst b/src/main/python/docs/source/api/operator/algorithms/gridSearch.rst new file mode 100644 index 00000000000..b59e4370045 --- /dev/null +++ b/src/main/python/docs/source/api/operator/algorithms/gridSearch.rst @@ -0,0 +1,4 @@ +gridSearch +========== + +.. autofunction:: systemds.operator.algorithm.gridSearch \ No newline at end of file diff --git a/src/main/python/docs/source/api/operator/algorithms/hospitalResidencyMatch.rst b/src/main/python/docs/source/api/operator/algorithms/hospitalResidencyMatch.rst new file mode 100644 index 00000000000..92d798e6442 --- /dev/null +++ b/src/main/python/docs/source/api/operator/algorithms/hospitalResidencyMatch.rst @@ -0,0 +1,4 @@ +hospitalResidencyMatch +====================== + +.. autofunction:: systemds.operator.algorithm.hospitalResidencyMatch \ No newline at end of file diff --git a/src/main/python/docs/source/api/operator/algorithms/hyperband.rst b/src/main/python/docs/source/api/operator/algorithms/hyperband.rst new file mode 100644 index 00000000000..270fd16601c --- /dev/null +++ b/src/main/python/docs/source/api/operator/algorithms/hyperband.rst @@ -0,0 +1,4 @@ +hyperband +========= + +.. autofunction:: systemds.operator.algorithm.hyperband \ No newline at end of file diff --git a/src/main/python/docs/source/api/operator/algorithms/img_brightness.rst b/src/main/python/docs/source/api/operator/algorithms/img_brightness.rst new file mode 100644 index 00000000000..b43686296b3 --- /dev/null +++ b/src/main/python/docs/source/api/operator/algorithms/img_brightness.rst @@ -0,0 +1,4 @@ +img_brightness +============== + +.. autofunction:: systemds.operator.algorithm.img_brightness \ No newline at end of file diff --git a/src/main/python/docs/source/api/operator/algorithms/img_brightness_linearized.rst b/src/main/python/docs/source/api/operator/algorithms/img_brightness_linearized.rst new file mode 100644 index 00000000000..a0ac5c1ea7e --- /dev/null +++ b/src/main/python/docs/source/api/operator/algorithms/img_brightness_linearized.rst @@ -0,0 +1,4 @@ +img_brightness_linearized +========================= + +.. autofunction:: systemds.operator.algorithm.img_brightness_linearized \ No newline at end of file diff --git a/src/main/python/docs/source/api/operator/algorithms/img_crop.rst b/src/main/python/docs/source/api/operator/algorithms/img_crop.rst new file mode 100644 index 00000000000..eab6bb2a07d --- /dev/null +++ b/src/main/python/docs/source/api/operator/algorithms/img_crop.rst @@ -0,0 +1,4 @@ +img_crop +======== + +.. autofunction:: systemds.operator.algorithm.img_crop \ No newline at end of file diff --git a/src/main/python/docs/source/api/operator/algorithms/img_crop_linearized.rst b/src/main/python/docs/source/api/operator/algorithms/img_crop_linearized.rst new file mode 100644 index 00000000000..e19d3a70a83 --- /dev/null +++ b/src/main/python/docs/source/api/operator/algorithms/img_crop_linearized.rst @@ -0,0 +1,4 @@ +img_crop_linearized +=================== + +.. autofunction:: systemds.operator.algorithm.img_crop_linearized \ No newline at end of file diff --git a/src/main/python/docs/source/api/operator/algorithms/img_cutout.rst b/src/main/python/docs/source/api/operator/algorithms/img_cutout.rst new file mode 100644 index 00000000000..b9b5e8467c9 --- /dev/null +++ b/src/main/python/docs/source/api/operator/algorithms/img_cutout.rst @@ -0,0 +1,4 @@ +img_cutout +========== + +.. autofunction:: systemds.operator.algorithm.img_cutout \ No newline at end of file diff --git a/src/main/python/docs/source/api/operator/algorithms/img_cutout_linearized.rst b/src/main/python/docs/source/api/operator/algorithms/img_cutout_linearized.rst new file mode 100644 index 00000000000..7ce7c76b1de --- /dev/null +++ b/src/main/python/docs/source/api/operator/algorithms/img_cutout_linearized.rst @@ -0,0 +1,4 @@ +img_cutout_linearized +===================== + +.. autofunction:: systemds.operator.algorithm.img_cutout_linearized \ No newline at end of file diff --git a/src/main/python/docs/source/api/operator/algorithms/img_invert.rst b/src/main/python/docs/source/api/operator/algorithms/img_invert.rst new file mode 100644 index 00000000000..be0af11a5af --- /dev/null +++ b/src/main/python/docs/source/api/operator/algorithms/img_invert.rst @@ -0,0 +1,4 @@ +img_invert +========== + +.. autofunction:: systemds.operator.algorithm.img_invert \ No newline at end of file diff --git a/src/main/python/docs/source/api/operator/algorithms/img_invert_linearized.rst b/src/main/python/docs/source/api/operator/algorithms/img_invert_linearized.rst new file mode 100644 index 00000000000..8d0a6cf5206 --- /dev/null +++ b/src/main/python/docs/source/api/operator/algorithms/img_invert_linearized.rst @@ -0,0 +1,4 @@ +img_invert_linearized +===================== + +.. autofunction:: systemds.operator.algorithm.img_invert_linearized \ No newline at end of file diff --git a/src/main/python/docs/source/api/operator/algorithms/img_mirror.rst b/src/main/python/docs/source/api/operator/algorithms/img_mirror.rst new file mode 100644 index 00000000000..d1d31fba230 --- /dev/null +++ b/src/main/python/docs/source/api/operator/algorithms/img_mirror.rst @@ -0,0 +1,4 @@ +img_mirror +========== + +.. autofunction:: systemds.operator.algorithm.img_mirror \ No newline at end of file diff --git a/src/main/python/docs/source/api/operator/algorithms/img_mirror_linearized.rst b/src/main/python/docs/source/api/operator/algorithms/img_mirror_linearized.rst new file mode 100644 index 00000000000..56076e239fa --- /dev/null +++ b/src/main/python/docs/source/api/operator/algorithms/img_mirror_linearized.rst @@ -0,0 +1,4 @@ +img_mirror_linearized +===================== + +.. autofunction:: systemds.operator.algorithm.img_mirror_linearized \ No newline at end of file diff --git a/src/main/python/docs/source/api/operator/algorithms/img_posterize.rst b/src/main/python/docs/source/api/operator/algorithms/img_posterize.rst new file mode 100644 index 00000000000..d2db21b114e --- /dev/null +++ b/src/main/python/docs/source/api/operator/algorithms/img_posterize.rst @@ -0,0 +1,4 @@ +img_posterize +============= + +.. autofunction:: systemds.operator.algorithm.img_posterize \ No newline at end of file diff --git a/src/main/python/docs/source/api/operator/algorithms/img_posterize_linearized.rst b/src/main/python/docs/source/api/operator/algorithms/img_posterize_linearized.rst new file mode 100644 index 00000000000..debac6d54df --- /dev/null +++ b/src/main/python/docs/source/api/operator/algorithms/img_posterize_linearized.rst @@ -0,0 +1,4 @@ +img_posterize_linearized +======================== + +.. autofunction:: systemds.operator.algorithm.img_posterize_linearized \ No newline at end of file diff --git a/src/main/python/docs/source/api/operator/algorithms/img_rotate.rst b/src/main/python/docs/source/api/operator/algorithms/img_rotate.rst new file mode 100644 index 00000000000..fca9801b32c --- /dev/null +++ b/src/main/python/docs/source/api/operator/algorithms/img_rotate.rst @@ -0,0 +1,4 @@ +img_rotate +========== + +.. autofunction:: systemds.operator.algorithm.img_rotate \ No newline at end of file diff --git a/src/main/python/docs/source/api/operator/algorithms/img_rotate_linearized.rst b/src/main/python/docs/source/api/operator/algorithms/img_rotate_linearized.rst new file mode 100644 index 00000000000..f40f086f84c --- /dev/null +++ b/src/main/python/docs/source/api/operator/algorithms/img_rotate_linearized.rst @@ -0,0 +1,4 @@ +img_rotate_linearized +===================== + +.. autofunction:: systemds.operator.algorithm.img_rotate_linearized \ No newline at end of file diff --git a/src/main/python/docs/source/api/operator/algorithms/img_sample_pairing.rst b/src/main/python/docs/source/api/operator/algorithms/img_sample_pairing.rst new file mode 100644 index 00000000000..7de25f74b52 --- /dev/null +++ b/src/main/python/docs/source/api/operator/algorithms/img_sample_pairing.rst @@ -0,0 +1,4 @@ +img_sample_pairing +================== + +.. autofunction:: systemds.operator.algorithm.img_sample_pairing \ No newline at end of file diff --git a/src/main/python/docs/source/api/operator/algorithms/img_sample_pairing_linearized.rst b/src/main/python/docs/source/api/operator/algorithms/img_sample_pairing_linearized.rst new file mode 100644 index 00000000000..8cfd5a1873f --- /dev/null +++ b/src/main/python/docs/source/api/operator/algorithms/img_sample_pairing_linearized.rst @@ -0,0 +1,4 @@ +img_sample_pairing_linearized +============================= + +.. autofunction:: systemds.operator.algorithm.img_sample_pairing_linearized \ No newline at end of file diff --git a/src/main/python/docs/source/api/operator/algorithms/img_shear.rst b/src/main/python/docs/source/api/operator/algorithms/img_shear.rst new file mode 100644 index 00000000000..8c8144d461c --- /dev/null +++ b/src/main/python/docs/source/api/operator/algorithms/img_shear.rst @@ -0,0 +1,4 @@ +img_shear +========= + +.. autofunction:: systemds.operator.algorithm.img_shear \ No newline at end of file diff --git a/src/main/python/docs/source/api/operator/algorithms/img_shear_linearized.rst b/src/main/python/docs/source/api/operator/algorithms/img_shear_linearized.rst new file mode 100644 index 00000000000..8907f6a90bb --- /dev/null +++ b/src/main/python/docs/source/api/operator/algorithms/img_shear_linearized.rst @@ -0,0 +1,4 @@ +img_shear_linearized +==================== + +.. autofunction:: systemds.operator.algorithm.img_shear_linearized \ No newline at end of file diff --git a/src/main/python/docs/source/api/operator/algorithms/img_transform.rst b/src/main/python/docs/source/api/operator/algorithms/img_transform.rst new file mode 100644 index 00000000000..6bb4676270a --- /dev/null +++ b/src/main/python/docs/source/api/operator/algorithms/img_transform.rst @@ -0,0 +1,4 @@ +img_transform +============= + +.. autofunction:: systemds.operator.algorithm.img_transform \ No newline at end of file diff --git a/src/main/python/docs/source/api/operator/algorithms/img_transform_linearized.rst b/src/main/python/docs/source/api/operator/algorithms/img_transform_linearized.rst new file mode 100644 index 00000000000..16d043af9b8 --- /dev/null +++ b/src/main/python/docs/source/api/operator/algorithms/img_transform_linearized.rst @@ -0,0 +1,4 @@ +img_transform_linearized +======================== + +.. autofunction:: systemds.operator.algorithm.img_transform_linearized \ No newline at end of file diff --git a/src/main/python/docs/source/api/operator/algorithms/img_translate.rst b/src/main/python/docs/source/api/operator/algorithms/img_translate.rst new file mode 100644 index 00000000000..8b5952f8d03 --- /dev/null +++ b/src/main/python/docs/source/api/operator/algorithms/img_translate.rst @@ -0,0 +1,4 @@ +img_translate +============= + +.. autofunction:: systemds.operator.algorithm.img_translate \ No newline at end of file diff --git a/src/main/python/docs/source/api/operator/algorithms/img_translate_linearized.rst b/src/main/python/docs/source/api/operator/algorithms/img_translate_linearized.rst new file mode 100644 index 00000000000..02f0ac813d9 --- /dev/null +++ b/src/main/python/docs/source/api/operator/algorithms/img_translate_linearized.rst @@ -0,0 +1,4 @@ +img_translate_linearized +======================== + +.. autofunction:: systemds.operator.algorithm.img_translate_linearized \ No newline at end of file diff --git a/src/main/python/docs/source/api/operator/algorithms/impurityMeasures.rst b/src/main/python/docs/source/api/operator/algorithms/impurityMeasures.rst new file mode 100644 index 00000000000..bbb6460a20e --- /dev/null +++ b/src/main/python/docs/source/api/operator/algorithms/impurityMeasures.rst @@ -0,0 +1,4 @@ +impurityMeasures +================ + +.. autofunction:: systemds.operator.algorithm.impurityMeasures \ No newline at end of file diff --git a/src/main/python/docs/source/api/operator/algorithms/imputeByFD.rst b/src/main/python/docs/source/api/operator/algorithms/imputeByFD.rst new file mode 100644 index 00000000000..f2378547e23 --- /dev/null +++ b/src/main/python/docs/source/api/operator/algorithms/imputeByFD.rst @@ -0,0 +1,4 @@ +imputeByFD +========== + +.. autofunction:: systemds.operator.algorithm.imputeByFD \ No newline at end of file diff --git a/src/main/python/docs/source/api/operator/algorithms/imputeByFDApply.rst b/src/main/python/docs/source/api/operator/algorithms/imputeByFDApply.rst new file mode 100644 index 00000000000..90d7b526013 --- /dev/null +++ b/src/main/python/docs/source/api/operator/algorithms/imputeByFDApply.rst @@ -0,0 +1,4 @@ +imputeByFDApply +=============== + +.. autofunction:: systemds.operator.algorithm.imputeByFDApply \ No newline at end of file diff --git a/src/main/python/docs/source/api/operator/algorithms/imputeByKNN.rst b/src/main/python/docs/source/api/operator/algorithms/imputeByKNN.rst new file mode 100644 index 00000000000..3f2e21eb59c --- /dev/null +++ b/src/main/python/docs/source/api/operator/algorithms/imputeByKNN.rst @@ -0,0 +1,4 @@ +imputeByKNN +=========== + +.. autofunction:: systemds.operator.algorithm.imputeByKNN \ No newline at end of file diff --git a/src/main/python/docs/source/api/operator/algorithms/imputeByMean.rst b/src/main/python/docs/source/api/operator/algorithms/imputeByMean.rst new file mode 100644 index 00000000000..cf7f4dc1283 --- /dev/null +++ b/src/main/python/docs/source/api/operator/algorithms/imputeByMean.rst @@ -0,0 +1,4 @@ +imputeByMean +============ + +.. autofunction:: systemds.operator.algorithm.imputeByMean \ No newline at end of file diff --git a/src/main/python/docs/source/api/operator/algorithms/imputeByMeanApply.rst b/src/main/python/docs/source/api/operator/algorithms/imputeByMeanApply.rst new file mode 100644 index 00000000000..688b48db94e --- /dev/null +++ b/src/main/python/docs/source/api/operator/algorithms/imputeByMeanApply.rst @@ -0,0 +1,4 @@ +imputeByMeanApply +================= + +.. autofunction:: systemds.operator.algorithm.imputeByMeanApply \ No newline at end of file diff --git a/src/main/python/docs/source/api/operator/algorithms/imputeByMedian.rst b/src/main/python/docs/source/api/operator/algorithms/imputeByMedian.rst new file mode 100644 index 00000000000..a6762253de2 --- /dev/null +++ b/src/main/python/docs/source/api/operator/algorithms/imputeByMedian.rst @@ -0,0 +1,4 @@ +imputeByMedian +============== + +.. autofunction:: systemds.operator.algorithm.imputeByMedian \ No newline at end of file diff --git a/src/main/python/docs/source/api/operator/algorithms/imputeByMedianApply.rst b/src/main/python/docs/source/api/operator/algorithms/imputeByMedianApply.rst new file mode 100644 index 00000000000..a6513410a3d --- /dev/null +++ b/src/main/python/docs/source/api/operator/algorithms/imputeByMedianApply.rst @@ -0,0 +1,4 @@ +imputeByMedianApply +=================== + +.. autofunction:: systemds.operator.algorithm.imputeByMedianApply \ No newline at end of file diff --git a/src/main/python/docs/source/api/operator/algorithms/imputeByMode.rst b/src/main/python/docs/source/api/operator/algorithms/imputeByMode.rst new file mode 100644 index 00000000000..817ec471d26 --- /dev/null +++ b/src/main/python/docs/source/api/operator/algorithms/imputeByMode.rst @@ -0,0 +1,4 @@ +imputeByMode +============ + +.. autofunction:: systemds.operator.algorithm.imputeByMode \ No newline at end of file diff --git a/src/main/python/docs/source/api/operator/algorithms/imputeByModeApply.rst b/src/main/python/docs/source/api/operator/algorithms/imputeByModeApply.rst new file mode 100644 index 00000000000..b7c78a95893 --- /dev/null +++ b/src/main/python/docs/source/api/operator/algorithms/imputeByModeApply.rst @@ -0,0 +1,4 @@ +imputeByModeApply +================= + +.. autofunction:: systemds.operator.algorithm.imputeByModeApply \ No newline at end of file diff --git a/src/main/python/docs/source/api/operator/algorithms/incSliceLine.rst b/src/main/python/docs/source/api/operator/algorithms/incSliceLine.rst new file mode 100644 index 00000000000..da2c1f58dee --- /dev/null +++ b/src/main/python/docs/source/api/operator/algorithms/incSliceLine.rst @@ -0,0 +1,4 @@ +incSliceLine +============ + +.. autofunction:: systemds.operator.algorithm.incSliceLine \ No newline at end of file diff --git a/src/main/python/docs/source/api/operator/algorithms/intersect.rst b/src/main/python/docs/source/api/operator/algorithms/intersect.rst new file mode 100644 index 00000000000..762d68708b5 --- /dev/null +++ b/src/main/python/docs/source/api/operator/algorithms/intersect.rst @@ -0,0 +1,4 @@ +intersect +========= + +.. autofunction:: systemds.operator.algorithm.intersect \ No newline at end of file diff --git a/src/main/python/docs/source/api/operator/algorithms/km.rst b/src/main/python/docs/source/api/operator/algorithms/km.rst new file mode 100644 index 00000000000..8392789508c --- /dev/null +++ b/src/main/python/docs/source/api/operator/algorithms/km.rst @@ -0,0 +1,4 @@ +km +== + +.. autofunction:: systemds.operator.algorithm.km \ No newline at end of file diff --git a/src/main/python/docs/source/api/operator/algorithms/kmeans.rst b/src/main/python/docs/source/api/operator/algorithms/kmeans.rst new file mode 100644 index 00000000000..7f4d1a92596 --- /dev/null +++ b/src/main/python/docs/source/api/operator/algorithms/kmeans.rst @@ -0,0 +1,4 @@ +kmeans +====== + +.. autofunction:: systemds.operator.algorithm.kmeans \ No newline at end of file diff --git a/src/main/python/docs/source/api/operator/algorithms/kmeansPredict.rst b/src/main/python/docs/source/api/operator/algorithms/kmeansPredict.rst new file mode 100644 index 00000000000..b0832ee4854 --- /dev/null +++ b/src/main/python/docs/source/api/operator/algorithms/kmeansPredict.rst @@ -0,0 +1,4 @@ +kmeansPredict +============= + +.. autofunction:: systemds.operator.algorithm.kmeansPredict \ No newline at end of file diff --git a/src/main/python/docs/source/api/operator/algorithms/knn.rst b/src/main/python/docs/source/api/operator/algorithms/knn.rst new file mode 100644 index 00000000000..c9bc1ed67d4 --- /dev/null +++ b/src/main/python/docs/source/api/operator/algorithms/knn.rst @@ -0,0 +1,4 @@ +knn +=== + +.. autofunction:: systemds.operator.algorithm.knn \ No newline at end of file diff --git a/src/main/python/docs/source/api/operator/algorithms/knnGraph.rst b/src/main/python/docs/source/api/operator/algorithms/knnGraph.rst new file mode 100644 index 00000000000..19c9c7adc25 --- /dev/null +++ b/src/main/python/docs/source/api/operator/algorithms/knnGraph.rst @@ -0,0 +1,4 @@ +knnGraph +======== + +.. autofunction:: systemds.operator.algorithm.knnGraph \ No newline at end of file diff --git a/src/main/python/docs/source/api/operator/algorithms/knnbf.rst b/src/main/python/docs/source/api/operator/algorithms/knnbf.rst new file mode 100644 index 00000000000..098af73b361 --- /dev/null +++ b/src/main/python/docs/source/api/operator/algorithms/knnbf.rst @@ -0,0 +1,4 @@ +knnbf +===== + +.. autofunction:: systemds.operator.algorithm.knnbf \ No newline at end of file diff --git a/src/main/python/docs/source/api/operator/algorithms/l2svm.rst b/src/main/python/docs/source/api/operator/algorithms/l2svm.rst new file mode 100644 index 00000000000..ab8155faaad --- /dev/null +++ b/src/main/python/docs/source/api/operator/algorithms/l2svm.rst @@ -0,0 +1,4 @@ +l2svm +===== + +.. autofunction:: systemds.operator.algorithm.l2svm \ No newline at end of file diff --git a/src/main/python/docs/source/api/operator/algorithms/l2svmPredict.rst b/src/main/python/docs/source/api/operator/algorithms/l2svmPredict.rst new file mode 100644 index 00000000000..0aede20fcb8 --- /dev/null +++ b/src/main/python/docs/source/api/operator/algorithms/l2svmPredict.rst @@ -0,0 +1,4 @@ +l2svmPredict +============ + +.. autofunction:: systemds.operator.algorithm.l2svmPredict \ No newline at end of file diff --git a/src/main/python/docs/source/api/operator/algorithms/lasso.rst b/src/main/python/docs/source/api/operator/algorithms/lasso.rst new file mode 100644 index 00000000000..c4d8f490939 --- /dev/null +++ b/src/main/python/docs/source/api/operator/algorithms/lasso.rst @@ -0,0 +1,4 @@ +lasso +===== + +.. autofunction:: systemds.operator.algorithm.lasso \ No newline at end of file diff --git a/src/main/python/docs/source/api/operator/algorithms/lenetPredict.rst b/src/main/python/docs/source/api/operator/algorithms/lenetPredict.rst new file mode 100644 index 00000000000..f7ca6d4cfad --- /dev/null +++ b/src/main/python/docs/source/api/operator/algorithms/lenetPredict.rst @@ -0,0 +1,4 @@ +lenetPredict +============ + +.. autofunction:: systemds.operator.algorithm.lenetPredict \ No newline at end of file diff --git a/src/main/python/docs/source/api/operator/algorithms/lenetTrain.rst b/src/main/python/docs/source/api/operator/algorithms/lenetTrain.rst new file mode 100644 index 00000000000..59a1463290c --- /dev/null +++ b/src/main/python/docs/source/api/operator/algorithms/lenetTrain.rst @@ -0,0 +1,4 @@ +lenetTrain +========== + +.. autofunction:: systemds.operator.algorithm.lenetTrain \ No newline at end of file diff --git a/src/main/python/docs/source/api/operator/algorithms/lm.rst b/src/main/python/docs/source/api/operator/algorithms/lm.rst new file mode 100644 index 00000000000..50eded5883c --- /dev/null +++ b/src/main/python/docs/source/api/operator/algorithms/lm.rst @@ -0,0 +1,4 @@ +lm +== + +.. autofunction:: systemds.operator.algorithm.lm \ No newline at end of file diff --git a/src/main/python/docs/source/api/operator/algorithms/lmCG.rst b/src/main/python/docs/source/api/operator/algorithms/lmCG.rst new file mode 100644 index 00000000000..e0242f13770 --- /dev/null +++ b/src/main/python/docs/source/api/operator/algorithms/lmCG.rst @@ -0,0 +1,4 @@ +lmCG +==== + +.. autofunction:: systemds.operator.algorithm.lmCG \ No newline at end of file diff --git a/src/main/python/docs/source/api/operator/algorithms/lmDS.rst b/src/main/python/docs/source/api/operator/algorithms/lmDS.rst new file mode 100644 index 00000000000..52255a93104 --- /dev/null +++ b/src/main/python/docs/source/api/operator/algorithms/lmDS.rst @@ -0,0 +1,4 @@ +lmDS +==== + +.. autofunction:: systemds.operator.algorithm.lmDS \ No newline at end of file diff --git a/src/main/python/docs/source/api/operator/algorithms/lmPredict.rst b/src/main/python/docs/source/api/operator/algorithms/lmPredict.rst new file mode 100644 index 00000000000..a7179cb84e5 --- /dev/null +++ b/src/main/python/docs/source/api/operator/algorithms/lmPredict.rst @@ -0,0 +1,4 @@ +lmPredict +========= + +.. autofunction:: systemds.operator.algorithm.lmPredict \ No newline at end of file diff --git a/src/main/python/docs/source/api/operator/algorithms/lmPredictStats.rst b/src/main/python/docs/source/api/operator/algorithms/lmPredictStats.rst new file mode 100644 index 00000000000..984fd907304 --- /dev/null +++ b/src/main/python/docs/source/api/operator/algorithms/lmPredictStats.rst @@ -0,0 +1,4 @@ +lmPredictStats +============== + +.. autofunction:: systemds.operator.algorithm.lmPredictStats \ No newline at end of file diff --git a/src/main/python/docs/source/api/operator/algorithms/logSumExp.rst b/src/main/python/docs/source/api/operator/algorithms/logSumExp.rst new file mode 100644 index 00000000000..aea539ced3b --- /dev/null +++ b/src/main/python/docs/source/api/operator/algorithms/logSumExp.rst @@ -0,0 +1,4 @@ +logSumExp +========= + +.. autofunction:: systemds.operator.algorithm.logSumExp \ No newline at end of file diff --git a/src/main/python/docs/source/api/operator/algorithms/mae.rst b/src/main/python/docs/source/api/operator/algorithms/mae.rst new file mode 100644 index 00000000000..36094918d88 --- /dev/null +++ b/src/main/python/docs/source/api/operator/algorithms/mae.rst @@ -0,0 +1,4 @@ +mae +=== + +.. autofunction:: systemds.operator.algorithm.mae \ No newline at end of file diff --git a/src/main/python/docs/source/api/operator/algorithms/mape.rst b/src/main/python/docs/source/api/operator/algorithms/mape.rst new file mode 100644 index 00000000000..a2284debb88 --- /dev/null +++ b/src/main/python/docs/source/api/operator/algorithms/mape.rst @@ -0,0 +1,4 @@ +mape +==== + +.. autofunction:: systemds.operator.algorithm.mape \ No newline at end of file diff --git a/src/main/python/docs/source/api/operator/algorithms/matrixProfile.rst b/src/main/python/docs/source/api/operator/algorithms/matrixProfile.rst new file mode 100644 index 00000000000..6dd5b4a7ddc --- /dev/null +++ b/src/main/python/docs/source/api/operator/algorithms/matrixProfile.rst @@ -0,0 +1,4 @@ +matrixProfile +============= + +.. autofunction:: systemds.operator.algorithm.matrixProfile \ No newline at end of file diff --git a/src/main/python/docs/source/api/operator/algorithms/mcc.rst b/src/main/python/docs/source/api/operator/algorithms/mcc.rst new file mode 100644 index 00000000000..918f849be8a --- /dev/null +++ b/src/main/python/docs/source/api/operator/algorithms/mcc.rst @@ -0,0 +1,4 @@ +mcc +=== + +.. autofunction:: systemds.operator.algorithm.mcc \ No newline at end of file diff --git a/src/main/python/docs/source/api/operator/algorithms/mdedup.rst b/src/main/python/docs/source/api/operator/algorithms/mdedup.rst new file mode 100644 index 00000000000..88fd7ed5be7 --- /dev/null +++ b/src/main/python/docs/source/api/operator/algorithms/mdedup.rst @@ -0,0 +1,4 @@ +mdedup +====== + +.. autofunction:: systemds.operator.algorithm.mdedup \ No newline at end of file diff --git a/src/main/python/docs/source/api/operator/algorithms/mice.rst b/src/main/python/docs/source/api/operator/algorithms/mice.rst new file mode 100644 index 00000000000..429337e782b --- /dev/null +++ b/src/main/python/docs/source/api/operator/algorithms/mice.rst @@ -0,0 +1,4 @@ +mice +==== + +.. autofunction:: systemds.operator.algorithm.mice \ No newline at end of file diff --git a/src/main/python/docs/source/api/operator/algorithms/miceApply.rst b/src/main/python/docs/source/api/operator/algorithms/miceApply.rst new file mode 100644 index 00000000000..2ecc09a5c25 --- /dev/null +++ b/src/main/python/docs/source/api/operator/algorithms/miceApply.rst @@ -0,0 +1,4 @@ +miceApply +========= + +.. autofunction:: systemds.operator.algorithm.miceApply \ No newline at end of file diff --git a/src/main/python/docs/source/api/operator/algorithms/mse.rst b/src/main/python/docs/source/api/operator/algorithms/mse.rst new file mode 100644 index 00000000000..b739bbda38f --- /dev/null +++ b/src/main/python/docs/source/api/operator/algorithms/mse.rst @@ -0,0 +1,4 @@ +mse +=== + +.. autofunction:: systemds.operator.algorithm.mse \ No newline at end of file diff --git a/src/main/python/docs/source/api/operator/algorithms/msmape.rst b/src/main/python/docs/source/api/operator/algorithms/msmape.rst new file mode 100644 index 00000000000..b71f226d93b --- /dev/null +++ b/src/main/python/docs/source/api/operator/algorithms/msmape.rst @@ -0,0 +1,4 @@ +msmape +====== + +.. autofunction:: systemds.operator.algorithm.msmape \ No newline at end of file diff --git a/src/main/python/docs/source/api/operator/algorithms/msvm.rst b/src/main/python/docs/source/api/operator/algorithms/msvm.rst new file mode 100644 index 00000000000..744f0b3dd7d --- /dev/null +++ b/src/main/python/docs/source/api/operator/algorithms/msvm.rst @@ -0,0 +1,4 @@ +msvm +==== + +.. autofunction:: systemds.operator.algorithm.msvm \ No newline at end of file diff --git a/src/main/python/docs/source/api/operator/algorithms/msvmPredict.rst b/src/main/python/docs/source/api/operator/algorithms/msvmPredict.rst new file mode 100644 index 00000000000..7900af7168a --- /dev/null +++ b/src/main/python/docs/source/api/operator/algorithms/msvmPredict.rst @@ -0,0 +1,4 @@ +msvmPredict +=========== + +.. autofunction:: systemds.operator.algorithm.msvmPredict \ No newline at end of file diff --git a/src/main/python/docs/source/api/operator/algorithms/multiLogReg.rst b/src/main/python/docs/source/api/operator/algorithms/multiLogReg.rst new file mode 100644 index 00000000000..603271a7026 --- /dev/null +++ b/src/main/python/docs/source/api/operator/algorithms/multiLogReg.rst @@ -0,0 +1,4 @@ +multiLogReg +=========== + +.. autofunction:: systemds.operator.algorithm.multiLogReg \ No newline at end of file diff --git a/src/main/python/docs/source/api/operator/algorithms/multiLogRegPredict.rst b/src/main/python/docs/source/api/operator/algorithms/multiLogRegPredict.rst new file mode 100644 index 00000000000..edaf7f83057 --- /dev/null +++ b/src/main/python/docs/source/api/operator/algorithms/multiLogRegPredict.rst @@ -0,0 +1,4 @@ +multiLogRegPredict +================== + +.. autofunction:: systemds.operator.algorithm.multiLogRegPredict \ No newline at end of file diff --git a/src/main/python/docs/source/api/operator/algorithms/na_locf.rst b/src/main/python/docs/source/api/operator/algorithms/na_locf.rst new file mode 100644 index 00000000000..bc1c11bbc1d --- /dev/null +++ b/src/main/python/docs/source/api/operator/algorithms/na_locf.rst @@ -0,0 +1,4 @@ +na_locf +======= + +.. autofunction:: systemds.operator.algorithm.na_locf \ No newline at end of file diff --git a/src/main/python/docs/source/api/operator/algorithms/naiveBayes.rst b/src/main/python/docs/source/api/operator/algorithms/naiveBayes.rst new file mode 100644 index 00000000000..0dcd81e0e2d --- /dev/null +++ b/src/main/python/docs/source/api/operator/algorithms/naiveBayes.rst @@ -0,0 +1,4 @@ +naiveBayes +========== + +.. autofunction:: systemds.operator.algorithm.naiveBayes \ No newline at end of file diff --git a/src/main/python/docs/source/api/operator/algorithms/naiveBayesPredict.rst b/src/main/python/docs/source/api/operator/algorithms/naiveBayesPredict.rst new file mode 100644 index 00000000000..96882aea3d9 --- /dev/null +++ b/src/main/python/docs/source/api/operator/algorithms/naiveBayesPredict.rst @@ -0,0 +1,4 @@ +naiveBayesPredict +================= + +.. autofunction:: systemds.operator.algorithm.naiveBayesPredict \ No newline at end of file diff --git a/src/main/python/docs/source/api/operator/algorithms/normalize.rst b/src/main/python/docs/source/api/operator/algorithms/normalize.rst new file mode 100644 index 00000000000..10ebc62538b --- /dev/null +++ b/src/main/python/docs/source/api/operator/algorithms/normalize.rst @@ -0,0 +1,4 @@ +normalize +========= + +.. autofunction:: systemds.operator.algorithm.normalize \ No newline at end of file diff --git a/src/main/python/docs/source/api/operator/algorithms/normalizeApply.rst b/src/main/python/docs/source/api/operator/algorithms/normalizeApply.rst new file mode 100644 index 00000000000..cc42b36e95f --- /dev/null +++ b/src/main/python/docs/source/api/operator/algorithms/normalizeApply.rst @@ -0,0 +1,4 @@ +normalizeApply +============== + +.. autofunction:: systemds.operator.algorithm.normalizeApply \ No newline at end of file diff --git a/src/main/python/docs/source/api/operator/algorithms/nrmse.rst b/src/main/python/docs/source/api/operator/algorithms/nrmse.rst new file mode 100644 index 00000000000..ad189fa0802 --- /dev/null +++ b/src/main/python/docs/source/api/operator/algorithms/nrmse.rst @@ -0,0 +1,4 @@ +nrmse +===== + +.. autofunction:: systemds.operator.algorithm.nrmse \ No newline at end of file diff --git a/src/main/python/docs/source/api/operator/algorithms/outlier.rst b/src/main/python/docs/source/api/operator/algorithms/outlier.rst new file mode 100644 index 00000000000..e79c68e3a32 --- /dev/null +++ b/src/main/python/docs/source/api/operator/algorithms/outlier.rst @@ -0,0 +1,4 @@ +outlier +======= + +.. autofunction:: systemds.operator.algorithm.outlier \ No newline at end of file diff --git a/src/main/python/docs/source/api/operator/algorithms/outlierByArima.rst b/src/main/python/docs/source/api/operator/algorithms/outlierByArima.rst new file mode 100644 index 00000000000..7ea56086d34 --- /dev/null +++ b/src/main/python/docs/source/api/operator/algorithms/outlierByArima.rst @@ -0,0 +1,4 @@ +outlierByArima +============== + +.. autofunction:: systemds.operator.algorithm.outlierByArima \ No newline at end of file diff --git a/src/main/python/docs/source/api/operator/algorithms/outlierByIQR.rst b/src/main/python/docs/source/api/operator/algorithms/outlierByIQR.rst new file mode 100644 index 00000000000..fe62f01994d --- /dev/null +++ b/src/main/python/docs/source/api/operator/algorithms/outlierByIQR.rst @@ -0,0 +1,4 @@ +outlierByIQR +============ + +.. autofunction:: systemds.operator.algorithm.outlierByIQR \ No newline at end of file diff --git a/src/main/python/docs/source/api/operator/algorithms/outlierByIQRApply.rst b/src/main/python/docs/source/api/operator/algorithms/outlierByIQRApply.rst new file mode 100644 index 00000000000..2681463d05c --- /dev/null +++ b/src/main/python/docs/source/api/operator/algorithms/outlierByIQRApply.rst @@ -0,0 +1,4 @@ +outlierByIQRApply +================= + +.. autofunction:: systemds.operator.algorithm.outlierByIQRApply \ No newline at end of file diff --git a/src/main/python/docs/source/api/operator/algorithms/outlierBySd.rst b/src/main/python/docs/source/api/operator/algorithms/outlierBySd.rst new file mode 100644 index 00000000000..bf7a66cf0d4 --- /dev/null +++ b/src/main/python/docs/source/api/operator/algorithms/outlierBySd.rst @@ -0,0 +1,4 @@ +outlierBySd +=========== + +.. autofunction:: systemds.operator.algorithm.outlierBySd \ No newline at end of file diff --git a/src/main/python/docs/source/api/operator/algorithms/outlierBySdApply.rst b/src/main/python/docs/source/api/operator/algorithms/outlierBySdApply.rst new file mode 100644 index 00000000000..ed34b6b6d2c --- /dev/null +++ b/src/main/python/docs/source/api/operator/algorithms/outlierBySdApply.rst @@ -0,0 +1,4 @@ +outlierBySdApply +================ + +.. autofunction:: systemds.operator.algorithm.outlierBySdApply \ No newline at end of file diff --git a/src/main/python/docs/source/api/operator/algorithms/pageRank.rst b/src/main/python/docs/source/api/operator/algorithms/pageRank.rst new file mode 100644 index 00000000000..f1bedc84190 --- /dev/null +++ b/src/main/python/docs/source/api/operator/algorithms/pageRank.rst @@ -0,0 +1,4 @@ +pageRank +======== + +.. autofunction:: systemds.operator.algorithm.pageRank \ No newline at end of file diff --git a/src/main/python/docs/source/api/operator/algorithms/pca.rst b/src/main/python/docs/source/api/operator/algorithms/pca.rst new file mode 100644 index 00000000000..1eda6ee503f --- /dev/null +++ b/src/main/python/docs/source/api/operator/algorithms/pca.rst @@ -0,0 +1,4 @@ +pca +=== + +.. autofunction:: systemds.operator.algorithm.pca \ No newline at end of file diff --git a/src/main/python/docs/source/api/operator/algorithms/pcaInverse.rst b/src/main/python/docs/source/api/operator/algorithms/pcaInverse.rst new file mode 100644 index 00000000000..6cc0cf1bff5 --- /dev/null +++ b/src/main/python/docs/source/api/operator/algorithms/pcaInverse.rst @@ -0,0 +1,4 @@ +pcaInverse +========== + +.. autofunction:: systemds.operator.algorithm.pcaInverse \ No newline at end of file diff --git a/src/main/python/docs/source/api/operator/algorithms/pcaTransform.rst b/src/main/python/docs/source/api/operator/algorithms/pcaTransform.rst new file mode 100644 index 00000000000..f78df2e4657 --- /dev/null +++ b/src/main/python/docs/source/api/operator/algorithms/pcaTransform.rst @@ -0,0 +1,4 @@ +pcaTransform +============ + +.. autofunction:: systemds.operator.algorithm.pcaTransform \ No newline at end of file diff --git a/src/main/python/docs/source/api/operator/algorithms/pnmf.rst b/src/main/python/docs/source/api/operator/algorithms/pnmf.rst new file mode 100644 index 00000000000..1dd0dcb367a --- /dev/null +++ b/src/main/python/docs/source/api/operator/algorithms/pnmf.rst @@ -0,0 +1,4 @@ +pnmf +==== + +.. autofunction:: systemds.operator.algorithm.pnmf \ No newline at end of file diff --git a/src/main/python/docs/source/api/operator/algorithms/ppca.rst b/src/main/python/docs/source/api/operator/algorithms/ppca.rst new file mode 100644 index 00000000000..38e6f4468b2 --- /dev/null +++ b/src/main/python/docs/source/api/operator/algorithms/ppca.rst @@ -0,0 +1,4 @@ +ppca +==== + +.. autofunction:: systemds.operator.algorithm.ppca \ No newline at end of file diff --git a/src/main/python/docs/source/api/operator/algorithms/psnr.rst b/src/main/python/docs/source/api/operator/algorithms/psnr.rst new file mode 100644 index 00000000000..c16bb02613d --- /dev/null +++ b/src/main/python/docs/source/api/operator/algorithms/psnr.rst @@ -0,0 +1,4 @@ +psnr +==== + +.. autofunction:: systemds.operator.algorithm.psnr \ No newline at end of file diff --git a/src/main/python/docs/source/api/operator/algorithms/quantizeByCluster.rst b/src/main/python/docs/source/api/operator/algorithms/quantizeByCluster.rst new file mode 100644 index 00000000000..1d4edf6d0d7 --- /dev/null +++ b/src/main/python/docs/source/api/operator/algorithms/quantizeByCluster.rst @@ -0,0 +1,4 @@ +quantizeByCluster +================= + +.. autofunction:: systemds.operator.algorithm.quantizeByCluster \ No newline at end of file diff --git a/src/main/python/docs/source/api/operator/algorithms/raGroupby.rst b/src/main/python/docs/source/api/operator/algorithms/raGroupby.rst new file mode 100644 index 00000000000..bdf7bc92942 --- /dev/null +++ b/src/main/python/docs/source/api/operator/algorithms/raGroupby.rst @@ -0,0 +1,4 @@ +raGroupby +========= + +.. autofunction:: systemds.operator.algorithm.raGroupby \ No newline at end of file diff --git a/src/main/python/docs/source/api/operator/algorithms/raJoin.rst b/src/main/python/docs/source/api/operator/algorithms/raJoin.rst new file mode 100644 index 00000000000..f26d443fdb3 --- /dev/null +++ b/src/main/python/docs/source/api/operator/algorithms/raJoin.rst @@ -0,0 +1,4 @@ +raJoin +====== + +.. autofunction:: systemds.operator.algorithm.raJoin \ No newline at end of file diff --git a/src/main/python/docs/source/api/operator/algorithms/raSelection.rst b/src/main/python/docs/source/api/operator/algorithms/raSelection.rst new file mode 100644 index 00000000000..7ce2f3805ee --- /dev/null +++ b/src/main/python/docs/source/api/operator/algorithms/raSelection.rst @@ -0,0 +1,4 @@ +raSelection +=========== + +.. autofunction:: systemds.operator.algorithm.raSelection \ No newline at end of file diff --git a/src/main/python/docs/source/api/operator/algorithms/randomForest.rst b/src/main/python/docs/source/api/operator/algorithms/randomForest.rst new file mode 100644 index 00000000000..e38646fabc6 --- /dev/null +++ b/src/main/python/docs/source/api/operator/algorithms/randomForest.rst @@ -0,0 +1,4 @@ +randomForest +============ + +.. autofunction:: systemds.operator.algorithm.randomForest \ No newline at end of file diff --git a/src/main/python/docs/source/api/operator/algorithms/randomForestPredict.rst b/src/main/python/docs/source/api/operator/algorithms/randomForestPredict.rst new file mode 100644 index 00000000000..e49e104980c --- /dev/null +++ b/src/main/python/docs/source/api/operator/algorithms/randomForestPredict.rst @@ -0,0 +1,4 @@ +randomForestPredict +=================== + +.. autofunction:: systemds.operator.algorithm.randomForestPredict \ No newline at end of file diff --git a/src/main/python/docs/source/api/operator/algorithms/rmse.rst b/src/main/python/docs/source/api/operator/algorithms/rmse.rst new file mode 100644 index 00000000000..9a4772dcbd1 --- /dev/null +++ b/src/main/python/docs/source/api/operator/algorithms/rmse.rst @@ -0,0 +1,4 @@ +rmse +==== + +.. autofunction:: systemds.operator.algorithm.rmse \ No newline at end of file diff --git a/src/main/python/docs/source/api/operator/algorithms/scale.rst b/src/main/python/docs/source/api/operator/algorithms/scale.rst new file mode 100644 index 00000000000..48a3d09415d --- /dev/null +++ b/src/main/python/docs/source/api/operator/algorithms/scale.rst @@ -0,0 +1,4 @@ +scale +===== + +.. autofunction:: systemds.operator.algorithm.scale \ No newline at end of file diff --git a/src/main/python/docs/source/api/operator/algorithms/scaleApply.rst b/src/main/python/docs/source/api/operator/algorithms/scaleApply.rst new file mode 100644 index 00000000000..4150e19e62f --- /dev/null +++ b/src/main/python/docs/source/api/operator/algorithms/scaleApply.rst @@ -0,0 +1,4 @@ +scaleApply +========== + +.. autofunction:: systemds.operator.algorithm.scaleApply \ No newline at end of file diff --git a/src/main/python/docs/source/api/operator/algorithms/scaleMinMax.rst b/src/main/python/docs/source/api/operator/algorithms/scaleMinMax.rst new file mode 100644 index 00000000000..a8a18237ca1 --- /dev/null +++ b/src/main/python/docs/source/api/operator/algorithms/scaleMinMax.rst @@ -0,0 +1,4 @@ +scaleMinMax +=========== + +.. autofunction:: systemds.operator.algorithm.scaleMinMax \ No newline at end of file diff --git a/src/main/python/docs/source/api/operator/algorithms/selectByVarThresh.rst b/src/main/python/docs/source/api/operator/algorithms/selectByVarThresh.rst new file mode 100644 index 00000000000..f84ccd06786 --- /dev/null +++ b/src/main/python/docs/source/api/operator/algorithms/selectByVarThresh.rst @@ -0,0 +1,4 @@ +selectByVarThresh +================= + +.. autofunction:: systemds.operator.algorithm.selectByVarThresh \ No newline at end of file diff --git a/src/main/python/docs/source/api/operator/algorithms/ses.rst b/src/main/python/docs/source/api/operator/algorithms/ses.rst new file mode 100644 index 00000000000..6e797bbfe7c --- /dev/null +++ b/src/main/python/docs/source/api/operator/algorithms/ses.rst @@ -0,0 +1,4 @@ +ses +=== + +.. autofunction:: systemds.operator.algorithm.ses \ No newline at end of file diff --git a/src/main/python/docs/source/api/operator/algorithms/setdiff.rst b/src/main/python/docs/source/api/operator/algorithms/setdiff.rst new file mode 100644 index 00000000000..abb64d899a2 --- /dev/null +++ b/src/main/python/docs/source/api/operator/algorithms/setdiff.rst @@ -0,0 +1,4 @@ +setdiff +======= + +.. autofunction:: systemds.operator.algorithm.setdiff \ No newline at end of file diff --git a/src/main/python/docs/source/api/operator/algorithms/shapExplainer.rst b/src/main/python/docs/source/api/operator/algorithms/shapExplainer.rst new file mode 100644 index 00000000000..b376f4a5f8d --- /dev/null +++ b/src/main/python/docs/source/api/operator/algorithms/shapExplainer.rst @@ -0,0 +1,4 @@ +shapExplainer +============= + +.. autofunction:: systemds.operator.algorithm.shapExplainer \ No newline at end of file diff --git a/src/main/python/docs/source/api/operator/algorithms/sherlock.rst b/src/main/python/docs/source/api/operator/algorithms/sherlock.rst new file mode 100644 index 00000000000..bec99c1dcc3 --- /dev/null +++ b/src/main/python/docs/source/api/operator/algorithms/sherlock.rst @@ -0,0 +1,4 @@ +sherlock +======== + +.. autofunction:: systemds.operator.algorithm.sherlock \ No newline at end of file diff --git a/src/main/python/docs/source/api/operator/algorithms/sherlockPredict.rst b/src/main/python/docs/source/api/operator/algorithms/sherlockPredict.rst new file mode 100644 index 00000000000..bb3c82835da --- /dev/null +++ b/src/main/python/docs/source/api/operator/algorithms/sherlockPredict.rst @@ -0,0 +1,4 @@ +sherlockPredict +=============== + +.. autofunction:: systemds.operator.algorithm.sherlockPredict \ No newline at end of file diff --git a/src/main/python/docs/source/api/operator/algorithms/shortestPath.rst b/src/main/python/docs/source/api/operator/algorithms/shortestPath.rst new file mode 100644 index 00000000000..8bba99bf797 --- /dev/null +++ b/src/main/python/docs/source/api/operator/algorithms/shortestPath.rst @@ -0,0 +1,4 @@ +shortestPath +============ + +.. autofunction:: systemds.operator.algorithm.shortestPath \ No newline at end of file diff --git a/src/main/python/docs/source/api/operator/algorithms/sigmoid.rst b/src/main/python/docs/source/api/operator/algorithms/sigmoid.rst new file mode 100644 index 00000000000..b2e613532f1 --- /dev/null +++ b/src/main/python/docs/source/api/operator/algorithms/sigmoid.rst @@ -0,0 +1,4 @@ +sigmoid +======= + +.. autofunction:: systemds.operator.algorithm.sigmoid \ No newline at end of file diff --git a/src/main/python/docs/source/api/operator/algorithms/skewness.rst b/src/main/python/docs/source/api/operator/algorithms/skewness.rst new file mode 100644 index 00000000000..7d4d80df498 --- /dev/null +++ b/src/main/python/docs/source/api/operator/algorithms/skewness.rst @@ -0,0 +1,4 @@ +skewness +======== + +.. autofunction:: systemds.operator.algorithm.skewness \ No newline at end of file diff --git a/src/main/python/docs/source/api/operator/algorithms/sliceLine.rst b/src/main/python/docs/source/api/operator/algorithms/sliceLine.rst new file mode 100644 index 00000000000..df6362d9f71 --- /dev/null +++ b/src/main/python/docs/source/api/operator/algorithms/sliceLine.rst @@ -0,0 +1,4 @@ +sliceLine +========= + +.. autofunction:: systemds.operator.algorithm.sliceLine \ No newline at end of file diff --git a/src/main/python/docs/source/api/operator/algorithms/sliceLineDebug.rst b/src/main/python/docs/source/api/operator/algorithms/sliceLineDebug.rst new file mode 100644 index 00000000000..0f5a3f19cfb --- /dev/null +++ b/src/main/python/docs/source/api/operator/algorithms/sliceLineDebug.rst @@ -0,0 +1,4 @@ +sliceLineDebug +============== + +.. autofunction:: systemds.operator.algorithm.sliceLineDebug \ No newline at end of file diff --git a/src/main/python/docs/source/api/operator/algorithms/sliceLineExtract.rst b/src/main/python/docs/source/api/operator/algorithms/sliceLineExtract.rst new file mode 100644 index 00000000000..94b0bbc0113 --- /dev/null +++ b/src/main/python/docs/source/api/operator/algorithms/sliceLineExtract.rst @@ -0,0 +1,4 @@ +sliceLineExtract +================ + +.. autofunction:: systemds.operator.algorithm.sliceLineExtract \ No newline at end of file diff --git a/src/main/python/docs/source/api/operator/algorithms/slicefinder.rst b/src/main/python/docs/source/api/operator/algorithms/slicefinder.rst new file mode 100644 index 00000000000..85c7fced018 --- /dev/null +++ b/src/main/python/docs/source/api/operator/algorithms/slicefinder.rst @@ -0,0 +1,4 @@ +slicefinder +=========== + +.. autofunction:: systemds.operator.algorithm.slicefinder \ No newline at end of file diff --git a/src/main/python/docs/source/api/operator/algorithms/smape.rst b/src/main/python/docs/source/api/operator/algorithms/smape.rst new file mode 100644 index 00000000000..6aadbd94205 --- /dev/null +++ b/src/main/python/docs/source/api/operator/algorithms/smape.rst @@ -0,0 +1,4 @@ +smape +===== + +.. autofunction:: systemds.operator.algorithm.smape \ No newline at end of file diff --git a/src/main/python/docs/source/api/operator/algorithms/smote.rst b/src/main/python/docs/source/api/operator/algorithms/smote.rst new file mode 100644 index 00000000000..729d9529d39 --- /dev/null +++ b/src/main/python/docs/source/api/operator/algorithms/smote.rst @@ -0,0 +1,4 @@ +smote +===== + +.. autofunction:: systemds.operator.algorithm.smote \ No newline at end of file diff --git a/src/main/python/docs/source/api/operator/algorithms/softmax.rst b/src/main/python/docs/source/api/operator/algorithms/softmax.rst new file mode 100644 index 00000000000..3d142077f96 --- /dev/null +++ b/src/main/python/docs/source/api/operator/algorithms/softmax.rst @@ -0,0 +1,4 @@ +softmax +======= + +.. autofunction:: systemds.operator.algorithm.softmax \ No newline at end of file diff --git a/src/main/python/docs/source/api/operator/algorithms/split.rst b/src/main/python/docs/source/api/operator/algorithms/split.rst new file mode 100644 index 00000000000..0c46d928195 --- /dev/null +++ b/src/main/python/docs/source/api/operator/algorithms/split.rst @@ -0,0 +1,4 @@ +split +===== + +.. autofunction:: systemds.operator.algorithm.split \ No newline at end of file diff --git a/src/main/python/docs/source/api/operator/algorithms/splitBalanced.rst b/src/main/python/docs/source/api/operator/algorithms/splitBalanced.rst new file mode 100644 index 00000000000..f9c6640a02f --- /dev/null +++ b/src/main/python/docs/source/api/operator/algorithms/splitBalanced.rst @@ -0,0 +1,4 @@ +splitBalanced +============= + +.. autofunction:: systemds.operator.algorithm.splitBalanced \ No newline at end of file diff --git a/src/main/python/docs/source/api/operator/algorithms/sqrtMatrix.rst b/src/main/python/docs/source/api/operator/algorithms/sqrtMatrix.rst new file mode 100644 index 00000000000..f59bbb3b4a5 --- /dev/null +++ b/src/main/python/docs/source/api/operator/algorithms/sqrtMatrix.rst @@ -0,0 +1,4 @@ +sqrtMatrix +========== + +.. autofunction:: systemds.operator.algorithm.sqrtMatrix \ No newline at end of file diff --git a/src/main/python/docs/source/api/operator/algorithms/stableMarriage.rst b/src/main/python/docs/source/api/operator/algorithms/stableMarriage.rst new file mode 100644 index 00000000000..ddddbae2b26 --- /dev/null +++ b/src/main/python/docs/source/api/operator/algorithms/stableMarriage.rst @@ -0,0 +1,4 @@ +stableMarriage +============== + +.. autofunction:: systemds.operator.algorithm.stableMarriage \ No newline at end of file diff --git a/src/main/python/docs/source/api/operator/algorithms/statsNA.rst b/src/main/python/docs/source/api/operator/algorithms/statsNA.rst new file mode 100644 index 00000000000..22724284872 --- /dev/null +++ b/src/main/python/docs/source/api/operator/algorithms/statsNA.rst @@ -0,0 +1,4 @@ +statsNA +======= + +.. autofunction:: systemds.operator.algorithm.statsNA \ No newline at end of file diff --git a/src/main/python/docs/source/api/operator/algorithms/steplm.rst b/src/main/python/docs/source/api/operator/algorithms/steplm.rst new file mode 100644 index 00000000000..35cb47cff1b --- /dev/null +++ b/src/main/python/docs/source/api/operator/algorithms/steplm.rst @@ -0,0 +1,4 @@ +steplm +====== + +.. autofunction:: systemds.operator.algorithm.steplm \ No newline at end of file diff --git a/src/main/python/docs/source/api/operator/algorithms/stratstats.rst b/src/main/python/docs/source/api/operator/algorithms/stratstats.rst new file mode 100644 index 00000000000..65ec9dd1b42 --- /dev/null +++ b/src/main/python/docs/source/api/operator/algorithms/stratstats.rst @@ -0,0 +1,4 @@ +stratstats +========== + +.. autofunction:: systemds.operator.algorithm.stratstats \ No newline at end of file diff --git a/src/main/python/docs/source/api/operator/algorithms/symmetricDifference.rst b/src/main/python/docs/source/api/operator/algorithms/symmetricDifference.rst new file mode 100644 index 00000000000..494e632383f --- /dev/null +++ b/src/main/python/docs/source/api/operator/algorithms/symmetricDifference.rst @@ -0,0 +1,4 @@ +symmetricDifference +=================== + +.. autofunction:: systemds.operator.algorithm.symmetricDifference \ No newline at end of file diff --git a/src/main/python/docs/source/api/operator/algorithms/tSNE.rst b/src/main/python/docs/source/api/operator/algorithms/tSNE.rst new file mode 100644 index 00000000000..1021a052198 --- /dev/null +++ b/src/main/python/docs/source/api/operator/algorithms/tSNE.rst @@ -0,0 +1,4 @@ +tSNE +==== + +.. autofunction:: systemds.operator.algorithm.tSNE \ No newline at end of file diff --git a/src/main/python/docs/source/api/operator/algorithms/toOneHot.rst b/src/main/python/docs/source/api/operator/algorithms/toOneHot.rst new file mode 100644 index 00000000000..8c220b3d866 --- /dev/null +++ b/src/main/python/docs/source/api/operator/algorithms/toOneHot.rst @@ -0,0 +1,4 @@ +toOneHot +======== + +.. autofunction:: systemds.operator.algorithm.toOneHot \ No newline at end of file diff --git a/src/main/python/docs/source/api/operator/algorithms/tomeklink.rst b/src/main/python/docs/source/api/operator/algorithms/tomeklink.rst new file mode 100644 index 00000000000..4fd0bd17f78 --- /dev/null +++ b/src/main/python/docs/source/api/operator/algorithms/tomeklink.rst @@ -0,0 +1,4 @@ +tomeklink +========= + +.. autofunction:: systemds.operator.algorithm.tomeklink \ No newline at end of file diff --git a/src/main/python/docs/source/api/operator/algorithms/topk_cleaning.rst b/src/main/python/docs/source/api/operator/algorithms/topk_cleaning.rst new file mode 100644 index 00000000000..b57c24f3a47 --- /dev/null +++ b/src/main/python/docs/source/api/operator/algorithms/topk_cleaning.rst @@ -0,0 +1,4 @@ +topk_cleaning +============= + +.. autofunction:: systemds.operator.algorithm.topk_cleaning \ No newline at end of file diff --git a/src/main/python/docs/source/api/operator/algorithms/underSampling.rst b/src/main/python/docs/source/api/operator/algorithms/underSampling.rst new file mode 100644 index 00000000000..0b7b146b0af --- /dev/null +++ b/src/main/python/docs/source/api/operator/algorithms/underSampling.rst @@ -0,0 +1,4 @@ +underSampling +============= + +.. autofunction:: systemds.operator.algorithm.underSampling \ No newline at end of file diff --git a/src/main/python/docs/source/api/operator/algorithms/union.rst b/src/main/python/docs/source/api/operator/algorithms/union.rst new file mode 100644 index 00000000000..24da31088ff --- /dev/null +++ b/src/main/python/docs/source/api/operator/algorithms/union.rst @@ -0,0 +1,4 @@ +union +===== + +.. autofunction:: systemds.operator.algorithm.union \ No newline at end of file diff --git a/src/main/python/docs/source/api/operator/algorithms/univar.rst b/src/main/python/docs/source/api/operator/algorithms/univar.rst new file mode 100644 index 00000000000..c4df0e4e758 --- /dev/null +++ b/src/main/python/docs/source/api/operator/algorithms/univar.rst @@ -0,0 +1,4 @@ +univar +====== + +.. autofunction:: systemds.operator.algorithm.univar \ No newline at end of file diff --git a/src/main/python/docs/source/api/operator/algorithms/vectorToCsv.rst b/src/main/python/docs/source/api/operator/algorithms/vectorToCsv.rst new file mode 100644 index 00000000000..c1c9e2cbc09 --- /dev/null +++ b/src/main/python/docs/source/api/operator/algorithms/vectorToCsv.rst @@ -0,0 +1,4 @@ +vectorToCsv +=========== + +.. autofunction:: systemds.operator.algorithm.vectorToCsv \ No newline at end of file diff --git a/src/main/python/docs/source/api/operator/algorithms/wer.rst b/src/main/python/docs/source/api/operator/algorithms/wer.rst new file mode 100644 index 00000000000..9ea55cbd1fe --- /dev/null +++ b/src/main/python/docs/source/api/operator/algorithms/wer.rst @@ -0,0 +1,4 @@ +wer +=== + +.. autofunction:: systemds.operator.algorithm.wer \ No newline at end of file diff --git a/src/main/python/docs/source/api/operator/algorithms/winsorize.rst b/src/main/python/docs/source/api/operator/algorithms/winsorize.rst new file mode 100644 index 00000000000..43c20fbc652 --- /dev/null +++ b/src/main/python/docs/source/api/operator/algorithms/winsorize.rst @@ -0,0 +1,4 @@ +winsorize +========= + +.. autofunction:: systemds.operator.algorithm.winsorize \ No newline at end of file diff --git a/src/main/python/docs/source/api/operator/algorithms/winsorizeApply.rst b/src/main/python/docs/source/api/operator/algorithms/winsorizeApply.rst new file mode 100644 index 00000000000..f4b3c9e983c --- /dev/null +++ b/src/main/python/docs/source/api/operator/algorithms/winsorizeApply.rst @@ -0,0 +1,4 @@ +winsorizeApply +============== + +.. autofunction:: systemds.operator.algorithm.winsorizeApply \ No newline at end of file diff --git a/src/main/python/docs/source/api/operator/algorithms/xdummy1.rst b/src/main/python/docs/source/api/operator/algorithms/xdummy1.rst new file mode 100644 index 00000000000..20e4eed2b36 --- /dev/null +++ b/src/main/python/docs/source/api/operator/algorithms/xdummy1.rst @@ -0,0 +1,4 @@ +xdummy1 +======= + +.. autofunction:: systemds.operator.algorithm.xdummy1 \ No newline at end of file diff --git a/src/main/python/docs/source/api/operator/algorithms/xdummy2.rst b/src/main/python/docs/source/api/operator/algorithms/xdummy2.rst new file mode 100644 index 00000000000..9254147e691 --- /dev/null +++ b/src/main/python/docs/source/api/operator/algorithms/xdummy2.rst @@ -0,0 +1,4 @@ +xdummy2 +======= + +.. autofunction:: systemds.operator.algorithm.xdummy2 \ No newline at end of file diff --git a/src/main/python/docs/source/api/operator/algorithms/xgboost.rst b/src/main/python/docs/source/api/operator/algorithms/xgboost.rst new file mode 100644 index 00000000000..506771aa6d8 --- /dev/null +++ b/src/main/python/docs/source/api/operator/algorithms/xgboost.rst @@ -0,0 +1,4 @@ +xgboost +======= + +.. autofunction:: systemds.operator.algorithm.xgboost \ No newline at end of file diff --git a/src/main/python/docs/source/api/operator/algorithms/xgboostPredictClassification.rst b/src/main/python/docs/source/api/operator/algorithms/xgboostPredictClassification.rst new file mode 100644 index 00000000000..b65b35c8ebd --- /dev/null +++ b/src/main/python/docs/source/api/operator/algorithms/xgboostPredictClassification.rst @@ -0,0 +1,4 @@ +xgboostPredictClassification +============================ + +.. autofunction:: systemds.operator.algorithm.xgboostPredictClassification \ No newline at end of file diff --git a/src/main/python/docs/source/api/operator/algorithms/xgboostPredictRegression.rst b/src/main/python/docs/source/api/operator/algorithms/xgboostPredictRegression.rst new file mode 100644 index 00000000000..1e331011902 --- /dev/null +++ b/src/main/python/docs/source/api/operator/algorithms/xgboostPredictRegression.rst @@ -0,0 +1,4 @@ +xgboostPredictRegression +======================== + +.. autofunction:: systemds.operator.algorithm.xgboostPredictRegression \ No newline at end of file diff --git a/src/main/python/generator/dml_parser.py b/src/main/python/generator/dml_parser.py index 8e835e96a12..78d91f5f429 100644 --- a/src/main/python/generator/dml_parser.py +++ b/src/main/python/generator/dml_parser.py @@ -23,7 +23,7 @@ import json import os import re - +import textwrap class FunctionParser(object): header_input_pattern = r"^[ \t\n]*[#]+[ \t\n]*input[ \t\n\w:;.,#]*[\s#\-]*[#]+[\w\s\d:,.()\" \t\n\-]*[\s#\-]*$" @@ -196,10 +196,30 @@ def parse_header(self, path: str): input_parameters = self.parse_input_output_string(h_input) output_parameters = self.parse_input_output_string(h_output) + code_block = None + with open(path, 'r') as f: + content = f.read() + pat = re.compile( + r""" + ^\s*\#\s*\.\.\s*code-block::\s*python # .. code-block:: python + (?:\s*\#.*\n)*? # optional adornments / blank lines + (.*?) # ← capture the actual example + (?= # stop just *before* … + (?:\s*\#\s*\n){2} # … two consecutive “blank” # lines + ) + """, + re.MULTILINE | re.DOTALL | re.VERBOSE, + ) + match = pat.search(content) + if match: + raw_block = match.group(1) + code_lines = [line.lstrip("#") for line in raw_block.splitlines()] # Remove leading # + code_block = textwrap.dedent("\n".join(code_lines)) data = {'description': description, 'parameters': input_parameters, - 'return_values': output_parameters} + 'return_values': output_parameters, + 'code_block': code_block} return data def parse_input_output_string(self, data: str): diff --git a/src/main/python/generator/generator.py b/src/main/python/generator/generator.py index b124feff190..abf80ccf04b 100644 --- a/src/main/python/generator/generator.py +++ b/src/main/python/generator/generator.py @@ -35,6 +35,10 @@ class PythonAPIFileGenerator(object): target_path = os.path.join(os.path.dirname(os.path.dirname( __file__)), 'systemds', 'operator', 'algorithm', 'builtin') + test_path = os.path.join(os.path.dirname(os.path.dirname( + __file__)), 'tests', 'auto_tests') + rst_path = os.path.join(os.path.dirname(os.path.dirname( + __file__)), 'docs', 'source', 'api', 'operator', 'algorithms') licence_path = os.path.join('resources', 'template_python_script_license') template_path = os.path.join('resources', 'template_python_script_imports') @@ -55,6 +59,8 @@ def __init__(self, source_path: str, extension: str = 'py'): self.extension = '.{extension}'.format(extension=extension) os.makedirs(self.__class__.target_path, exist_ok=True) + os.makedirs(self.__class__.test_path, exist_ok=True) + os.makedirs(self.__class__.rst_path, exist_ok=True) self.function_names = list() for name in manually_added_algorithm_builtins: # only add files which actually exist, to avoid breaking @@ -89,13 +95,61 @@ def generate_file(self, filename: str, file_content: str, dml_file: str): with open(target_file, "w") as new_script: new_script.write(self.licence) new_script.write(self.generated_by) - new_script.write((self.generated_from + dml_file.replace("\\", "/") + "\n").replace( - "../", "").replace("src/main/python/generator/", "")) + relative_path = os.path.relpath(dml_file, start=self.source_path) + new_script.write(f"{self.generated_from}scripts/builtin/{relative_path}\n") new_script.write(self.imports) new_script.write(file_content) self.function_names.append(filename) + def generate_test_file(self, function_name: str, code_block: str = None): + """ + Generates a test file for the given function + """ + target_file = os.path.join(self.test_path, f"test_{function_name}") + self.extension + with open(target_file, "w") as test_script: + test_script.write(self.licence) + test_script.write(self.generated_by) + test_script.write("import unittest, contextlib, io\n") + test_script.write(f"from systemds.context import SystemDSContext\n") + test_script.write(f"from systemds.operator.algorithm.builtin.{function_name} import {function_name}\n\n\n") + + test_script.write(f"class Test{function_name.upper()}(unittest.TestCase):\n") + test_script.write(f" def test_{function_name}(self):\n") + if code_block: + test_script.write(" # Example test case provided in python the code block\n") + test_script.write(" buf = io.StringIO()\n") + test_script.write(" with contextlib.redirect_stdout(buf):\n") + + expected ="" + for raw_line in code_block.splitlines(keepends=True): # keepends=True → ‘\n’ is preserved + stripped = raw_line.lstrip() + if stripped.startswith((">>>", "...")): + code_line = stripped[4:] + if code_line.strip(): + test_script.write(f" {code_line}") + else: + test_script.write("\n") + else: + expected += raw_line + expected = expected.lstrip("\n") + test_script.write(f'\n expected="""{expected}"""\n') + test_script.write(f" self.assertEqual(buf.getvalue().strip(), expected)\n") + + test_script.write("\nif __name__ == '__main__':\n") + test_script.write(" unittest.main()\n") + + def generate_rst_file(self, function_name: str): + """ + Generates an rst file for the given function + """ + target_file = os.path.join(self.rst_path, f"{function_name}") + ".rst" + with open(target_file, "w") as rst_script: + # rst_script.write(self.licence) + rst_script.write(function_name + "\n") + rst_script.write("=" * len(function_name) + "\n\n") + rst_script.write(f".. autofunction:: systemds.operator.algorithm.{function_name}") + def generate_init_file(self): with open(self.init_path, "w") as init_file: init_file.write(self.licence) @@ -390,6 +444,8 @@ def format_exception(e): try: header_data = f_parser.parse_header(dml_file) data = f_parser.parse_function(dml_file) + if not data: + continue f_parser.check_parameters(header_data, data) doc_generator.generate_documentation(header_data, data) @@ -404,5 +460,13 @@ def format_exception(e): continue file_generator.generate_file( data["function_name"], script_content, dml_file) + # TODO: multiple code blocks -> multiple test_files + test_example = header_data.get("code_block", None) + if test_example: + # TODO: dml test file + # TODO: logs should have funcs without test cases + # TODO: imports should be exlicitly added to the examples + file_generator.generate_test_file(data["function_name"], test_example) + file_generator.generate_rst_file(data["function_name"]) file_generator.function_names.sort() file_generator.generate_init_file() From 931e3de1de44b5c7e3289cff625640d7a39f970d Mon Sep 17 00:00:00 2001 From: anuunchin <88698977+anuunchin@users.noreply.github.com> Date: Mon, 14 Jul 2025 21:01:18 +0200 Subject: [PATCH 3/4] example code blocks --- scripts/builtin/dist.dml | 15 ++++ scripts/builtin/img_brightness.dml | 19 ++++- scripts/builtin/img_brightness_linearized.dml | 19 ++++- scripts/builtin/img_crop.dml | 19 ++++- scripts/builtin/img_crop_linearized.dml | 19 ++++- scripts/builtin/img_cutout.dml | 20 ++++++ scripts/builtin/img_cutout_linearized.dml | 23 +++++- scripts/builtin/img_invert.dml | 21 +++++- scripts/builtin/img_invert_linearized.dml | 21 +++++- scripts/builtin/img_mirror.dml | 23 +++++- scripts/builtin/img_mirror_linearized.dml | 22 +++++- scripts/builtin/img_posterize.dml | 23 +++++- scripts/builtin/img_posterize_linearized.dml | 23 +++++- scripts/builtin/img_rotate.dml | 19 +++++ scripts/builtin/img_rotate_linearized.dml | 25 ++++++- scripts/builtin/img_sample_pairing.dml | 28 +++++++- .../builtin/img_sample_pairing_linearized.dml | 26 ++++++- scripts/builtin/img_shear.dml | 19 +++++ scripts/builtin/img_shear_linearized.dml | 23 +++++- scripts/builtin/img_transform.dml | 19 +++++ scripts/builtin/img_transform_linearized.dml | 25 ++++++- scripts/builtin/img_translate.dml | 19 +++++ scripts/builtin/img_translate_linearized.dml | 22 +++++- scripts/builtin/lm.dml | 30 ++++++++ scripts/builtin/normalize.dml | 14 ++++ scripts/builtin/randomForest.dml | 41 +++++++++++ scripts/builtin/randomForestPredict.dml | 43 +++++++++++ scripts/builtin/toOneHot.dml | 16 +++++ src/main/python/generator/dml_parser.py | 11 ++- src/main/python/generator/generator.py | 2 - .../operator/algorithm/builtin/dist.py | 15 ++++ .../algorithm/builtin/img_brightness.py | 19 ++++- .../builtin/img_brightness_linearized.py | 19 ++++- .../operator/algorithm/builtin/img_crop.py | 19 ++++- .../algorithm/builtin/img_crop_linearized.py | 19 ++++- .../operator/algorithm/builtin/img_cutout.py | 20 ++++++ .../builtin/img_cutout_linearized.py | 21 +++++- .../operator/algorithm/builtin/img_invert.py | 21 +++++- .../builtin/img_invert_linearized.py | 21 +++++- .../operator/algorithm/builtin/img_mirror.py | 23 +++++- .../builtin/img_mirror_linearized.py | 22 +++++- .../algorithm/builtin/img_posterize.py | 23 +++++- .../builtin/img_posterize_linearized.py | 23 +++++- .../operator/algorithm/builtin/img_rotate.py | 19 +++++ .../builtin/img_rotate_linearized.py | 23 +++++- .../algorithm/builtin/img_sample_pairing.py | 28 +++++++- .../builtin/img_sample_pairing_linearized.py | 26 ++++++- .../operator/algorithm/builtin/img_shear.py | 19 +++++ .../algorithm/builtin/img_shear_linearized.py | 23 +++++- .../algorithm/builtin/img_transform.py | 19 +++++ .../builtin/img_transform_linearized.py | 23 +++++- .../algorithm/builtin/img_translate.py | 19 +++++ .../builtin/img_translate_linearized.py | 22 +++++- .../systemds/operator/algorithm/builtin/lm.py | 30 ++++++++ .../operator/algorithm/builtin/normalize.py | 14 ++++ .../algorithm/builtin/randomForest.py | 41 +++++++++++ .../algorithm/builtin/randomForestPredict.py | 43 +++++++++++ .../operator/algorithm/builtin/toOneHot.py | 16 +++++ src/main/python/tests/algorithms/test_lm.py | 65 ----------------- src/main/python/tests/auto_tests/test_dist.py | 44 ++++++++++++ .../tests/auto_tests/test_img_brightness.py | 46 ++++++++++++ .../test_img_brightness_linearized.py | 46 ++++++++++++ .../python/tests/auto_tests/test_img_crop.py | 46 ++++++++++++ .../auto_tests/test_img_crop_linearized.py | 46 ++++++++++++ .../tests/auto_tests/test_img_cutout.py | 48 +++++++++++++ .../auto_tests/test_img_cutout_linearized.py | 48 +++++++++++++ .../tests/auto_tests/test_img_invert.py | 48 +++++++++++++ .../auto_tests/test_img_invert_linearized.py | 48 +++++++++++++ .../tests/auto_tests/test_img_mirror.py | 48 +++++++++++++ .../auto_tests/test_img_mirror_linearized.py | 48 +++++++++++++ .../tests/auto_tests/test_img_posterize.py | 48 +++++++++++++ .../test_img_posterize_linearized.py | 48 +++++++++++++ .../tests/auto_tests/test_img_rotate.py | 48 +++++++++++++ .../auto_tests/test_img_rotate_linearized.py | 48 +++++++++++++ .../auto_tests/test_img_sample_pairing.py | 53 ++++++++++++++ .../test_img_sample_pairing_linearized.py | 53 ++++++++++++++ .../python/tests/auto_tests/test_img_shear.py | 48 +++++++++++++ .../auto_tests/test_img_shear_linearized.py | 48 +++++++++++++ .../tests/auto_tests/test_img_transform.py | 48 +++++++++++++ .../test_img_transform_linearized.py | 48 +++++++++++++ .../tests/auto_tests/test_img_translate.py | 48 +++++++++++++ .../test_img_translate_linearized.py | 48 +++++++++++++ src/main/python/tests/auto_tests/test_lm.py | 59 +++++++++++++++ .../python/tests/auto_tests/test_normalize.py | 43 +++++++++++ .../tests/auto_tests/test_randomForest.py | 71 +++++++++++++++++++ .../auto_tests/test_randomForestPredict.py | 71 +++++++++++++++++++ .../python/tests/auto_tests/test_toOneHot.py | 45 ++++++++++++ 87 files changed, 2617 insertions(+), 118 deletions(-) delete mode 100644 src/main/python/tests/algorithms/test_lm.py create mode 100644 src/main/python/tests/auto_tests/test_dist.py create mode 100644 src/main/python/tests/auto_tests/test_img_brightness.py create mode 100644 src/main/python/tests/auto_tests/test_img_brightness_linearized.py create mode 100644 src/main/python/tests/auto_tests/test_img_crop.py create mode 100644 src/main/python/tests/auto_tests/test_img_crop_linearized.py create mode 100644 src/main/python/tests/auto_tests/test_img_cutout.py create mode 100644 src/main/python/tests/auto_tests/test_img_cutout_linearized.py create mode 100644 src/main/python/tests/auto_tests/test_img_invert.py create mode 100644 src/main/python/tests/auto_tests/test_img_invert_linearized.py create mode 100644 src/main/python/tests/auto_tests/test_img_mirror.py create mode 100644 src/main/python/tests/auto_tests/test_img_mirror_linearized.py create mode 100644 src/main/python/tests/auto_tests/test_img_posterize.py create mode 100644 src/main/python/tests/auto_tests/test_img_posterize_linearized.py create mode 100644 src/main/python/tests/auto_tests/test_img_rotate.py create mode 100644 src/main/python/tests/auto_tests/test_img_rotate_linearized.py create mode 100644 src/main/python/tests/auto_tests/test_img_sample_pairing.py create mode 100644 src/main/python/tests/auto_tests/test_img_sample_pairing_linearized.py create mode 100644 src/main/python/tests/auto_tests/test_img_shear.py create mode 100644 src/main/python/tests/auto_tests/test_img_shear_linearized.py create mode 100644 src/main/python/tests/auto_tests/test_img_transform.py create mode 100644 src/main/python/tests/auto_tests/test_img_transform_linearized.py create mode 100644 src/main/python/tests/auto_tests/test_img_translate.py create mode 100644 src/main/python/tests/auto_tests/test_img_translate_linearized.py create mode 100644 src/main/python/tests/auto_tests/test_lm.py create mode 100644 src/main/python/tests/auto_tests/test_normalize.py create mode 100644 src/main/python/tests/auto_tests/test_randomForest.py create mode 100644 src/main/python/tests/auto_tests/test_randomForestPredict.py create mode 100644 src/main/python/tests/auto_tests/test_toOneHot.py diff --git a/scripts/builtin/dist.dml b/scripts/builtin/dist.dml index f296fd717bc..831992b192b 100644 --- a/scripts/builtin/dist.dml +++ b/scripts/builtin/dist.dml @@ -21,6 +21,21 @@ # Returns Euclidean distance matrix (distances between N n-dimensional points) # +# .. code-block:: python +# +# >>> import numpy as np +# >>> from systemds.context import SystemDSContext +# >>> from systemds.operator.algorithm import dist +# >>> +# >>> with SystemDSContext() as sds: +# ... X = sds.from_numpy(np.array([[0], [3], [4]])) +# ... out = dist(X).compute() +# ... print(out) +# [[0. 3. 4.] +# [3. 0. 1.] +# [4. 1. 0.]] +# +# # INPUT: # -------------------------------------------------------------------------------- # X Matrix to calculate the distance inside diff --git a/scripts/builtin/img_brightness.dml b/scripts/builtin/img_brightness.dml index 100ccb7588b..dda7eba23dc 100644 --- a/scripts/builtin/img_brightness.dml +++ b/scripts/builtin/img_brightness.dml @@ -21,9 +21,26 @@ # The img_brightness-function is an image data augmentation function. It changes the brightness of the image. # +# .. code-block:: python +# +# >>> import numpy as np +# >>> from systemds.context import SystemDSContext +# >>> from systemds.operator.algorithm import img_brightness +# >>> +# >>> with SystemDSContext() as sds: +# ... img = sds.from_numpy( +# ... np.array([[ 50, 100, +# ... 150, 200 ]], dtype=np.float32) +# ... ) +# ... result_img = img_brightness(img, 30.0, 150).compute() +# ... print(result_img.reshape(2, 2)) +# [[ 80. 130.] +# [150. 150.]] +# +# # INPUT: # ----------------------------------------------------------------------------------------- -# img_in Input matrix/image +# img_in Input image as 2D matrix with top left corner at [1, 1] # value The amount of brightness to be changed for the image # channel_max Maximum value of the brightness of the image # ----------------------------------------------------------------------------------------- diff --git a/scripts/builtin/img_brightness_linearized.dml b/scripts/builtin/img_brightness_linearized.dml index 8c5e72d13f9..061b4597c28 100644 --- a/scripts/builtin/img_brightness_linearized.dml +++ b/scripts/builtin/img_brightness_linearized.dml @@ -21,9 +21,26 @@ # The img_brightness_linearized-function is an image data augmentation function. It changes the brightness of one or multiple images. # +# .. code-block:: python +# +# >>> import numpy as np +# >>> from systemds.context import SystemDSContext +# >>> from systemds.operator.algorithm import img_brightness_linearized +# >>> +# >>> with SystemDSContext() as sds: +# ... img = sds.from_numpy( +# ... np.array([[ 50, 100, +# ... 150, 200 ]], dtype=np.float32) +# ... ) +# ... result_img = img_brightness_linearized(img, 30.0, 255).compute() +# ... print(result_img.reshape(2, 2)) +# [[ 80. 130.] +# [180. 230.]] +# +# # INPUT: # ----------------------------------------------------------------------------------------- -# img_in Input matrix/image (can represent multiple images every row of the matrix represents a linearized image) +# img_in Input images as linearized 2D matrix with top left corner at [1, 1] (every row represents a linearized matrix/image) # value The amount of brightness to be changed for the image # channel_max Maximum value of the brightness of the image # ----------------------------------------------------------------------------------------- diff --git a/scripts/builtin/img_crop.dml b/scripts/builtin/img_crop.dml index e85301f8bb6..be183347b35 100644 --- a/scripts/builtin/img_crop.dml +++ b/scripts/builtin/img_crop.dml @@ -21,9 +21,26 @@ # The img_crop-function is an image data augmentation function. It cuts out a subregion of an image. # +# .. code-block:: python +# +# >>> import numpy as np +# >>> from systemds.context import SystemDSContext +# >>> from systemds.operator.algorithm import img_crop +# >>> +# >>> with SystemDSContext() as sds: +# ... img = sds.from_numpy( +# ... np.array([[ 50., 100., 150.], +# ... [150., 200., 250.], +# ... [250., 200., 200.]], dtype=np.float32) +# ... ) +# ... result_img = img_crop(img, 1, 1, 1, 1).compute() +# ... print(result_img) +# [[200.]] +# +# # INPUT: # ---------------------------------------------------------------------------------------- -# img_in Input matrix/image +# img_in Input image as 2D matrix with top left corner at [1, 1] # w The width of the subregion required # h The height of the subregion required # x_offset The horizontal coordinate in the image to begin the crop operation diff --git a/scripts/builtin/img_crop_linearized.dml b/scripts/builtin/img_crop_linearized.dml index b2c2c03fd18..c79da14c141 100644 --- a/scripts/builtin/img_crop_linearized.dml +++ b/scripts/builtin/img_crop_linearized.dml @@ -21,9 +21,26 @@ # The img_crop_linearized cuts out a rectangular section of multiple linearized images. # +# .. code-block:: python +# +# >>> import numpy as np +# >>> from systemds.context import SystemDSContext +# >>> from systemds.operator.algorithm import img_crop_linearized +# >>> +# >>> with SystemDSContext() as sds: +# ... img = sds.from_numpy( +# ... np.array([[ 50., 100., 150., +# ... 150., 200., 250., +# ... 250., 200., 200. ]], dtype=np.float32) +# ... ) +# ... result_img = img_crop_linearized(img, 1, 1, 1, 1, 3, 3).compute() +# ... print(result_img) +# [[200.]] +# +# # INPUT: # ---------------------------------------------------------------------------------------- -# img_in Linearized input images as 2D matrix +# img_in Input images as linearized 2D matrix with top left corner at [1, 1] (every row represents a linearized matrix/image) # w The width of the subregion required # h The height of the subregion required # x_offset The horizontal offset for the center of the crop region diff --git a/scripts/builtin/img_cutout.dml b/scripts/builtin/img_cutout.dml index cd3f432cd0f..efe194045fc 100644 --- a/scripts/builtin/img_cutout.dml +++ b/scripts/builtin/img_cutout.dml @@ -21,6 +21,26 @@ # Image Cutout function replaces a rectangular section of an image with a constant value. # +# +# .. code-block:: python +# +# >>> import numpy as np +# >>> from systemds.context import SystemDSContext +# >>> from systemds.operator.algorithm import img_cutout +# >>> +# >>> with SystemDSContext() as sds: +# ... img = sds.from_numpy( +# ... np.array([[ 50., 100., 150.], +# ... [150., 200., 250.], +# ... [250., 200., 200.]], dtype=np.float32) +# ... ) +# ... result_img = img_cutout(img, 2, 2, 1, 1, 49.).compute() +# ... print(result_img) +# [[ 50. 100. 150.] +# [150. 49. 250.] +# [250. 200. 200.]] +# +# # INPUT: # --------------------------------------------------------------------------------------------- # img_in Input image as 2D matrix with top left corner at [1, 1] diff --git a/scripts/builtin/img_cutout_linearized.dml b/scripts/builtin/img_cutout_linearized.dml index cb923e31ba4..9e10908904c 100644 --- a/scripts/builtin/img_cutout_linearized.dml +++ b/scripts/builtin/img_cutout_linearized.dml @@ -21,14 +21,33 @@ # Image Cutout function replaces a rectangular section of an image with a constant value. # +# .. code-block:: python +# +# >>> import numpy as np +# >>> from systemds.context import SystemDSContext +# >>> from systemds.operator.algorithm import img_cutout_linearized +# >>> +# >>> with SystemDSContext() as sds: +# ... img = sds.from_numpy( +# ... np.array([[ 50., 100., 150., +# ... 150., 200., 250., +# ... 250., 200., 200. ]], dtype=np.float32) +# ... ) +# ... result_img = img_cutout_linearized(img, 2, 2, 1, 1, 25.0, 3, 3).compute() +# ... print(result_img.reshape(3, 3)) +# [[ 50. 100. 150.] +# [150. 25. 250.] +# [250. 200. 200.]] +# +# # INPUT: # --------------------------------------------------------------------------------------------- -# img_in Input images as linearized 2D matrix with top left corner at [1, 1] +# img_in Input images as linearized 2D matrix with top left corner at [1, 1] (every row represents a linearized matrix/image) # x Column index of the top left corner of the rectangle (starting at 1) # y Row index of the top left corner of the rectangle (starting at 1) # width Width of the rectangle (must be positive) # height Height of the rectangle (must be positive) -# fill_value The value to set for the rectangle +# fill_value The value to set for the rectangle # s_cols Width of a single image # s_rows Height of a single image # --------------------------------------------------------------------------------------------- diff --git a/scripts/builtin/img_invert.dml b/scripts/builtin/img_invert.dml index c52f5bed3a7..2475535aadd 100644 --- a/scripts/builtin/img_invert.dml +++ b/scripts/builtin/img_invert.dml @@ -21,9 +21,28 @@ # This is an image data augmentation function. It inverts an image. # +# .. code-block:: python +# +# >>> import numpy as np +# >>> from systemds.context import SystemDSContext +# >>> from systemds.operator.algorithm import img_invert +# >>> +# >>> with SystemDSContext() as sds: +# ... img = sds.from_numpy( +# ... np.array([[ 10., 20., 30.], +# ... [ 40., 50., 60.], +# ... [ 70., 80., 90.]], dtype=np.float32) +# ... ) +# ... result_img = img_invert(img, 210.).compute() +# ... print(result_img) +# [[200. 190. 180.] +# [170. 160. 150.] +# [140. 130. 120.]] +# +# # INPUT: # --------------------------------------------------------------------------------------------- -# img_in Input image +# img_in Input image as 2D matrix with top left corner at [1, 1] # max_value The maximum value pixels can have # --------------------------------------------------------------------------------------------- # diff --git a/scripts/builtin/img_invert_linearized.dml b/scripts/builtin/img_invert_linearized.dml index 68b245492c0..ce4c45219f1 100644 --- a/scripts/builtin/img_invert_linearized.dml +++ b/scripts/builtin/img_invert_linearized.dml @@ -21,9 +21,28 @@ # This is an image data augmentation function. It inverts an image.It can handle one or multiple images # +# .. code-block:: python +# +# >>> import numpy as np +# >>> from systemds.context import SystemDSContext +# >>> from systemds.operator.algorithm import img_invert_linearized +# >>> +# >>> with SystemDSContext() as sds: +# ... img = sds.from_numpy( +# ... np.array([[ 10., 20., 30., +# ... 40., 50., 60., +# ... 70., 80., 90. ]], dtype=np.float32) +# ... ) +# ... result_img = img_invert_linearized(img, 200.).compute() +# ... print(result_img.reshape(3, 3)) +# [[190. 180. 170.] +# [160. 150. 140.] +# [130. 120. 110.]] +# +# # INPUT: # --------------------------------------------------------------------------------------------- -# img_in Input matrix/image (every row of the matrix represents a linearized image) +# img_in Input images as linearized 2D matrix with top left corner at [1, 1] (every row represents a linearized matrix/image) # max_value The maximum value pixels can have # --------------------------------------------------------------------------------------------- # diff --git a/scripts/builtin/img_mirror.dml b/scripts/builtin/img_mirror.dml index a8836f6fd25..1b79e385a52 100644 --- a/scripts/builtin/img_mirror.dml +++ b/scripts/builtin/img_mirror.dml @@ -22,10 +22,29 @@ # This function is an image data augmentation function. # It flips an image on the X (horizontal) or Y (vertical) axis. # +# .. code-block:: python +# +# >>> import numpy as np +# >>> from systemds.context import SystemDSContext +# >>> from systemds.operator.algorithm import img_mirror +# >>> +# >>> with SystemDSContext() as sds: +# ... img = sds.from_numpy( +# ... np.array([[ 10., 20., 30.], +# ... [ 40., 50., 60.], +# ... [ 70., 80., 90.]], dtype=np.float32) +# ... ) +# ... result_img = img_mirror(img, False).compute() +# ... print(result_img) +# [[30. 20. 10.] +# [60. 50. 40.] +# [90. 80. 70.]] +# +# # INPUT: # --------------------------------------------------------------------------------------------- -# img_in Input matrix/image -# max_value The maximum value pixels can have +# img_in Input image as 2D matrix with top left corner at [1, 1] +# horizontal_axis Flip either in X or Y axis # --------------------------------------------------------------------------------------------- # # OUTPUT: diff --git a/scripts/builtin/img_mirror_linearized.dml b/scripts/builtin/img_mirror_linearized.dml index 08b3fe539f9..33dd7efa451 100644 --- a/scripts/builtin/img_mirror_linearized.dml +++ b/scripts/builtin/img_mirror_linearized.dml @@ -22,9 +22,29 @@ # This function has the same functionality with img_mirror but it handles multiple images at # the same time. Each row of the input and output matrix represents a linearized image/matrix # It flips an image on the X (horizontal) or Y (vertical) axis. +# +# .. code-block:: python +# +# >>> import numpy as np +# >>> from systemds.context import SystemDSContext +# >>> from systemds.operator.algorithm import img_mirror_linearized +# >>> +# >>> with SystemDSContext() as sds: +# ... img = sds.from_numpy( +# ... np.array([[ 10., 20., 30., +# ... 40., 50., 60., +# ... 70., 80., 90. ]], dtype=np.float32) +# ... ) +# ... result_img = img_mirror_linearized(img, True, 3, 3).compute() +# ... print(result_img.reshape(3, 3)) +# [[70. 80. 90.] +# [40. 50. 60.] +# [10. 20. 30.]] +# +# # INPUT: # ----------------------------------------------------------------------------------------- -# img_matrix Input matrix/image (every row represents a linearized matrix/image) +# img_matrix Input images as linearized 2D matrix with top left corner at [1, 1] (every row represents a linearized matrix/image) # horizontal_axis flip either in X or Y axis # original_rows number of rows in the original 2-D images # original_cols number of cols in the original 2-D images diff --git a/scripts/builtin/img_posterize.dml b/scripts/builtin/img_posterize.dml index 91578b9c766..3916c828ba4 100644 --- a/scripts/builtin/img_posterize.dml +++ b/scripts/builtin/img_posterize.dml @@ -22,10 +22,29 @@ # The Image Posterize function limits pixel values to 2^bits different values in the range [0, 255]. # Assumes the input image can attain values in the range [0, 255]. # +# .. code-block:: python +# +# >>> import numpy as np +# >>> from systemds.context import SystemDSContext +# >>> from systemds.operator.algorithm import img_posterize +# >>> +# >>> with SystemDSContext() as sds: +# ... img = sds.from_numpy( +# ... np.array([[ 10., 20., 30.], +# ... [ 40., 255., 60.], +# ... [ 70., 80., 90.]], dtype=np.float32) +# ... ) +# ... result_img = img_posterize(img, 1).compute() +# ... print(result_img) +# [[ 0. 0. 0.] +# [ 0. 128. 0.] +# [ 0. 0. 0.]] +# +# # INPUT: # ------------------------------------------------------------------------------------------- -# img_in Input image -# bits The number of bits keep for the values. +# img_in Input image as 2D matrix with top left corner at [1, 1] +# bits The number of bits to keep for the values. # 1 means black and white, 8 means every integer between 0 and 255. # ------------------------------------------------------------------------------------------- # diff --git a/scripts/builtin/img_posterize_linearized.dml b/scripts/builtin/img_posterize_linearized.dml index a0edcf3ed4f..0ff833bf67f 100644 --- a/scripts/builtin/img_posterize_linearized.dml +++ b/scripts/builtin/img_posterize_linearized.dml @@ -22,10 +22,29 @@ # The Linearized Image Posterize function limits pixel values to 2^bits different values in the range [0, 255]. # Assumes the input image can attain values in the range [0, 255]. # +# .. code-block:: python +# +# >>> import numpy as np +# >>> from systemds.context import SystemDSContext +# >>> from systemds.operator.algorithm import img_posterize_linearized +# >>> +# >>> with SystemDSContext() as sds: +# ... img = sds.from_numpy( +# ... np.array([[ 10., 20., 30., +# ... 40., 255., 60., +# ... 70., 80., 90. ]], dtype=np.float32) +# ... ) +# ... result_img = img_posterize_linearized(img, 1).compute() +# ... print(result_img.reshape(3, 3)) +# [[ 0. 0. 0.] +# [ 0. 128. 0.] +# [ 0. 0. 0.]] +# +# # INPUT: # ------------------------------------------------------------------------------------------- -# img_in Row linearized input images as 2D matrix -# bits The number of bits keep for the values. +# img_in Input images as linearized 2D matrix with top left corner at [1, 1] (every row represents a linearized matrix/image) +# bits The number of bits to keep for the values. # 1 means black and white, 8 means every integer between 0 and 255. # ------------------------------------------------------------------------------------------- # diff --git a/scripts/builtin/img_rotate.dml b/scripts/builtin/img_rotate.dml index c49826c2104..d1e6da7bdac 100644 --- a/scripts/builtin/img_rotate.dml +++ b/scripts/builtin/img_rotate.dml @@ -22,6 +22,25 @@ # The Image Rotate function rotates the input image counter-clockwise around the center. # Uses nearest neighbor sampling. # +# .. code-block:: python +# +# >>> import numpy as np +# >>> from systemds.context import SystemDSContext +# >>> from systemds.operator.algorithm import img_rotate +# >>> +# >>> with SystemDSContext() as sds: +# ... img = sds.from_numpy( +# ... np.array([[ 10., 20., 30.], +# ... [ 40., 50., 60.], +# ... [ 70., 80., 90.]], dtype=np.float32) +# ... ) +# ... result_img = img_rotate(img, 3.14159, 255.).compute() +# ... print(result_img) +# [[90. 80. 70.] +# [60. 50. 40.] +# [30. 20. 10.]] +# +# # INPUT: # ----------------------------------------------------------------------------------------------- # img_in Input image as 2D matrix with top left corner at [1, 1] diff --git a/scripts/builtin/img_rotate_linearized.dml b/scripts/builtin/img_rotate_linearized.dml index f5ac43625d5..45ad077aae4 100644 --- a/scripts/builtin/img_rotate_linearized.dml +++ b/scripts/builtin/img_rotate_linearized.dml @@ -22,11 +22,32 @@ # The Linearized Image Rotate function rotates the linearized input images counter-clockwise around the center. # Uses nearest neighbor sampling. # +# .. code-block:: python +# +# >>> import numpy as np +# >>> from systemds.context import SystemDSContext +# >>> from systemds.operator.algorithm import img_rotate_linearized +# >>> +# >>> with SystemDSContext() as sds: +# ... img = sds.from_numpy( +# ... np.array([[ 10., 20., 30., +# ... 40., 50., 60., +# ... 70., 80., 90. ]], dtype=np.float32) +# ... ) +# ... result_img = img_rotate_linearized(img, 3.14159, 255., 3, 3).compute() +# ... print(result_img.reshape(3, 3)) +# [[90. 80. 70.] +# [60. 50. 40.] +# [30. 20. 10.]] +# +# # INPUT: # ----------------------------------------------------------------------------------------------- -# img_in Linearized input images as 2D matrix with top left corner at [1, 1] +# img_in Input images as linearized 2D matrix with top left corner at [1, 1] (every row represents a linearized matrix/image) # radians The value by which to rotate in radian. -# fill_value The background color revealed by the rotation +# fill_value The background color revealed by the rotation +# s_cols Width of a single image +# s_rows Height of a single image # ----------------------------------------------------------------------------------------------- # # OUTPUT: diff --git a/scripts/builtin/img_sample_pairing.dml b/scripts/builtin/img_sample_pairing.dml index 99147b25550..bac412a511e 100644 --- a/scripts/builtin/img_sample_pairing.dml +++ b/scripts/builtin/img_sample_pairing.dml @@ -21,10 +21,34 @@ # The image sample pairing function blends two images together. # +# .. code-block:: python +# +# >>> import numpy as np +# >>> from systemds.context import SystemDSContext +# >>> from systemds.operator.algorithm import img_sample_pairing +# >>> +# >>> with SystemDSContext() as sds: +# ... img_in1 = sds.from_numpy( +# ... np.array([[ 10., 20., 30.], +# ... [ 40., 50., 60.], +# ... [ 70., 80., 90.]], dtype=np.float32) +# ... ) +# ... img_in2 = sds.from_numpy( +# ... np.array([[ 30., 40., 50.], +# ... [ 60., 70., 80.], +# ... [ 90., 100., 110.]], dtype=np.float32) +# ... ) +# ... result_img = img_sample_pairing(img_in1, img_in2, 0.5).compute() +# ... print(result_img) +# [[ 20. 30. 40.] +# [ 50. 60. 70.] +# [ 80. 90. 100.]] +# +# # INPUT: # ------------------------------------------------------------------------------------------- -# img_in1 First input image -# img_in2 Second input image +# img_in1 First input image as 2D matrix with top left corner at [1, 1] +# img_in2 Second input image as 2D matrix with top left corner at [1, 1] # weight The weight given to the second image. # 0 means only img_in1, 1 means only img_in2 will be visible # ------------------------------------------------------------------------------------------- diff --git a/scripts/builtin/img_sample_pairing_linearized.dml b/scripts/builtin/img_sample_pairing_linearized.dml index f09046cc181..975d12d2b8d 100644 --- a/scripts/builtin/img_sample_pairing_linearized.dml +++ b/scripts/builtin/img_sample_pairing_linearized.dml @@ -21,9 +21,33 @@ # The image sample pairing function blends two images together. # +# .. code-block:: python +# +# >>> import numpy as np +# >>> from systemds.context import SystemDSContext +# >>> from systemds.operator.algorithm import img_sample_pairing_linearized +# >>> +# >>> with SystemDSContext() as sds: +# ... img_in1 = sds.from_numpy( +# ... np.array([[ 10., 20., 30., +# ... 40., 50., 60., +# ... 70., 80., 90. ]], dtype=np.float32) +# ... ) +# ... img_in2 = sds.from_numpy( +# ... np.array([[ 30., 40., 50., +# ... 60., 70., 80., +# ... 90., 100., 110. ]], dtype=np.float32) +# ... ) +# ... result_img = img_sample_pairing_linearized(img_in1, img_in2, 0.5).compute() +# ... print(result_img.reshape(3, 3)) +# [[ 20. 30. 40.] +# [ 50. 60. 70.] +# [ 80. 90. 100.]] +# +# # INPUT: # ------------------------------------------------------------------------------------------- -# img_in1 input matrix/image (every row is a linearized image) +# img_in1 Input images as linearized 2D matrix with top left corner at [1, 1] (every row represents a linearized matrix/image) # img_in2 Second input image (one image represented as a single row linearized matrix) # weight The weight given to the second image. # 0 means only img_in1, 1 means only img_in2 will be visible diff --git a/scripts/builtin/img_shear.dml b/scripts/builtin/img_shear.dml index 2cf00592a63..bce106d80ef 100644 --- a/scripts/builtin/img_shear.dml +++ b/scripts/builtin/img_shear.dml @@ -22,6 +22,25 @@ # This function applies a shearing transformation to an image. # Uses nearest neighbor sampling. # +# .. code-block:: python +# +# >>> import numpy as np +# >>> from systemds.context import SystemDSContext +# >>> from systemds.operator.algorithm import img_shear +# >>> +# >>> with SystemDSContext() as sds: +# ... img = sds.from_numpy( +# ... np.array([[ 10., 20., 30.], +# ... [ 40., 50., 60.], +# ... [ 70., 80., 90.]], dtype=np.float32) +# ... ) +# ... result_img = img_shear(img, 1., 0., 255).compute() +# ... print(result_img) +# [[ 10. 20. 30.] +# [255. 40. 50.] +# [255. 255. 70.]] +# +# # INPUT: # --------------------------------------------------------------------------------------------- # img_in Input image as 2D matrix with top left corner at [1, 1] diff --git a/scripts/builtin/img_shear_linearized.dml b/scripts/builtin/img_shear_linearized.dml index 79471f358b6..4bac774b28b 100644 --- a/scripts/builtin/img_shear_linearized.dml +++ b/scripts/builtin/img_shear_linearized.dml @@ -22,12 +22,33 @@ # This function applies a shearing transformation to linearized images. # Uses nearest neighbor sampling. # +# .. code-block:: python +# +# >>> import numpy as np +# >>> from systemds.context import SystemDSContext +# >>> from systemds.operator.algorithm import img_shear_linearized +# >>> +# >>> with SystemDSContext() as sds: +# ... img = sds.from_numpy( +# ... np.array([[ 10., 20., 30., +# ... 40., 50., 60., +# ... 70., 80., 90. ]], dtype=np.float32) +# ... ) +# ... result_img = img_shear_linearized(img, 1., 0., 0., 3, 3).compute() +# ... print(result_img.reshape(3, 3)) +# [[10. 20. 30.] +# [ 0. 40. 50.] +# [ 0. 0. 70.]] +# +# # INPUT: # --------------------------------------------------------------------------------------------- -# img_in Linearized input images as 2D matrix with top left corner at [1, 1] +# img_in Input images as linearized 2D matrix with top left corner at [1, 1] (every row represents a linearized matrix/image) # shear_x Shearing factor for horizontal shearing # shear_y Shearing factor for vertical shearing # fill_value The background color revealed by the shearing +# s_cols Width of a single image +# s_rows Height of a single image # --------------------------------------------------------------------------------------------- # # OUTPUT: diff --git a/scripts/builtin/img_transform.dml b/scripts/builtin/img_transform.dml index f65e2f4a5f5..d6636c0bc31 100644 --- a/scripts/builtin/img_transform.dml +++ b/scripts/builtin/img_transform.dml @@ -23,6 +23,25 @@ # Optionally resizes the image (without scaling). # Uses nearest neighbor sampling. # +# .. code-block:: python +# +# >>> import numpy as np +# >>> from systemds.context import SystemDSContext +# >>> from systemds.operator.algorithm import img_transform +# >>> +# >>> with SystemDSContext() as sds: +# ... img = sds.from_numpy( +# ... np.array([[ 10., 20., 30.], +# ... [ 40., 50., 60.], +# ... [ 70., 80., 90.]], dtype=np.float32) +# ... ) +# ... result_img = img_transform(img, 3, 3, -1., 0., 2., 0., 1., 0., 255.).compute() +# ... print(result_img) +# [[ 20. 10. 255.] +# [ 50. 40. 255.] +# [ 80. 70. 255.]] +# +# # INPUT: # ------------------------------------------------------------------------------------------- # img_in Input image as 2D matrix with top left corner at [1, 1] diff --git a/scripts/builtin/img_transform_linearized.dml b/scripts/builtin/img_transform_linearized.dml index 06867d61b2e..dc4c650c58f 100644 --- a/scripts/builtin/img_transform_linearized.dml +++ b/scripts/builtin/img_transform_linearized.dml @@ -23,13 +23,34 @@ # Optionally resizes the image (without scaling). # Uses nearest neighbor sampling. # +# .. code-block:: python +# +# >>> import numpy as np +# >>> from systemds.context import SystemDSContext +# >>> from systemds.operator.algorithm import img_transform_linearized +# >>> +# >>> with SystemDSContext() as sds: +# ... img = sds.from_numpy( +# ... np.array([[ 10., 20., 30., +# ... 40., 50., 60., +# ... 70., 80., 90. ]], dtype=np.float32) +# ... ) +# ... result_img = img_transform_linearized(img, 3, 3, -1., 0., 2., 0., 1., 0., 255., 3, 3).compute() +# ... print(result_img.reshape(3, 3)) +# [[ 20. 10. 255.] +# [ 50. 40. 255.] +# [ 80. 70. 255.]] +# +# # INPUT: # ------------------------------------------------------------------------------------------- -# img_in Linearized input images as 2D matrix with top left corner at [1, 1] +# img_in Input images as linearized 2D matrix with top left corner at [1, 1] (every row represents a linearized matrix/image) # out_w Width of the output matrix # out_h Height of the output matrix # a,b,c,d,e,f The first two rows of the affine matrix in row-major order -# fill_value The background of an image +# fill_value The background of an image +# s_cols Width of a single image +# s_rows Height of a single image # ------------------------------------------------------------------------------------------- # # OUTPUT: diff --git a/scripts/builtin/img_translate.dml b/scripts/builtin/img_translate.dml index 9bf2664d33c..687f8f04150 100644 --- a/scripts/builtin/img_translate.dml +++ b/scripts/builtin/img_translate.dml @@ -23,6 +23,25 @@ # Optionally resizes the image (without scaling). # Uses nearest neighbor sampling. # +# .. code-block:: python +# +# >>> import numpy as np +# >>> from systemds.context import SystemDSContext +# >>> from systemds.operator.algorithm import img_translate +# >>> +# >>> with SystemDSContext() as sds: +# ... img = sds.from_numpy( +# ... np.array([[ 10., 20., 30.], +# ... [ 40., 50., 60.], +# ... [ 70., 80., 90.]], dtype=np.float32) +# ... ) +# ... result_img = img_translate(img, 1., 1., 3, 3, 255.).compute() +# ... print(result_img) +# [[255. 255. 255.] +# [255. 10. 20.] +# [255. 40. 50.]] +# +# # INPUT: # ---------------------------------------------------------------------------------------------- # img_in Input image as 2D matrix with top left corner at [1, 1] diff --git a/scripts/builtin/img_translate_linearized.dml b/scripts/builtin/img_translate_linearized.dml index c2c898d21c9..2feec938007 100644 --- a/scripts/builtin/img_translate_linearized.dml +++ b/scripts/builtin/img_translate_linearized.dml @@ -22,9 +22,29 @@ # This function has the same functionality with img_translate but it handles multiple images at # the same time. Each row of the input and output matrix represents a linearized image/matrix # It translates the image and Optionally resizes the image (without scaling). +# +# .. code-block:: python +# +# >>> import numpy as np +# >>> from systemds.context import SystemDSContext +# >>> from systemds.operator.algorithm import img_translate_linearized +# >>> +# >>> with SystemDSContext() as sds: +# ... img = sds.from_numpy( +# ... np.array([[ 10., 20., 30., +# ... 40., 50., 60., +# ... 70., 80., 90. ]], dtype=np.float32) +# ... ) +# ... result_img = img_translate_linearized(img, 1., 1., 3, 3, 255.0, 3, 3).compute() +# ... print(result_img.reshape(3, 3)) +# [[255. 255. 255.] +# [255. 10. 20.] +# [255. 40. 50.]] +# +# # INPUT: # ---------------------------------------------------------------------------------------------- -# img_in Input matrix/image (every row represents a linearized matrix/image) +# img_in Input images as linearized 2D matrix with top left corner at [1, 1] (every row represents a linearized matrix/image) # offset_x The distance to move the image in x direction # offset_y The distance to move the image in y direction # out_w Width of the output image diff --git a/scripts/builtin/lm.dml b/scripts/builtin/lm.dml index b7fc55e0ff9..58f161a3ce2 100644 --- a/scripts/builtin/lm.dml +++ b/scripts/builtin/lm.dml @@ -23,6 +23,36 @@ # method or the conjugate gradient algorithm depending on the input size # of the matrices (See lmDS-function and lmCG-function respectively). # +# .. code-block:: python +# +# >>> import numpy as np +# >>> from systemds.context import SystemDSContext +# >>> from systemds.operator.algorithm import lm +# >>> +# >>> np.random.seed(0) +# >>> features = np.random.rand(10, 15) +# >>> y = np.random.rand(10, 1) +# >>> +# >>> with SystemDSContext() as sds: +# ... weights = lm(sds.from_numpy(features), sds.from_numpy(y)).compute() +# ... print(weights) +# [[-0.11538199] +# [-0.20386541] +# [-0.39956034] +# [ 1.04078623] +# [ 0.43270839] +# [ 0.18954599] +# [ 0.49858969] +# [-0.26812763] +# [ 0.09961844] +# [-0.57000751] +# [-0.43386048] +# [ 0.55358873] +# [-0.54638565] +# [ 0.2205885 ] +# [ 0.37957689]] +# +# # INPUT: # -------------------------------------------------------------------- # X Matrix of feature vectors. diff --git a/scripts/builtin/normalize.dml b/scripts/builtin/normalize.dml index 1f136757aa4..9496431ee84 100644 --- a/scripts/builtin/normalize.dml +++ b/scripts/builtin/normalize.dml @@ -22,6 +22,20 @@ # Min-max normalization (a.k.a. min-max scaling) to range [0,1]. For matrices # of positive values, this normalization preserves the input sparsity. # +# .. code-block:: python +# +# >>> import numpy as np +# >>> from systemds.context import SystemDSContext +# >>> from systemds.operator.algorithm import normalize +# >>> +# >>> with SystemDSContext() as sds: +# ... X = sds.from_numpy(np.array([[1, 2], [3, 4]])) +# ... Y, cmin, cmax = normalize(X).compute() +# ... print(Y) +# [[0. 0.] +# [1. 1.]] +# +# # INPUT: # --------------------------------------------------------------------------------------- # X Input feature matrix of shape n-by-m diff --git a/scripts/builtin/randomForest.dml b/scripts/builtin/randomForest.dml index 53529afb9a9..cd298265417 100644 --- a/scripts/builtin/randomForest.dml +++ b/scripts/builtin/randomForest.dml @@ -46,6 +46,47 @@ # prefixed by a one-hot vector of sampled features # (e.g., [1,1,1,0] if we sampled a,b,c of the four features) # +# .. code-block:: python +# +# >>> import numpy as np +# >>> from systemds.context import SystemDSContext +# >>> from systemds.operator.algorithm import randomForest, randomForestPredict +# >>> +# >>> # tiny toy dataset +# >>> X = np.array([[1], +# ... [2], +# ... [10], +# ... [11]], dtype=np.int64) +# >>> y = np.array([[1], +# ... [1], +# ... [2], +# ... [2]], dtype=np.int64) +# >>> +# >>> with SystemDSContext() as sds: +# ... X_sds = sds.from_numpy(X) +# ... y_sds = sds.from_numpy(y) +# ... +# ... ctypes = sds.from_numpy(np.array([[1, 2]], dtype=np.int64)) +# ... +# ... # train a 4-tree forest (no sampling) +# ... M = randomForest( +# ... X_sds, y_sds, ctypes, +# ... num_trees = 4, +# ... sample_frac = 1.0, +# ... feature_frac = 1.0, +# ... max_depth = 3, +# ... min_leaf = 1, +# ... min_split = 2, +# ... seed = 42 +# ... ) +# ... +# ... preds = randomForestPredict(X_sds, ctypes, M).compute() +# ... print(preds) +# [[1.] +# [1.] +# [2.] +# [2.]] +# # # INPUT: # ------------------------------------------------------------------------------ diff --git a/scripts/builtin/randomForestPredict.dml b/scripts/builtin/randomForestPredict.dml index a003f26f7d4..3a42c7f4870 100644 --- a/scripts/builtin/randomForestPredict.dml +++ b/scripts/builtin/randomForestPredict.dml @@ -22,6 +22,49 @@ # This script implements random forest prediction for recoded and binned # categorical and numerical input features. # +# +# .. code-block:: python +# +# >>> import numpy as np +# >>> from systemds.context import SystemDSContext +# >>> from systemds.operator.algorithm import randomForest, randomForestPredict +# >>> +# >>> # tiny toy dataset +# >>> X = np.array([[1], +# ... [2], +# ... [10], +# ... [11]], dtype=np.int64) +# >>> y = np.array([[1], +# ... [1], +# ... [2], +# ... [2]], dtype=np.int64) +# >>> +# >>> with SystemDSContext() as sds: +# ... X_sds = sds.from_numpy(X) +# ... y_sds = sds.from_numpy(y) +# ... +# ... ctypes = sds.from_numpy(np.array([[1, 2]], dtype=np.int64)) +# ... +# ... # train a 4-tree forest (no sampling) +# ... M = randomForest( +# ... X_sds, y_sds, ctypes, +# ... num_trees = 4, +# ... sample_frac = 1.0, +# ... feature_frac = 1.0, +# ... max_depth = 3, +# ... min_leaf = 1, +# ... min_split = 2, +# ... seed = 42 +# ... ) +# ... +# ... preds = randomForestPredict(X_sds, ctypes, M).compute() +# ... print(preds) +# [[1.] +# [1.] +# [2.] +# [2.]] +# +# # INPUT: # ------------------------------------------------------------------------------ # X Feature matrix in recoded/binned representation diff --git a/scripts/builtin/toOneHot.dml b/scripts/builtin/toOneHot.dml index 2232cdc97bc..aec58c49994 100644 --- a/scripts/builtin/toOneHot.dml +++ b/scripts/builtin/toOneHot.dml @@ -21,6 +21,22 @@ # The toOneHot-function encodes unordered categorical vector to multiple binary vectors. # +# .. code-block:: python +# +# >>> import numpy as np +# >>> from systemds.context import SystemDSContext +# >>> from systemds.operator.algorithm import toOneHot +# >>> +# >>> with SystemDSContext() as sds: +# ... X = sds.from_numpy(np.array([[1], [3], [2], [3]])) +# ... Y = toOneHot(X, numClasses=3).compute() +# ... print(Y) +# [[1. 0. 0.] +# [0. 0. 1.] +# [0. 1. 0.] +# [0. 0. 1.]] +# +# # INPUT: # ------------------------------------------------------------------------------------------ # X Vector with N integer entries between 1 and numClasses diff --git a/src/main/python/generator/dml_parser.py b/src/main/python/generator/dml_parser.py index 78d91f5f429..866c3aea0f2 100644 --- a/src/main/python/generator/dml_parser.py +++ b/src/main/python/generator/dml_parser.py @@ -201,11 +201,10 @@ def parse_header(self, path: str): content = f.read() pat = re.compile( r""" - ^\s*\#\s*\.\.\s*code-block::\s*python # .. code-block:: python - (?:\s*\#.*\n)*? # optional adornments / blank lines - (.*?) # ← capture the actual example - (?= # stop just *before* … - (?:\s*\#\s*\n){2} # … two consecutive “blank” # lines + ^\#\s*\.\.\s*code-block::\s*python # .. code-block:: python + (.*?) # ← capture the actual example + (?= # stop just before + ^\#\s*INPUT: # INPUT: ) """, re.MULTILINE | re.DOTALL | re.VERBOSE, @@ -214,7 +213,7 @@ def parse_header(self, path: str): if match: raw_block = match.group(1) code_lines = [line.lstrip("#") for line in raw_block.splitlines()] # Remove leading # - code_block = textwrap.dedent("\n".join(code_lines)) + code_block = textwrap.dedent("\n".join([code_line for code_line in code_lines if code_line != ""])) data = {'description': description, 'parameters': input_parameters, diff --git a/src/main/python/generator/generator.py b/src/main/python/generator/generator.py index abf80ccf04b..fe3e7ce9c30 100644 --- a/src/main/python/generator/generator.py +++ b/src/main/python/generator/generator.py @@ -111,8 +111,6 @@ def generate_test_file(self, function_name: str, code_block: str = None): test_script.write(self.licence) test_script.write(self.generated_by) test_script.write("import unittest, contextlib, io\n") - test_script.write(f"from systemds.context import SystemDSContext\n") - test_script.write(f"from systemds.operator.algorithm.builtin.{function_name} import {function_name}\n\n\n") test_script.write(f"class Test{function_name.upper()}(unittest.TestCase):\n") test_script.write(f" def test_{function_name}(self):\n") diff --git a/src/main/python/systemds/operator/algorithm/builtin/dist.py b/src/main/python/systemds/operator/algorithm/builtin/dist.py index 5f33d96bc22..c04f6fb1f4e 100644 --- a/src/main/python/systemds/operator/algorithm/builtin/dist.py +++ b/src/main/python/systemds/operator/algorithm/builtin/dist.py @@ -32,6 +32,21 @@ def dist(X: Matrix): """ Returns Euclidean distance matrix (distances between N n-dimensional points) + .. code-block:: python + + >>> import numpy as np + >>> from systemds.context import SystemDSContext + >>> from systemds.operator.algorithm import dist + >>> + >>> with SystemDSContext() as sds: + ... X = sds.from_numpy(np.array([[0], [3], [4]])) + ... out = dist(X).compute() + ... print(out) + [[0. 3. 4.] + [3. 0. 1.] + [4. 1. 0.]] + + :param X: Matrix to calculate the distance inside diff --git a/src/main/python/systemds/operator/algorithm/builtin/img_brightness.py b/src/main/python/systemds/operator/algorithm/builtin/img_brightness.py index 63a472716ab..a7e93f13a1b 100644 --- a/src/main/python/systemds/operator/algorithm/builtin/img_brightness.py +++ b/src/main/python/systemds/operator/algorithm/builtin/img_brightness.py @@ -34,9 +34,26 @@ def img_brightness(img_in: Matrix, """ The img_brightness-function is an image data augmentation function. It changes the brightness of the image. + .. code-block:: python + >>> import numpy as np + >>> from systemds.context import SystemDSContext + >>> from systemds.operator.algorithm import img_brightness + >>> + >>> with SystemDSContext() as sds: + ... img = sds.from_numpy( + ... np.array([[ 50, 100, + ... 150, 200 ]], dtype=np.float32) + ... ) + ... result_img = img_brightness(img, 30.0, 150).compute() + ... print(result_img.reshape(2, 2)) + [[ 80. 130.] + [150. 150.]] - :param img_in: Input matrix/image + + + + :param img_in: Input image as 2D matrix with top left corner at [1, 1] :param value: The amount of brightness to be changed for the image :param channel_max: Maximum value of the brightness of the image :return: Output matrix/image diff --git a/src/main/python/systemds/operator/algorithm/builtin/img_brightness_linearized.py b/src/main/python/systemds/operator/algorithm/builtin/img_brightness_linearized.py index a5b62012f50..c5c107526e5 100644 --- a/src/main/python/systemds/operator/algorithm/builtin/img_brightness_linearized.py +++ b/src/main/python/systemds/operator/algorithm/builtin/img_brightness_linearized.py @@ -34,9 +34,26 @@ def img_brightness_linearized(img_in: Matrix, """ The img_brightness_linearized-function is an image data augmentation function. It changes the brightness of one or multiple images. + .. code-block:: python + >>> import numpy as np + >>> from systemds.context import SystemDSContext + >>> from systemds.operator.algorithm import img_brightness_linearized + >>> + >>> with SystemDSContext() as sds: + ... img = sds.from_numpy( + ... np.array([[ 50, 100, + ... 150, 200 ]], dtype=np.float32) + ... ) + ... result_img = img_brightness_linearized(img, 30.0, 255).compute() + ... print(result_img.reshape(2, 2)) + [[ 80. 130.] + [180. 230.]] - :param img_in: Input matrix/image (can represent multiple images every row of the matrix represents a linearized image) + + + + :param img_in: Input images as linearized 2D matrix with top left corner at [1, 1] (every row represents a linearized matrix/image) :param value: The amount of brightness to be changed for the image :param channel_max: Maximum value of the brightness of the image :return: Output matrix/images (every row of the matrix represents a linearized image) diff --git a/src/main/python/systemds/operator/algorithm/builtin/img_crop.py b/src/main/python/systemds/operator/algorithm/builtin/img_crop.py index f0432c7578a..f5eed9a2ea2 100644 --- a/src/main/python/systemds/operator/algorithm/builtin/img_crop.py +++ b/src/main/python/systemds/operator/algorithm/builtin/img_crop.py @@ -36,9 +36,26 @@ def img_crop(img_in: Matrix, """ The img_crop-function is an image data augmentation function. It cuts out a subregion of an image. + .. code-block:: python + >>> import numpy as np + >>> from systemds.context import SystemDSContext + >>> from systemds.operator.algorithm import img_crop + >>> + >>> with SystemDSContext() as sds: + ... img = sds.from_numpy( + ... np.array([[ 50., 100., 150.], + ... [150., 200., 250.], + ... [250., 200., 200.]], dtype=np.float32) + ... ) + ... result_img = img_crop(img, 1, 1, 1, 1).compute() + ... print(result_img) + [[200.]] - :param img_in: Input matrix/image + + + + :param img_in: Input image as 2D matrix with top left corner at [1, 1] :param w: The width of the subregion required :param h: The height of the subregion required :param x_offset: The horizontal coordinate in the image to begin the crop operation diff --git a/src/main/python/systemds/operator/algorithm/builtin/img_crop_linearized.py b/src/main/python/systemds/operator/algorithm/builtin/img_crop_linearized.py index 4321a8af0db..46fcf802f45 100644 --- a/src/main/python/systemds/operator/algorithm/builtin/img_crop_linearized.py +++ b/src/main/python/systemds/operator/algorithm/builtin/img_crop_linearized.py @@ -38,9 +38,26 @@ def img_crop_linearized(img_in: Matrix, """ The img_crop_linearized cuts out a rectangular section of multiple linearized images. + .. code-block:: python + >>> import numpy as np + >>> from systemds.context import SystemDSContext + >>> from systemds.operator.algorithm import img_crop_linearized + >>> + >>> with SystemDSContext() as sds: + ... img = sds.from_numpy( + ... np.array([[ 50., 100., 150., + ... 150., 200., 250., + ... 250., 200., 200. ]], dtype=np.float32) + ... ) + ... result_img = img_crop_linearized(img, 1, 1, 1, 1, 3, 3).compute() + ... print(result_img) + [[200.]] - :param img_in: Linearized input images as 2D matrix + + + + :param img_in: Input images as linearized 2D matrix with top left corner at [1, 1] (every row represents a linearized matrix/image) :param w: The width of the subregion required :param h: The height of the subregion required :param x_offset: The horizontal offset for the center of the crop region diff --git a/src/main/python/systemds/operator/algorithm/builtin/img_cutout.py b/src/main/python/systemds/operator/algorithm/builtin/img_cutout.py index 93befbd7366..110e91e9e9d 100644 --- a/src/main/python/systemds/operator/algorithm/builtin/img_cutout.py +++ b/src/main/python/systemds/operator/algorithm/builtin/img_cutout.py @@ -38,6 +38,26 @@ def img_cutout(img_in: Matrix, Image Cutout function replaces a rectangular section of an image with a constant value. + .. code-block:: python + + >>> import numpy as np + >>> from systemds.context import SystemDSContext + >>> from systemds.operator.algorithm import img_cutout + >>> + >>> with SystemDSContext() as sds: + ... img = sds.from_numpy( + ... np.array([[ 50., 100., 150.], + ... [150., 200., 250.], + ... [250., 200., 200.]], dtype=np.float32) + ... ) + ... result_img = img_cutout(img, 2, 2, 1, 1, 49.).compute() + ... print(result_img) + [[ 50. 100. 150.] + [150. 49. 250.] + [250. 200. 200.]] + + + :param img_in: Input image as 2D matrix with top left corner at [1, 1] :param x: Column index of the top left corner of the rectangle (starting at 1) diff --git a/src/main/python/systemds/operator/algorithm/builtin/img_cutout_linearized.py b/src/main/python/systemds/operator/algorithm/builtin/img_cutout_linearized.py index 2dd0c52239f..765885c663b 100644 --- a/src/main/python/systemds/operator/algorithm/builtin/img_cutout_linearized.py +++ b/src/main/python/systemds/operator/algorithm/builtin/img_cutout_linearized.py @@ -39,9 +39,28 @@ def img_cutout_linearized(img_in: Matrix, """ Image Cutout function replaces a rectangular section of an image with a constant value. + .. code-block:: python + >>> import numpy as np + >>> from systemds.context import SystemDSContext + >>> from systemds.operator.algorithm import img_cutout_linearized + >>> + >>> with SystemDSContext() as sds: + ... img = sds.from_numpy( + ... np.array([[ 50., 100., 150., + ... 150., 200., 250., + ... 250., 200., 200. ]], dtype=np.float32) + ... ) + ... result_img = img_cutout_linearized(img, 2, 2, 1, 1, 25.0, 3, 3).compute() + ... print(result_img.reshape(3, 3)) + [[ 50. 100. 150.] + [150. 25. 250.] + [250. 200. 200.]] - :param img_in: Input images as linearized 2D matrix with top left corner at [1, 1] + + + + :param img_in: Input images as linearized 2D matrix with top left corner at [1, 1] (every row represents a linearized matrix/image) :param x: Column index of the top left corner of the rectangle (starting at 1) :param y: Row index of the top left corner of the rectangle (starting at 1) :param width: Width of the rectangle (must be positive) diff --git a/src/main/python/systemds/operator/algorithm/builtin/img_invert.py b/src/main/python/systemds/operator/algorithm/builtin/img_invert.py index a555f23708c..32ad67c4059 100644 --- a/src/main/python/systemds/operator/algorithm/builtin/img_invert.py +++ b/src/main/python/systemds/operator/algorithm/builtin/img_invert.py @@ -33,9 +33,28 @@ def img_invert(img_in: Matrix, """ This is an image data augmentation function. It inverts an image. + .. code-block:: python + >>> import numpy as np + >>> from systemds.context import SystemDSContext + >>> from systemds.operator.algorithm import img_invert + >>> + >>> with SystemDSContext() as sds: + ... img = sds.from_numpy( + ... np.array([[ 10., 20., 30.], + ... [ 40., 50., 60.], + ... [ 70., 80., 90.]], dtype=np.float32) + ... ) + ... result_img = img_invert(img, 210.).compute() + ... print(result_img) + [[200. 190. 180.] + [170. 160. 150.] + [140. 130. 120.]] - :param img_in: Input image + + + + :param img_in: Input image as 2D matrix with top left corner at [1, 1] :param max_value: The maximum value pixels can have :return: Output image """ diff --git a/src/main/python/systemds/operator/algorithm/builtin/img_invert_linearized.py b/src/main/python/systemds/operator/algorithm/builtin/img_invert_linearized.py index 2f66a0b8be2..59e30a3c77d 100644 --- a/src/main/python/systemds/operator/algorithm/builtin/img_invert_linearized.py +++ b/src/main/python/systemds/operator/algorithm/builtin/img_invert_linearized.py @@ -33,9 +33,28 @@ def img_invert_linearized(img_in: Matrix, """ This is an image data augmentation function. It inverts an image.It can handle one or multiple images + .. code-block:: python + >>> import numpy as np + >>> from systemds.context import SystemDSContext + >>> from systemds.operator.algorithm import img_invert_linearized + >>> + >>> with SystemDSContext() as sds: + ... img = sds.from_numpy( + ... np.array([[ 10., 20., 30., + ... 40., 50., 60., + ... 70., 80., 90. ]], dtype=np.float32) + ... ) + ... result_img = img_invert_linearized(img, 200.).compute() + ... print(result_img.reshape(3, 3)) + [[190. 180. 170.] + [160. 150. 140.] + [130. 120. 110.]] - :param img_in: Input matrix/image (every row of the matrix represents a linearized image) + + + + :param img_in: Input images as linearized 2D matrix with top left corner at [1, 1] (every row represents a linearized matrix/image) :param max_value: The maximum value pixels can have :return: Output images (every row of the matrix represents a linearized image) """ diff --git a/src/main/python/systemds/operator/algorithm/builtin/img_mirror.py b/src/main/python/systemds/operator/algorithm/builtin/img_mirror.py index 285d25fbf29..94d95438fb5 100644 --- a/src/main/python/systemds/operator/algorithm/builtin/img_mirror.py +++ b/src/main/python/systemds/operator/algorithm/builtin/img_mirror.py @@ -34,10 +34,29 @@ def img_mirror(img_in: Matrix, This function is an image data augmentation function. It flips an image on the X (horizontal) or Y (vertical) axis. + .. code-block:: python + >>> import numpy as np + >>> from systemds.context import SystemDSContext + >>> from systemds.operator.algorithm import img_mirror + >>> + >>> with SystemDSContext() as sds: + ... img = sds.from_numpy( + ... np.array([[ 10., 20., 30.], + ... [ 40., 50., 60.], + ... [ 70., 80., 90.]], dtype=np.float32) + ... ) + ... result_img = img_mirror(img, False).compute() + ... print(result_img) + [[30. 20. 10.] + [60. 50. 40.] + [90. 80. 70.]] - :param img_in: Input matrix/image - :param max_value: The maximum value pixels can have + + + + :param img_in: Input image as 2D matrix with top left corner at [1, 1] + :param horizontal_axis: Flip either in X or Y axis :return: Flipped matrix/image """ diff --git a/src/main/python/systemds/operator/algorithm/builtin/img_mirror_linearized.py b/src/main/python/systemds/operator/algorithm/builtin/img_mirror_linearized.py index 1c6ae58ad03..d8f8948347c 100644 --- a/src/main/python/systemds/operator/algorithm/builtin/img_mirror_linearized.py +++ b/src/main/python/systemds/operator/algorithm/builtin/img_mirror_linearized.py @@ -37,8 +37,28 @@ def img_mirror_linearized(img_matrix: Matrix, the same time. Each row of the input and output matrix represents a linearized image/matrix It flips an image on the X (horizontal) or Y (vertical) axis. + .. code-block:: python - :param img_matrix: Input matrix/image (every row represents a linearized matrix/image) + >>> import numpy as np + >>> from systemds.context import SystemDSContext + >>> from systemds.operator.algorithm import img_mirror_linearized + >>> + >>> with SystemDSContext() as sds: + ... img = sds.from_numpy( + ... np.array([[ 10., 20., 30., + ... 40., 50., 60., + ... 70., 80., 90. ]], dtype=np.float32) + ... ) + ... result_img = img_mirror_linearized(img, True, 3, 3).compute() + ... print(result_img.reshape(3, 3)) + [[70. 80. 90.] + [40. 50. 60.] + [10. 20. 30.]] + + + + + :param img_matrix: Input images as linearized 2D matrix with top left corner at [1, 1] (every row represents a linearized matrix/image) :param horizontal_axis: flip either in X or Y axis :param original_rows: number of rows in the original 2-D images :param original_cols: number of cols in the original 2-D images diff --git a/src/main/python/systemds/operator/algorithm/builtin/img_posterize.py b/src/main/python/systemds/operator/algorithm/builtin/img_posterize.py index e314b0e7ca0..ae1b367ded5 100644 --- a/src/main/python/systemds/operator/algorithm/builtin/img_posterize.py +++ b/src/main/python/systemds/operator/algorithm/builtin/img_posterize.py @@ -34,10 +34,29 @@ def img_posterize(img_in: Matrix, The Image Posterize function limits pixel values to 2^bits different values in the range [0, 255]. Assumes the input image can attain values in the range [0, 255]. + .. code-block:: python + >>> import numpy as np + >>> from systemds.context import SystemDSContext + >>> from systemds.operator.algorithm import img_posterize + >>> + >>> with SystemDSContext() as sds: + ... img = sds.from_numpy( + ... np.array([[ 10., 20., 30.], + ... [ 40., 255., 60.], + ... [ 70., 80., 90.]], dtype=np.float32) + ... ) + ... result_img = img_posterize(img, 1).compute() + ... print(result_img) + [[ 0. 0. 0.] + [ 0. 128. 0.] + [ 0. 0. 0.]] - :param img_in: Input image - :param bits: The number of bits keep for the values. + + + + :param img_in: Input image as 2D matrix with top left corner at [1, 1] + :param bits: The number of bits to keep for the values. 1 means black and white, 8 means every integer between 0 and 255. :return: Output image """ diff --git a/src/main/python/systemds/operator/algorithm/builtin/img_posterize_linearized.py b/src/main/python/systemds/operator/algorithm/builtin/img_posterize_linearized.py index 286ce0222df..508145943fb 100644 --- a/src/main/python/systemds/operator/algorithm/builtin/img_posterize_linearized.py +++ b/src/main/python/systemds/operator/algorithm/builtin/img_posterize_linearized.py @@ -34,10 +34,29 @@ def img_posterize_linearized(img_in: Matrix, The Linearized Image Posterize function limits pixel values to 2^bits different values in the range [0, 255]. Assumes the input image can attain values in the range [0, 255]. + .. code-block:: python + >>> import numpy as np + >>> from systemds.context import SystemDSContext + >>> from systemds.operator.algorithm import img_posterize_linearized + >>> + >>> with SystemDSContext() as sds: + ... img = sds.from_numpy( + ... np.array([[ 10., 20., 30., + ... 40., 255., 60., + ... 70., 80., 90. ]], dtype=np.float32) + ... ) + ... result_img = img_posterize_linearized(img, 1).compute() + ... print(result_img.reshape(3, 3)) + [[ 0. 0. 0.] + [ 0. 128. 0.] + [ 0. 0. 0.]] - :param img_in: Row linearized input images as 2D matrix - :param bits: The number of bits keep for the values. + + + + :param img_in: Input images as linearized 2D matrix with top left corner at [1, 1] (every row represents a linearized matrix/image) + :param bits: The number of bits to keep for the values. 1 means black and white, 8 means every integer between 0 and 255. :return: Row linearized output images as 2D matrix """ diff --git a/src/main/python/systemds/operator/algorithm/builtin/img_rotate.py b/src/main/python/systemds/operator/algorithm/builtin/img_rotate.py index b8ab1ec0687..55f7f28864b 100644 --- a/src/main/python/systemds/operator/algorithm/builtin/img_rotate.py +++ b/src/main/python/systemds/operator/algorithm/builtin/img_rotate.py @@ -35,6 +35,25 @@ def img_rotate(img_in: Matrix, The Image Rotate function rotates the input image counter-clockwise around the center. Uses nearest neighbor sampling. + .. code-block:: python + + >>> import numpy as np + >>> from systemds.context import SystemDSContext + >>> from systemds.operator.algorithm import img_rotate + >>> + >>> with SystemDSContext() as sds: + ... img = sds.from_numpy( + ... np.array([[ 10., 20., 30.], + ... [ 40., 50., 60.], + ... [ 70., 80., 90.]], dtype=np.float32) + ... ) + ... result_img = img_rotate(img, 3.14159, 255.).compute() + ... print(result_img) + [[90. 80. 70.] + [60. 50. 40.] + [30. 20. 10.]] + + :param img_in: Input image as 2D matrix with top left corner at [1, 1] diff --git a/src/main/python/systemds/operator/algorithm/builtin/img_rotate_linearized.py b/src/main/python/systemds/operator/algorithm/builtin/img_rotate_linearized.py index 94e7ecbff2b..9b1c1725de9 100644 --- a/src/main/python/systemds/operator/algorithm/builtin/img_rotate_linearized.py +++ b/src/main/python/systemds/operator/algorithm/builtin/img_rotate_linearized.py @@ -37,11 +37,32 @@ def img_rotate_linearized(img_in: Matrix, The Linearized Image Rotate function rotates the linearized input images counter-clockwise around the center. Uses nearest neighbor sampling. + .. code-block:: python + >>> import numpy as np + >>> from systemds.context import SystemDSContext + >>> from systemds.operator.algorithm import img_rotate_linearized + >>> + >>> with SystemDSContext() as sds: + ... img = sds.from_numpy( + ... np.array([[ 10., 20., 30., + ... 40., 50., 60., + ... 70., 80., 90. ]], dtype=np.float32) + ... ) + ... result_img = img_rotate_linearized(img, 3.14159, 255., 3, 3).compute() + ... print(result_img.reshape(3, 3)) + [[90. 80. 70.] + [60. 50. 40.] + [30. 20. 10.]] - :param img_in: Linearized input images as 2D matrix with top left corner at [1, 1] + + + + :param img_in: Input images as linearized 2D matrix with top left corner at [1, 1] (every row represents a linearized matrix/image) :param radians: The value by which to rotate in radian. :param fill_value: The background color revealed by the rotation + :param s_cols: Width of a single image + :param s_rows: Height of a single image :return: Output images in linearized form as 2D matrix with top left corner at [1, 1] """ diff --git a/src/main/python/systemds/operator/algorithm/builtin/img_sample_pairing.py b/src/main/python/systemds/operator/algorithm/builtin/img_sample_pairing.py index 892c524baf7..c757d626bd9 100644 --- a/src/main/python/systemds/operator/algorithm/builtin/img_sample_pairing.py +++ b/src/main/python/systemds/operator/algorithm/builtin/img_sample_pairing.py @@ -34,10 +34,34 @@ def img_sample_pairing(img_in1: Matrix, """ The image sample pairing function blends two images together. + .. code-block:: python + >>> import numpy as np + >>> from systemds.context import SystemDSContext + >>> from systemds.operator.algorithm import img_sample_pairing + >>> + >>> with SystemDSContext() as sds: + ... img_in1 = sds.from_numpy( + ... np.array([[ 10., 20., 30.], + ... [ 40., 50., 60.], + ... [ 70., 80., 90.]], dtype=np.float32) + ... ) + ... img_in2 = sds.from_numpy( + ... np.array([[ 30., 40., 50.], + ... [ 60., 70., 80.], + ... [ 90., 100., 110.]], dtype=np.float32) + ... ) + ... result_img = img_sample_pairing(img_in1, img_in2, 0.5).compute() + ... print(result_img) + [[ 20. 30. 40.] + [ 50. 60. 70.] + [ 80. 90. 100.]] - :param img_in1: First input image - :param img_in2: Second input image + + + + :param img_in1: First input image as 2D matrix with top left corner at [1, 1] + :param img_in2: Second input image as 2D matrix with top left corner at [1, 1] :param weight: The weight given to the second image. 0 means only img_in1, 1 means only img_in2 will be visible :return: Output image diff --git a/src/main/python/systemds/operator/algorithm/builtin/img_sample_pairing_linearized.py b/src/main/python/systemds/operator/algorithm/builtin/img_sample_pairing_linearized.py index a7f08c74f41..4368e2ddc38 100644 --- a/src/main/python/systemds/operator/algorithm/builtin/img_sample_pairing_linearized.py +++ b/src/main/python/systemds/operator/algorithm/builtin/img_sample_pairing_linearized.py @@ -34,9 +34,33 @@ def img_sample_pairing_linearized(img_in1: Matrix, """ The image sample pairing function blends two images together. + .. code-block:: python + >>> import numpy as np + >>> from systemds.context import SystemDSContext + >>> from systemds.operator.algorithm import img_sample_pairing_linearized + >>> + >>> with SystemDSContext() as sds: + ... img_in1 = sds.from_numpy( + ... np.array([[ 10., 20., 30., + ... 40., 50., 60., + ... 70., 80., 90. ]], dtype=np.float32) + ... ) + ... img_in2 = sds.from_numpy( + ... np.array([[ 30., 40., 50., + ... 60., 70., 80., + ... 90., 100., 110. ]], dtype=np.float32) + ... ) + ... result_img = img_sample_pairing_linearized(img_in1, img_in2, 0.5).compute() + ... print(result_img.reshape(3, 3)) + [[ 20. 30. 40.] + [ 50. 60. 70.] + [ 80. 90. 100.]] - :param img_in1: input matrix/image (every row is a linearized image) + + + + :param img_in1: Input images as linearized 2D matrix with top left corner at [1, 1] (every row represents a linearized matrix/image) :param img_in2: Second input image (one image represented as a single row linearized matrix) :param weight: The weight given to the second image. 0 means only img_in1, 1 means only img_in2 will be visible diff --git a/src/main/python/systemds/operator/algorithm/builtin/img_shear.py b/src/main/python/systemds/operator/algorithm/builtin/img_shear.py index 44ad9f6883a..b9c6a22cabb 100644 --- a/src/main/python/systemds/operator/algorithm/builtin/img_shear.py +++ b/src/main/python/systemds/operator/algorithm/builtin/img_shear.py @@ -36,6 +36,25 @@ def img_shear(img_in: Matrix, This function applies a shearing transformation to an image. Uses nearest neighbor sampling. + .. code-block:: python + + >>> import numpy as np + >>> from systemds.context import SystemDSContext + >>> from systemds.operator.algorithm import img_shear + >>> + >>> with SystemDSContext() as sds: + ... img = sds.from_numpy( + ... np.array([[ 10., 20., 30.], + ... [ 40., 50., 60.], + ... [ 70., 80., 90.]], dtype=np.float32) + ... ) + ... result_img = img_shear(img, 1., 0., 255).compute() + ... print(result_img) + [[ 10. 20. 30.] + [255. 40. 50.] + [255. 255. 70.]] + + :param img_in: Input image as 2D matrix with top left corner at [1, 1] diff --git a/src/main/python/systemds/operator/algorithm/builtin/img_shear_linearized.py b/src/main/python/systemds/operator/algorithm/builtin/img_shear_linearized.py index a470c8a08c2..96899e6d06a 100644 --- a/src/main/python/systemds/operator/algorithm/builtin/img_shear_linearized.py +++ b/src/main/python/systemds/operator/algorithm/builtin/img_shear_linearized.py @@ -38,12 +38,33 @@ def img_shear_linearized(img_in: Matrix, This function applies a shearing transformation to linearized images. Uses nearest neighbor sampling. + .. code-block:: python + >>> import numpy as np + >>> from systemds.context import SystemDSContext + >>> from systemds.operator.algorithm import img_shear_linearized + >>> + >>> with SystemDSContext() as sds: + ... img = sds.from_numpy( + ... np.array([[ 10., 20., 30., + ... 40., 50., 60., + ... 70., 80., 90. ]], dtype=np.float32) + ... ) + ... result_img = img_shear_linearized(img, 1., 0., 0., 3, 3).compute() + ... print(result_img.reshape(3, 3)) + [[10. 20. 30.] + [ 0. 40. 50.] + [ 0. 0. 70.]] - :param img_in: Linearized input images as 2D matrix with top left corner at [1, 1] + + + + :param img_in: Input images as linearized 2D matrix with top left corner at [1, 1] (every row represents a linearized matrix/image) :param shear_x: Shearing factor for horizontal shearing :param shear_y: Shearing factor for vertical shearing :param fill_value: The background color revealed by the shearing + :param s_cols: Width of a single image + :param s_rows: Height of a single image :return: Output images in linearized form as 2D matrix with top left corner at [1, 1] """ diff --git a/src/main/python/systemds/operator/algorithm/builtin/img_transform.py b/src/main/python/systemds/operator/algorithm/builtin/img_transform.py index 73095816bdf..51a2468e1ad 100644 --- a/src/main/python/systemds/operator/algorithm/builtin/img_transform.py +++ b/src/main/python/systemds/operator/algorithm/builtin/img_transform.py @@ -43,6 +43,25 @@ def img_transform(img_in: Matrix, Optionally resizes the image (without scaling). Uses nearest neighbor sampling. + .. code-block:: python + + >>> import numpy as np + >>> from systemds.context import SystemDSContext + >>> from systemds.operator.algorithm import img_transform + >>> + >>> with SystemDSContext() as sds: + ... img = sds.from_numpy( + ... np.array([[ 10., 20., 30.], + ... [ 40., 50., 60.], + ... [ 70., 80., 90.]], dtype=np.float32) + ... ) + ... result_img = img_transform(img, 3, 3, -1., 0., 2., 0., 1., 0., 255.).compute() + ... print(result_img) + [[ 20. 10. 255.] + [ 50. 40. 255.] + [ 80. 70. 255.]] + + :param img_in: Input image as 2D matrix with top left corner at [1, 1] diff --git a/src/main/python/systemds/operator/algorithm/builtin/img_transform_linearized.py b/src/main/python/systemds/operator/algorithm/builtin/img_transform_linearized.py index 8020b22a54e..fb706fc2f93 100644 --- a/src/main/python/systemds/operator/algorithm/builtin/img_transform_linearized.py +++ b/src/main/python/systemds/operator/algorithm/builtin/img_transform_linearized.py @@ -45,13 +45,34 @@ def img_transform_linearized(img_in: Matrix, Optionally resizes the image (without scaling). Uses nearest neighbor sampling. + .. code-block:: python + >>> import numpy as np + >>> from systemds.context import SystemDSContext + >>> from systemds.operator.algorithm import img_transform_linearized + >>> + >>> with SystemDSContext() as sds: + ... img = sds.from_numpy( + ... np.array([[ 10., 20., 30., + ... 40., 50., 60., + ... 70., 80., 90. ]], dtype=np.float32) + ... ) + ... result_img = img_transform_linearized(img, 3, 3, -1., 0., 2., 0., 1., 0., 255., 3, 3).compute() + ... print(result_img.reshape(3, 3)) + [[ 20. 10. 255.] + [ 50. 40. 255.] + [ 80. 70. 255.]] - :param img_in: Linearized input images as 2D matrix with top left corner at [1, 1] + + + + :param img_in: Input images as linearized 2D matrix with top left corner at [1, 1] (every row represents a linearized matrix/image) :param out_w: Width of the output matrix :param out_h: Height of the output matrix :param a,b,c,d,e,f: The first two rows of the affine matrix in row-major order :param fill_value: The background of an image + :param s_cols: Width of a single image + :param s_rows: Height of a single image :return: Output images in linearized form as 2D matrix with top left corner at [1, 1] """ diff --git a/src/main/python/systemds/operator/algorithm/builtin/img_translate.py b/src/main/python/systemds/operator/algorithm/builtin/img_translate.py index 9cfc991ca57..e184bdbadfe 100644 --- a/src/main/python/systemds/operator/algorithm/builtin/img_translate.py +++ b/src/main/python/systemds/operator/algorithm/builtin/img_translate.py @@ -39,6 +39,25 @@ def img_translate(img_in: Matrix, Optionally resizes the image (without scaling). Uses nearest neighbor sampling. + .. code-block:: python + + >>> import numpy as np + >>> from systemds.context import SystemDSContext + >>> from systemds.operator.algorithm import img_translate + >>> + >>> with SystemDSContext() as sds: + ... img = sds.from_numpy( + ... np.array([[ 10., 20., 30.], + ... [ 40., 50., 60.], + ... [ 70., 80., 90.]], dtype=np.float32) + ... ) + ... result_img = img_translate(img, 1., 1., 3, 3, 255.).compute() + ... print(result_img) + [[255. 255. 255.] + [255. 10. 20.] + [255. 40. 50.]] + + :param img_in: Input image as 2D matrix with top left corner at [1, 1] diff --git a/src/main/python/systemds/operator/algorithm/builtin/img_translate_linearized.py b/src/main/python/systemds/operator/algorithm/builtin/img_translate_linearized.py index 92098dab2c7..9b3e2eab92f 100644 --- a/src/main/python/systemds/operator/algorithm/builtin/img_translate_linearized.py +++ b/src/main/python/systemds/operator/algorithm/builtin/img_translate_linearized.py @@ -41,8 +41,28 @@ def img_translate_linearized(img_in: Matrix, the same time. Each row of the input and output matrix represents a linearized image/matrix It translates the image and Optionally resizes the image (without scaling). + .. code-block:: python - :param img_in: Input matrix/image (every row represents a linearized matrix/image) + >>> import numpy as np + >>> from systemds.context import SystemDSContext + >>> from systemds.operator.algorithm import img_translate_linearized + >>> + >>> with SystemDSContext() as sds: + ... img = sds.from_numpy( + ... np.array([[ 10., 20., 30., + ... 40., 50., 60., + ... 70., 80., 90. ]], dtype=np.float32) + ... ) + ... result_img = img_translate_linearized(img, 1., 1., 3, 3, 255.0, 3, 3).compute() + ... print(result_img.reshape(3, 3)) + [[255. 255. 255.] + [255. 10. 20.] + [255. 40. 50.]] + + + + + :param img_in: Input images as linearized 2D matrix with top left corner at [1, 1] (every row represents a linearized matrix/image) :param offset_x: The distance to move the image in x direction :param offset_y: The distance to move the image in y direction :param out_w: Width of the output image diff --git a/src/main/python/systemds/operator/algorithm/builtin/lm.py b/src/main/python/systemds/operator/algorithm/builtin/lm.py index e67bbb366fc..fa317fb22c2 100644 --- a/src/main/python/systemds/operator/algorithm/builtin/lm.py +++ b/src/main/python/systemds/operator/algorithm/builtin/lm.py @@ -36,6 +36,36 @@ def lm(X: Matrix, method or the conjugate gradient algorithm depending on the input size of the matrices (See lmDS-function and lmCG-function respectively). + .. code-block:: python + + >>> import numpy as np + >>> from systemds.context import SystemDSContext + >>> from systemds.operator.algorithm import lm + >>> + >>> np.random.seed(0) + >>> features = np.random.rand(10, 15) + >>> y = np.random.rand(10, 1) + >>> + >>> with SystemDSContext() as sds: + ... weights = lm(sds.from_numpy(features), sds.from_numpy(y)).compute() + ... print(weights) + [[-0.11538199] + [-0.20386541] + [-0.39956034] + [ 1.04078623] + [ 0.43270839] + [ 0.18954599] + [ 0.49858969] + [-0.26812763] + [ 0.09961844] + [-0.57000751] + [-0.43386048] + [ 0.55358873] + [-0.54638565] + [ 0.2205885 ] + [ 0.37957689]] + + :param X: Matrix of feature vectors. diff --git a/src/main/python/systemds/operator/algorithm/builtin/normalize.py b/src/main/python/systemds/operator/algorithm/builtin/normalize.py index 9353f75b8bf..f4dd5bbd123 100644 --- a/src/main/python/systemds/operator/algorithm/builtin/normalize.py +++ b/src/main/python/systemds/operator/algorithm/builtin/normalize.py @@ -33,6 +33,20 @@ def normalize(X: Matrix): Min-max normalization (a.k.a. min-max scaling) to range [0,1]. For matrices of positive values, this normalization preserves the input sparsity. + .. code-block:: python + + >>> import numpy as np + >>> from systemds.context import SystemDSContext + >>> from systemds.operator.algorithm import normalize + >>> + >>> with SystemDSContext() as sds: + ... X = sds.from_numpy(np.array([[1, 2], [3, 4]])) + ... Y, cmin, cmax = normalize(X).compute() + ... print(Y) + [[0. 0.] + [1. 1.]] + + :param X: Input feature matrix of shape n-by-m diff --git a/src/main/python/systemds/operator/algorithm/builtin/randomForest.py b/src/main/python/systemds/operator/algorithm/builtin/randomForest.py index 177ebd3fd38..f100d28092f 100644 --- a/src/main/python/systemds/operator/algorithm/builtin/randomForest.py +++ b/src/main/python/systemds/operator/algorithm/builtin/randomForest.py @@ -60,6 +60,47 @@ def randomForest(X: Matrix, prefixed by a one-hot vector of sampled features (e.g., [1,1,1,0] if we sampled a,b,c of the four features) + .. code-block:: python + + >>> import numpy as np + >>> from systemds.context import SystemDSContext + >>> from systemds.operator.algorithm import randomForest, randomForestPredict + >>> + >>> # tiny toy dataset + >>> X = np.array([[1], + ... [2], + ... [10], + ... [11]], dtype=np.int64) + >>> y = np.array([[1], + ... [1], + ... [2], + ... [2]], dtype=np.int64) + >>> + >>> with SystemDSContext() as sds: + ... X_sds = sds.from_numpy(X) + ... y_sds = sds.from_numpy(y) + ... + ... ctypes = sds.from_numpy(np.array([[1, 2]], dtype=np.int64)) + ... + ... # train a 4-tree forest (no sampling) + ... M = randomForest( + ... X_sds, y_sds, ctypes, + ... num_trees = 4, + ... sample_frac = 1.0, + ... feature_frac = 1.0, + ... max_depth = 3, + ... min_leaf = 1, + ... min_split = 2, + ... seed = 42 + ... ) + ... + ... preds = randomForestPredict(X_sds, ctypes, M).compute() + ... print(preds) + [[1.] + [1.] + [2.] + [2.]] + diff --git a/src/main/python/systemds/operator/algorithm/builtin/randomForestPredict.py b/src/main/python/systemds/operator/algorithm/builtin/randomForestPredict.py index bff00217e27..78dc1bda3f1 100644 --- a/src/main/python/systemds/operator/algorithm/builtin/randomForestPredict.py +++ b/src/main/python/systemds/operator/algorithm/builtin/randomForestPredict.py @@ -37,6 +37,49 @@ def randomForestPredict(X: Matrix, categorical and numerical input features. + .. code-block:: python + + >>> import numpy as np + >>> from systemds.context import SystemDSContext + >>> from systemds.operator.algorithm import randomForest, randomForestPredict + >>> + >>> # tiny toy dataset + >>> X = np.array([[1], + ... [2], + ... [10], + ... [11]], dtype=np.int64) + >>> y = np.array([[1], + ... [1], + ... [2], + ... [2]], dtype=np.int64) + >>> + >>> with SystemDSContext() as sds: + ... X_sds = sds.from_numpy(X) + ... y_sds = sds.from_numpy(y) + ... + ... ctypes = sds.from_numpy(np.array([[1, 2]], dtype=np.int64)) + ... + ... # train a 4-tree forest (no sampling) + ... M = randomForest( + ... X_sds, y_sds, ctypes, + ... num_trees = 4, + ... sample_frac = 1.0, + ... feature_frac = 1.0, + ... max_depth = 3, + ... min_leaf = 1, + ... min_split = 2, + ... seed = 42 + ... ) + ... + ... preds = randomForestPredict(X_sds, ctypes, M).compute() + ... print(preds) + [[1.] + [1.] + [2.] + [2.]] + + + :param X: Feature matrix in recoded/binned representation :param y: Label matrix in recoded/binned representation, diff --git a/src/main/python/systemds/operator/algorithm/builtin/toOneHot.py b/src/main/python/systemds/operator/algorithm/builtin/toOneHot.py index 0ad76ec5f70..436ed64df72 100644 --- a/src/main/python/systemds/operator/algorithm/builtin/toOneHot.py +++ b/src/main/python/systemds/operator/algorithm/builtin/toOneHot.py @@ -33,6 +33,22 @@ def toOneHot(X: Matrix, """ The toOneHot-function encodes unordered categorical vector to multiple binary vectors. + .. code-block:: python + + >>> import numpy as np + >>> from systemds.context import SystemDSContext + >>> from systemds.operator.algorithm import toOneHot + >>> + >>> with SystemDSContext() as sds: + ... X = sds.from_numpy(np.array([[1], [3], [2], [3]])) + ... Y = toOneHot(X, numClasses=3).compute() + ... print(Y) + [[1. 0. 0.] + [0. 0. 1.] + [0. 1. 0.] + [0. 0. 1.]] + + :param X: Vector with N integer entries between 1 and numClasses diff --git a/src/main/python/tests/algorithms/test_lm.py b/src/main/python/tests/algorithms/test_lm.py deleted file mode 100644 index aad8abac66a..00000000000 --- a/src/main/python/tests/algorithms/test_lm.py +++ /dev/null @@ -1,65 +0,0 @@ -# ------------------------------------------------------------- -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# -# ------------------------------------------------------------- - -import unittest - -import numpy as np -from sklearn.linear_model import LinearRegression -from systemds.context import SystemDSContext -from systemds.operator.algorithm import lm - -np.random.seed(7) - - -class TestLm(unittest.TestCase): - - sds: SystemDSContext = None - - @classmethod - def setUpClass(cls): - cls.sds = SystemDSContext() - - @classmethod - def tearDownClass(cls): - cls.sds.close() - - def test_lm_simple(self): - # if the dimensions of the input is 1, then the - X = np.random.rand(30, 1) - Y = np.random.rand(30, 1) - regressor = LinearRegression(fit_intercept=False) - model = regressor.fit(X, Y).coef_ - - X_sds = self.sds.from_numpy(X) - Y_sds = self.sds.from_numpy(Y) - - sds_model_weights = lm(X_sds, Y_sds, verbose=False).compute() - model = model.reshape(sds_model_weights.shape) - - eps = 1e-03 - - self.assertTrue( - np.allclose(sds_model_weights, model, eps), "All elements are not close" - ) - - -if __name__ == "__main__": - unittest.main(exit=False) diff --git a/src/main/python/tests/auto_tests/test_dist.py b/src/main/python/tests/auto_tests/test_dist.py new file mode 100644 index 00000000000..6a3e39f05da --- /dev/null +++ b/src/main/python/tests/auto_tests/test_dist.py @@ -0,0 +1,44 @@ +# ------------------------------------------------------------- +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# ------------------------------------------------------------- + +# Autogenerated By : src/main/python/generator/generator.py +import unittest, contextlib, io +class TestDIST(unittest.TestCase): + def test_dist(self): + # Example test case provided in python the code block + buf = io.StringIO() + with contextlib.redirect_stdout(buf): + import numpy as np + from systemds.context import SystemDSContext + from systemds.operator.algorithm import dist + + with SystemDSContext() as sds: + X = sds.from_numpy(np.array([[0], [3], [4]])) + out = dist(X).compute() + print(out) + + expected="""[[0. 3. 4.] + [3. 0. 1.] + [4. 1. 0.]]""" + self.assertEqual(buf.getvalue().strip(), expected) + +if __name__ == '__main__': + unittest.main() diff --git a/src/main/python/tests/auto_tests/test_img_brightness.py b/src/main/python/tests/auto_tests/test_img_brightness.py new file mode 100644 index 00000000000..f30545cd769 --- /dev/null +++ b/src/main/python/tests/auto_tests/test_img_brightness.py @@ -0,0 +1,46 @@ +# ------------------------------------------------------------- +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# ------------------------------------------------------------- + +# Autogenerated By : src/main/python/generator/generator.py +import unittest, contextlib, io +class TestIMG_BRIGHTNESS(unittest.TestCase): + def test_img_brightness(self): + # Example test case provided in python the code block + buf = io.StringIO() + with contextlib.redirect_stdout(buf): + import numpy as np + from systemds.context import SystemDSContext + from systemds.operator.algorithm import img_brightness + + with SystemDSContext() as sds: + img = sds.from_numpy( + np.array([[ 50, 100, + 150, 200 ]], dtype=np.float32) + ) + result_img = img_brightness(img, 30.0, 150).compute() + print(result_img.reshape(2, 2)) + + expected="""[[ 80. 130.] + [150. 150.]]""" + self.assertEqual(buf.getvalue().strip(), expected) + +if __name__ == '__main__': + unittest.main() diff --git a/src/main/python/tests/auto_tests/test_img_brightness_linearized.py b/src/main/python/tests/auto_tests/test_img_brightness_linearized.py new file mode 100644 index 00000000000..a82bab31536 --- /dev/null +++ b/src/main/python/tests/auto_tests/test_img_brightness_linearized.py @@ -0,0 +1,46 @@ +# ------------------------------------------------------------- +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# ------------------------------------------------------------- + +# Autogenerated By : src/main/python/generator/generator.py +import unittest, contextlib, io +class TestIMG_BRIGHTNESS_LINEARIZED(unittest.TestCase): + def test_img_brightness_linearized(self): + # Example test case provided in python the code block + buf = io.StringIO() + with contextlib.redirect_stdout(buf): + import numpy as np + from systemds.context import SystemDSContext + from systemds.operator.algorithm import img_brightness_linearized + + with SystemDSContext() as sds: + img = sds.from_numpy( + np.array([[ 50, 100, + 150, 200 ]], dtype=np.float32) + ) + result_img = img_brightness_linearized(img, 30.0, 255).compute() + print(result_img.reshape(2, 2)) + + expected="""[[ 80. 130.] + [180. 230.]]""" + self.assertEqual(buf.getvalue().strip(), expected) + +if __name__ == '__main__': + unittest.main() diff --git a/src/main/python/tests/auto_tests/test_img_crop.py b/src/main/python/tests/auto_tests/test_img_crop.py new file mode 100644 index 00000000000..ac94a9f8338 --- /dev/null +++ b/src/main/python/tests/auto_tests/test_img_crop.py @@ -0,0 +1,46 @@ +# ------------------------------------------------------------- +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# ------------------------------------------------------------- + +# Autogenerated By : src/main/python/generator/generator.py +import unittest, contextlib, io +class TestIMG_CROP(unittest.TestCase): + def test_img_crop(self): + # Example test case provided in python the code block + buf = io.StringIO() + with contextlib.redirect_stdout(buf): + import numpy as np + from systemds.context import SystemDSContext + from systemds.operator.algorithm import img_crop + + with SystemDSContext() as sds: + img = sds.from_numpy( + np.array([[ 50., 100., 150.], + [150., 200., 250.], + [250., 200., 200.]], dtype=np.float32) + ) + result_img = img_crop(img, 1, 1, 1, 1).compute() + print(result_img) + + expected="""[[200.]]""" + self.assertEqual(buf.getvalue().strip(), expected) + +if __name__ == '__main__': + unittest.main() diff --git a/src/main/python/tests/auto_tests/test_img_crop_linearized.py b/src/main/python/tests/auto_tests/test_img_crop_linearized.py new file mode 100644 index 00000000000..ae223721886 --- /dev/null +++ b/src/main/python/tests/auto_tests/test_img_crop_linearized.py @@ -0,0 +1,46 @@ +# ------------------------------------------------------------- +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# ------------------------------------------------------------- + +# Autogenerated By : src/main/python/generator/generator.py +import unittest, contextlib, io +class TestIMG_CROP_LINEARIZED(unittest.TestCase): + def test_img_crop_linearized(self): + # Example test case provided in python the code block + buf = io.StringIO() + with contextlib.redirect_stdout(buf): + import numpy as np + from systemds.context import SystemDSContext + from systemds.operator.algorithm import img_crop_linearized + + with SystemDSContext() as sds: + img = sds.from_numpy( + np.array([[ 50., 100., 150., + 150., 200., 250., + 250., 200., 200. ]], dtype=np.float32) + ) + result_img = img_crop_linearized(img, 1, 1, 1, 1, 3, 3).compute() + print(result_img) + + expected="""[[200.]]""" + self.assertEqual(buf.getvalue().strip(), expected) + +if __name__ == '__main__': + unittest.main() diff --git a/src/main/python/tests/auto_tests/test_img_cutout.py b/src/main/python/tests/auto_tests/test_img_cutout.py new file mode 100644 index 00000000000..e66f0aed6f2 --- /dev/null +++ b/src/main/python/tests/auto_tests/test_img_cutout.py @@ -0,0 +1,48 @@ +# ------------------------------------------------------------- +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# ------------------------------------------------------------- + +# Autogenerated By : src/main/python/generator/generator.py +import unittest, contextlib, io +class TestIMG_CUTOUT(unittest.TestCase): + def test_img_cutout(self): + # Example test case provided in python the code block + buf = io.StringIO() + with contextlib.redirect_stdout(buf): + import numpy as np + from systemds.context import SystemDSContext + from systemds.operator.algorithm import img_cutout + + with SystemDSContext() as sds: + img = sds.from_numpy( + np.array([[ 50., 100., 150.], + [150., 200., 250.], + [250., 200., 200.]], dtype=np.float32) + ) + result_img = img_cutout(img, 2, 2, 1, 1, 49.).compute() + print(result_img) + + expected="""[[ 50. 100. 150.] + [150. 49. 250.] + [250. 200. 200.]]""" + self.assertEqual(buf.getvalue().strip(), expected) + +if __name__ == '__main__': + unittest.main() diff --git a/src/main/python/tests/auto_tests/test_img_cutout_linearized.py b/src/main/python/tests/auto_tests/test_img_cutout_linearized.py new file mode 100644 index 00000000000..f553b140393 --- /dev/null +++ b/src/main/python/tests/auto_tests/test_img_cutout_linearized.py @@ -0,0 +1,48 @@ +# ------------------------------------------------------------- +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# ------------------------------------------------------------- + +# Autogenerated By : src/main/python/generator/generator.py +import unittest, contextlib, io +class TestIMG_CUTOUT_LINEARIZED(unittest.TestCase): + def test_img_cutout_linearized(self): + # Example test case provided in python the code block + buf = io.StringIO() + with contextlib.redirect_stdout(buf): + import numpy as np + from systemds.context import SystemDSContext + from systemds.operator.algorithm import img_cutout_linearized + + with SystemDSContext() as sds: + img = sds.from_numpy( + np.array([[ 50., 100., 150., + 150., 200., 250., + 250., 200., 200. ]], dtype=np.float32) + ) + result_img = img_cutout_linearized(img, 2, 2, 1, 1, 25.0, 3, 3).compute() + print(result_img.reshape(3, 3)) + + expected="""[[ 50. 100. 150.] + [150. 25. 250.] + [250. 200. 200.]]""" + self.assertEqual(buf.getvalue().strip(), expected) + +if __name__ == '__main__': + unittest.main() diff --git a/src/main/python/tests/auto_tests/test_img_invert.py b/src/main/python/tests/auto_tests/test_img_invert.py new file mode 100644 index 00000000000..a11ff19e99a --- /dev/null +++ b/src/main/python/tests/auto_tests/test_img_invert.py @@ -0,0 +1,48 @@ +# ------------------------------------------------------------- +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# ------------------------------------------------------------- + +# Autogenerated By : src/main/python/generator/generator.py +import unittest, contextlib, io +class TestIMG_INVERT(unittest.TestCase): + def test_img_invert(self): + # Example test case provided in python the code block + buf = io.StringIO() + with contextlib.redirect_stdout(buf): + import numpy as np + from systemds.context import SystemDSContext + from systemds.operator.algorithm import img_invert + + with SystemDSContext() as sds: + img = sds.from_numpy( + np.array([[ 10., 20., 30.], + [ 40., 50., 60.], + [ 70., 80., 90.]], dtype=np.float32) + ) + result_img = img_invert(img, 210.).compute() + print(result_img) + + expected="""[[200. 190. 180.] + [170. 160. 150.] + [140. 130. 120.]]""" + self.assertEqual(buf.getvalue().strip(), expected) + +if __name__ == '__main__': + unittest.main() diff --git a/src/main/python/tests/auto_tests/test_img_invert_linearized.py b/src/main/python/tests/auto_tests/test_img_invert_linearized.py new file mode 100644 index 00000000000..65db0866c9c --- /dev/null +++ b/src/main/python/tests/auto_tests/test_img_invert_linearized.py @@ -0,0 +1,48 @@ +# ------------------------------------------------------------- +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# ------------------------------------------------------------- + +# Autogenerated By : src/main/python/generator/generator.py +import unittest, contextlib, io +class TestIMG_INVERT_LINEARIZED(unittest.TestCase): + def test_img_invert_linearized(self): + # Example test case provided in python the code block + buf = io.StringIO() + with contextlib.redirect_stdout(buf): + import numpy as np + from systemds.context import SystemDSContext + from systemds.operator.algorithm import img_invert_linearized + + with SystemDSContext() as sds: + img = sds.from_numpy( + np.array([[ 10., 20., 30., + 40., 50., 60., + 70., 80., 90. ]], dtype=np.float32) + ) + result_img = img_invert_linearized(img, 200.).compute() + print(result_img.reshape(3, 3)) + + expected="""[[190. 180. 170.] + [160. 150. 140.] + [130. 120. 110.]]""" + self.assertEqual(buf.getvalue().strip(), expected) + +if __name__ == '__main__': + unittest.main() diff --git a/src/main/python/tests/auto_tests/test_img_mirror.py b/src/main/python/tests/auto_tests/test_img_mirror.py new file mode 100644 index 00000000000..241635199c8 --- /dev/null +++ b/src/main/python/tests/auto_tests/test_img_mirror.py @@ -0,0 +1,48 @@ +# ------------------------------------------------------------- +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# ------------------------------------------------------------- + +# Autogenerated By : src/main/python/generator/generator.py +import unittest, contextlib, io +class TestIMG_MIRROR(unittest.TestCase): + def test_img_mirror(self): + # Example test case provided in python the code block + buf = io.StringIO() + with contextlib.redirect_stdout(buf): + import numpy as np + from systemds.context import SystemDSContext + from systemds.operator.algorithm import img_mirror + + with SystemDSContext() as sds: + img = sds.from_numpy( + np.array([[ 10., 20., 30.], + [ 40., 50., 60.], + [ 70., 80., 90.]], dtype=np.float32) + ) + result_img = img_mirror(img, False).compute() + print(result_img) + + expected="""[[30. 20. 10.] + [60. 50. 40.] + [90. 80. 70.]]""" + self.assertEqual(buf.getvalue().strip(), expected) + +if __name__ == '__main__': + unittest.main() diff --git a/src/main/python/tests/auto_tests/test_img_mirror_linearized.py b/src/main/python/tests/auto_tests/test_img_mirror_linearized.py new file mode 100644 index 00000000000..582c1361639 --- /dev/null +++ b/src/main/python/tests/auto_tests/test_img_mirror_linearized.py @@ -0,0 +1,48 @@ +# ------------------------------------------------------------- +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# ------------------------------------------------------------- + +# Autogenerated By : src/main/python/generator/generator.py +import unittest, contextlib, io +class TestIMG_MIRROR_LINEARIZED(unittest.TestCase): + def test_img_mirror_linearized(self): + # Example test case provided in python the code block + buf = io.StringIO() + with contextlib.redirect_stdout(buf): + import numpy as np + from systemds.context import SystemDSContext + from systemds.operator.algorithm import img_mirror_linearized + + with SystemDSContext() as sds: + img = sds.from_numpy( + np.array([[ 10., 20., 30., + 40., 50., 60., + 70., 80., 90. ]], dtype=np.float32) + ) + result_img = img_mirror_linearized(img, True, 3, 3).compute() + print(result_img.reshape(3, 3)) + + expected="""[[70. 80. 90.] + [40. 50. 60.] + [10. 20. 30.]]""" + self.assertEqual(buf.getvalue().strip(), expected) + +if __name__ == '__main__': + unittest.main() diff --git a/src/main/python/tests/auto_tests/test_img_posterize.py b/src/main/python/tests/auto_tests/test_img_posterize.py new file mode 100644 index 00000000000..d0f1cd7b6cc --- /dev/null +++ b/src/main/python/tests/auto_tests/test_img_posterize.py @@ -0,0 +1,48 @@ +# ------------------------------------------------------------- +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# ------------------------------------------------------------- + +# Autogenerated By : src/main/python/generator/generator.py +import unittest, contextlib, io +class TestIMG_POSTERIZE(unittest.TestCase): + def test_img_posterize(self): + # Example test case provided in python the code block + buf = io.StringIO() + with contextlib.redirect_stdout(buf): + import numpy as np + from systemds.context import SystemDSContext + from systemds.operator.algorithm import img_posterize + + with SystemDSContext() as sds: + img = sds.from_numpy( + np.array([[ 10., 20., 30.], + [ 40., 255., 60.], + [ 70., 80., 90.]], dtype=np.float32) + ) + result_img = img_posterize(img, 1).compute() + print(result_img) + + expected="""[[ 0. 0. 0.] + [ 0. 128. 0.] + [ 0. 0. 0.]]""" + self.assertEqual(buf.getvalue().strip(), expected) + +if __name__ == '__main__': + unittest.main() diff --git a/src/main/python/tests/auto_tests/test_img_posterize_linearized.py b/src/main/python/tests/auto_tests/test_img_posterize_linearized.py new file mode 100644 index 00000000000..fb2605998f8 --- /dev/null +++ b/src/main/python/tests/auto_tests/test_img_posterize_linearized.py @@ -0,0 +1,48 @@ +# ------------------------------------------------------------- +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# ------------------------------------------------------------- + +# Autogenerated By : src/main/python/generator/generator.py +import unittest, contextlib, io +class TestIMG_POSTERIZE_LINEARIZED(unittest.TestCase): + def test_img_posterize_linearized(self): + # Example test case provided in python the code block + buf = io.StringIO() + with contextlib.redirect_stdout(buf): + import numpy as np + from systemds.context import SystemDSContext + from systemds.operator.algorithm import img_posterize_linearized + + with SystemDSContext() as sds: + img = sds.from_numpy( + np.array([[ 10., 20., 30., + 40., 255., 60., + 70., 80., 90. ]], dtype=np.float32) + ) + result_img = img_posterize_linearized(img, 1).compute() + print(result_img.reshape(3, 3)) + + expected="""[[ 0. 0. 0.] + [ 0. 128. 0.] + [ 0. 0. 0.]]""" + self.assertEqual(buf.getvalue().strip(), expected) + +if __name__ == '__main__': + unittest.main() diff --git a/src/main/python/tests/auto_tests/test_img_rotate.py b/src/main/python/tests/auto_tests/test_img_rotate.py new file mode 100644 index 00000000000..ed7e8bcc435 --- /dev/null +++ b/src/main/python/tests/auto_tests/test_img_rotate.py @@ -0,0 +1,48 @@ +# ------------------------------------------------------------- +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# ------------------------------------------------------------- + +# Autogenerated By : src/main/python/generator/generator.py +import unittest, contextlib, io +class TestIMG_ROTATE(unittest.TestCase): + def test_img_rotate(self): + # Example test case provided in python the code block + buf = io.StringIO() + with contextlib.redirect_stdout(buf): + import numpy as np + from systemds.context import SystemDSContext + from systemds.operator.algorithm import img_rotate + + with SystemDSContext() as sds: + img = sds.from_numpy( + np.array([[ 10., 20., 30.], + [ 40., 50., 60.], + [ 70., 80., 90.]], dtype=np.float32) + ) + result_img = img_rotate(img, 3.14159, 255.).compute() + print(result_img) + + expected="""[[90. 80. 70.] + [60. 50. 40.] + [30. 20. 10.]]""" + self.assertEqual(buf.getvalue().strip(), expected) + +if __name__ == '__main__': + unittest.main() diff --git a/src/main/python/tests/auto_tests/test_img_rotate_linearized.py b/src/main/python/tests/auto_tests/test_img_rotate_linearized.py new file mode 100644 index 00000000000..1f2d7ba1f6d --- /dev/null +++ b/src/main/python/tests/auto_tests/test_img_rotate_linearized.py @@ -0,0 +1,48 @@ +# ------------------------------------------------------------- +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# ------------------------------------------------------------- + +# Autogenerated By : src/main/python/generator/generator.py +import unittest, contextlib, io +class TestIMG_ROTATE_LINEARIZED(unittest.TestCase): + def test_img_rotate_linearized(self): + # Example test case provided in python the code block + buf = io.StringIO() + with contextlib.redirect_stdout(buf): + import numpy as np + from systemds.context import SystemDSContext + from systemds.operator.algorithm import img_rotate_linearized + + with SystemDSContext() as sds: + img = sds.from_numpy( + np.array([[ 10., 20., 30., + 40., 50., 60., + 70., 80., 90. ]], dtype=np.float32) + ) + result_img = img_rotate_linearized(img, 3.14159, 255., 3, 3).compute() + print(result_img.reshape(3, 3)) + + expected="""[[90. 80. 70.] + [60. 50. 40.] + [30. 20. 10.]]""" + self.assertEqual(buf.getvalue().strip(), expected) + +if __name__ == '__main__': + unittest.main() diff --git a/src/main/python/tests/auto_tests/test_img_sample_pairing.py b/src/main/python/tests/auto_tests/test_img_sample_pairing.py new file mode 100644 index 00000000000..87c50493c41 --- /dev/null +++ b/src/main/python/tests/auto_tests/test_img_sample_pairing.py @@ -0,0 +1,53 @@ +# ------------------------------------------------------------- +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# ------------------------------------------------------------- + +# Autogenerated By : src/main/python/generator/generator.py +import unittest, contextlib, io +class TestIMG_SAMPLE_PAIRING(unittest.TestCase): + def test_img_sample_pairing(self): + # Example test case provided in python the code block + buf = io.StringIO() + with contextlib.redirect_stdout(buf): + import numpy as np + from systemds.context import SystemDSContext + from systemds.operator.algorithm import img_sample_pairing + + with SystemDSContext() as sds: + img_in1 = sds.from_numpy( + np.array([[ 10., 20., 30.], + [ 40., 50., 60.], + [ 70., 80., 90.]], dtype=np.float32) + ) + img_in2 = sds.from_numpy( + np.array([[ 30., 40., 50.], + [ 60., 70., 80.], + [ 90., 100., 110.]], dtype=np.float32) + ) + result_img = img_sample_pairing(img_in1, img_in2, 0.5).compute() + print(result_img) + + expected="""[[ 20. 30. 40.] + [ 50. 60. 70.] + [ 80. 90. 100.]]""" + self.assertEqual(buf.getvalue().strip(), expected) + +if __name__ == '__main__': + unittest.main() diff --git a/src/main/python/tests/auto_tests/test_img_sample_pairing_linearized.py b/src/main/python/tests/auto_tests/test_img_sample_pairing_linearized.py new file mode 100644 index 00000000000..5b550735e35 --- /dev/null +++ b/src/main/python/tests/auto_tests/test_img_sample_pairing_linearized.py @@ -0,0 +1,53 @@ +# ------------------------------------------------------------- +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# ------------------------------------------------------------- + +# Autogenerated By : src/main/python/generator/generator.py +import unittest, contextlib, io +class TestIMG_SAMPLE_PAIRING_LINEARIZED(unittest.TestCase): + def test_img_sample_pairing_linearized(self): + # Example test case provided in python the code block + buf = io.StringIO() + with contextlib.redirect_stdout(buf): + import numpy as np + from systemds.context import SystemDSContext + from systemds.operator.algorithm import img_sample_pairing_linearized + + with SystemDSContext() as sds: + img_in1 = sds.from_numpy( + np.array([[ 10., 20., 30., + 40., 50., 60., + 70., 80., 90. ]], dtype=np.float32) + ) + img_in2 = sds.from_numpy( + np.array([[ 30., 40., 50., + 60., 70., 80., + 90., 100., 110. ]], dtype=np.float32) + ) + result_img = img_sample_pairing_linearized(img_in1, img_in2, 0.5).compute() + print(result_img.reshape(3, 3)) + + expected="""[[ 20. 30. 40.] + [ 50. 60. 70.] + [ 80. 90. 100.]]""" + self.assertEqual(buf.getvalue().strip(), expected) + +if __name__ == '__main__': + unittest.main() diff --git a/src/main/python/tests/auto_tests/test_img_shear.py b/src/main/python/tests/auto_tests/test_img_shear.py new file mode 100644 index 00000000000..5571e9d3cb5 --- /dev/null +++ b/src/main/python/tests/auto_tests/test_img_shear.py @@ -0,0 +1,48 @@ +# ------------------------------------------------------------- +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# ------------------------------------------------------------- + +# Autogenerated By : src/main/python/generator/generator.py +import unittest, contextlib, io +class TestIMG_SHEAR(unittest.TestCase): + def test_img_shear(self): + # Example test case provided in python the code block + buf = io.StringIO() + with contextlib.redirect_stdout(buf): + import numpy as np + from systemds.context import SystemDSContext + from systemds.operator.algorithm import img_shear + + with SystemDSContext() as sds: + img = sds.from_numpy( + np.array([[ 10., 20., 30.], + [ 40., 50., 60.], + [ 70., 80., 90.]], dtype=np.float32) + ) + result_img = img_shear(img, 1., 0., 255).compute() + print(result_img) + + expected="""[[ 10. 20. 30.] + [255. 40. 50.] + [255. 255. 70.]]""" + self.assertEqual(buf.getvalue().strip(), expected) + +if __name__ == '__main__': + unittest.main() diff --git a/src/main/python/tests/auto_tests/test_img_shear_linearized.py b/src/main/python/tests/auto_tests/test_img_shear_linearized.py new file mode 100644 index 00000000000..d8963beaebd --- /dev/null +++ b/src/main/python/tests/auto_tests/test_img_shear_linearized.py @@ -0,0 +1,48 @@ +# ------------------------------------------------------------- +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# ------------------------------------------------------------- + +# Autogenerated By : src/main/python/generator/generator.py +import unittest, contextlib, io +class TestIMG_SHEAR_LINEARIZED(unittest.TestCase): + def test_img_shear_linearized(self): + # Example test case provided in python the code block + buf = io.StringIO() + with contextlib.redirect_stdout(buf): + import numpy as np + from systemds.context import SystemDSContext + from systemds.operator.algorithm import img_shear_linearized + + with SystemDSContext() as sds: + img = sds.from_numpy( + np.array([[ 10., 20., 30., + 40., 50., 60., + 70., 80., 90. ]], dtype=np.float32) + ) + result_img = img_shear_linearized(img, 1., 0., 0., 3, 3).compute() + print(result_img.reshape(3, 3)) + + expected="""[[10. 20. 30.] + [ 0. 40. 50.] + [ 0. 0. 70.]]""" + self.assertEqual(buf.getvalue().strip(), expected) + +if __name__ == '__main__': + unittest.main() diff --git a/src/main/python/tests/auto_tests/test_img_transform.py b/src/main/python/tests/auto_tests/test_img_transform.py new file mode 100644 index 00000000000..f1a91bc6d8e --- /dev/null +++ b/src/main/python/tests/auto_tests/test_img_transform.py @@ -0,0 +1,48 @@ +# ------------------------------------------------------------- +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# ------------------------------------------------------------- + +# Autogenerated By : src/main/python/generator/generator.py +import unittest, contextlib, io +class TestIMG_TRANSFORM(unittest.TestCase): + def test_img_transform(self): + # Example test case provided in python the code block + buf = io.StringIO() + with contextlib.redirect_stdout(buf): + import numpy as np + from systemds.context import SystemDSContext + from systemds.operator.algorithm import img_transform + + with SystemDSContext() as sds: + img = sds.from_numpy( + np.array([[ 10., 20., 30.], + [ 40., 50., 60.], + [ 70., 80., 90.]], dtype=np.float32) + ) + result_img = img_transform(img, 3, 3, -1., 0., 2., 0., 1., 0., 255.).compute() + print(result_img) + + expected="""[[ 20. 10. 255.] + [ 50. 40. 255.] + [ 80. 70. 255.]]""" + self.assertEqual(buf.getvalue().strip(), expected) + +if __name__ == '__main__': + unittest.main() diff --git a/src/main/python/tests/auto_tests/test_img_transform_linearized.py b/src/main/python/tests/auto_tests/test_img_transform_linearized.py new file mode 100644 index 00000000000..6992968486b --- /dev/null +++ b/src/main/python/tests/auto_tests/test_img_transform_linearized.py @@ -0,0 +1,48 @@ +# ------------------------------------------------------------- +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# ------------------------------------------------------------- + +# Autogenerated By : src/main/python/generator/generator.py +import unittest, contextlib, io +class TestIMG_TRANSFORM_LINEARIZED(unittest.TestCase): + def test_img_transform_linearized(self): + # Example test case provided in python the code block + buf = io.StringIO() + with contextlib.redirect_stdout(buf): + import numpy as np + from systemds.context import SystemDSContext + from systemds.operator.algorithm import img_transform_linearized + + with SystemDSContext() as sds: + img = sds.from_numpy( + np.array([[ 10., 20., 30., + 40., 50., 60., + 70., 80., 90. ]], dtype=np.float32) + ) + result_img = img_transform_linearized(img, 3, 3, -1., 0., 2., 0., 1., 0., 255., 3, 3).compute() + print(result_img.reshape(3, 3)) + + expected="""[[ 20. 10. 255.] + [ 50. 40. 255.] + [ 80. 70. 255.]]""" + self.assertEqual(buf.getvalue().strip(), expected) + +if __name__ == '__main__': + unittest.main() diff --git a/src/main/python/tests/auto_tests/test_img_translate.py b/src/main/python/tests/auto_tests/test_img_translate.py new file mode 100644 index 00000000000..22f63e25f43 --- /dev/null +++ b/src/main/python/tests/auto_tests/test_img_translate.py @@ -0,0 +1,48 @@ +# ------------------------------------------------------------- +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# ------------------------------------------------------------- + +# Autogenerated By : src/main/python/generator/generator.py +import unittest, contextlib, io +class TestIMG_TRANSLATE(unittest.TestCase): + def test_img_translate(self): + # Example test case provided in python the code block + buf = io.StringIO() + with contextlib.redirect_stdout(buf): + import numpy as np + from systemds.context import SystemDSContext + from systemds.operator.algorithm import img_translate + + with SystemDSContext() as sds: + img = sds.from_numpy( + np.array([[ 10., 20., 30.], + [ 40., 50., 60.], + [ 70., 80., 90.]], dtype=np.float32) + ) + result_img = img_translate(img, 1., 1., 3, 3, 255.).compute() + print(result_img) + + expected="""[[255. 255. 255.] + [255. 10. 20.] + [255. 40. 50.]]""" + self.assertEqual(buf.getvalue().strip(), expected) + +if __name__ == '__main__': + unittest.main() diff --git a/src/main/python/tests/auto_tests/test_img_translate_linearized.py b/src/main/python/tests/auto_tests/test_img_translate_linearized.py new file mode 100644 index 00000000000..86c894b1c55 --- /dev/null +++ b/src/main/python/tests/auto_tests/test_img_translate_linearized.py @@ -0,0 +1,48 @@ +# ------------------------------------------------------------- +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# ------------------------------------------------------------- + +# Autogenerated By : src/main/python/generator/generator.py +import unittest, contextlib, io +class TestIMG_TRANSLATE_LINEARIZED(unittest.TestCase): + def test_img_translate_linearized(self): + # Example test case provided in python the code block + buf = io.StringIO() + with contextlib.redirect_stdout(buf): + import numpy as np + from systemds.context import SystemDSContext + from systemds.operator.algorithm import img_translate_linearized + + with SystemDSContext() as sds: + img = sds.from_numpy( + np.array([[ 10., 20., 30., + 40., 50., 60., + 70., 80., 90. ]], dtype=np.float32) + ) + result_img = img_translate_linearized(img, 1., 1., 3, 3, 255.0, 3, 3).compute() + print(result_img.reshape(3, 3)) + + expected="""[[255. 255. 255.] + [255. 10. 20.] + [255. 40. 50.]]""" + self.assertEqual(buf.getvalue().strip(), expected) + +if __name__ == '__main__': + unittest.main() diff --git a/src/main/python/tests/auto_tests/test_lm.py b/src/main/python/tests/auto_tests/test_lm.py new file mode 100644 index 00000000000..5c4e83bc8b7 --- /dev/null +++ b/src/main/python/tests/auto_tests/test_lm.py @@ -0,0 +1,59 @@ +# ------------------------------------------------------------- +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# ------------------------------------------------------------- + +# Autogenerated By : src/main/python/generator/generator.py +import unittest, contextlib, io +class TestLM(unittest.TestCase): + def test_lm(self): + # Example test case provided in python the code block + buf = io.StringIO() + with contextlib.redirect_stdout(buf): + import numpy as np + from systemds.context import SystemDSContext + from systemds.operator.algorithm import lm + + np.random.seed(0) + features = np.random.rand(10, 15) + y = np.random.rand(10, 1) + + with SystemDSContext() as sds: + weights = lm(sds.from_numpy(features), sds.from_numpy(y)).compute() + print(weights) + + expected="""[[-0.11538199] + [-0.20386541] + [-0.39956034] + [ 1.04078623] + [ 0.43270839] + [ 0.18954599] + [ 0.49858969] + [-0.26812763] + [ 0.09961844] + [-0.57000751] + [-0.43386048] + [ 0.55358873] + [-0.54638565] + [ 0.2205885 ] + [ 0.37957689]]""" + self.assertEqual(buf.getvalue().strip(), expected) + +if __name__ == '__main__': + unittest.main() diff --git a/src/main/python/tests/auto_tests/test_normalize.py b/src/main/python/tests/auto_tests/test_normalize.py new file mode 100644 index 00000000000..8d6e910a429 --- /dev/null +++ b/src/main/python/tests/auto_tests/test_normalize.py @@ -0,0 +1,43 @@ +# ------------------------------------------------------------- +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# ------------------------------------------------------------- + +# Autogenerated By : src/main/python/generator/generator.py +import unittest, contextlib, io +class TestNORMALIZE(unittest.TestCase): + def test_normalize(self): + # Example test case provided in python the code block + buf = io.StringIO() + with contextlib.redirect_stdout(buf): + import numpy as np + from systemds.context import SystemDSContext + from systemds.operator.algorithm import normalize + + with SystemDSContext() as sds: + X = sds.from_numpy(np.array([[1, 2], [3, 4]])) + Y, cmin, cmax = normalize(X).compute() + print(Y) + + expected="""[[0. 0.] + [1. 1.]]""" + self.assertEqual(buf.getvalue().strip(), expected) + +if __name__ == '__main__': + unittest.main() diff --git a/src/main/python/tests/auto_tests/test_randomForest.py b/src/main/python/tests/auto_tests/test_randomForest.py new file mode 100644 index 00000000000..917e6524d3f --- /dev/null +++ b/src/main/python/tests/auto_tests/test_randomForest.py @@ -0,0 +1,71 @@ +# ------------------------------------------------------------- +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# ------------------------------------------------------------- + +# Autogenerated By : src/main/python/generator/generator.py +import unittest, contextlib, io +class TestRANDOMFOREST(unittest.TestCase): + def test_randomForest(self): + # Example test case provided in python the code block + buf = io.StringIO() + with contextlib.redirect_stdout(buf): + import numpy as np + from systemds.context import SystemDSContext + from systemds.operator.algorithm import randomForest, randomForestPredict + + # tiny toy dataset + X = np.array([[1], + [2], + [10], + [11]], dtype=np.int64) + y = np.array([[1], + [1], + [2], + [2]], dtype=np.int64) + + with SystemDSContext() as sds: + X_sds = sds.from_numpy(X) + y_sds = sds.from_numpy(y) + + ctypes = sds.from_numpy(np.array([[1, 2]], dtype=np.int64)) + + # train a 4-tree forest (no sampling) + M = randomForest( + X_sds, y_sds, ctypes, + num_trees = 4, + sample_frac = 1.0, + feature_frac = 1.0, + max_depth = 3, + min_leaf = 1, + min_split = 2, + seed = 42 + ) + + preds = randomForestPredict(X_sds, ctypes, M).compute() + print(preds) + + expected="""[[1.] + [1.] + [2.] + [2.]]""" + self.assertEqual(buf.getvalue().strip(), expected) + +if __name__ == '__main__': + unittest.main() diff --git a/src/main/python/tests/auto_tests/test_randomForestPredict.py b/src/main/python/tests/auto_tests/test_randomForestPredict.py new file mode 100644 index 00000000000..48b8e06bdf9 --- /dev/null +++ b/src/main/python/tests/auto_tests/test_randomForestPredict.py @@ -0,0 +1,71 @@ +# ------------------------------------------------------------- +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# ------------------------------------------------------------- + +# Autogenerated By : src/main/python/generator/generator.py +import unittest, contextlib, io +class TestRANDOMFORESTPREDICT(unittest.TestCase): + def test_randomForestPredict(self): + # Example test case provided in python the code block + buf = io.StringIO() + with contextlib.redirect_stdout(buf): + import numpy as np + from systemds.context import SystemDSContext + from systemds.operator.algorithm import randomForest, randomForestPredict + + # tiny toy dataset + X = np.array([[1], + [2], + [10], + [11]], dtype=np.int64) + y = np.array([[1], + [1], + [2], + [2]], dtype=np.int64) + + with SystemDSContext() as sds: + X_sds = sds.from_numpy(X) + y_sds = sds.from_numpy(y) + + ctypes = sds.from_numpy(np.array([[1, 2]], dtype=np.int64)) + + # train a 4-tree forest (no sampling) + M = randomForest( + X_sds, y_sds, ctypes, + num_trees = 4, + sample_frac = 1.0, + feature_frac = 1.0, + max_depth = 3, + min_leaf = 1, + min_split = 2, + seed = 42 + ) + + preds = randomForestPredict(X_sds, ctypes, M).compute() + print(preds) + + expected="""[[1.] + [1.] + [2.] + [2.]]""" + self.assertEqual(buf.getvalue().strip(), expected) + +if __name__ == '__main__': + unittest.main() diff --git a/src/main/python/tests/auto_tests/test_toOneHot.py b/src/main/python/tests/auto_tests/test_toOneHot.py new file mode 100644 index 00000000000..4d6b15e79fa --- /dev/null +++ b/src/main/python/tests/auto_tests/test_toOneHot.py @@ -0,0 +1,45 @@ +# ------------------------------------------------------------- +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# ------------------------------------------------------------- + +# Autogenerated By : src/main/python/generator/generator.py +import unittest, contextlib, io +class TestTOONEHOT(unittest.TestCase): + def test_toOneHot(self): + # Example test case provided in python the code block + buf = io.StringIO() + with contextlib.redirect_stdout(buf): + import numpy as np + from systemds.context import SystemDSContext + from systemds.operator.algorithm import toOneHot + + with SystemDSContext() as sds: + X = sds.from_numpy(np.array([[1], [3], [2], [3]])) + Y = toOneHot(X, numClasses=3).compute() + print(Y) + + expected="""[[1. 0. 0.] + [0. 0. 1.] + [0. 1. 0.] + [0. 0. 1.]]""" + self.assertEqual(buf.getvalue().strip(), expected) + +if __name__ == '__main__': + unittest.main() From 3fb4a61a84820dff32788b8fdeef5f06ea0ac7de Mon Sep 17 00:00:00 2001 From: anuunchin <88698977+anuunchin@users.noreply.github.com> Date: Tue, 15 Jul 2025 20:16:17 +0200 Subject: [PATCH 4/4] Handling of multiple code blocks --- scripts/builtin/img_mirror_linearized.dml | 26 +++++++++ src/main/python/generator/dml_parser.py | 11 ++-- src/main/python/generator/generator.py | 23 +++++--- .../builtin/img_mirror_linearized.py | 26 +++++++++ ...zed.py => test_img_mirror_linearized_0.py} | 4 +- .../test_img_mirror_linearized_1.py | 55 +++++++++++++++++++ 6 files changed, 131 insertions(+), 14 deletions(-) rename src/main/python/tests/auto_tests/{test_img_mirror_linearized.py => test_img_mirror_linearized_0.py} (95%) create mode 100644 src/main/python/tests/auto_tests/test_img_mirror_linearized_1.py diff --git a/scripts/builtin/img_mirror_linearized.dml b/scripts/builtin/img_mirror_linearized.dml index 33dd7efa451..f56f17dabb3 100644 --- a/scripts/builtin/img_mirror_linearized.dml +++ b/scripts/builtin/img_mirror_linearized.dml @@ -42,6 +42,32 @@ # [10. 20. 30.]] # # +# .. code-block:: python +# +# >>> import numpy as np +# >>> from systemds.context import SystemDSContext +# >>> from systemds.operator.algorithm import img_mirror_linearized +# >>> +# >>> with SystemDSContext() as sds: +# ... imgs = sds.from_numpy( +# ... np.array([[ 10., 20., 30., +# ... 40., 50., 60., +# ... 70., 80., 90. ], +# ... [ 70., 80., 90., +# ... 40., 50., 60., +# ... 10., 20., 30. ]], dtype=np.float32) +# ... ) +# ... result_imgs = img_mirror_linearized(imgs, True, 3, 3).compute() +# ... print(result_imgs[0].reshape(3, 3)) +# ... print(result_imgs[1].reshape(3, 3)) +# [[70. 80. 90.] +# [40. 50. 60.] +# [10. 20. 30.]] +# [[10. 20. 30.] +# [40. 50. 60.] +# [70. 80. 90.]] +# +# # INPUT: # ----------------------------------------------------------------------------------------- # img_matrix Input images as linearized 2D matrix with top left corner at [1, 1] (every row represents a linearized matrix/image) diff --git a/src/main/python/generator/dml_parser.py b/src/main/python/generator/dml_parser.py index 866c3aea0f2..e9079b95d63 100644 --- a/src/main/python/generator/dml_parser.py +++ b/src/main/python/generator/dml_parser.py @@ -196,7 +196,6 @@ def parse_header(self, path: str): input_parameters = self.parse_input_output_string(h_input) output_parameters = self.parse_input_output_string(h_output) - code_block = None with open(path, 'r') as f: content = f.read() pat = re.compile( @@ -204,21 +203,23 @@ def parse_header(self, path: str): ^\#\s*\.\.\s*code-block::\s*python # .. code-block:: python (.*?) # ← capture the actual example (?= # stop just before - ^\#\s*INPUT: # INPUT: + ^\#\s*(?:INPUT:| # → “# INPUT:” OR + \.\.\s*code-block::\s*python) # → another “# .. code-block:: python” ) """, re.MULTILINE | re.DOTALL | re.VERBOSE, ) - match = pat.search(content) - if match: + code_blocks = [] + for match in pat.finditer(content): raw_block = match.group(1) code_lines = [line.lstrip("#") for line in raw_block.splitlines()] # Remove leading # code_block = textwrap.dedent("\n".join([code_line for code_line in code_lines if code_line != ""])) + code_blocks.append(code_block) data = {'description': description, 'parameters': input_parameters, 'return_values': output_parameters, - 'code_block': code_block} + 'code_blocks': code_blocks} return data def parse_input_output_string(self, data: str): diff --git a/src/main/python/generator/generator.py b/src/main/python/generator/generator.py index fe3e7ce9c30..e321d8336c4 100644 --- a/src/main/python/generator/generator.py +++ b/src/main/python/generator/generator.py @@ -458,13 +458,22 @@ def format_exception(e): continue file_generator.generate_file( data["function_name"], script_content, dml_file) - # TODO: multiple code blocks -> multiple test_files - test_example = header_data.get("code_block", None) - if test_example: - # TODO: dml test file - # TODO: logs should have funcs without test cases - # TODO: imports should be exlicitly added to the examples - file_generator.generate_test_file(data["function_name"], test_example) + + # Generate test cases using the code blocks + test_examples = header_data.get("code_blocks", None) + if test_examples: + for i, test_example in enumerate(test_examples): + test_example_name = data["function_name"] + # Don't add test number if only one example + if len(test_examples) > 1: + test_example_name += f"_{i}" + file_generator.generate_test_file(test_example_name, test_example) + # TODO: dml test case files should also be created + else: + print(f"[INFO] Skipping python test case creation for '{data['function_name']}': No code example.") + + # Generate rst file file_generator.generate_rst_file(data["function_name"]) + file_generator.function_names.sort() file_generator.generate_init_file() diff --git a/src/main/python/systemds/operator/algorithm/builtin/img_mirror_linearized.py b/src/main/python/systemds/operator/algorithm/builtin/img_mirror_linearized.py index d8f8948347c..fdde6506c32 100644 --- a/src/main/python/systemds/operator/algorithm/builtin/img_mirror_linearized.py +++ b/src/main/python/systemds/operator/algorithm/builtin/img_mirror_linearized.py @@ -56,6 +56,32 @@ def img_mirror_linearized(img_matrix: Matrix, [10. 20. 30.]] + .. code-block:: python + + >>> import numpy as np + >>> from systemds.context import SystemDSContext + >>> from systemds.operator.algorithm import img_mirror_linearized + >>> + >>> with SystemDSContext() as sds: + ... imgs = sds.from_numpy( + ... np.array([[ 10., 20., 30., + ... 40., 50., 60., + ... 70., 80., 90. ], + ... [ 70., 80., 90., + ... 40., 50., 60., + ... 10., 20., 30. ]], dtype=np.float32) + ... ) + ... result_imgs = img_mirror_linearized(imgs, True, 3, 3).compute() + ... print(result_imgs[0].reshape(3, 3)) + ... print(result_imgs[1].reshape(3, 3)) + [[70. 80. 90.] + [40. 50. 60.] + [10. 20. 30.]] + [[10. 20. 30.] + [40. 50. 60.] + [70. 80. 90.]] + + :param img_matrix: Input images as linearized 2D matrix with top left corner at [1, 1] (every row represents a linearized matrix/image) diff --git a/src/main/python/tests/auto_tests/test_img_mirror_linearized.py b/src/main/python/tests/auto_tests/test_img_mirror_linearized_0.py similarity index 95% rename from src/main/python/tests/auto_tests/test_img_mirror_linearized.py rename to src/main/python/tests/auto_tests/test_img_mirror_linearized_0.py index 582c1361639..f5b8ba4733e 100644 --- a/src/main/python/tests/auto_tests/test_img_mirror_linearized.py +++ b/src/main/python/tests/auto_tests/test_img_mirror_linearized_0.py @@ -21,8 +21,8 @@ # Autogenerated By : src/main/python/generator/generator.py import unittest, contextlib, io -class TestIMG_MIRROR_LINEARIZED(unittest.TestCase): - def test_img_mirror_linearized(self): +class TestIMG_MIRROR_LINEARIZED_0(unittest.TestCase): + def test_img_mirror_linearized_0(self): # Example test case provided in python the code block buf = io.StringIO() with contextlib.redirect_stdout(buf): diff --git a/src/main/python/tests/auto_tests/test_img_mirror_linearized_1.py b/src/main/python/tests/auto_tests/test_img_mirror_linearized_1.py new file mode 100644 index 00000000000..a9884ab19eb --- /dev/null +++ b/src/main/python/tests/auto_tests/test_img_mirror_linearized_1.py @@ -0,0 +1,55 @@ +# ------------------------------------------------------------- +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# ------------------------------------------------------------- + +# Autogenerated By : src/main/python/generator/generator.py +import unittest, contextlib, io +class TestIMG_MIRROR_LINEARIZED_1(unittest.TestCase): + def test_img_mirror_linearized_1(self): + # Example test case provided in python the code block + buf = io.StringIO() + with contextlib.redirect_stdout(buf): + import numpy as np + from systemds.context import SystemDSContext + from systemds.operator.algorithm import img_mirror_linearized + + with SystemDSContext() as sds: + imgs = sds.from_numpy( + np.array([[ 10., 20., 30., + 40., 50., 60., + 70., 80., 90. ], + [ 70., 80., 90., + 40., 50., 60., + 10., 20., 30. ]], dtype=np.float32) + ) + result_imgs = img_mirror_linearized(imgs, True, 3, 3).compute() + print(result_imgs[0].reshape(3, 3)) + print(result_imgs[1].reshape(3, 3)) + + expected="""[[70. 80. 90.] + [40. 50. 60.] + [10. 20. 30.]] +[[10. 20. 30.] + [40. 50. 60.] + [70. 80. 90.]]""" + self.assertEqual(buf.getvalue().strip(), expected) + +if __name__ == '__main__': + unittest.main()