diff --git a/src/Microsoft.ML.Core/EntryPoints/ModuleArgs.cs b/src/Microsoft.ML.Core/EntryPoints/ModuleArgs.cs index 99cfec0dd9..2c4f877c1b 100644 --- a/src/Microsoft.ML.Core/EntryPoints/ModuleArgs.cs +++ b/src/Microsoft.ML.Core/EntryPoints/ModuleArgs.cs @@ -527,6 +527,11 @@ public sealed class EntryPointAttribute : Attribute /// Short name of the Entry Point /// public string ShortName { get; set; } + + /// + /// Remarks on the Entry Point, for more extensive XML documentation on the C#API + /// + public string Remarks { get; set; } } /// diff --git a/src/Microsoft.ML.Core/EntryPoints/ModuleCatalog.cs b/src/Microsoft.ML.Core/EntryPoints/ModuleCatalog.cs index 498a75c9e5..af45202937 100644 --- a/src/Microsoft.ML.Core/EntryPoints/ModuleCatalog.cs +++ b/src/Microsoft.ML.Core/EntryPoints/ModuleCatalog.cs @@ -44,6 +44,7 @@ public sealed class EntryPointInfo public readonly string Description; public readonly string ShortName; public readonly string FriendlyName; + public readonly string Remarks; public readonly MethodInfo Method; public readonly Type InputType; public readonly Type OutputType; @@ -63,6 +64,7 @@ internal EntryPointInfo(IExceptionContext ectx, MethodInfo method, Method = method; ShortName = attribute.ShortName; FriendlyName = attribute.UserName; + Remarks = attribute.Remarks; ObsoleteAttribute = obsoleteAttribute; // There are supposed to be 2 parameters, env and input for non-macro nodes. diff --git a/src/Microsoft.ML.FastTree/FastTree.cs b/src/Microsoft.ML.FastTree/FastTree.cs index 654735c4b6..9f00fb70e0 100644 --- a/src/Microsoft.ML.FastTree/FastTree.cs +++ b/src/Microsoft.ML.FastTree/FastTree.cs @@ -82,6 +82,31 @@ public abstract class FastTreeTrainerBase : protected string InnerArgs => CmdParser.GetSettings(Host, Args, new TArgs()); + internal const string Remarks = @" +FastTrees is an efficient implementation of the MART gradient boosting algorithm. +Gradient boosting is a machine learning technique for regression problems. +It builds each regression tree in a step-wise fashion, using a predefined loss function to measure the error for each step and corrects for it in the next. +So this prediction model is actually an ensemble of weaker prediction models. In regression problems, boosting builds a series of of such trees in a step-wise fashion and then selects the optimal tree using an arbitrary differentiable loss function. + + +MART learns an ensemble of regression trees, which is a decision tree with scalar values in its leaves. +A decision (or regression) tree is a binary tree-like flow chart, where at each interior node one decides which of the two child nodes to continue to based on one of the feature values from the input. +At each leaf node, a value is returned. In the interior nodes, the decision is based on the test 'x <= v' where x is the value of the feature in the input sample and v is one of the possible values of this feature. +The functions that can be produced by a regression tree are all the piece-wise constant functions. + + +The ensemble of trees is produced by computing, in each step, a regression tree that approximates the gradient of the loss function, and adding it to the previous tree with coefficients that minimize the loss of the new tree. +The output of the ensemble produced by MART on a given instance is the sum of the tree outputs. + + +In case of a binary classification problem, the output is converted to a probability by using some form of calibration. +In case of a regression problem, the output is the predicted value of the function. +In case of a ranking problem, the instances are ordered by the output value of the ensemble. + +Wikipedia: Gradient boosting (Gradient tree boosting). +Greedy function approximation: A gradient boosting machine.. +"; + public override bool NeedNormalization => false; public override bool WantCaching => false; diff --git a/src/Microsoft.ML.FastTree/FastTreeClassification.cs b/src/Microsoft.ML.FastTree/FastTreeClassification.cs index 43409dadd3..edbdd47a03 100644 --- a/src/Microsoft.ML.FastTree/FastTreeClassification.cs +++ b/src/Microsoft.ML.FastTree/FastTreeClassification.cs @@ -338,7 +338,11 @@ public void AdjustTreeOutputs(IChannel ch, RegressionTree tree, public static partial class FastTree { - [TlcModule.EntryPoint(Name = "Trainers.FastTreeBinaryClassifier", Desc = FastTreeBinaryClassificationTrainer.Summary, UserName = FastTreeBinaryClassificationTrainer.UserNameValue, ShortName = FastTreeBinaryClassificationTrainer.ShortName)] + [TlcModule.EntryPoint(Name = "Trainers.FastTreeBinaryClassifier", + Desc = FastTreeBinaryClassificationTrainer.Summary, + Remarks = FastTreeBinaryClassificationTrainer.Remarks, + UserName = FastTreeBinaryClassificationTrainer.UserNameValue, + ShortName = FastTreeBinaryClassificationTrainer.ShortName)] public static CommonOutputs.BinaryClassificationOutput TrainBinary(IHostEnvironment env, FastTreeBinaryClassificationTrainer.Arguments input) { Contracts.CheckValue(env, nameof(env)); diff --git a/src/Microsoft.ML.FastTree/FastTreeRanking.cs b/src/Microsoft.ML.FastTree/FastTreeRanking.cs index 2263d2541e..a689408748 100644 --- a/src/Microsoft.ML.FastTree/FastTreeRanking.cs +++ b/src/Microsoft.ML.FastTree/FastTreeRanking.cs @@ -1096,7 +1096,11 @@ public static FastTreeRankingPredictor Create(IHostEnvironment env, ModelLoadCon public static partial class FastTree { - [TlcModule.EntryPoint(Name = "Trainers.FastTreeRanker", Desc = FastTreeRankingTrainer.Summary, UserName = FastTreeRankingTrainer.UserNameValue, ShortName = FastTreeRankingTrainer.ShortName)] + [TlcModule.EntryPoint(Name = "Trainers.FastTreeRanker", + Desc = FastTreeRankingTrainer.Summary, + Remarks = FastTreeRankingTrainer.Remarks, + UserName = FastTreeRankingTrainer.UserNameValue, + ShortName = FastTreeRankingTrainer.ShortName)] public static CommonOutputs.RankingOutput TrainRanking(IHostEnvironment env, FastTreeRankingTrainer.Arguments input) { Contracts.CheckValue(env, nameof(env)); diff --git a/src/Microsoft.ML.FastTree/FastTreeRegression.cs b/src/Microsoft.ML.FastTree/FastTreeRegression.cs index ae7f4cfdbd..40ee906b5b 100644 --- a/src/Microsoft.ML.FastTree/FastTreeRegression.cs +++ b/src/Microsoft.ML.FastTree/FastTreeRegression.cs @@ -448,7 +448,11 @@ public static FastTreeRegressionPredictor Create(IHostEnvironment env, ModelLoad public static partial class FastTree { - [TlcModule.EntryPoint(Name = "Trainers.FastTreeRegressor", Desc = FastTreeRegressionTrainer.Summary, UserName = FastTreeRegressionTrainer.UserNameValue, ShortName = FastTreeRegressionTrainer.ShortName)] + [TlcModule.EntryPoint(Name = "Trainers.FastTreeRegressor", + Desc = FastTreeRegressionTrainer.Summary, + Remarks = FastTreeRegressionTrainer.Remarks, + UserName = FastTreeRegressionTrainer.UserNameValue, + ShortName = FastTreeRegressionTrainer.ShortName)] public static CommonOutputs.RegressionOutput TrainRegression(IHostEnvironment env, FastTreeRegressionTrainer.Arguments input) { Contracts.CheckValue(env, nameof(env)); diff --git a/src/Microsoft.ML.FastTree/FastTreeTweedie.cs b/src/Microsoft.ML.FastTree/FastTreeTweedie.cs index 19a026df20..00c29afd94 100644 --- a/src/Microsoft.ML.FastTree/FastTreeTweedie.cs +++ b/src/Microsoft.ML.FastTree/FastTreeTweedie.cs @@ -36,8 +36,11 @@ public sealed partial class FastTreeTweedieTrainer : BoostingFastTreeTrainerBase { public const string LoadNameValue = "FastTreeTweedieRegression"; public const string UserNameValue = "FastTree (Boosted Trees) Tweedie Regression"; - public const string Summary = "Trains gradient boosted decision trees to fit target values using a Tweedie loss function. This learner " + - "is a generalization of Poisson, compound Poisson, and gamma regression."; + public const string Summary = "Trains gradient boosted decision trees to fit target values using a Tweedie loss function. This learner is a generalization of Poisson, compound Poisson, and gamma regression."; + new public const string Remarks = @" +Wikipedia: Gradient boosting (Gradient tree boosting) +Greedy function approximation: A gradient boosting machine +"; public const string ShortName = "fttweedie"; @@ -460,7 +463,10 @@ protected override void Map(ref VBuffer src, ref float dst) public static partial class FastTree { - [TlcModule.EntryPoint(Name = "Trainers.FastTreeTweedieRegressor", Desc = FastTreeTweedieTrainer.Summary, UserName = FastTreeTweedieTrainer.UserNameValue, ShortName = FastTreeTweedieTrainer.ShortName)] + [TlcModule.EntryPoint(Name = "Trainers.FastTreeTweedieRegressor", + Desc = FastTreeTweedieTrainer.Summary, + UserName = FastTreeTweedieTrainer.UserNameValue, + ShortName = FastTreeTweedieTrainer.ShortName)] public static CommonOutputs.RegressionOutput TrainTweedieRegression(IHostEnvironment env, FastTreeTweedieTrainer.Arguments input) { Contracts.CheckValue(env, nameof(env)); diff --git a/src/Microsoft.ML.FastTree/RandomForest.cs b/src/Microsoft.ML.FastTree/RandomForest.cs index 88676754d5..2f670539b4 100644 --- a/src/Microsoft.ML.FastTree/RandomForest.cs +++ b/src/Microsoft.ML.FastTree/RandomForest.cs @@ -12,6 +12,28 @@ public abstract class RandomForestTrainerBase : FastTreeTrain where TArgs : FastForestArgumentsBase, new() where TPredictor : IPredictorProducing { + new internal const string Remarks = @" +Decision trees are non-parametric models that perform a sequence of simple tests on inputs. +This decision procedure maps them to outputs found in the training dataset whose inputs were similar to the instance being processed. +A decision is made at each node of the binary tree data structure based on a measure of similarity that maps each instance recursively through the branches of the tree until the appropriate leaf node is reached and the output decision returned. +Decision trees have several advantages: + +They are efficient in both computation and memory usage during training and prediction. +They can represent non-linear decision boundaries. +They perform integrated feature selection and classification. +They are resilient in the presence of noisy features. + +Fast forest is a random forest implementation. +The model consists of an ensemble of decision trees. Each tree in a decision forest outputs a Gaussian distribution by way of prediction. +An aggregation is performed over the ensemble of trees to find a Gaussian distribution closest to the combined distribution for all trees in the model. +This decision forest classifier consists of an ensemble of decision trees. +Generally, ensemble models provide better coverage and accuracy than single decision trees. +Each tree in a decision forest outputs a Gaussian distribution. +Wikipedia: Random forest +Quantile regression forest +From Stumps to Trees to Forests +"; + private readonly bool _quantileEnabled; protected RandomForestTrainerBase(IHostEnvironment env, TArgs args, bool quantileEnabled = false) diff --git a/src/Microsoft.ML.FastTree/RandomForestClassification.cs b/src/Microsoft.ML.FastTree/RandomForestClassification.cs index 54a05d5b11..e085996747 100644 --- a/src/Microsoft.ML.FastTree/RandomForestClassification.cs +++ b/src/Microsoft.ML.FastTree/RandomForestClassification.cs @@ -208,7 +208,11 @@ protected override void GetGradientInOneQuery(int query, int threadIndex) public static partial class FastForest { - [TlcModule.EntryPoint(Name = "Trainers.FastForestBinaryClassifier", Desc = FastForestClassification.Summary, UserName = FastForestClassification.UserNameValue, ShortName = FastForestClassification.ShortName)] + [TlcModule.EntryPoint(Name = "Trainers.FastForestBinaryClassifier", + Desc = FastForestClassification.Summary, + Remarks = FastForestClassification.Remarks, + UserName = FastForestClassification.UserNameValue, + ShortName = FastForestClassification.ShortName)] public static CommonOutputs.BinaryClassificationOutput TrainBinary(IHostEnvironment env, FastForestClassification.Arguments input) { Contracts.CheckValue(env, nameof(env)); diff --git a/src/Microsoft.ML.FastTree/RandomForestRegression.cs b/src/Microsoft.ML.FastTree/RandomForestRegression.cs index 3fd97afb32..74bf8c2a1c 100644 --- a/src/Microsoft.ML.FastTree/RandomForestRegression.cs +++ b/src/Microsoft.ML.FastTree/RandomForestRegression.cs @@ -280,7 +280,11 @@ public BasicImpl(Dataset trainData, Arguments args) public static partial class FastForest { - [TlcModule.EntryPoint(Name = "Trainers.FastForestRegressor", Desc = FastForestRegression.Summary, UserName = FastForestRegression.LoadNameValue, ShortName = FastForestRegression.ShortName)] + [TlcModule.EntryPoint(Name = "Trainers.FastForestRegressor", + Desc = FastForestRegression.Summary, + Remarks = FastForestRegression.Remarks, + UserName = FastForestRegression.LoadNameValue, + ShortName = FastForestRegression.ShortName)] public static CommonOutputs.RegressionOutput TrainRegression(IHostEnvironment env, FastForestRegression.Arguments input) { Contracts.CheckValue(env, nameof(env)); diff --git a/src/Microsoft.ML.KMeansClustering/KMeansPlusPlusTrainer.cs b/src/Microsoft.ML.KMeansClustering/KMeansPlusPlusTrainer.cs index 1f09ec850f..3e47c595cd 100644 --- a/src/Microsoft.ML.KMeansClustering/KMeansPlusPlusTrainer.cs +++ b/src/Microsoft.ML.KMeansClustering/KMeansPlusPlusTrainer.cs @@ -36,6 +36,14 @@ public class KMeansPlusPlusTrainer : TrainerBase +K-means++ improves upon K-means by using the Yinyang K-Means method for choosing the initial cluster centers. +YYK-Means accelerates K-Means up to an order of magnitude while producing exactly the same clustering results (modulo floating point precision issues). +YYK-Means observes that there is a lot of redundancy across iterations in the KMeans algorithms and most points do not change their clusters during an iteration. +It uses various bounding techniques to identify this redundancy and eliminate many distance computations and optimize centroid computations. +K-means. +K-means++ +"; public enum InitAlgorithm { @@ -225,7 +233,11 @@ private static int ComputeNumThreads(IHost host, int? argNumThreads) return Math.Max(1, maxThreads); } - [TlcModule.EntryPoint(Name = "Trainers.KMeansPlusPlusClusterer", Desc = KMeansPlusPlusTrainer.Summary, UserName = UserNameValue, ShortName = ShortName)] + [TlcModule.EntryPoint(Name = "Trainers.KMeansPlusPlusClusterer", + Desc = Summary, + Remarks = Remarks, + UserName = UserNameValue, + ShortName = ShortName)] public static CommonOutputs.ClusteringOutput TrainKMeans(IHostEnvironment env, Arguments input) { Contracts.CheckValue(env, nameof(env)); diff --git a/src/Microsoft.ML.LightGBM/LightGbmBinaryTrainer.cs b/src/Microsoft.ML.LightGBM/LightGbmBinaryTrainer.cs index 4d7e067f66..0b71bfa70e 100644 --- a/src/Microsoft.ML.LightGBM/LightGbmBinaryTrainer.cs +++ b/src/Microsoft.ML.LightGBM/LightGbmBinaryTrainer.cs @@ -13,7 +13,7 @@ [assembly: LoadableClass(LightGbmBinaryTrainer.Summary, typeof(LightGbmBinaryTrainer), typeof(LightGbmArguments), new[] { typeof(SignatureBinaryClassifierTrainer), typeof(SignatureTrainer), typeof(SignatureTreeEnsembleTrainer) }, - "LightGBM Binary Classification", LightGbmBinaryTrainer.LoadNameValue, LightGbmBinaryTrainer.ShortName, DocName = "trainer/LightGBM.md")] + LightGbmBinaryTrainer.UserName, LightGbmBinaryTrainer.LoadNameValue, LightGbmBinaryTrainer.ShortName, DocName = "trainer/LightGBM.md")] [assembly: LoadableClass(typeof(IPredictorProducing), typeof(LightGbmBinaryPredictor), null, typeof(SignatureLoadModel), "LightGBM Binary Executor", @@ -27,6 +27,7 @@ public sealed class LightGbmBinaryPredictor : FastTreePredictionWrapper { public const string LoaderSignature = "LightGBMBinaryExec"; public const string RegistrationName = "LightGBMBinaryPredictor"; + private static VersionInfo GetVersionInfo() { // REVIEW: can we decouple the version from FastTree predictor version ? @@ -82,9 +83,10 @@ public static IPredictorProducing Create(IHostEnvironment env, ModelLoadC public sealed class LightGbmBinaryTrainer : LightGbmTrainerBase> { - public const string Summary = "LightGBM Binary Classifier"; - public const string LoadNameValue = "LightGBMBinary"; - public const string ShortName = "LightGBM"; + internal const string UserName = "LightGBM Binary Classifier"; + internal const string LoadNameValue = "LightGBMBinary"; + internal const string ShortName = "LightGBM"; + internal const string Summary = "Train a LightGBM binary classification model."; public LightGbmBinaryTrainer(IHostEnvironment env, LightGbmArguments args) : base(env, args, PredictionKind.BinaryClassification, "LGBBINCL") @@ -122,14 +124,15 @@ protected override void CheckAndUpdateParametersBeforeTraining(IChannel ch, Role } /// - /// A component to train an LightGBM model. + /// A component to train a LightGBM model. /// public static partial class LightGbm { [TlcModule.EntryPoint( Name = "Trainers.LightGbmBinaryClassifier", - Desc = "Train a LightGBM binary class model.", - UserName = LightGbmBinaryTrainer.Summary, + Desc = LightGbmBinaryTrainer.Summary, + Remarks = LightGbmBinaryTrainer.Remarks, + UserName = LightGbmBinaryTrainer.UserName, ShortName = LightGbmBinaryTrainer.ShortName)] public static CommonOutputs.BinaryClassificationOutput TrainBinary(IHostEnvironment env, LightGbmArguments input) { diff --git a/src/Microsoft.ML.LightGBM/LightGbmMulticlassTrainer.cs b/src/Microsoft.ML.LightGBM/LightGbmMulticlassTrainer.cs index 48a208e05c..2966c428e3 100644 --- a/src/Microsoft.ML.LightGBM/LightGbmMulticlassTrainer.cs +++ b/src/Microsoft.ML.LightGBM/LightGbmMulticlassTrainer.cs @@ -174,13 +174,14 @@ protected override void CheckAndUpdateParametersBeforeTraining(IChannel ch, Role } /// - /// A component to train an LightGBM model. + /// A component to train a LightGBM model. /// public static partial class LightGbm { [TlcModule.EntryPoint( Name = "Trainers.LightGbmClassifier", Desc = "Train a LightGBM multi class model.", + Remarks = LightGbmMulticlassTrainer.Remarks, UserName = LightGbmMulticlassTrainer.Summary, ShortName = LightGbmMulticlassTrainer.ShortName)] public static CommonOutputs.MulticlassClassificationOutput TrainMultiClass(IHostEnvironment env, LightGbmArguments input) diff --git a/src/Microsoft.ML.LightGBM/LightGbmRankingTrainer.cs b/src/Microsoft.ML.LightGBM/LightGbmRankingTrainer.cs index 64579b3315..2ed436b4eb 100644 --- a/src/Microsoft.ML.LightGBM/LightGbmRankingTrainer.cs +++ b/src/Microsoft.ML.LightGBM/LightGbmRankingTrainer.cs @@ -10,7 +10,7 @@ using Microsoft.ML.Runtime.LightGBM; using Microsoft.ML.Runtime.Model; -[assembly: LoadableClass(LightGbmRankingTrainer.Summary, typeof(LightGbmRankingTrainer), typeof(LightGbmArguments), +[assembly: LoadableClass(LightGbmRankingTrainer.UserName, typeof(LightGbmRankingTrainer), typeof(LightGbmArguments), new[] { typeof(SignatureRankerTrainer), typeof(SignatureTrainer), typeof(SignatureTreeEnsembleTrainer) }, "LightGBM Ranking", LightGbmRankingTrainer.LoadNameValue, LightGbmRankingTrainer.ShortName, DocName = "trainer/LightGBM.md")] @@ -73,7 +73,7 @@ public static LightGbmRankingPredictor Create(IHostEnvironment env, ModelLoadCon public sealed class LightGbmRankingTrainer : LightGbmTrainerBase { - public const string Summary = "LightGBM Ranking"; + public const string UserName = "LightGBM Ranking"; public const string LoadNameValue = "LightGBMRanking"; public const string ShortName = "LightGBMRank"; @@ -123,14 +123,14 @@ protected override void CheckAndUpdateParametersBeforeTraining(IChannel ch, Role } /// - /// A component to train an LightGBM model. + /// A component to train a LightGBM model. /// public static partial class LightGbm { - [TlcModule.EntryPoint( - Name = "Trainers.LightGbmRanker", + [TlcModule.EntryPoint(Name = "Trainers.LightGbmRanker", + Remarks = LightGbmMulticlassTrainer.Remarks, Desc = "Train a LightGBM ranking model.", - UserName = LightGbmRankingTrainer.Summary, + UserName = LightGbmRankingTrainer.UserName, ShortName = LightGbmRankingTrainer.ShortName)] public static CommonOutputs.RankingOutput TrainRanking(IHostEnvironment env, LightGbmArguments input) { diff --git a/src/Microsoft.ML.LightGBM/LightGbmRegressionTrainer.cs b/src/Microsoft.ML.LightGBM/LightGbmRegressionTrainer.cs index 461110ae59..36c82aa79a 100644 --- a/src/Microsoft.ML.LightGBM/LightGbmRegressionTrainer.cs +++ b/src/Microsoft.ML.LightGBM/LightGbmRegressionTrainer.cs @@ -116,13 +116,13 @@ protected override void CheckAndUpdateParametersBeforeTraining(IChannel ch, Role } /// - /// A component to train an LightGBM model. + /// A component to train a LightGBM model. /// public static partial class LightGbm { - [TlcModule.EntryPoint( - Name = "Trainers.LightGbmRegressor", + [TlcModule.EntryPoint(Name = "Trainers.LightGbmRegressor", Desc = LightGbmRegressorTrainer.Summary, + Remarks = LightGbmRegressorTrainer.Remarks, UserName = LightGbmRegressorTrainer.UserNameValue, ShortName = LightGbmRegressorTrainer.ShortName)] public static CommonOutputs.RegressionOutput TrainRegression(IHostEnvironment env, LightGbmArguments input) diff --git a/src/Microsoft.ML.LightGBM/LightGbmTrainerBase.cs b/src/Microsoft.ML.LightGBM/LightGbmTrainerBase.cs index aff3befa0d..58872f27c3 100644 --- a/src/Microsoft.ML.LightGBM/LightGbmTrainerBase.cs +++ b/src/Microsoft.ML.LightGBM/LightGbmTrainerBase.cs @@ -56,6 +56,9 @@ private sealed class CategoricalMetaData protected int FeatureCount; protected FastTree.Internal.Ensemble TrainedEnsemble; + internal const string Remarks = @"Light GBM is an open source implementation of boosted trees. +GitHub: LightGBM"; + #endregion protected LightGbmTrainerBase(IHostEnvironment env, LightGbmArguments args, PredictionKind predictionKind, string name) diff --git a/src/Microsoft.ML.PCA/PcaTrainer.cs b/src/Microsoft.ML.PCA/PcaTrainer.cs index 0945b04041..6a6efd1ea6 100644 --- a/src/Microsoft.ML.PCA/PcaTrainer.cs +++ b/src/Microsoft.ML.PCA/PcaTrainer.cs @@ -284,7 +284,11 @@ private static void PostProcess(VBuffer[] y, Float[] sigma, Float[] z, in } } - [TlcModule.EntryPoint(Name = "Trainers.PcaAnomalyDetector", Desc = "Train an PCA Anomaly model.", UserName = UserNameValue, ShortName = ShortName)] + [TlcModule.EntryPoint(Name = "Trainers.PcaAnomalyDetector", + Desc = "Train an PCA Anomaly model.", + Remarks = PcaPredictor.Remarks, + UserName = UserNameValue, + ShortName = ShortName)] public static CommonOutputs.AnomalyDetectionOutput TrainPcaAnomaly(IHostEnvironment env, Arguments input) { Contracts.CheckValue(env, nameof(env)); @@ -312,6 +316,14 @@ public sealed class PcaPredictor : PredictorBase, { public const string LoaderSignature = "pcaAnomExec"; public const string RegistrationName = "PCAPredictor"; + internal const string Remarks = @" +Principle Component Analysis (PCA) is a dimensionality-reduction transform which computes the projection of the feature vector to onto a low-rank subspace. +Its training is done using the technique described in the paper: Combining Structured and Unstructured Randomness in Large Scale PCA, +and the paper Finding Structure with Randomness: Probabilistic Algorithms for Constructing Approximate Matrix Decompositions +Randomized Methods for Computing the Singular Value Decomposition (SVD) of very large matrices +A randomized algorithm for principal component analysis +Finding Structure with Randomness: Probabilistic Algorithms for Constructing Approximate Matrix Decompositions +"; private static VersionInfo GetVersionInfo() { diff --git a/src/Microsoft.ML.StandardLearners/FactorizationMachine/FactorizationMachineTrainer.cs b/src/Microsoft.ML.StandardLearners/FactorizationMachine/FactorizationMachineTrainer.cs index 87d72471d9..08190acb74 100644 --- a/src/Microsoft.ML.StandardLearners/FactorizationMachine/FactorizationMachineTrainer.cs +++ b/src/Microsoft.ML.StandardLearners/FactorizationMachine/FactorizationMachineTrainer.cs @@ -39,6 +39,15 @@ public sealed class FieldAwareFactorizationMachineTrainer : TrainerBase +Field Aware Factorization Machines use, in addition to the input variables, factorized parameters to model the interaction between pairs of variables. +The algorithm is particularly useful for high dimensional datasets which can be very sparse (e.g. click-prediction for advertising systems). +An advantage of FFM over SVMs is that the training data does not need to be stored in memory, and the coefficients can be optimized directly. +Field Aware Factorization Machines +Field-aware Factorization Machines for CTR Prediction +Adaptive Subgradient Methods for Online Learning and Stochastic Optimization +An Improved Stochastic Gradient Method for Training Large-scale Field-aware Factorization Machine. +"; public sealed class Arguments : LearnerInputBaseWithLabel { @@ -404,7 +413,11 @@ public override FieldAwareFactorizationMachinePredictor CreatePredictor() return _pred; } - [TlcModule.EntryPoint(Name = "Trainers.FieldAwareFactorizationMachineBinaryClassifier", Desc = FieldAwareFactorizationMachineTrainer.Summary, UserName = FieldAwareFactorizationMachineTrainer.UserName, ShortName = FieldAwareFactorizationMachineTrainer.ShortName)] + [TlcModule.EntryPoint(Name = "Trainers.FieldAwareFactorizationMachineBinaryClassifier", + Desc = Summary, + Remarks = Remarks, + UserName = UserName, + ShortName = ShortName)] public static CommonOutputs.BinaryClassificationOutput TrainBinary(IHostEnvironment env, Arguments input) { Contracts.CheckValue(env, nameof(env)); diff --git a/src/Microsoft.ML.StandardLearners/Standard/LinearClassificationTrainer.cs b/src/Microsoft.ML.StandardLearners/Standard/LinearClassificationTrainer.cs index 6a2e18dbda..93a7cc2b32 100644 --- a/src/Microsoft.ML.StandardLearners/Standard/LinearClassificationTrainer.cs +++ b/src/Microsoft.ML.StandardLearners/Standard/LinearClassificationTrainer.cs @@ -222,21 +222,25 @@ internal virtual void Check(IHostEnvironment env) } } - internal const string SDCADetailedSummary = @"This classifier is a trainer based on the Stochastic DualCoordinate -Ascent(SDCA) method, a state-of-the-art optimization technique for convex objective functions. + internal const string Remarks = @" +This classifier is a trainer based on the Stochastic DualCoordinate Ascent(SDCA) method, a state-of-the-art optimization technique for convex objective functions. The algorithm can be scaled for use on large out-of-memory data sets due to a semi-asynchronized implementation that supports multi-threading. + Convergence is underwritten by periodically enforcing synchronization between primal and dual updates in a separate thread. Several choices of loss functions are also provided. The SDCA method combines several of the best properties and capabilities of logistic regression and SVM algorithms. -For more information on SDCA, see: -Scaling Up Stochastic Dual Coordinate Ascent. -Stochastic Dual Coordinate Ascent Methods for Regularized Loss Minimization. + + Note that SDCA is a stochastic and streaming optimization algorithm. -The results depends on the order of the training data. For reproducible results, it is recommended that one sets `shuffle` to -`False` and `NumThreads` to `1`. -Elastic net regularization can be specified by the l2_weight and l1_weight parameters. Note that the l2_weight has an effect on the rate of convergence. -In general, the larger the l2_weight, the faster SDCA converges."; +The results depends on the order of the training data. For reproducible results, it is recommended that one sets to +False and to 1. +Elastic net regularization can be specified by the and parameters. Note that the has an effect on the rate of convergence. +In general, the larger the , the faster SDCA converges. + +Scaling Up Stochastic Dual Coordinate Ascent. +Stochastic Dual Coordinate Ascent Methods for Regularized Loss Minimization. +"; // The order of these matter, since they are used as indices into arrays. protected enum MetricKind @@ -1791,7 +1795,11 @@ public static CommonOutputs.BinaryClassificationOutput TrainBinary(IHostEnvironm /// public static partial class Sdca { - [TlcModule.EntryPoint(Name = "Trainers.StochasticDualCoordinateAscentBinaryClassifier", Desc = "Train an SDCA binary model.", UserName = LinearClassificationTrainer.UserNameValue, ShortName = LinearClassificationTrainer.LoadNameValue)] + [TlcModule.EntryPoint(Name = "Trainers.StochasticDualCoordinateAscentBinaryClassifier", + Desc = "Train an SDCA binary model.", + Remarks = LinearClassificationTrainer.Remarks, + UserName = LinearClassificationTrainer.UserNameValue, + ShortName = LinearClassificationTrainer.LoadNameValue)] public static CommonOutputs.BinaryClassificationOutput TrainBinary(IHostEnvironment env, LinearClassificationTrainer.Arguments input) { Contracts.CheckValue(env, nameof(env)); diff --git a/src/Microsoft.ML.StandardLearners/Standard/LogisticRegression/LbfgsPredictorBase.cs b/src/Microsoft.ML.StandardLearners/Standard/LogisticRegression/LbfgsPredictorBase.cs index 5fe70de2f0..95982047aa 100644 --- a/src/Microsoft.ML.StandardLearners/Standard/LogisticRegression/LbfgsPredictorBase.cs +++ b/src/Microsoft.ML.StandardLearners/Standard/LogisticRegression/LbfgsPredictorBase.cs @@ -94,27 +94,34 @@ public abstract class ArgumentsBase : LearnerInputBaseWithWeight public bool EnforceNonNegativity = false; } - internal const string DetailedSummary = @"Logistic Regression is a classification method used to predict the value of a categorical dependent variable from its relationship to one or more independent variables assumed to have a logistic distribution. -If the dependent variable has only two possible values (success/failure), then the logistic regression is binary. + internal const string Remarks = @" If the dependent variable has more than two possible values (blood type given diagnostic test results), then the logistic regression is multinomial. + The optimization technique used for LogisticRegressionBinaryClassifier is the limited memory Broyden-Fletcher-Goldfarb-Shanno (L-BFGS). Both the L-BFGS and regular BFGS algorithms use quasi-Newtonian methods to estimate the computationally intensive Hessian matrix in the equation used by Newton's method to calculate steps. -But the L-BFGS approximation uses only a limited amount of memory to compute the next step direction, so that it is especially suited for problems with a large number of variables. -The memory_size parameter specifies the number of past positions and gradients to store for use in the computation of the next step. +But the L-BFGS approximation uses only a limited amount of memory to compute the next step direction, +so that it is especially suited for problems with a large number of variables. +The MemorySize parameter specifies the number of past positions and gradients to store for use in the computation of the next step. + + This learner can use elastic net regularization: a linear combination of L1 (lasso) and L2 (ridge) regularizations. Regularization is a method that can render an ill-posed problem more tractable by imposing constraints that provide information to supplement the data and that prevents overfitting by penalizing models with extreme coefficient values. -This can improve the generalization of the model learned by selecting the optimal complexity in the bias-variance tradeoff. Regularization works by adding the penalty that is associated with coefficient values to the error of the hypothesis. +This can improve the generalization of the model learned by selecting the optimal complexity in the bias-variance tradeoff. +Regularization works by adding the penalty that is associated with coefficient values to the error of the hypothesis. An accurate model with extreme coefficient values would be penalized more, but a less accurate model with more conservative values would be penalized less. L1 and L2 regularization have different effects and uses that are complementary in certain respects. -l1_weight: can be applied to sparse models, when working with high-dimensional data. It pulls small weights associated features that are relatively unimportant towards 0. -l2_weight: is preferable for data that is not sparse. It pulls large weights towards zero. + +L1Weight: can be applied to sparse models, when working with high-dimensional data. +It pulls small weights associated features that are relatively unimportant towards 0. +L2Weight: is preferable for data that is not sparse. It pulls large weights towards zero. + Adding the ridge penalty to the regularization overcomes some of lasso's limitations. It can improve its predictive accuracy, for example, when the number of predictors is greater than the sample size. If x = l1_weight and y = l2_weight, ax + by = c defines the linear span of the regularization terms. The default values of x and y are both 1. An agressive regularization can harm predictive capacity by excluding important variables out of the model. So choosing the optimal values for the regularization parameters is important for the performance of the logistic regression model. -Wikipedia: L-BFGS. -Wikipedia: Logistic regression. -Scalable Training of L1-Regularized Log-Linear Models. -Test Run - L1 and L2 Regularization for Machine Learning. -"; +Scalable Training of L1-Regularized Log-Linear Models. +Test Run - L1 and L2 Regularization for Machine Learning. +Wikipedia: L-BFGS. +Wikipedia: Logistic regression. +"; protected int NumFeatures; protected VBuffer CurrentWeights; diff --git a/src/Microsoft.ML.StandardLearners/Standard/LogisticRegression/LogisticRegression.cs b/src/Microsoft.ML.StandardLearners/Standard/LogisticRegression/LogisticRegression.cs index 6f4a1d9617..0e85cd6712 100644 --- a/src/Microsoft.ML.StandardLearners/Standard/LogisticRegression/LogisticRegression.cs +++ b/src/Microsoft.ML.StandardLearners/Standard/LogisticRegression/LogisticRegression.cs @@ -386,7 +386,11 @@ public override ParameterMixingCalibratedPredictor CreatePredictor() new PlattCalibrator(Host, -1, 0)); } - [TlcModule.EntryPoint(Name = "Trainers.LogisticRegressionBinaryClassifier", Desc = DetailedSummary, UserName = UserNameValue, ShortName = ShortName)] + [TlcModule.EntryPoint(Name = "Trainers.LogisticRegressionBinaryClassifier", + Desc = Summary, + Remarks = Remarks, + UserName = UserNameValue, + ShortName = ShortName)] public static CommonOutputs.BinaryClassificationOutput TrainBinary(IHostEnvironment env, Arguments input) { Contracts.CheckValue(env, nameof(env)); diff --git a/src/Microsoft.ML.StandardLearners/Standard/LogisticRegression/MulticlassLogisticRegression.cs b/src/Microsoft.ML.StandardLearners/Standard/LogisticRegression/MulticlassLogisticRegression.cs index 8e9b03b831..f2fad63794 100644 --- a/src/Microsoft.ML.StandardLearners/Standard/LogisticRegression/MulticlassLogisticRegression.cs +++ b/src/Microsoft.ML.StandardLearners/Standard/LogisticRegression/MulticlassLogisticRegression.cs @@ -961,7 +961,11 @@ public IRow GetStatsIRowOrNull(RoleMappedSchema schema) /// public partial class LogisticRegression { - [TlcModule.EntryPoint(Name = "Trainers.LogisticRegressionClassifier", Desc = DetailedSummary, UserName = MulticlassLogisticRegression.UserNameValue, ShortName = MulticlassLogisticRegression.ShortName)] + [TlcModule.EntryPoint(Name = "Trainers.LogisticRegressionClassifier", + Desc = Summary, + Remarks = MulticlassLogisticRegression.Remarks, + UserName = MulticlassLogisticRegression.UserNameValue, + ShortName = MulticlassLogisticRegression.ShortName)] public static CommonOutputs.MulticlassClassificationOutput TrainMultiClass(IHostEnvironment env, MulticlassLogisticRegression.Arguments input) { Contracts.CheckValue(env, nameof(env)); diff --git a/src/Microsoft.ML.StandardLearners/Standard/MultiClass/MultiClassNaiveBayesTrainer.cs b/src/Microsoft.ML.StandardLearners/Standard/MultiClass/MultiClassNaiveBayesTrainer.cs index a06b54fc26..239392b085 100644 --- a/src/Microsoft.ML.StandardLearners/Standard/MultiClass/MultiClassNaiveBayesTrainer.cs +++ b/src/Microsoft.ML.StandardLearners/Standard/MultiClass/MultiClassNaiveBayesTrainer.cs @@ -32,6 +32,12 @@ public sealed class MultiClassNaiveBayesTrainer : TrainerBase +Naive Bayes is a probabilistic classifier that can be used for multiclass problems. +Using Bayes' theorem, the conditional probability for a sample belonging to a class can be calculated based on the sample count for each feature combination groups. +However, Naive Bayes Classifier is feasible only if the number of features and the values each feature can take is relatively small. +It also assumes that the features are strictly independent. +"; public sealed class Arguments : LearnerInputBaseWithLabel { @@ -124,7 +130,9 @@ public override MultiClassNaiveBayesPredictor CreatePredictor() return _predictor; } - [TlcModule.EntryPoint(Name = "Trainers.NaiveBayesClassifier", Desc = "Train a MultiClassNaiveBayesTrainer.", UserName = UserName, ShortName = ShortName)] + [TlcModule.EntryPoint(Name = "Trainers.NaiveBayesClassifier", + Desc = "Train a MultiClassNaiveBayesTrainer.", + UserName = UserName, ShortName = ShortName)] public static CommonOutputs.MulticlassClassificationOutput TrainMultiClassNaiveBayesTrainer(IHostEnvironment env, Arguments input) { Contracts.CheckValue(env, nameof(env)); diff --git a/src/Microsoft.ML.StandardLearners/Standard/OlsLinearRegression.cs b/src/Microsoft.ML.StandardLearners/Standard/OlsLinearRegression.cs index 7ea557159e..db271ff858 100644 --- a/src/Microsoft.ML.StandardLearners/Standard/OlsLinearRegression.cs +++ b/src/Microsoft.ML.StandardLearners/Standard/OlsLinearRegression.cs @@ -51,6 +51,11 @@ public sealed class Arguments : LearnerInputBaseWithWeight public const string ShortName = "ols"; internal const string Summary = "The ordinary least square regression fits the target function as a linear function of the numerical features " + "that minimizes the square loss function."; + internal const string Remarks = @" +Ordinary least squares (OLS) is a parameterized regression method. +It assumes that the conditional mean of the dependent variable follows a linear function of the dependent variables. +By minimizing the squares of the difference between observed values and the predictions, the parameters of the regressor can be estimated. +"; private VBuffer _weights; private Float _bias; diff --git a/src/Microsoft.ML.StandardLearners/Standard/Online/AveragedPerceptron.cs b/src/Microsoft.ML.StandardLearners/Standard/Online/AveragedPerceptron.cs index 1164cbb5ae..138b3f0485 100644 --- a/src/Microsoft.ML.StandardLearners/Standard/Online/AveragedPerceptron.cs +++ b/src/Microsoft.ML.StandardLearners/Standard/Online/AveragedPerceptron.cs @@ -36,18 +36,26 @@ public sealed class AveragedPerceptronTrainer : public const string LoadNameValue = "AveragedPerceptron"; internal const string UserNameValue = "Averaged Perceptron"; internal const string ShortName = "ap"; - internal const string Summary = "Perceptron is a binary classification algorithm that makes its predictions based on a linear function."; - internal const string DetailedSummary = @"Perceptron is a classification algorithm that makes its predictions based on a linear function. + internal const string Summary = "Averaged Perceptron Binary Classifier."; + internal const string Remarks = @" +Perceptron is a classification algorithm that makes its predictions based on a linear function. I.e., for an instance with feature values f0, f1,..., f_D-1, , the prediction is given by the sign of sigma[0,D-1] ( w_i * f_i), where w_0, w_1,...,w_D-1 are the weights computed by the algorithm. + Perceptron is an online algorithm, i.e., it processes the instances in the training set one at a time. The weights are initialized to be 0, or some random values. Then, for each example in the training set, the value of sigma[0, D-1] (w_i * f_i) is computed. If this value has the same sign as the label of the current example, the weights remain the same. If they have opposite signs, the weights vector is updated by either subtracting or adding (if the label is negative or positive, respectively) the feature vector of the current example, multiplied by a factor 0 < a <= 1, called the learning rate. In a generalization of this algorithm, the weights are updated by adding the feature vector multiplied by the learning rate, and by the gradient of some loss function (in the specific case described above, the loss is hinge-loss, whose gradient is 1 when it is non-zero). + + In Averaged Perceptron (AKA voted-perceptron), the weight vectors are stored, together with a weight that counts the number of iterations it survived (this is equivalent to storing the weight vector after every iteration, regardless of whether it was updated or not). -The prediction is then calculated by taking the weighted average of all the sums sigma[0, D-1] (w_i * f_i) or the different weight vectors."; +The prediction is then calculated by taking the weighted average of all the sums sigma[0, D-1] (w_i * f_i) or the different weight vectors. + +Wikipedia entry for Perceptron +Large Margin Classification Using the Perceptron Algorithm +"; public class Arguments : AveragedLinearArguments { @@ -102,7 +110,11 @@ public override LinearBinaryPredictor CreatePredictor() return new LinearBinaryPredictor(Host, ref weights, bias); } - [TlcModule.EntryPoint(Name = "Trainers.AveragedPerceptronBinaryClassifier", Desc = DetailedSummary, UserName = UserNameValue, ShortName = ShortName)] + [TlcModule.EntryPoint(Name = "Trainers.AveragedPerceptronBinaryClassifier", + Desc = Summary, + Remarks = Remarks, + UserName = UserNameValue, + ShortName = ShortName)] public static CommonOutputs.BinaryClassificationOutput TrainBinary(IHostEnvironment env, Arguments input) { Contracts.CheckValue(env, nameof(env)); diff --git a/src/Microsoft.ML.StandardLearners/Standard/Online/OnlineGradientDescent.cs b/src/Microsoft.ML.StandardLearners/Standard/Online/OnlineGradientDescent.cs index 6910267759..1af080a76b 100644 --- a/src/Microsoft.ML.StandardLearners/Standard/Online/OnlineGradientDescent.cs +++ b/src/Microsoft.ML.StandardLearners/Standard/Online/OnlineGradientDescent.cs @@ -32,8 +32,13 @@ public sealed class OnlineGradientDescentTrainer : AveragedLinearTrainer +Stochastic gradient descent uses a simple yet efficient iterative technique to fit model coefficients using error gradients for convex loss functions. +The OnlineGradientDescentRegressor implements the standard (non-batch) SGD, with a choice of loss functions, +and an option to update the weight vector using the average of the vectors seen over time (averaged argument is set to True by default). +"; public sealed class Arguments : AveragedLinearArguments { @@ -89,7 +94,11 @@ public override TPredictor CreatePredictor() return new LinearRegressionPredictor(Host, ref weights, bias); } - [TlcModule.EntryPoint(Name = "Trainers.OnlineGradientDescentRegressor", Desc = "Train a Online gradient descent perceptron.", UserName = UserNameValue, ShortName = OnlineGradientDescentTrainer.ShortName)] + [TlcModule.EntryPoint(Name = "Trainers.OnlineGradientDescentRegressor", + Desc = "Train a Online gradient descent perceptron.", + Remarks = Remarks, + UserName = UserNameValue, + ShortName = ShortName)] public static CommonOutputs.RegressionOutput TrainRegression(IHostEnvironment env, Arguments input) { Contracts.CheckValue(env, nameof(env)); diff --git a/src/Microsoft.ML.StandardLearners/Standard/PoissonRegression/PoissonRegression.cs b/src/Microsoft.ML.StandardLearners/Standard/PoissonRegression/PoissonRegression.cs index a3ef06cf4e..c5ad4b4495 100644 --- a/src/Microsoft.ML.StandardLearners/Standard/PoissonRegression/PoissonRegression.cs +++ b/src/Microsoft.ML.StandardLearners/Standard/PoissonRegression/PoissonRegression.cs @@ -32,6 +32,11 @@ public sealed class PoissonRegression : LbfgsTrainerBase +Poisson regression is a parameterized regression method. +It assumes that the log of the conditional mean of the dependent variable follows a linear function of the dependent variables. +Assuming that the dependent variable follows a Poisson distribution, the parameters of the regressor can be estimated by maximizing the likelihood of the obtained observations. +"; public sealed class Arguments : ArgumentsBase { diff --git a/src/Microsoft.ML.StandardLearners/Standard/SdcaMultiClass.cs b/src/Microsoft.ML.StandardLearners/Standard/SdcaMultiClass.cs index b00bc1a4c5..f0a250e4b8 100644 --- a/src/Microsoft.ML.StandardLearners/Standard/SdcaMultiClass.cs +++ b/src/Microsoft.ML.StandardLearners/Standard/SdcaMultiClass.cs @@ -386,7 +386,11 @@ protected override Float GetInstanceWeight(FloatLabelCursor cursor) /// public static partial class Sdca { - [TlcModule.EntryPoint(Name = "Trainers.StochasticDualCoordinateAscentClassifier", Desc = SdcaMultiClassTrainer.SDCADetailedSummary, UserName = SdcaMultiClassTrainer.UserNameValue, ShortName = SdcaMultiClassTrainer.ShortName)] + [TlcModule.EntryPoint(Name = "Trainers.StochasticDualCoordinateAscentClassifier", + Desc = SdcaMultiClassTrainer.Summary, + Remarks = SdcaMultiClassTrainer.Remarks, + UserName = SdcaMultiClassTrainer.UserNameValue, + ShortName = SdcaMultiClassTrainer.ShortName)] public static CommonOutputs.MulticlassClassificationOutput TrainMultiClass(IHostEnvironment env, SdcaMultiClassTrainer.Arguments input) { Contracts.CheckValue(env, nameof(env)); diff --git a/src/Microsoft.ML.StandardLearners/Standard/SdcaRegression.cs b/src/Microsoft.ML.StandardLearners/Standard/SdcaRegression.cs index 516c2c7fcb..55a021ebb7 100644 --- a/src/Microsoft.ML.StandardLearners/Standard/SdcaRegression.cs +++ b/src/Microsoft.ML.StandardLearners/Standard/SdcaRegression.cs @@ -131,7 +131,11 @@ protected override Float TuneDefaultL2(IChannel ch, int maxIterations, long rowC /// public static partial class Sdca { - [TlcModule.EntryPoint(Name = "Trainers.StochasticDualCoordinateAscentRegressor", Desc = SdcaRegressionTrainer.SDCADetailedSummary, UserName = SdcaRegressionTrainer.UserNameValue, ShortName = SdcaRegressionTrainer.ShortName)] + [TlcModule.EntryPoint(Name = "Trainers.StochasticDualCoordinateAscentRegressor", + Desc = SdcaRegressionTrainer.Summary, + Remarks = SdcaRegressionTrainer.Remarks, + UserName = SdcaRegressionTrainer.UserNameValue, + ShortName = SdcaRegressionTrainer.ShortName)] public static CommonOutputs.RegressionOutput TrainRegression(IHostEnvironment env, SdcaRegressionTrainer.Arguments input) { Contracts.CheckValue(env, nameof(env)); diff --git a/src/Microsoft.ML/CSharpApi.cs b/src/Microsoft.ML/CSharpApi.cs index 8080deac26..6819f81543 100644 --- a/src/Microsoft.ML/CSharpApi.cs +++ b/src/Microsoft.ML/CSharpApi.cs @@ -4081,18 +4081,27 @@ namespace Trainers { /// + /// Averaged Perceptron Binary Classifier. + /// + /// /// Perceptron is a classification algorithm that makes its predictions based on a linear function. /// I.e., for an instance with feature values f0, f1,..., f_D-1, , the prediction is given by the sign of sigma[0,D-1] ( w_i * f_i), where w_0, w_1,...,w_D-1 are the weights computed by the algorithm. + /// /// Perceptron is an online algorithm, i.e., it processes the instances in the training set one at a time. /// The weights are initialized to be 0, or some random values. Then, for each example in the training set, the value of sigma[0, D-1] (w_i * f_i) is computed. /// If this value has the same sign as the label of the current example, the weights remain the same. If they have opposite signs, /// the weights vector is updated by either subtracting or adding (if the label is negative or positive, respectively) the feature vector of the current example, /// multiplied by a factor 0 < a <= 1, called the learning rate. In a generalization of this algorithm, the weights are updated by adding the feature vector multiplied by the learning rate, /// and by the gradient of some loss function (in the specific case described above, the loss is hinge-loss, whose gradient is 1 when it is non-zero). + /// + /// /// In Averaged Perceptron (AKA voted-perceptron), the weight vectors are stored, /// together with a weight that counts the number of iterations it survived (this is equivalent to storing the weight vector after every iteration, regardless of whether it was updated or not). /// The prediction is then calculated by taking the weighted average of all the sums sigma[0, D-1] (w_i * f_i) or the different weight vectors. - /// + /// + /// Wikipedia entry for Perceptron + /// Large Margin Classification Using the Perceptron Algorithm + /// public sealed partial class AveragedPerceptronBinaryClassifier : Microsoft.ML.Runtime.EntryPoints.CommonInputs.ITrainerInputWithLabel, Microsoft.ML.Runtime.EntryPoints.CommonInputs.ITrainerInput, Microsoft.ML.ILearningPipelineItem { @@ -4599,6 +4608,27 @@ public enum Bundle : byte /// /// Uses a random forest learner to perform binary classification. /// + /// + /// Decision trees are non-parametric models that perform a sequence of simple tests on inputs. + /// This decision procedure maps them to outputs found in the training dataset whose inputs were similar to the instance being processed. + /// A decision is made at each node of the binary tree data structure based on a measure of similarity that maps each instance recursively through the branches of the tree until the appropriate leaf node is reached and the output decision returned. + /// Decision trees have several advantages: + /// + /// They are efficient in both computation and memory usage during training and prediction. + /// They can represent non-linear decision boundaries. + /// They perform integrated feature selection and classification. + /// They are resilient in the presence of noisy features. + /// + /// Fast forest is a random forest implementation. + /// The model consists of an ensemble of decision trees. Each tree in a decision forest outputs a Gaussian distribution by way of prediction. + /// An aggregation is performed over the ensemble of trees to find a Gaussian distribution closest to the combined distribution for all trees in the model. + /// This decision forest classifier consists of an ensemble of decision trees. + /// Generally, ensemble models provide better coverage and accuracy than single decision trees. + /// Each tree in a decision forest outputs a Gaussian distribution. + /// Wikipedia: Random forest + /// Quantile regression forest + /// From Stumps to Trees to Forests + /// public sealed partial class FastForestBinaryClassifier : Microsoft.ML.Runtime.EntryPoints.CommonInputs.ITrainerInputWithGroupId, Microsoft.ML.Runtime.EntryPoints.CommonInputs.ITrainerInputWithWeight, Microsoft.ML.Runtime.EntryPoints.CommonInputs.ITrainerInputWithLabel, Microsoft.ML.Runtime.EntryPoints.CommonInputs.ITrainerInput, Microsoft.ML.ILearningPipelineItem { @@ -4892,6 +4922,27 @@ namespace Trainers /// /// Trains a random forest to fit target values using least-squares. /// + /// + /// Decision trees are non-parametric models that perform a sequence of simple tests on inputs. + /// This decision procedure maps them to outputs found in the training dataset whose inputs were similar to the instance being processed. + /// A decision is made at each node of the binary tree data structure based on a measure of similarity that maps each instance recursively through the branches of the tree until the appropriate leaf node is reached and the output decision returned. + /// Decision trees have several advantages: + /// + /// They are efficient in both computation and memory usage during training and prediction. + /// They can represent non-linear decision boundaries. + /// They perform integrated feature selection and classification. + /// They are resilient in the presence of noisy features. + /// + /// Fast forest is a random forest implementation. + /// The model consists of an ensemble of decision trees. Each tree in a decision forest outputs a Gaussian distribution by way of prediction. + /// An aggregation is performed over the ensemble of trees to find a Gaussian distribution closest to the combined distribution for all trees in the model. + /// This decision forest classifier consists of an ensemble of decision trees. + /// Generally, ensemble models provide better coverage and accuracy than single decision trees. + /// Each tree in a decision forest outputs a Gaussian distribution. + /// Wikipedia: Random forest + /// Quantile regression forest + /// From Stumps to Trees to Forests + /// public sealed partial class FastForestRegressor : Microsoft.ML.Runtime.EntryPoints.CommonInputs.ITrainerInputWithGroupId, Microsoft.ML.Runtime.EntryPoints.CommonInputs.ITrainerInputWithWeight, Microsoft.ML.Runtime.EntryPoints.CommonInputs.ITrainerInputWithLabel, Microsoft.ML.Runtime.EntryPoints.CommonInputs.ITrainerInput, Microsoft.ML.ILearningPipelineItem { @@ -5181,6 +5232,30 @@ public enum BoostedTreeArgsOptimizationAlgorithmType /// /// Uses a logit-boost boosted tree learner to perform binary classification. /// + /// + /// FastTrees is an efficient implementation of the MART gradient boosting algorithm. + /// Gradient boosting is a machine learning technique for regression problems. + /// It builds each regression tree in a step-wise fashion, using a predefined loss function to measure the error for each step and corrects for it in the next. + /// So this prediction model is actually an ensemble of weaker prediction models. In regression problems, boosting builds a series of of such trees in a step-wise fashion and then selects the optimal tree using an arbitrary differentiable loss function. + /// + /// + /// MART learns an ensemble of regression trees, which is a decision tree with scalar values in its leaves. + /// A decision (or regression) tree is a binary tree-like flow chart, where at each interior node one decides which of the two child nodes to continue to based on one of the feature values from the input. + /// At each leaf node, a value is returned. In the interior nodes, the decision is based on the test 'x <= v' where x is the value of the feature in the input sample and v is one of the possible values of this feature. + /// The functions that can be produced by a regression tree are all the piece-wise constant functions. + /// + /// + /// The ensemble of trees is produced by computing, in each step, a regression tree that approximates the gradient of the loss function, and adding it to the previous tree with coefficients that minimize the loss of the new tree. + /// The output of the ensemble produced by MART on a given instance is the sum of the tree outputs. + /// + /// + /// In case of a binary classification problem, the output is converted to a probability by using some form of calibration. + /// In case of a regression problem, the output is the predicted value of the function. + /// In case of a ranking problem, the instances are ordered by the output value of the ensemble. + /// + /// Wikipedia: Gradient boosting (Gradient tree boosting). + /// Greedy function approximation: A gradient boosting machine.. + /// public sealed partial class FastTreeBinaryClassifier : Microsoft.ML.Runtime.EntryPoints.CommonInputs.ITrainerInputWithGroupId, Microsoft.ML.Runtime.EntryPoints.CommonInputs.ITrainerInputWithWeight, Microsoft.ML.Runtime.EntryPoints.CommonInputs.ITrainerInputWithLabel, Microsoft.ML.Runtime.EntryPoints.CommonInputs.ITrainerInput, Microsoft.ML.ILearningPipelineItem { @@ -5572,6 +5647,30 @@ namespace Trainers /// /// Trains gradient boosted decision trees to the LambdaRank quasi-gradient. /// + /// + /// FastTrees is an efficient implementation of the MART gradient boosting algorithm. + /// Gradient boosting is a machine learning technique for regression problems. + /// It builds each regression tree in a step-wise fashion, using a predefined loss function to measure the error for each step and corrects for it in the next. + /// So this prediction model is actually an ensemble of weaker prediction models. In regression problems, boosting builds a series of of such trees in a step-wise fashion and then selects the optimal tree using an arbitrary differentiable loss function. + /// + /// + /// MART learns an ensemble of regression trees, which is a decision tree with scalar values in its leaves. + /// A decision (or regression) tree is a binary tree-like flow chart, where at each interior node one decides which of the two child nodes to continue to based on one of the feature values from the input. + /// At each leaf node, a value is returned. In the interior nodes, the decision is based on the test 'x <= v' where x is the value of the feature in the input sample and v is one of the possible values of this feature. + /// The functions that can be produced by a regression tree are all the piece-wise constant functions. + /// + /// + /// The ensemble of trees is produced by computing, in each step, a regression tree that approximates the gradient of the loss function, and adding it to the previous tree with coefficients that minimize the loss of the new tree. + /// The output of the ensemble produced by MART on a given instance is the sum of the tree outputs. + /// + /// + /// In case of a binary classification problem, the output is converted to a probability by using some form of calibration. + /// In case of a regression problem, the output is the predicted value of the function. + /// In case of a ranking problem, the instances are ordered by the output value of the ensemble. + /// + /// Wikipedia: Gradient boosting (Gradient tree boosting). + /// Greedy function approximation: A gradient boosting machine.. + /// public sealed partial class FastTreeRanker : Microsoft.ML.Runtime.EntryPoints.CommonInputs.ITrainerInputWithGroupId, Microsoft.ML.Runtime.EntryPoints.CommonInputs.ITrainerInputWithWeight, Microsoft.ML.Runtime.EntryPoints.CommonInputs.ITrainerInputWithLabel, Microsoft.ML.Runtime.EntryPoints.CommonInputs.ITrainerInput, Microsoft.ML.ILearningPipelineItem { @@ -5998,6 +6097,30 @@ namespace Trainers /// /// Trains gradient boosted decision trees to fit target values using least-squares. /// + /// + /// FastTrees is an efficient implementation of the MART gradient boosting algorithm. + /// Gradient boosting is a machine learning technique for regression problems. + /// It builds each regression tree in a step-wise fashion, using a predefined loss function to measure the error for each step and corrects for it in the next. + /// So this prediction model is actually an ensemble of weaker prediction models. In regression problems, boosting builds a series of of such trees in a step-wise fashion and then selects the optimal tree using an arbitrary differentiable loss function. + /// + /// + /// MART learns an ensemble of regression trees, which is a decision tree with scalar values in its leaves. + /// A decision (or regression) tree is a binary tree-like flow chart, where at each interior node one decides which of the two child nodes to continue to based on one of the feature values from the input. + /// At each leaf node, a value is returned. In the interior nodes, the decision is based on the test 'x <= v' where x is the value of the feature in the input sample and v is one of the possible values of this feature. + /// The functions that can be produced by a regression tree are all the piece-wise constant functions. + /// + /// + /// The ensemble of trees is produced by computing, in each step, a regression tree that approximates the gradient of the loss function, and adding it to the previous tree with coefficients that minimize the loss of the new tree. + /// The output of the ensemble produced by MART on a given instance is the sum of the tree outputs. + /// + /// + /// In case of a binary classification problem, the output is converted to a probability by using some form of calibration. + /// In case of a regression problem, the output is the predicted value of the function. + /// In case of a ranking problem, the instances are ordered by the output value of the ensemble. + /// + /// Wikipedia: Gradient boosting (Gradient tree boosting). + /// Greedy function approximation: A gradient boosting machine.. + /// public sealed partial class FastTreeRegressor : Microsoft.ML.Runtime.EntryPoints.CommonInputs.ITrainerInputWithGroupId, Microsoft.ML.Runtime.EntryPoints.CommonInputs.ITrainerInputWithWeight, Microsoft.ML.Runtime.EntryPoints.CommonInputs.ITrainerInputWithLabel, Microsoft.ML.Runtime.EntryPoints.CommonInputs.ITrainerInput, Microsoft.ML.ILearningPipelineItem { @@ -6775,6 +6898,15 @@ namespace Trainers /// /// Train a field-aware factorization machine for binary classification /// + /// + /// Field Aware Factorization Machines use, in addition to the input variables, factorized parameters to model the interaction between pairs of variables. + /// The algorithm is particularly useful for high dimensional datasets which can be very sparse (e.g. click-prediction for advertising systems). + /// An advantage of FFM over SVMs is that the training data does not need to be stored in memory, and the coefficients can be optimized directly. + /// Field Aware Factorization Machines + /// Field-aware Factorization Machines for CTR Prediction + /// Adaptive Subgradient Methods for Online Learning and Stochastic Optimization + /// An Improved Stochastic Gradient Method for Training Large-scale Field-aware Factorization Machine. + /// public sealed partial class FieldAwareFactorizationMachineBinaryClassifier : Microsoft.ML.Runtime.EntryPoints.CommonInputs.ITrainerInputWithLabel, Microsoft.ML.Runtime.EntryPoints.CommonInputs.ITrainerInput, Microsoft.ML.ILearningPipelineItem { @@ -7204,6 +7336,14 @@ public enum KMeansPlusPlusTrainerInitAlgorithm /// /// K-means is a popular clustering algorithm. With K-means, the data is clustered into a specified number of clusters in order to minimize the within-cluster sum of squares. K-means++ improves upon K-means by using a better method for choosing the initial cluster centers. /// + /// + /// K-means++ improves upon K-means by using the Yinyang K-Means method for choosing the initial cluster centers. + /// YYK-Means accelerates K-Means up to an order of magnitude while producing exactly the same clustering results (modulo floating point precision issues). + /// YYK-Means observes that there is a lot of redundancy across iterations in the KMeans algorithms and most points do not change their clusters during an iteration. + /// It uses various bounding techniques to identify this redundancy and eliminate many distance computations and optimize centroid computations. + /// K-means. + /// K-means++ + /// public sealed partial class KMeansPlusPlusClusterer : Microsoft.ML.Runtime.EntryPoints.CommonInputs.IUnsupervisedTrainerWithWeight, Microsoft.ML.Runtime.EntryPoints.CommonInputs.ITrainerInput, Microsoft.ML.ILearningPipelineItem { @@ -7320,8 +7460,10 @@ public enum LightGbmArgumentsEvalMetricType /// - /// Train a LightGBM binary class model. + /// Train a LightGBM binary classification model. /// + /// Light GBM is an open source implementation of boosted trees. + /// GitHub: LightGBM public sealed partial class LightGbmBinaryClassifier : Microsoft.ML.Runtime.EntryPoints.CommonInputs.ITrainerInputWithGroupId, Microsoft.ML.Runtime.EntryPoints.CommonInputs.ITrainerInputWithWeight, Microsoft.ML.Runtime.EntryPoints.CommonInputs.ITrainerInputWithLabel, Microsoft.ML.Runtime.EntryPoints.CommonInputs.ITrainerInput, Microsoft.ML.ILearningPipelineItem { @@ -7527,6 +7669,8 @@ namespace Trainers /// /// Train a LightGBM multi class model. /// + /// Light GBM is an open source implementation of boosted trees. + /// GitHub: LightGBM public sealed partial class LightGbmClassifier : Microsoft.ML.Runtime.EntryPoints.CommonInputs.ITrainerInputWithGroupId, Microsoft.ML.Runtime.EntryPoints.CommonInputs.ITrainerInputWithWeight, Microsoft.ML.Runtime.EntryPoints.CommonInputs.ITrainerInputWithLabel, Microsoft.ML.Runtime.EntryPoints.CommonInputs.ITrainerInput, Microsoft.ML.ILearningPipelineItem { @@ -7732,6 +7876,8 @@ namespace Trainers /// /// Train a LightGBM ranking model. /// + /// Light GBM is an open source implementation of boosted trees. + /// GitHub: LightGBM public sealed partial class LightGbmRanker : Microsoft.ML.Runtime.EntryPoints.CommonInputs.ITrainerInputWithGroupId, Microsoft.ML.Runtime.EntryPoints.CommonInputs.ITrainerInputWithWeight, Microsoft.ML.Runtime.EntryPoints.CommonInputs.ITrainerInputWithLabel, Microsoft.ML.Runtime.EntryPoints.CommonInputs.ITrainerInput, Microsoft.ML.ILearningPipelineItem { @@ -7937,6 +8083,8 @@ namespace Trainers /// /// LightGBM Regression /// + /// Light GBM is an open source implementation of boosted trees. + /// GitHub: LightGBM public sealed partial class LightGbmRegressor : Microsoft.ML.Runtime.EntryPoints.CommonInputs.ITrainerInputWithGroupId, Microsoft.ML.Runtime.EntryPoints.CommonInputs.ITrainerInputWithWeight, Microsoft.ML.Runtime.EntryPoints.CommonInputs.ITrainerInputWithLabel, Microsoft.ML.Runtime.EntryPoints.CommonInputs.ITrainerInput, Microsoft.ML.ILearningPipelineItem { @@ -8275,27 +8423,36 @@ namespace Trainers { /// - /// Logistic Regression is a classification method used to predict the value of a categorical dependent variable from its relationship to one or more independent variables assumed to have a logistic distribution. - /// If the dependent variable has only two possible values (success/failure), then the logistic regression is binary. + /// Logistic Regression is a method in statistics used to predict the probability of occurrence of an event and can be used as a classification algorithm. The algorithm predicts the probability of occurrence of an event by fitting data to a logistical function. + /// + /// /// If the dependent variable has more than two possible values (blood type given diagnostic test results), then the logistic regression is multinomial. + /// /// The optimization technique used for LogisticRegressionBinaryClassifier is the limited memory Broyden-Fletcher-Goldfarb-Shanno (L-BFGS). /// Both the L-BFGS and regular BFGS algorithms use quasi-Newtonian methods to estimate the computationally intensive Hessian matrix in the equation used by Newton's method to calculate steps. - /// But the L-BFGS approximation uses only a limited amount of memory to compute the next step direction, so that it is especially suited for problems with a large number of variables. - /// The memory_size parameter specifies the number of past positions and gradients to store for use in the computation of the next step. + /// But the L-BFGS approximation uses only a limited amount of memory to compute the next step direction, + /// so that it is especially suited for problems with a large number of variables. + /// The MemorySize parameter specifies the number of past positions and gradients to store for use in the computation of the next step. + /// + /// /// This learner can use elastic net regularization: a linear combination of L1 (lasso) and L2 (ridge) regularizations. /// Regularization is a method that can render an ill-posed problem more tractable by imposing constraints that provide information to supplement the data and that prevents overfitting by penalizing models with extreme coefficient values. - /// This can improve the generalization of the model learned by selecting the optimal complexity in the bias-variance tradeoff. Regularization works by adding the penalty that is associated with coefficient values to the error of the hypothesis. + /// This can improve the generalization of the model learned by selecting the optimal complexity in the bias-variance tradeoff. + /// Regularization works by adding the penalty that is associated with coefficient values to the error of the hypothesis. /// An accurate model with extreme coefficient values would be penalized more, but a less accurate model with more conservative values would be penalized less. L1 and L2 regularization have different effects and uses that are complementary in certain respects. - /// l1_weight: can be applied to sparse models, when working with high-dimensional data. It pulls small weights associated features that are relatively unimportant towards 0. - /// l2_weight: is preferable for data that is not sparse. It pulls large weights towards zero. + /// + /// L1Weight: can be applied to sparse models, when working with high-dimensional data. + /// It pulls small weights associated features that are relatively unimportant towards 0. + /// L2Weight: is preferable for data that is not sparse. It pulls large weights towards zero. + /// /// Adding the ridge penalty to the regularization overcomes some of lasso's limitations. It can improve its predictive accuracy, for example, when the number of predictors is greater than the sample size. If x = l1_weight and y = l2_weight, ax + by = c defines the linear span of the regularization terms. /// The default values of x and y are both 1. /// An agressive regularization can harm predictive capacity by excluding important variables out of the model. So choosing the optimal values for the regularization parameters is important for the performance of the logistic regression model. - /// Wikipedia: L-BFGS. - /// Wikipedia: Logistic regression. - /// Scalable Training of L1-Regularized Log-Linear Models. - /// Test Run - L1 and L2 Regularization for Machine Learning. - /// + /// Scalable Training of L1-Regularized Log-Linear Models. + /// Test Run - L1 and L2 Regularization for Machine Learning. + /// Wikipedia: L-BFGS. + /// Wikipedia: Logistic regression. + /// public sealed partial class LogisticRegressionBinaryClassifier : Microsoft.ML.Runtime.EntryPoints.CommonInputs.ITrainerInputWithWeight, Microsoft.ML.Runtime.EntryPoints.CommonInputs.ITrainerInputWithLabel, Microsoft.ML.Runtime.EntryPoints.CommonInputs.ITrainerInput, Microsoft.ML.ILearningPipelineItem { @@ -8444,27 +8601,36 @@ namespace Trainers { /// - /// Logistic Regression is a classification method used to predict the value of a categorical dependent variable from its relationship to one or more independent variables assumed to have a logistic distribution. - /// If the dependent variable has only two possible values (success/failure), then the logistic regression is binary. + /// Logistic Regression is a method in statistics used to predict the probability of occurrence of an event and can be used as a classification algorithm. The algorithm predicts the probability of occurrence of an event by fitting data to a logistical function. + /// + /// /// If the dependent variable has more than two possible values (blood type given diagnostic test results), then the logistic regression is multinomial. + /// /// The optimization technique used for LogisticRegressionBinaryClassifier is the limited memory Broyden-Fletcher-Goldfarb-Shanno (L-BFGS). /// Both the L-BFGS and regular BFGS algorithms use quasi-Newtonian methods to estimate the computationally intensive Hessian matrix in the equation used by Newton's method to calculate steps. - /// But the L-BFGS approximation uses only a limited amount of memory to compute the next step direction, so that it is especially suited for problems with a large number of variables. - /// The memory_size parameter specifies the number of past positions and gradients to store for use in the computation of the next step. + /// But the L-BFGS approximation uses only a limited amount of memory to compute the next step direction, + /// so that it is especially suited for problems with a large number of variables. + /// The MemorySize parameter specifies the number of past positions and gradients to store for use in the computation of the next step. + /// + /// /// This learner can use elastic net regularization: a linear combination of L1 (lasso) and L2 (ridge) regularizations. /// Regularization is a method that can render an ill-posed problem more tractable by imposing constraints that provide information to supplement the data and that prevents overfitting by penalizing models with extreme coefficient values. - /// This can improve the generalization of the model learned by selecting the optimal complexity in the bias-variance tradeoff. Regularization works by adding the penalty that is associated with coefficient values to the error of the hypothesis. + /// This can improve the generalization of the model learned by selecting the optimal complexity in the bias-variance tradeoff. + /// Regularization works by adding the penalty that is associated with coefficient values to the error of the hypothesis. /// An accurate model with extreme coefficient values would be penalized more, but a less accurate model with more conservative values would be penalized less. L1 and L2 regularization have different effects and uses that are complementary in certain respects. - /// l1_weight: can be applied to sparse models, when working with high-dimensional data. It pulls small weights associated features that are relatively unimportant towards 0. - /// l2_weight: is preferable for data that is not sparse. It pulls large weights towards zero. + /// + /// L1Weight: can be applied to sparse models, when working with high-dimensional data. + /// It pulls small weights associated features that are relatively unimportant towards 0. + /// L2Weight: is preferable for data that is not sparse. It pulls large weights towards zero. + /// /// Adding the ridge penalty to the regularization overcomes some of lasso's limitations. It can improve its predictive accuracy, for example, when the number of predictors is greater than the sample size. If x = l1_weight and y = l2_weight, ax + by = c defines the linear span of the regularization terms. /// The default values of x and y are both 1. /// An agressive regularization can harm predictive capacity by excluding important variables out of the model. So choosing the optimal values for the regularization parameters is important for the performance of the logistic regression model. - /// Wikipedia: L-BFGS. - /// Wikipedia: Logistic regression. - /// Scalable Training of L1-Regularized Log-Linear Models. - /// Test Run - L1 and L2 Regularization for Machine Learning. - /// + /// Scalable Training of L1-Regularized Log-Linear Models. + /// Test Run - L1 and L2 Regularization for Machine Learning. + /// Wikipedia: L-BFGS. + /// Wikipedia: Logistic regression. + /// public sealed partial class LogisticRegressionClassifier : Microsoft.ML.Runtime.EntryPoints.CommonInputs.ITrainerInputWithWeight, Microsoft.ML.Runtime.EntryPoints.CommonInputs.ITrainerInputWithLabel, Microsoft.ML.Runtime.EntryPoints.CommonInputs.ITrainerInput, Microsoft.ML.ILearningPipelineItem { @@ -8688,6 +8854,11 @@ namespace Trainers /// /// Train a Online gradient descent perceptron. /// + /// + /// Stochastic gradient descent uses a simple yet efficient iterative technique to fit model coefficients using error gradients for convex loss functions. + /// The OnlineGradientDescentRegressor implements the standard (non-batch) SGD, with a choice of loss functions, + /// and an option to update the weight vector using the average of the vectors seen over time (averaged argument is set to True by default). + /// public sealed partial class OnlineGradientDescentRegressor : Microsoft.ML.Runtime.EntryPoints.CommonInputs.ITrainerInputWithLabel, Microsoft.ML.Runtime.EntryPoints.CommonInputs.ITrainerInput, Microsoft.ML.ILearningPipelineItem { @@ -8843,6 +9014,14 @@ namespace Trainers /// /// Train an PCA Anomaly model. /// + /// + /// Principle Component Analysis (PCA) is a dimensionality-reduction transform which computes the projection of the feature vector to onto a low-rank subspace. + /// Its training is done using the technique described in the paper: Combining Structured and Unstructured Randomness in Large Scale PCA, + /// and the paper Finding Structure with Randomness: Probabilistic Algorithms for Constructing Approximate Matrix Decompositions + /// Randomized Methods for Computing the Singular Value Decomposition (SVD) of very large matrices + /// A randomized algorithm for principal component analysis + /// Finding Structure with Randomness: Probabilistic Algorithms for Constructing Approximate Matrix Decompositions + /// public sealed partial class PcaAnomalyDetector : Microsoft.ML.Runtime.EntryPoints.CommonInputs.IUnsupervisedTrainerWithWeight, Microsoft.ML.Runtime.EntryPoints.CommonInputs.ITrainerInput, Microsoft.ML.ILearningPipelineItem { @@ -9084,6 +9263,25 @@ namespace Trainers /// /// Train an SDCA binary model. /// + /// + /// This classifier is a trainer based on the Stochastic DualCoordinate Ascent(SDCA) method, a state-of-the-art optimization technique for convex objective functions. + /// The algorithm can be scaled for use on large out-of-memory data sets due to a semi-asynchronized implementation + /// that supports multi-threading. + /// + /// Convergence is underwritten by periodically enforcing synchronization between primal and dual updates in a separate thread. + /// Several choices of loss functions are also provided. + /// The SDCA method combines several of the best properties and capabilities of logistic regression and SVM algorithms. + /// + /// + /// Note that SDCA is a stochastic and streaming optimization algorithm. + /// The results depends on the order of the training data. For reproducible results, it is recommended that one sets to + /// False and to 1. + /// Elastic net regularization can be specified by the and parameters. Note that the has an effect on the rate of convergence. + /// In general, the larger the , the faster SDCA converges. + /// + /// Scaling Up Stochastic Dual Coordinate Ascent. + /// Stochastic Dual Coordinate Ascent Methods for Regularized Loss Minimization. + /// public sealed partial class StochasticDualCoordinateAscentBinaryClassifier : Microsoft.ML.Runtime.EntryPoints.CommonInputs.ITrainerInputWithLabel, Microsoft.ML.Runtime.EntryPoints.CommonInputs.ITrainerInput, Microsoft.ML.ILearningPipelineItem { @@ -9223,22 +9421,27 @@ namespace Trainers { /// - /// This classifier is a trainer based on the Stochastic DualCoordinate - /// Ascent(SDCA) method, a state-of-the-art optimization technique for convex objective functions. + /// The SDCA linear multi-class classification trainer. + /// + /// + /// This classifier is a trainer based on the Stochastic DualCoordinate Ascent(SDCA) method, a state-of-the-art optimization technique for convex objective functions. /// The algorithm can be scaled for use on large out-of-memory data sets due to a semi-asynchronized implementation /// that supports multi-threading. + /// /// Convergence is underwritten by periodically enforcing synchronization between primal and dual updates in a separate thread. /// Several choices of loss functions are also provided. /// The SDCA method combines several of the best properties and capabilities of logistic regression and SVM algorithms. - /// For more information on SDCA, see: - /// Scaling Up Stochastic Dual Coordinate Ascent. - /// Stochastic Dual Coordinate Ascent Methods for Regularized Loss Minimization. + /// + /// /// Note that SDCA is a stochastic and streaming optimization algorithm. - /// The results depends on the order of the training data. For reproducible results, it is recommended that one sets `shuffle` to - /// `False` and `NumThreads` to `1`. - /// Elastic net regularization can be specified by the l2_weight and l1_weight parameters. Note that the l2_weight has an effect on the rate of convergence. - /// In general, the larger the l2_weight, the faster SDCA converges. - /// + /// The results depends on the order of the training data. For reproducible results, it is recommended that one sets to + /// False and to 1. + /// Elastic net regularization can be specified by the and parameters. Note that the has an effect on the rate of convergence. + /// In general, the larger the , the faster SDCA converges. + /// + /// Scaling Up Stochastic Dual Coordinate Ascent. + /// Stochastic Dual Coordinate Ascent Methods for Regularized Loss Minimization. + /// public sealed partial class StochasticDualCoordinateAscentClassifier : Microsoft.ML.Runtime.EntryPoints.CommonInputs.ITrainerInputWithLabel, Microsoft.ML.Runtime.EntryPoints.CommonInputs.ITrainerInput, Microsoft.ML.ILearningPipelineItem { @@ -9362,22 +9565,27 @@ namespace Trainers { /// - /// This classifier is a trainer based on the Stochastic DualCoordinate - /// Ascent(SDCA) method, a state-of-the-art optimization technique for convex objective functions. + /// The SDCA linear regression trainer. + /// + /// + /// This classifier is a trainer based on the Stochastic DualCoordinate Ascent(SDCA) method, a state-of-the-art optimization technique for convex objective functions. /// The algorithm can be scaled for use on large out-of-memory data sets due to a semi-asynchronized implementation /// that supports multi-threading. + /// /// Convergence is underwritten by periodically enforcing synchronization between primal and dual updates in a separate thread. /// Several choices of loss functions are also provided. /// The SDCA method combines several of the best properties and capabilities of logistic regression and SVM algorithms. - /// For more information on SDCA, see: - /// Scaling Up Stochastic Dual Coordinate Ascent. - /// Stochastic Dual Coordinate Ascent Methods for Regularized Loss Minimization. + /// + /// /// Note that SDCA is a stochastic and streaming optimization algorithm. - /// The results depends on the order of the training data. For reproducible results, it is recommended that one sets `shuffle` to - /// `False` and `NumThreads` to `1`. - /// Elastic net regularization can be specified by the l2_weight and l1_weight parameters. Note that the l2_weight has an effect on the rate of convergence. - /// In general, the larger the l2_weight, the faster SDCA converges. - /// + /// The results depends on the order of the training data. For reproducible results, it is recommended that one sets to + /// False and to 1. + /// Elastic net regularization can be specified by the and parameters. Note that the has an effect on the rate of convergence. + /// In general, the larger the , the faster SDCA converges. + /// + /// Scaling Up Stochastic Dual Coordinate Ascent. + /// Stochastic Dual Coordinate Ascent Methods for Regularized Loss Minimization. + /// public sealed partial class StochasticDualCoordinateAscentRegressor : Microsoft.ML.Runtime.EntryPoints.CommonInputs.ITrainerInputWithLabel, Microsoft.ML.Runtime.EntryPoints.CommonInputs.ITrainerInput, Microsoft.ML.ILearningPipelineItem { diff --git a/src/Microsoft.ML/Runtime/Internal/Tools/CSharpApiGenerator.cs b/src/Microsoft.ML/Runtime/Internal/Tools/CSharpApiGenerator.cs index db7c1d490d..29884fe620 100644 --- a/src/Microsoft.ML/Runtime/Internal/Tools/CSharpApiGenerator.cs +++ b/src/Microsoft.ML/Runtime/Internal/Tools/CSharpApiGenerator.cs @@ -382,7 +382,7 @@ private void GenerateInput(IndentingTextWriter writer, ModuleCatalog.EntryPointI GenerateEnums(writer, entryPointInfo.InputType, _defaultNamespace + entryPointMetadata.Namespace); writer.WriteLine(); GenerateClasses(writer, entryPointInfo.InputType, catalog, _defaultNamespace + entryPointMetadata.Namespace); - CSharpGeneratorUtils.GenerateSummary(writer, entryPointInfo.Description); + CSharpGeneratorUtils.GenerateSummary(writer, entryPointInfo.Description, entryPointInfo.Remarks); if (entryPointInfo.ObsoleteAttribute != null) writer.WriteLine($"[Obsolete(\"{entryPointInfo.ObsoleteAttribute.Message}\")]"); diff --git a/src/Microsoft.ML/Runtime/Internal/Tools/CSharpGeneratorUtils.cs b/src/Microsoft.ML/Runtime/Internal/Tools/CSharpGeneratorUtils.cs index cca73a21f9..1cab5cc35c 100644 --- a/src/Microsoft.ML/Runtime/Internal/Tools/CSharpGeneratorUtils.cs +++ b/src/Microsoft.ML/Runtime/Internal/Tools/CSharpGeneratorUtils.cs @@ -349,7 +349,7 @@ public static string GetComponentName(ModuleCatalog.ComponentInfo component) return $"{Capitalize(component.Name)}{component.Kind}"; } - public static void GenerateSummary(IndentingTextWriter writer, string summary) + public static void GenerateSummary(IndentingTextWriter writer, string summary, string remarks = null) { if (string.IsNullOrEmpty(summary)) return; @@ -357,6 +357,10 @@ public static void GenerateSummary(IndentingTextWriter writer, string summary) foreach (var line in summary.Split(new[] { Environment.NewLine }, StringSplitOptions.RemoveEmptyEntries)) writer.WriteLine($"/// {line}"); writer.WriteLine("/// "); + + if(!string.IsNullOrEmpty(remarks)) + foreach (var line in remarks.Split(new[] { Environment.NewLine }, StringSplitOptions.None)) + writer.WriteLine($"/// {line}"); } public static void GenerateHeader(IndentingTextWriter writer) diff --git a/test/BaselineOutput/Common/EntryPoints/core_ep-list.tsv b/test/BaselineOutput/Common/EntryPoints/core_ep-list.tsv index 34e91f2d3b..6b65317c62 100644 --- a/test/BaselineOutput/Common/EntryPoints/core_ep-list.tsv +++ b/test/BaselineOutput/Common/EntryPoints/core_ep-list.tsv @@ -36,7 +36,7 @@ Models.Summarizer Summarize a linear regression predictor. Microsoft.ML.Runtime. Models.SweepResultExtractor Extracts the sweep result. Microsoft.ML.Runtime.EntryPoints.PipelineSweeperMacro ExtractSweepResult Microsoft.ML.Runtime.EntryPoints.PipelineSweeperMacro+ResultInput Microsoft.ML.Runtime.EntryPoints.PipelineSweeperMacro+Output Models.TrainTestBinaryEvaluator Train test for binary classification Microsoft.ML.Runtime.EntryPoints.TrainTestBinaryMacro TrainTestBinary Microsoft.ML.Runtime.EntryPoints.TrainTestBinaryMacro+Arguments Microsoft.ML.Runtime.EntryPoints.CommonOutputs+MacroOutput`1[Microsoft.ML.Runtime.EntryPoints.TrainTestBinaryMacro+Output] Models.TrainTestEvaluator General train test for any supported evaluator Microsoft.ML.Runtime.EntryPoints.TrainTestMacro TrainTest Microsoft.ML.Runtime.EntryPoints.TrainTestMacro+Arguments Microsoft.ML.Runtime.EntryPoints.CommonOutputs+MacroOutput`1[Microsoft.ML.Runtime.EntryPoints.TrainTestMacro+Output] -Trainers.AveragedPerceptronBinaryClassifier Perceptron is a classification algorithm that makes its predictions based on a linear function.I.e., for an instance with feature values f0, f1,..., f_D-1, , the prediction is given by the sign of sigma[0,D-1] ( w_i * f_i), where w_0, w_1,...,w_D-1 are the weights computed by the algorithm.Perceptron is an online algorithm, i.e., it processes the instances in the training set one at a time.The weights are initialized to be 0, or some random values. Then, for each example in the training set, the value of sigma[0, D-1] (w_i * f_i) is computed. If this value has the same sign as the label of the current example, the weights remain the same. If they have opposite signs,the weights vector is updated by either subtracting or adding (if the label is negative or positive, respectively) the feature vector of the current example,multiplied by a factor 0 < a <= 1, called the learning rate. In a generalization of this algorithm, the weights are updated by adding the feature vector multiplied by the learning rate, and by the gradient of some loss function (in the specific case described above, the loss is hinge-loss, whose gradient is 1 when it is non-zero).In Averaged Perceptron (AKA voted-perceptron), the weight vectors are stored, together with a weight that counts the number of iterations it survived (this is equivalent to storing the weight vector after every iteration, regardless of whether it was updated or not).The prediction is then calculated by taking the weighted average of all the sums sigma[0, D-1] (w_i * f_i) or the different weight vectors. Microsoft.ML.Runtime.Learners.AveragedPerceptronTrainer TrainBinary Microsoft.ML.Runtime.Learners.AveragedPerceptronTrainer+Arguments Microsoft.ML.Runtime.EntryPoints.CommonOutputs+BinaryClassificationOutput +Trainers.AveragedPerceptronBinaryClassifier Averaged Perceptron Binary Classifier. Microsoft.ML.Runtime.Learners.AveragedPerceptronTrainer TrainBinary Microsoft.ML.Runtime.Learners.AveragedPerceptronTrainer+Arguments Microsoft.ML.Runtime.EntryPoints.CommonOutputs+BinaryClassificationOutput Trainers.EnsembleBinaryClassifier Train binary ensemble. Microsoft.ML.Ensemble.EntryPoints.Ensemble CreateBinaryEnsemble Microsoft.ML.Runtime.Ensemble.EnsembleTrainer+Arguments Microsoft.ML.Runtime.EntryPoints.CommonOutputs+BinaryClassificationOutput Trainers.EnsembleClassification Train multiclass ensemble. Microsoft.ML.Ensemble.EntryPoints.Ensemble CreateMultiClassEnsemble Microsoft.ML.Runtime.Ensemble.MulticlassDataPartitionEnsembleTrainer+Arguments Microsoft.ML.Runtime.EntryPoints.CommonOutputs+MulticlassClassificationOutput Trainers.EnsembleRegression Train regression ensemble. Microsoft.ML.Ensemble.EntryPoints.Ensemble CreateRegressionEnsemble Microsoft.ML.Runtime.Ensemble.RegressionEnsembleTrainer+Arguments Microsoft.ML.Runtime.EntryPoints.CommonOutputs+RegressionOutput @@ -50,20 +50,20 @@ Trainers.FieldAwareFactorizationMachineBinaryClassifier Train a field-aware fact Trainers.GeneralizedAdditiveModelBinaryClassifier Trains a gradient boosted stump per feature, on all features simultaneously, to fit target values using least-squares. It mantains no interactions between features. Microsoft.ML.Runtime.FastTree.Gam TrainBinary Microsoft.ML.Runtime.FastTree.BinaryClassificationGamTrainer+Arguments Microsoft.ML.Runtime.EntryPoints.CommonOutputs+BinaryClassificationOutput Trainers.GeneralizedAdditiveModelRegressor Trains a gradient boosted stump per feature, on all features simultaneously, to fit target values using least-squares. It mantains no interactions between features. Microsoft.ML.Runtime.FastTree.Gam TrainRegression Microsoft.ML.Runtime.FastTree.RegressionGamTrainer+Arguments Microsoft.ML.Runtime.EntryPoints.CommonOutputs+RegressionOutput Trainers.KMeansPlusPlusClusterer K-means is a popular clustering algorithm. With K-means, the data is clustered into a specified number of clusters in order to minimize the within-cluster sum of squares. K-means++ improves upon K-means by using a better method for choosing the initial cluster centers. Microsoft.ML.Runtime.KMeans.KMeansPlusPlusTrainer TrainKMeans Microsoft.ML.Runtime.KMeans.KMeansPlusPlusTrainer+Arguments Microsoft.ML.Runtime.EntryPoints.CommonOutputs+ClusteringOutput -Trainers.LightGbmBinaryClassifier Train a LightGBM binary class model. Microsoft.ML.Runtime.LightGBM.LightGbm TrainBinary Microsoft.ML.Runtime.LightGBM.LightGbmArguments Microsoft.ML.Runtime.EntryPoints.CommonOutputs+BinaryClassificationOutput +Trainers.LightGbmBinaryClassifier Train a LightGBM binary classification model. Microsoft.ML.Runtime.LightGBM.LightGbm TrainBinary Microsoft.ML.Runtime.LightGBM.LightGbmArguments Microsoft.ML.Runtime.EntryPoints.CommonOutputs+BinaryClassificationOutput Trainers.LightGbmClassifier Train a LightGBM multi class model. Microsoft.ML.Runtime.LightGBM.LightGbm TrainMultiClass Microsoft.ML.Runtime.LightGBM.LightGbmArguments Microsoft.ML.Runtime.EntryPoints.CommonOutputs+MulticlassClassificationOutput Trainers.LightGbmRanker Train a LightGBM ranking model. Microsoft.ML.Runtime.LightGBM.LightGbm TrainRanking Microsoft.ML.Runtime.LightGBM.LightGbmArguments Microsoft.ML.Runtime.EntryPoints.CommonOutputs+RankingOutput Trainers.LightGbmRegressor LightGBM Regression Microsoft.ML.Runtime.LightGBM.LightGbm TrainRegression Microsoft.ML.Runtime.LightGBM.LightGbmArguments Microsoft.ML.Runtime.EntryPoints.CommonOutputs+RegressionOutput Trainers.LinearSvmBinaryClassifier Train a linear SVM. Microsoft.ML.Runtime.Learners.LinearSvm TrainLinearSvm Microsoft.ML.Runtime.Learners.LinearSvm+Arguments Microsoft.ML.Runtime.EntryPoints.CommonOutputs+BinaryClassificationOutput -Trainers.LogisticRegressionBinaryClassifier Logistic Regression is a classification method used to predict the value of a categorical dependent variable from its relationship to one or more independent variables assumed to have a logistic distribution. If the dependent variable has only two possible values (success/failure), then the logistic regression is binary. If the dependent variable has more than two possible values (blood type given diagnostic test results), then the logistic regression is multinomial.The optimization technique used for LogisticRegressionBinaryClassifier is the limited memory Broyden-Fletcher-Goldfarb-Shanno (L-BFGS). Both the L-BFGS and regular BFGS algorithms use quasi-Newtonian methods to estimate the computationally intensive Hessian matrix in the equation used by Newton's method to calculate steps. But the L-BFGS approximation uses only a limited amount of memory to compute the next step direction, so that it is especially suited for problems with a large number of variables. The memory_size parameter specifies the number of past positions and gradients to store for use in the computation of the next step.This learner can use elastic net regularization: a linear combination of L1 (lasso) and L2 (ridge) regularizations. Regularization is a method that can render an ill-posed problem more tractable by imposing constraints that provide information to supplement the data and that prevents overfitting by penalizing models with extreme coefficient values. This can improve the generalization of the model learned by selecting the optimal complexity in the bias-variance tradeoff. Regularization works by adding the penalty that is associated with coefficient values to the error of the hypothesis. An accurate model with extreme coefficient values would be penalized more, but a less accurate model with more conservative values would be penalized less. L1 and L2 regularization have different effects and uses that are complementary in certain respects.l1_weight: can be applied to sparse models, when working with high-dimensional data. It pulls small weights associated features that are relatively unimportant towards 0. l2_weight: is preferable for data that is not sparse. It pulls large weights towards zero. Adding the ridge penalty to the regularization overcomes some of lasso's limitations. It can improve its predictive accuracy, for example, when the number of predictors is greater than the sample size. If x = l1_weight and y = l2_weight, ax + by = c defines the linear span of the regularization terms. The default values of x and y are both 1. An agressive regularization can harm predictive capacity by excluding important variables out of the model. So choosing the optimal values for the regularization parameters is important for the performance of the logistic regression model.Wikipedia: L-BFGS.Wikipedia: Logistic regression.Scalable Training of L1-Regularized Log-Linear Models.Test Run - L1 and L2 Regularization for Machine Learning. Microsoft.ML.Runtime.Learners.LogisticRegression TrainBinary Microsoft.ML.Runtime.Learners.LogisticRegression+Arguments Microsoft.ML.Runtime.EntryPoints.CommonOutputs+BinaryClassificationOutput -Trainers.LogisticRegressionClassifier Logistic Regression is a classification method used to predict the value of a categorical dependent variable from its relationship to one or more independent variables assumed to have a logistic distribution. If the dependent variable has only two possible values (success/failure), then the logistic regression is binary. If the dependent variable has more than two possible values (blood type given diagnostic test results), then the logistic regression is multinomial.The optimization technique used for LogisticRegressionBinaryClassifier is the limited memory Broyden-Fletcher-Goldfarb-Shanno (L-BFGS). Both the L-BFGS and regular BFGS algorithms use quasi-Newtonian methods to estimate the computationally intensive Hessian matrix in the equation used by Newton's method to calculate steps. But the L-BFGS approximation uses only a limited amount of memory to compute the next step direction, so that it is especially suited for problems with a large number of variables. The memory_size parameter specifies the number of past positions and gradients to store for use in the computation of the next step.This learner can use elastic net regularization: a linear combination of L1 (lasso) and L2 (ridge) regularizations. Regularization is a method that can render an ill-posed problem more tractable by imposing constraints that provide information to supplement the data and that prevents overfitting by penalizing models with extreme coefficient values. This can improve the generalization of the model learned by selecting the optimal complexity in the bias-variance tradeoff. Regularization works by adding the penalty that is associated with coefficient values to the error of the hypothesis. An accurate model with extreme coefficient values would be penalized more, but a less accurate model with more conservative values would be penalized less. L1 and L2 regularization have different effects and uses that are complementary in certain respects.l1_weight: can be applied to sparse models, when working with high-dimensional data. It pulls small weights associated features that are relatively unimportant towards 0. l2_weight: is preferable for data that is not sparse. It pulls large weights towards zero. Adding the ridge penalty to the regularization overcomes some of lasso's limitations. It can improve its predictive accuracy, for example, when the number of predictors is greater than the sample size. If x = l1_weight and y = l2_weight, ax + by = c defines the linear span of the regularization terms. The default values of x and y are both 1. An agressive regularization can harm predictive capacity by excluding important variables out of the model. So choosing the optimal values for the regularization parameters is important for the performance of the logistic regression model.Wikipedia: L-BFGS.Wikipedia: Logistic regression.Scalable Training of L1-Regularized Log-Linear Models.Test Run - L1 and L2 Regularization for Machine Learning. Microsoft.ML.Runtime.Learners.LogisticRegression TrainMultiClass Microsoft.ML.Runtime.Learners.MulticlassLogisticRegression+Arguments Microsoft.ML.Runtime.EntryPoints.CommonOutputs+MulticlassClassificationOutput +Trainers.LogisticRegressionBinaryClassifier Logistic Regression is a method in statistics used to predict the probability of occurrence of an event and can be used as a classification algorithm. The algorithm predicts the probability of occurrence of an event by fitting data to a logistical function. Microsoft.ML.Runtime.Learners.LogisticRegression TrainBinary Microsoft.ML.Runtime.Learners.LogisticRegression+Arguments Microsoft.ML.Runtime.EntryPoints.CommonOutputs+BinaryClassificationOutput +Trainers.LogisticRegressionClassifier Logistic Regression is a method in statistics used to predict the probability of occurrence of an event and can be used as a classification algorithm. The algorithm predicts the probability of occurrence of an event by fitting data to a logistical function. Microsoft.ML.Runtime.Learners.LogisticRegression TrainMultiClass Microsoft.ML.Runtime.Learners.MulticlassLogisticRegression+Arguments Microsoft.ML.Runtime.EntryPoints.CommonOutputs+MulticlassClassificationOutput Trainers.NaiveBayesClassifier Train a MultiClassNaiveBayesTrainer. Microsoft.ML.Runtime.Learners.MultiClassNaiveBayesTrainer TrainMultiClassNaiveBayesTrainer Microsoft.ML.Runtime.Learners.MultiClassNaiveBayesTrainer+Arguments Microsoft.ML.Runtime.EntryPoints.CommonOutputs+MulticlassClassificationOutput Trainers.OnlineGradientDescentRegressor Train a Online gradient descent perceptron. Microsoft.ML.Runtime.Learners.OnlineGradientDescentTrainer TrainRegression Microsoft.ML.Runtime.Learners.OnlineGradientDescentTrainer+Arguments Microsoft.ML.Runtime.EntryPoints.CommonOutputs+RegressionOutput Trainers.PcaAnomalyDetector Train an PCA Anomaly model. Microsoft.ML.Runtime.PCA.RandomizedPcaTrainer TrainPcaAnomaly Microsoft.ML.Runtime.PCA.RandomizedPcaTrainer+Arguments Microsoft.ML.Runtime.EntryPoints.CommonOutputs+AnomalyDetectionOutput Trainers.PoissonRegressor Train an Poisson regression model. Microsoft.ML.Runtime.Learners.PoissonRegression TrainRegression Microsoft.ML.Runtime.Learners.PoissonRegression+Arguments Microsoft.ML.Runtime.EntryPoints.CommonOutputs+RegressionOutput Trainers.StochasticDualCoordinateAscentBinaryClassifier Train an SDCA binary model. Microsoft.ML.Runtime.Learners.Sdca TrainBinary Microsoft.ML.Runtime.Learners.LinearClassificationTrainer+Arguments Microsoft.ML.Runtime.EntryPoints.CommonOutputs+BinaryClassificationOutput -Trainers.StochasticDualCoordinateAscentClassifier This classifier is a trainer based on the Stochastic DualCoordinate Ascent(SDCA) method, a state-of-the-art optimization technique for convex objective functions.The algorithm can be scaled for use on large out-of-memory data sets due to a semi-asynchronized implementation that supports multi-threading.Convergence is underwritten by periodically enforcing synchronization between primal and dual updates in a separate thread.Several choices of loss functions are also provided.The SDCA method combines several of the best properties and capabilities of logistic regression and SVM algorithms.For more information on SDCA, see:Scaling Up Stochastic Dual Coordinate Ascent.Stochastic Dual Coordinate Ascent Methods for Regularized Loss Minimization.Note that SDCA is a stochastic and streaming optimization algorithm. The results depends on the order of the training data. For reproducible results, it is recommended that one sets `shuffle` to`False` and `NumThreads` to `1`.Elastic net regularization can be specified by the l2_weight and l1_weight parameters. Note that the l2_weight has an effect on the rate of convergence. In general, the larger the l2_weight, the faster SDCA converges. Microsoft.ML.Runtime.Learners.Sdca TrainMultiClass Microsoft.ML.Runtime.Learners.SdcaMultiClassTrainer+Arguments Microsoft.ML.Runtime.EntryPoints.CommonOutputs+MulticlassClassificationOutput -Trainers.StochasticDualCoordinateAscentRegressor This classifier is a trainer based on the Stochastic DualCoordinate Ascent(SDCA) method, a state-of-the-art optimization technique for convex objective functions.The algorithm can be scaled for use on large out-of-memory data sets due to a semi-asynchronized implementation that supports multi-threading.Convergence is underwritten by periodically enforcing synchronization between primal and dual updates in a separate thread.Several choices of loss functions are also provided.The SDCA method combines several of the best properties and capabilities of logistic regression and SVM algorithms.For more information on SDCA, see:Scaling Up Stochastic Dual Coordinate Ascent.Stochastic Dual Coordinate Ascent Methods for Regularized Loss Minimization.Note that SDCA is a stochastic and streaming optimization algorithm. The results depends on the order of the training data. For reproducible results, it is recommended that one sets `shuffle` to`False` and `NumThreads` to `1`.Elastic net regularization can be specified by the l2_weight and l1_weight parameters. Note that the l2_weight has an effect on the rate of convergence. In general, the larger the l2_weight, the faster SDCA converges. Microsoft.ML.Runtime.Learners.Sdca TrainRegression Microsoft.ML.Runtime.Learners.SdcaRegressionTrainer+Arguments Microsoft.ML.Runtime.EntryPoints.CommonOutputs+RegressionOutput +Trainers.StochasticDualCoordinateAscentClassifier The SDCA linear multi-class classification trainer. Microsoft.ML.Runtime.Learners.Sdca TrainMultiClass Microsoft.ML.Runtime.Learners.SdcaMultiClassTrainer+Arguments Microsoft.ML.Runtime.EntryPoints.CommonOutputs+MulticlassClassificationOutput +Trainers.StochasticDualCoordinateAscentRegressor The SDCA linear regression trainer. Microsoft.ML.Runtime.Learners.Sdca TrainRegression Microsoft.ML.Runtime.Learners.SdcaRegressionTrainer+Arguments Microsoft.ML.Runtime.EntryPoints.CommonOutputs+RegressionOutput Trainers.StochasticGradientDescentBinaryClassifier Train an Hogwild SGD binary model. Microsoft.ML.Runtime.Learners.StochasticGradientDescentClassificationTrainer TrainBinary Microsoft.ML.Runtime.Learners.StochasticGradientDescentClassificationTrainer+Arguments Microsoft.ML.Runtime.EntryPoints.CommonOutputs+BinaryClassificationOutput Transforms.ApproximateBootstrapSampler Approximate bootstrap sampling. Microsoft.ML.Runtime.Data.BootstrapSample GetSample Microsoft.ML.Runtime.Data.BootstrapSampleTransform+Arguments Microsoft.ML.Runtime.EntryPoints.CommonOutputs+TransformOutput Transforms.BinaryPredictionScoreColumnsRenamer For binary prediction, it renames the PredictedLabel and Score columns to include the name of the positive class. Microsoft.ML.Runtime.EntryPoints.ScoreModel RenameBinaryPredictionScoreColumns Microsoft.ML.Runtime.EntryPoints.ScoreModel+RenameBinaryPredictionScoreColumnsInput Microsoft.ML.Runtime.EntryPoints.CommonOutputs+TransformOutput diff --git a/test/BaselineOutput/Common/EntryPoints/core_manifest.json b/test/BaselineOutput/Common/EntryPoints/core_manifest.json index d27cd1684a..b4c4229cdf 100644 --- a/test/BaselineOutput/Common/EntryPoints/core_manifest.json +++ b/test/BaselineOutput/Common/EntryPoints/core_manifest.json @@ -3719,7 +3719,7 @@ }, { "Name": "Trainers.AveragedPerceptronBinaryClassifier", - "Desc": "Perceptron is a classification algorithm that makes its predictions based on a linear function.I.e., for an instance with feature values f0, f1,..., f_D-1, , the prediction is given by the sign of sigma[0,D-1] ( w_i * f_i), where w_0, w_1,...,w_D-1 are the weights computed by the algorithm.Perceptron is an online algorithm, i.e., it processes the instances in the training set one at a time.The weights are initialized to be 0, or some random values. Then, for each example in the training set, the value of sigma[0, D-1] (w_i * f_i) is computed. If this value has the same sign as the label of the current example, the weights remain the same. If they have opposite signs,the weights vector is updated by either subtracting or adding (if the label is negative or positive, respectively) the feature vector of the current example,multiplied by a factor 0 < a <= 1, called the learning rate. In a generalization of this algorithm, the weights are updated by adding the feature vector multiplied by the learning rate, and by the gradient of some loss function (in the specific case described above, the loss is hinge-loss, whose gradient is 1 when it is non-zero).In Averaged Perceptron (AKA voted-perceptron), the weight vectors are stored, together with a weight that counts the number of iterations it survived (this is equivalent to storing the weight vector after every iteration, regardless of whether it was updated or not).The prediction is then calculated by taking the weighted average of all the sums sigma[0, D-1] (w_i * f_i) or the different weight vectors.", + "Desc": "Averaged Perceptron Binary Classifier.", "FriendlyName": "Averaged Perceptron", "ShortName": "ap", "Inputs": [ @@ -10675,7 +10675,7 @@ }, { "Name": "Trainers.LightGbmBinaryClassifier", - "Desc": "Train a LightGBM binary class model.", + "Desc": "Train a LightGBM binary classification model.", "FriendlyName": "LightGBM Binary Classifier", "ShortName": "LightGBM", "Inputs": [ @@ -12874,7 +12874,7 @@ }, { "Name": "Trainers.LogisticRegressionBinaryClassifier", - "Desc": "Logistic Regression is a classification method used to predict the value of a categorical dependent variable from its relationship to one or more independent variables assumed to have a logistic distribution. If the dependent variable has only two possible values (success/failure), then the logistic regression is binary. If the dependent variable has more than two possible values (blood type given diagnostic test results), then the logistic regression is multinomial.The optimization technique used for LogisticRegressionBinaryClassifier is the limited memory Broyden-Fletcher-Goldfarb-Shanno (L-BFGS). Both the L-BFGS and regular BFGS algorithms use quasi-Newtonian methods to estimate the computationally intensive Hessian matrix in the equation used by Newton's method to calculate steps. But the L-BFGS approximation uses only a limited amount of memory to compute the next step direction, so that it is especially suited for problems with a large number of variables. The memory_size parameter specifies the number of past positions and gradients to store for use in the computation of the next step.This learner can use elastic net regularization: a linear combination of L1 (lasso) and L2 (ridge) regularizations. Regularization is a method that can render an ill-posed problem more tractable by imposing constraints that provide information to supplement the data and that prevents overfitting by penalizing models with extreme coefficient values. This can improve the generalization of the model learned by selecting the optimal complexity in the bias-variance tradeoff. Regularization works by adding the penalty that is associated with coefficient values to the error of the hypothesis. An accurate model with extreme coefficient values would be penalized more, but a less accurate model with more conservative values would be penalized less. L1 and L2 regularization have different effects and uses that are complementary in certain respects.l1_weight: can be applied to sparse models, when working with high-dimensional data. It pulls small weights associated features that are relatively unimportant towards 0. l2_weight: is preferable for data that is not sparse. It pulls large weights towards zero. Adding the ridge penalty to the regularization overcomes some of lasso's limitations. It can improve its predictive accuracy, for example, when the number of predictors is greater than the sample size. If x = l1_weight and y = l2_weight, ax + by = c defines the linear span of the regularization terms. The default values of x and y are both 1. An agressive regularization can harm predictive capacity by excluding important variables out of the model. So choosing the optimal values for the regularization parameters is important for the performance of the logistic regression model.Wikipedia: L-BFGS.Wikipedia: Logistic regression.Scalable Training of L1-Regularized Log-Linear Models.Test Run - L1 and L2 Regularization for Machine Learning.", + "Desc": "Logistic Regression is a method in statistics used to predict the probability of occurrence of an event and can be used as a classification algorithm. The algorithm predicts the probability of occurrence of an event by fitting data to a logistical function.", "FriendlyName": "Logistic Regression", "ShortName": "lr", "Inputs": [ @@ -13186,7 +13186,7 @@ }, { "Name": "Trainers.LogisticRegressionClassifier", - "Desc": "Logistic Regression is a classification method used to predict the value of a categorical dependent variable from its relationship to one or more independent variables assumed to have a logistic distribution. If the dependent variable has only two possible values (success/failure), then the logistic regression is binary. If the dependent variable has more than two possible values (blood type given diagnostic test results), then the logistic regression is multinomial.The optimization technique used for LogisticRegressionBinaryClassifier is the limited memory Broyden-Fletcher-Goldfarb-Shanno (L-BFGS). Both the L-BFGS and regular BFGS algorithms use quasi-Newtonian methods to estimate the computationally intensive Hessian matrix in the equation used by Newton's method to calculate steps. But the L-BFGS approximation uses only a limited amount of memory to compute the next step direction, so that it is especially suited for problems with a large number of variables. The memory_size parameter specifies the number of past positions and gradients to store for use in the computation of the next step.This learner can use elastic net regularization: a linear combination of L1 (lasso) and L2 (ridge) regularizations. Regularization is a method that can render an ill-posed problem more tractable by imposing constraints that provide information to supplement the data and that prevents overfitting by penalizing models with extreme coefficient values. This can improve the generalization of the model learned by selecting the optimal complexity in the bias-variance tradeoff. Regularization works by adding the penalty that is associated with coefficient values to the error of the hypothesis. An accurate model with extreme coefficient values would be penalized more, but a less accurate model with more conservative values would be penalized less. L1 and L2 regularization have different effects and uses that are complementary in certain respects.l1_weight: can be applied to sparse models, when working with high-dimensional data. It pulls small weights associated features that are relatively unimportant towards 0. l2_weight: is preferable for data that is not sparse. It pulls large weights towards zero. Adding the ridge penalty to the regularization overcomes some of lasso's limitations. It can improve its predictive accuracy, for example, when the number of predictors is greater than the sample size. If x = l1_weight and y = l2_weight, ax + by = c defines the linear span of the regularization terms. The default values of x and y are both 1. An agressive regularization can harm predictive capacity by excluding important variables out of the model. So choosing the optimal values for the regularization parameters is important for the performance of the logistic regression model.Wikipedia: L-BFGS.Wikipedia: Logistic regression.Scalable Training of L1-Regularized Log-Linear Models.Test Run - L1 and L2 Regularization for Machine Learning.", + "Desc": "Logistic Regression is a method in statistics used to predict the probability of occurrence of an event and can be used as a classification algorithm. The algorithm predicts the probability of occurrence of an event by fitting data to a logistical function.", "FriendlyName": "Multi-class Logistic Regression", "ShortName": "mlr", "Inputs": [ @@ -14692,7 +14692,7 @@ }, { "Name": "Trainers.StochasticDualCoordinateAscentClassifier", - "Desc": "This classifier is a trainer based on the Stochastic DualCoordinate Ascent(SDCA) method, a state-of-the-art optimization technique for convex objective functions.The algorithm can be scaled for use on large out-of-memory data sets due to a semi-asynchronized implementation that supports multi-threading.Convergence is underwritten by periodically enforcing synchronization between primal and dual updates in a separate thread.Several choices of loss functions are also provided.The SDCA method combines several of the best properties and capabilities of logistic regression and SVM algorithms.For more information on SDCA, see:Scaling Up Stochastic Dual Coordinate Ascent.Stochastic Dual Coordinate Ascent Methods for Regularized Loss Minimization.Note that SDCA is a stochastic and streaming optimization algorithm. The results depends on the order of the training data. For reproducible results, it is recommended that one sets `shuffle` to`False` and `NumThreads` to `1`.Elastic net regularization can be specified by the l2_weight and l1_weight parameters. Note that the l2_weight has an effect on the rate of convergence. In general, the larger the l2_weight, the faster SDCA converges.", + "Desc": "The SDCA linear multi-class classification trainer.", "FriendlyName": "Fast Linear Multi-class Classification (SA-SDCA)", "ShortName": "sasdcamc", "Inputs": [ @@ -14962,7 +14962,7 @@ }, { "Name": "Trainers.StochasticDualCoordinateAscentRegressor", - "Desc": "This classifier is a trainer based on the Stochastic DualCoordinate Ascent(SDCA) method, a state-of-the-art optimization technique for convex objective functions.The algorithm can be scaled for use on large out-of-memory data sets due to a semi-asynchronized implementation that supports multi-threading.Convergence is underwritten by periodically enforcing synchronization between primal and dual updates in a separate thread.Several choices of loss functions are also provided.The SDCA method combines several of the best properties and capabilities of logistic regression and SVM algorithms.For more information on SDCA, see:Scaling Up Stochastic Dual Coordinate Ascent.Stochastic Dual Coordinate Ascent Methods for Regularized Loss Minimization.Note that SDCA is a stochastic and streaming optimization algorithm. The results depends on the order of the training data. For reproducible results, it is recommended that one sets `shuffle` to`False` and `NumThreads` to `1`.Elastic net regularization can be specified by the l2_weight and l1_weight parameters. Note that the l2_weight has an effect on the rate of convergence. In general, the larger the l2_weight, the faster SDCA converges.", + "Desc": "The SDCA linear regression trainer.", "FriendlyName": "Fast Linear Regression (SA-SDCA)", "ShortName": "sasdcar", "Inputs": [