diff --git a/docs/api-reference/time-series-pvalue.md b/docs/api-reference/time-series-pvalue.md index 6cf532b18b..05c69cb489 100644 --- a/docs/api-reference/time-series-pvalue.md +++ b/docs/api-reference/time-series-pvalue.md @@ -7,7 +7,7 @@ The lower its value, the more likely it is a spike. The p-value score is always This score is the p-value of the current computed raw score according to a distribution of raw scores. Here, the distribution is estimated based on the most recent raw score values up to certain depth back in the history. -More specifically, this distribution is estimated using [kernel density estimation (https://en.wikipedia.org/wiki/Kernel_density_estimation) with the Gaussian [kernels](https://en.wikipedia.org/wiki/Kernel_(statistics)#In_non-parametric_statistics) of adaptive bandwidth. +More specifically, this distribution is estimated using [kernel density estimation](https://en.wikipedia.org/wiki/Kernel_density_estimation) with the Gaussian [kernels](https://en.wikipedia.org/wiki/Kernel_(statistics)#In_non-parametric_statistics) of adaptive bandwidth. If the p-value score exceeds $1 - \frac{\text{confidence}}{100}$, the associated timestamp may get a non-zero alert value in spike detection, which means a spike point is detected. Note that $\text{confidence}$ is defined in the signatures of [DetectIidSpike](xref:Microsoft.ML.TimeSeriesCatalog.DetectIidSpike(Microsoft.ML.TransformsCatalog,System.String,System.String,System.Int32,System.Int32,Microsoft.ML.Transforms.TimeSeries.AnomalySide)) diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/DataOperations/LoadingSvmLight.cs b/docs/samples/Microsoft.ML.Samples/Dynamic/DataOperations/LoadingSvmLight.cs index 4e6d4bc524..83d45c08ba 100644 --- a/docs/samples/Microsoft.ML.Samples/Dynamic/DataOperations/LoadingSvmLight.cs +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/DataOperations/LoadingSvmLight.cs @@ -1,12 +1,8 @@ using System; -using System.Collections.Generic; using System.IO; using System.Text; using Microsoft.ML; using Microsoft.ML.Data; -using Microsoft.ML.Transforms; -using Microsoft.VisualBasic.CompilerServices; -using Tensorflow; namespace Samples.Dynamic.DataOperations { @@ -31,9 +27,9 @@ public static void Example() else sb.Append("-1 "); if (line % 2 == 0) - sb.Append("cost:1"); + sb.Append("cost:1 "); else - sb.Append("cost:2"); + sb.Append("cost:2 "); for (int i = 1; i <= 10; i++) { if (random.NextDouble() > 0.5) diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/DataOperations/LoadingText.cs b/docs/samples/Microsoft.ML.Samples/Dynamic/DataOperations/LoadingText.cs index 60538ffc49..4f0a32dc9f 100644 --- a/docs/samples/Microsoft.ML.Samples/Dynamic/DataOperations/LoadingText.cs +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/DataOperations/LoadingText.cs @@ -14,7 +14,7 @@ public static void Example() { // Create 5 data files to illustrate different loading methods. var dataFiles = new List(); - var random = new Random(); + var random = new Random(1); var dataDirectoryName = "DataDir"; Directory.CreateDirectory(dataDirectoryName); for (int i = 0; i < 5; i++) @@ -22,18 +22,29 @@ public static void Example() var fileName = Path.Combine(dataDirectoryName, $"Data_{i}.csv"); dataFiles.Add(fileName); using (var fs = File.CreateText(fileName)) - // Write random lines without header + { + // Write without header with 10 random columns, forcing + // approximately 80% of values to be 0. for (int line = 0; line < 10; line++) - fs.WriteLine(random.NextDouble().ToString()); + { + var sb = new StringBuilder(); + for (int pos = 0; pos < 10; pos++) + { + var value = random.NextDouble(); + sb.Append((value < 0.8 ? 0 : value).ToString() + '\t'); + } + fs.WriteLine(sb.ToString(0, sb.Length - 1)); + } + } } // Create a TextLoader. var mlContext = new MLContext(); var loader = mlContext.Data.CreateTextLoader( columns: new[] - { - new TextLoader.Column("RandomFeature", DataKind.Single, 0) - }, + { + new TextLoader.Column("Features", DataKind.Single, 0, 9) + }, hasHeader: false ); @@ -55,11 +66,119 @@ public static void Example() // Load all files using path wildcard. var multipleFilesWildcardData = - loader.Load(Path.Combine(dataDirectoryName, "*")); + loader.Load(Path.Combine(dataDirectoryName, "Data_*.csv")); PrintRowCount(multipleFilesWildcardData); // Expected Output: // 50 + + + // Create a TextLoader with user defined type. + var loaderWithCustomType = + mlContext.Data.CreateTextLoader(hasHeader: false); + + // Load a single file from path. + var singleFileCustomTypeData = loaderWithCustomType.Load(dataFiles[0]); + PrintRowCount(singleFileCustomTypeData); + + // Expected Output: + // 10 + + + // Create a TextLoader with unknown column length to illustrate + // how a data sample may be used to infer column size. + var dataSample = new MultiFileSource(dataFiles[0]); + var loaderWithUnknownLength = mlContext.Data.CreateTextLoader( + columns: new[] + { + new TextLoader.Column("Features", + DataKind.Single, + new[] { new TextLoader.Range(0, null) }) + }, + dataSample: dataSample + ); + + var dataWithInferredLength = loaderWithUnknownLength.Load(dataFiles[0]); + var featuresColumn = dataWithInferredLength.Schema.GetColumnOrNull("Features"); + if (featuresColumn.HasValue) + Console.WriteLine(featuresColumn.Value.ToString()); + + // Expected Output: + // Features: Vector + // + // ML.NET infers the correct length of 10 for the Features column, + // which is of type Vector. + + PrintRowCount(dataWithInferredLength); + + // Expected Output: + // 10 + + + // Save the data with 10 rows to a text file to illustrate the use of + // sparse format. + var sparseDataFileName = Path.Combine(dataDirectoryName, "saved_data.tsv"); + using (FileStream stream = new FileStream(sparseDataFileName, FileMode.Create)) + mlContext.Data.SaveAsText(singleFileData, stream); + + // Since there are many zeroes in the data, it will be saved in a sparse + // representation to save disk space. The data may be forced to be saved + // in a dense representation by setting forceDense to true. The sparse + // data will look like the following: + // + // 10 7:0.943862259 + // 10 3:0.989767134 + // 10 0:0.949778438 8:0.823028445 9:0.886469543 + // + // The sparse representation of the first row indicates that there are + // 10 columns, the column 7 (8-th column) has value 0.943862259, and other + // omitted columns have value 0. + + // Create a TextLoader that allows sparse input. + var sparseLoader = mlContext.Data.CreateTextLoader( + columns: new[] + { + new TextLoader.Column("Features", DataKind.Single, 0, 9) + }, + allowSparse: true + ); + + // Load the saved sparse data. + var sparseData = sparseLoader.Load(sparseDataFileName); + PrintRowCount(sparseData); + + // Expected Output: + // 10 + + + // Create a TextLoader without any column schema using TextLoader.Options. + // Since the sparse data file was saved with ML.NET, it has the schema + // enoded in its header that the loader can understand: + // + // #@ TextLoader{ + // #@ sep=tab + // #@ col=Features:R4:0-9 + // #@ } + // + // The schema syntax is unimportant since it is only used internally. In + // short, it tells the loader that the values are separated by tabs, and + // that columns 0-9 in the text file are to be read into one column named + // "Features" of type Single (internal type R4). + + var options = new TextLoader.Options() + { + AllowSparse = true, + }; + var dataSampleWithSchema = new MultiFileSource(sparseDataFileName); + var sparseLoaderWithSchema = + mlContext.Data.CreateTextLoader(options, dataSample: dataSampleWithSchema); + + // Load the saved sparse data. + var sparseDataWithSchema = sparseLoaderWithSchema.Load(sparseDataFileName); + PrintRowCount(sparseDataWithSchema); + + // Expected Output: + // 10 } private static void PrintRowCount(IDataView idv) @@ -73,5 +192,11 @@ private static void PrintRowCount(IDataView idv) Console.WriteLine(rowCount); } + + private class Data + { + [LoadColumn(0, 9)] + public float[] Features { get; set; } + } } } diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/CustomMapping.cs b/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/CustomMapping.cs index c3ad19caaf..8bfdef80a0 100644 --- a/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/CustomMapping.cs +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/CustomMapping.cs @@ -6,6 +6,10 @@ namespace Samples.Dynamic { public static class CustomMapping { + // This example shows how to define and apply a custom mapping of input + // columns to output columns without defining a contract. Since a contract + // is not defined, the pipeline containing this mapping cannot be saved and + // loaded back. public static void Example() { // Create a new ML context, for ML.NET operations. It can be used for diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/CustomMappingSaveAndLoad.cs b/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/CustomMappingSaveAndLoad.cs index 745169ef4e..f2d97b70eb 100644 --- a/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/CustomMappingSaveAndLoad.cs +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/CustomMappingSaveAndLoad.cs @@ -7,6 +7,12 @@ namespace Samples.Dynamic { public static class CustomMappingSaveAndLoad { + // This example shows how to define and apply a custom mapping of input + // columns to output columns with a contract name. The contract name is + // used in the CustomMappingFactoryAttribute that decorates the custom + // mapping action. The pipeline containing the custom mapping can then be + // saved to disk, and it can be loaded back after the assembly containing + // the custom mapping action is registered. public static void Example() { // Create a new ML context, for ML.NET operations. It can be used for @@ -24,7 +30,11 @@ public static void Example() var data = mlContext.Data.LoadFromEnumerable(samples); // Custom transformations can be used to transform data directly, or as - // part of a pipeline of estimators. + // part of a pipeline of estimators. The contractName must be provided + // in order for a pipeline containing a CustomMapping estimator to be + // saved and loaded back. The contractName must be the same as in the + // CustomMappingFactoryAttribute used to decorate the custom action + // defined by the user. var pipeline = mlContext.Transforms.CustomMapping(new IsUnderThirtyCustomAction().GetMapping(), contractName: "IsUnderThirty"); diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/CustomMappingWithInMemoryCustomType.cs b/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/CustomMappingWithInMemoryCustomType.cs index a4b5ae30e3..eddf2dcebe 100644 --- a/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/CustomMappingWithInMemoryCustomType.cs +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/CustomMappingWithInMemoryCustomType.cs @@ -8,6 +8,16 @@ namespace Samples.Dynamic { class CustomMappingWithInMemoryCustomType { + // This example shows how custom mapping actions can be performed on custom data + // types that ML.NET doesn't know yet. The example tells a story of how two alien + // bodies are merged to form a super alien with a single body. + // + // Here, the type AlienHero represents a single alien entity with a member "Name" + // of type string and members "One" and "Two" of type AlienBody. It defines a custom + // mapping action AlienFusionProcess that takes an AlienHero and "fuses" its two + // AlienBody members to produce a SuperAlienHero entity with a "Name" member of type + // string and a single "Merged" member of type AlienBody, where the merger is just + // the addition of the various members of AlienBody. static public void Example() { var mlContext = new MLContext(); @@ -33,7 +43,7 @@ static public void Example() + firstAlien.Merged.HandCount + " hands."); // Expected output: - // We got a super alien with name Super Unknown, age 4002, height 6000, weight 8000, and 10000 hands. + // We got a super alien with name Super ML.NET, age 4002, height 6000, weight 8000, and 10000 hands. // Create a prediction engine and print out its prediction. var engine = mlContext.Model.CreatePredictionEngine and Two" would be mapped to different types inside + // The members One and Two would be mapped to different types inside // ML.NET type system because they have different // AlienTypeAttributeAttribute's. For example, the column type of One would - // be DataViewAlienBodyType - // with RaceId=100. - // + // be DataViewAlienBodyType with RaceId=100. + // + // This type represents a "Hero" Alien that is a single entity with two bodies. + // The "Hero" undergoes a fusion process defined in AlienFusionProcess to + // become a SuperAlienHero with a single body that is a merger of the two + // bodies. private class AlienHero { public string Name { get; set; } @@ -129,14 +149,16 @@ public AlienHero(string name, int anotherAge, float anotherHeight, float anotherWeight, int anotherHandCount) { - Name = "Unknown"; + Name = name; One = new AlienBody(age, height, weight, handCount); Two = new AlienBody(anotherAge, anotherHeight, anotherWeight, anotherHandCount); } } - // Type of AlienBody in ML.NET's type system. + // Type of AlienBody in ML.NET's type system. This is the data view type that + // will represent AlienBody in ML.NET's type system when it is registered as + // such in AlienTypeAttributeAttribute. // It usually shows up as DataViewSchema.Column.Type among IDataView.Schema. private class DataViewAlienBodyType : StructuredDataViewType { @@ -162,6 +184,8 @@ public override int GetHashCode() // The output type of processing AlienHero using AlienFusionProcess // .MergeBody(AlienHero, SuperAlienHero). + // This is a "fused" alien whose body is a merger of the two bodies + // of AlienHero. private class SuperAlienHero { public string Name { get; set; } @@ -194,6 +218,5 @@ public static Action GetMapping() return MergeBody; } } - } } diff --git a/src/Microsoft.ML.Core/Data/DataKind.cs b/src/Microsoft.ML.Core/Data/DataKind.cs index 8f22596ef2..fe3e69b5b1 100644 --- a/src/Microsoft.ML.Core/Data/DataKind.cs +++ b/src/Microsoft.ML.Core/Data/DataKind.cs @@ -16,7 +16,7 @@ namespace Microsoft.ML.Data /// /// | Type | Default Value | IsDefault Indicator | /// | -- | -- | -- | - /// | or [text](xref:Microsoft.ML.Data.TextDataViewType) | Empty or `null` string (both result in empty `System.ReadOnlyMemory` | | + /// | or [text](xref:Microsoft.ML.Data.TextDataViewType) | Empty or `null` string (both result in empty `System.ReadOnlyMemory` | | /// | [Key](xref:Microsoft.ML.Data.KeyDataViewType) type (supported by the unsigned integer types in `DataKind`) | Not defined | Always `false` | /// | All other types | Default value of the corresponding system type as defined by .NET standard. In C#, default value expression `default(T)` provides that value. | Equality test with the default value | /// diff --git a/src/Microsoft.ML.Data/DataLoadSave/EstimatorChain.cs b/src/Microsoft.ML.Data/DataLoadSave/EstimatorChain.cs index ceae3fbe10..d880f74d95 100644 --- a/src/Microsoft.ML.Data/DataLoadSave/EstimatorChain.cs +++ b/src/Microsoft.ML.Data/DataLoadSave/EstimatorChain.cs @@ -93,7 +93,12 @@ public EstimatorChain Append(IEstimator estimat /// /// Append a 'caching checkpoint' to the estimator chain. This will ensure that the downstream estimators will be trained against - /// cached data. It is helpful to have a caching checkpoint before trainers that take multiple data passes. + /// cached data. It is helpful to have a caching checkpoint before trainers or feature engineering that take multiple data passes. + /// It is also helpful to have after a slow operation, for example after dataset loading from a slow source or after feature + /// engineering that is slow on its apply phase, if downstream estimators will do multiple passes over the output of this operation. + /// Adding a cache checkpoint at the end of an is meaningless and should be avoided. + /// Cache checkpoints should be removed if disk thrashing or OutOfMemory exceptions are seen, which can occur on when the featured + /// dataset immediately prior to the checkpoint is larger than available RAM. /// /// The host environment to use for caching. public EstimatorChain AppendCacheCheckpoint(IHostEnvironment env) diff --git a/src/Microsoft.ML.Data/DataLoadSave/Text/TextLoader.cs b/src/Microsoft.ML.Data/DataLoadSave/Text/TextLoader.cs index c83e9e4d9f..4a95a83ea5 100644 --- a/src/Microsoft.ML.Data/DataLoadSave/Text/TextLoader.cs +++ b/src/Microsoft.ML.Data/DataLoadSave/Text/TextLoader.cs @@ -427,9 +427,9 @@ internal bool TryUnparse(StringBuilder sb) public class Options { /// - /// Whether the input may include quoted values, which can contain separator characters, colons, - /// and distinguish empty values from missing values. When true, consecutive separators denote a - /// missing value and an empty value is denoted by \"\". When false, consecutive separators denote an empty value. + /// Whether the input may include double-quoted values. This parameter is used to distinguish separator characters + /// in an input value from actual separators. When , separators within double quotes are treated as part of the + /// input value. When , all separators, even those within quotes, are treated as delimiting a new column. /// [Argument(ArgumentType.AtMostOnce, HelpText = @@ -441,7 +441,17 @@ public class Options public bool AllowQuoting = Defaults.AllowQuoting; /// - /// Whether the input may include sparse representations. + /// Whether the input may include sparse representations. For example, a row containing + /// "5 2:6 4:3" means that there are 5 columns, and the only non-zero are columns 2 and 4, which have values 6 and 3, + /// respectively. Column indices are zero-based, so columns 2 and 4 represent the 3rd and 5th columns. + /// A column may also have dense values followed by sparse values represented in this fashion. For example, + /// a row containing "1 2 5 2:6 4:3" represents two dense columns with values 1 and 2, followed by 5 sparsely represented + /// columns with values 0, 0, 6, 0, and 3. The indices of the sparse columns start from 0, even though 0 represents the third column. + /// + /// In addition, should be used when the number of sparse elements (5 in this example) is not present in each line. + /// It should specify the total size, not just the size of the sparse part. However, indices of the spars part are relative to where the sparse part begins. + /// If is set to 7, the line "1 2 2:6 4:3" will be mapped to "1 2 0 0 6 0 4", but if set to 10, the same line will + /// be mapped to "1 2 0 0 6 0 4 0 0 0". /// [Argument(ArgumentType.AtMostOnce, HelpText = "Whether the input may include sparse representations", ShortName = "sparse")] public bool AllowSparse = Defaults.AllowSparse; @@ -478,11 +488,8 @@ public class Options public bool TrimWhitespace = Defaults.TrimWhitespace; /// - /// Whether the data file has a header with feature names. - /// Note: If a TextLoader is created with hasHeader = true but without a dataSample, then vector columns made by TextLoader will not contain slot name - /// annotations (slots being the elements of the given vector column), because the output schema is made when the TextLoader is made, and not when - /// TextLoader.Load(IMultiStreamSource source) is called. In addition, the case where dataSample = null and hasHeader = true indicates to the - /// loader that when it is given a file when is called, it needs to skip the first line. + /// Whether the file has a header with feature names. When , the loader will skip the first line when + /// is called. The sample can be used to infer slot name annotations if present. /// [Argument(ArgumentType.AtMostOnce, ShortName = "header", HelpText = "Data file has header with feature names. Header is read only if options 'hs' and 'hf' are not specified.")] diff --git a/src/Microsoft.ML.Data/DataLoadSave/Text/TextLoaderSaverCatalog.cs b/src/Microsoft.ML.Data/DataLoadSave/Text/TextLoaderSaverCatalog.cs index bc1bd789fe..2c527c4d55 100644 --- a/src/Microsoft.ML.Data/DataLoadSave/Text/TextLoaderSaverCatalog.cs +++ b/src/Microsoft.ML.Data/DataLoadSave/Text/TextLoaderSaverCatalog.cs @@ -21,15 +21,33 @@ public static class TextLoaderSaverCatalog /// The catalog. /// Array of columns defining the schema. /// The character used as separator between data points in a row. By default the tab character is used as separator. - /// Whether the file has a header with feature names. Note: If a TextLoader is created with hasHeader = true but without a - /// , then vector columns made by TextLoader will not contain slot name annotations (slots being the elements of the given vector column), - /// because the output schema is made when the TextLoader is made, and not when is called. - /// In addition, the case where dataSample = null and hasHeader = true indicates to the loader that when it is given a file when Load() - /// is called, it needs to skip the first line. - /// The optional location of a data sample. The sample can be used to infer column names and number of slots in each column. - /// Whether the file can contain columns defined by a quoted string. - /// Remove trailing whitespace from lines - /// Whether the file can contain numerical vectors in sparse format. + /// Whether the file has a header with feature names. When a is provided, + /// indicates that the first line in the will be used for feature names, and that when + /// is called, the first line will be skipped. When there is no provided, just indicates that the loader should + /// skip the first line when is called, but columns will not have slot names annotations. This is + /// because the output schema is made when the loader is created, and not when is called. + /// The optional location of a data sample. The sample can be used to infer slot name annotations if present, and also the number + /// of slots in a column defined with with maximum index. + /// If the sample has been saved with ML.NET's , + /// it will also contain the schema information in the header that the loader can read even if is . + /// In order to use the schema defined in the file, all other arguments sould be left with their default values. + /// Whether the input may include double-quoted values. This parameter is used to distinguish separator characters + /// in an input value from actual separators. When , separators within double quotes are treated as part of the + /// input value. When , all separators, even those within quotes, are treated as delimiting a new column. + /// Remove trailing whitespace from lines. + /// Whether the input may include sparse representations. For example, a row containing + /// "5 2:6 4:3" means that there are 5 columns, and the only non-zero are columns 2 and 4, which have values 6 and 3, + /// respectively. Column indices are zero-based, so columns 2 and 4 represent the 3rd and 5th columns. + /// A column may also have dense values followed by sparse values represented in this fashion. For example, + /// a row containing "1 2 5 2:6 4:3" represents two dense columns with values 1 and 2, followed by 5 sparsely represented + /// columns with values 0, 0, 6, 0, and 3. The indices of the sparse columns start from 0, even though 0 represents the third column. + /// + /// + /// + /// + /// public static TextLoader CreateTextLoader(this DataOperationsCatalog catalog, TextLoader.Column[] columns, char separatorChar = TextLoader.Defaults.Separator, @@ -57,7 +75,11 @@ public static TextLoader CreateTextLoader(this DataOperationsCatalog catalog, /// /// The catalog. /// Defines the settings of the load operation. - /// The optional location of a data sample. The sample can be used to infer column names and number of slots in each column. + /// The optional location of a data sample. The sample can be used to infer slot name annotations if present, and also the number + /// of slots in defined with with maximum index. + /// If the sample has been saved with ML.NET's , + /// it will also contain the schema information in the header that the loader can read even if are not specified. + /// In order to use the schema defined in the file, all other sould be left with their default values. public static TextLoader CreateTextLoader(this DataOperationsCatalog catalog, TextLoader.Options options, IMultiStreamSource dataSample = null) @@ -71,22 +93,22 @@ public static TextLoader CreateTextLoader(this DataOperationsCatalog catalog, /// names and their data types in the schema of the loaded data. /// The catalog. /// Column separator character. Default is '\t' - /// Whether the file has a header with feature names. Note: If a TextLoader is created with hasHeader = true but without a - /// , then vector columns made by TextLoader will not contain slot name annotations (slots being the elements of the given vector column), - /// because the output schema is made when the TextLoader is made, and not when is called. - /// In addition, the case where dataSample = null and hasHeader = true indicates to the loader that when it is given a file when Load() - /// is called, it needs to skip the first line. - /// The optional location of a data sample. The sample can be used to infer information - /// about the columns, such as slot names. - /// Whether the input may include quoted values, - /// which can contain separator characters, colons, - /// and distinguish empty values from missing values. When true, consecutive separators - /// denote a missing value and an empty value is denoted by \"\". - /// When false, consecutive separators denote an empty value. - /// Remove trailing whitespace from lines - /// Whether the input may include sparse representations for example, - /// if one of the row contains "5 2:6 4:3" that's mean there are 5 columns all zero - /// except for 3rd and 5th columns which have values 6 and 3 + /// Whether the file has a header with feature names. When a is provided, + /// indicates that the first line in the will be used for feature names, and that when + /// is called, the first line will be skipped. When there is no provided, just indicates that the loader should + /// skip the first line when is called, but columns will not have slot names annotations. This is + /// because the output schema is made when the loader is created, and not when is called. + /// The optional location of a data sample. The sample can be used to infer slot name annotations if present. + /// Whether the input may include double-quoted values. This parameter is used to distinguish separator characters + /// in an input value from actual separators. When , separators within double quotes are treated as part of the + /// input value. When , all separators, even those whitin quotes, are treated as delimiting a new column. + /// Remove trailing whitespace from lines. + /// Whether the input may include sparse representations. For example, a row containing + /// "5 2:6 4:3" means that there are 5 columns, and the only non-zero are columns 2 and 4, which have values 6 and 3, + /// respectively. Column indices are zero-based, so columns 2 and 4 represent the 3rd and 5th columns. + /// A column may also have dense values followed by sparse values represented in this fashion. For example, + /// a row containing "1 2 5 2:6 4:3" represents two dense columns with values 1 and 2, followed by 5 sparsely represented + /// columns with values 0, 0, 6, 0, and 3. The indices of the sparse columns start from 0, even though 0 represents the third column. public static TextLoader CreateTextLoader(this DataOperationsCatalog catalog, char separatorChar = TextLoader.Defaults.Separator, bool hasHeader = TextLoader.Defaults.HasHeader, @@ -105,14 +127,21 @@ public static TextLoader CreateTextLoader(this DataOperationsCatalog cat /// The path to the file. /// The columns of the schema. /// The character used as separator between data points in a row. By default the tab character is used as separator. - /// Whether the file has a header with feature names. Note: If a TextLoader is created with hasHeader = true but without a - /// dataSample, then vector columns made by TextLoader will not contain slot name annotations (slots being the elements of the given vector column), - /// because the output schema is made when the TextLoader is made, and not when is called. - /// In addition, the case where dataSample = null and hasHeader = true indicates to the loader that when it is given a file when Load() - /// is called, it needs to skip the first line. - /// Whether the file can contain columns defined by a quoted string. - /// Remove trailing whitespace from lines - /// Whether the file can contain numerical vectors in sparse format. + /// Whether the file has a header. When , the loader will skip the first line when + /// is called. + /// Whether the input may include double-quoted values. This parameter is used to distinguish separator characters + /// in an input value from actual separators. When , separators within double quotes are treated as part of the + /// input value. When , all separators, even those whitin quotes, are treated as delimiting a new column. + /// It is also used to distinguish empty values from missing values. When , missing value are denoted by consecutive + /// separators and empty values by \"\". When , empty values are denoted by consecutive separators and missing + /// values by the default missing value for each type documented in . + /// Remove trailing whitespace from lines. + /// Whether the input may include sparse representations. For example, a row containing + /// "5 2:6 4:3" means that there are 5 columns, and the only non-zero are columns 2 and 4, which have values 6 and 3, + /// respectively. Column indices are zero-based, so columns 2 and 4 represent the 3rd and 5th columns. + /// A column may also have dense values followed by sparse values represented in this fashion. For example, + /// a row containing "1 2 5 2:6 4:3" represents two dense columns with values 1 and 2, followed by 5 sparsely represented + /// columns with values 0, 0, 6, 0, and 3. The indices of the sparse columns start from 0, even though 0 represents the third column. /// The data view. public static IDataView LoadFromTextFile(this DataOperationsCatalog catalog, string path, @@ -150,20 +179,21 @@ public static IDataView LoadFromTextFile(this DataOperationsCatalog catalog, /// The catalog. /// The path to the file. /// Column separator character. Default is '\t' - /// Whether the file has a header with feature names. Note: If a TextLoader is created with hasHeader = true but without a - /// dataSample, then vector columns made by TextLoader will not contain slot name annotations (slots being the elements of the given vector column), - /// because the output schema is made when the TextLoader is made, and not when is called. - /// In addition, the case where dataSample = null and hasHeader = true indicates to the loader that when it is given a file when Load() - /// is called, it needs to skip the first line. - /// Whether the input may include quoted values, - /// which can contain separator characters, colons, - /// and distinguish empty values from missing values. When true, consecutive separators - /// denote a missing value and an empty value is denoted by \"\". - /// When false, consecutive separators denote an empty value. - /// Remove trailing whitespace from lines - /// Whether the input may include sparse representations for example, - /// if one of the row contains "5 2:6 4:3" that's mean there are 5 columns all zero - /// except for 3rd and 5th columns which have values 6 and 3 + /// Whether the file has a header. When , the loader will skip the first line when + /// is called. + /// Whether the input may include double-quoted values. This parameter is used to distinguish separator characters + /// in an input value from actual separators. When , separators within double quotes are treated as part of the + /// input value. When , all separators, even those whitin quotes, are treated as delimiting a new column. + /// It is also used to distinguish empty values from missing values. When , missing value are denoted by consecutive + /// separators and empty values by \"\". When , empty values are denoted by consecutive separators and missing + /// values by the default missing value for each type documented in . + /// Remove trailing whitespace from lines. + /// Whether the input may include sparse representations. For example, a row containing + /// "5 2:6 4:3" means that there are 5 columns, and the only non-zero are columns 2 and 4, which have values 6 and 3, + /// respectively. Column indices are zero-based, so columns 2 and 4 represent the 3rd and 5th columns. + /// A column may also have dense values followed by sparse values represented in this fashion. For example, + /// a row containing "1 2 5 2:6 4:3" represents two dense columns with values 1 and 2, followed by 5 sparsely represented + /// columns with values 0, 0, 6, 0, and 3. The indices of the sparse columns start from 0, even though 0 represents the third column. /// The data view. public static IDataView LoadFromTextFile(this DataOperationsCatalog catalog, string path, diff --git a/src/Microsoft.ML.Data/Evaluators/Metrics/AnomalyDetectionMetrics.cs b/src/Microsoft.ML.Data/Evaluators/Metrics/AnomalyDetectionMetrics.cs index 8bb3a5d84b..cae4f6ead2 100644 --- a/src/Microsoft.ML.Data/Evaluators/Metrics/AnomalyDetectionMetrics.cs +++ b/src/Microsoft.ML.Data/Evaluators/Metrics/AnomalyDetectionMetrics.cs @@ -17,12 +17,14 @@ public sealed class AnomalyDetectionMetrics /// /// The area under the ROC curve is equal to the probability that the algorithm ranks /// a randomly chosen positive instance higher than a randomly chosen negative one - /// (assuming 'positive' ranks higher than 'negative'). + /// (assuming 'positive' ranks higher than 'negative'). Area under the ROC curve ranges between + /// 0 and 1, with a value closer to 1 indicating a better model. /// public double AreaUnderRocCurve { get; } /// - /// Detection rate at K false positives. + /// Detection rate at K false positives. This gives the ratio of correctly identified anomalies given + /// the specified number of false positives. A value closer to 1 indicates a better model. /// /// /// This is computed as follows: diff --git a/src/Microsoft.ML.Data/Evaluators/Metrics/BinaryClassificationMetrics.cs b/src/Microsoft.ML.Data/Evaluators/Metrics/BinaryClassificationMetrics.cs index 81cac683f1..8a621b9de5 100644 --- a/src/Microsoft.ML.Data/Evaluators/Metrics/BinaryClassificationMetrics.cs +++ b/src/Microsoft.ML.Data/Evaluators/Metrics/BinaryClassificationMetrics.cs @@ -17,7 +17,10 @@ public class BinaryClassificationMetrics /// /// The area under the ROC curve is equal to the probability that the classifier ranks /// a randomly chosen positive instance higher than a randomly chosen negative one - /// (assuming 'positive' ranks higher than 'negative'). + /// (assuming 'positive' ranks higher than 'negative'). Area under the ROC curve ranges between + /// 0 and 1, with a value closer to 1 indicating a better model. + /// + /// Area Under ROC Curve /// public double AreaUnderRocCurve { get; } @@ -55,10 +58,12 @@ public class BinaryClassificationMetrics public double NegativeRecall { get; } /// - /// Gets the F1 score of the classifier. + /// Gets the F1 score of the classifier, which is a measure of the classifier's quality considering + /// both precision and recall. /// /// /// F1 score is the harmonic mean of precision and recall: 2 * precision * recall / (precision + recall). + /// F1 ranges between 0 and 1, with a value of 1 indicating perfect precision and recall. /// public double F1Score { get; } diff --git a/src/Microsoft.ML.Data/Evaluators/Metrics/CalibratedBinaryClassificationMetrics.cs b/src/Microsoft.ML.Data/Evaluators/Metrics/CalibratedBinaryClassificationMetrics.cs index a2d193deed..e116d94025 100644 --- a/src/Microsoft.ML.Data/Evaluators/Metrics/CalibratedBinaryClassificationMetrics.cs +++ b/src/Microsoft.ML.Data/Evaluators/Metrics/CalibratedBinaryClassificationMetrics.cs @@ -12,33 +12,53 @@ namespace Microsoft.ML.Data public sealed class CalibratedBinaryClassificationMetrics : BinaryClassificationMetrics { /// - /// Gets the log-loss of the classifier. + /// Gets the log-loss of the classifier. Log-loss measures the performance of a classifier + /// with respect to how much the predicted probabilities diverge from the true class label. Lower + /// log-loss indicates a better model. A perfect model, which predicts a probability of 1 for the + /// true class, will have a log-loss of 0. /// /// + /// + /// /// public double LogLoss { get; } /// /// Gets the log-loss reduction (also known as relative log-loss, or reduction in information gain - RIG) - /// of the classifier. + /// of the classifier. It gives a measure of how much a model improves on a model that gives random predictions. + /// Log-loss reduction closer to 1 indicates a better model. /// /// + /// + /// /// public double LogLossReduction { get; } /// - /// Gets the test-set entropy (prior Log-Loss/instance) of the classifier. + /// Gets the test-set entropy, which is the prior log-loss based on the proportion of positive + /// and negative instances in the test set. A classifier's lower than + /// the entropy indicates that a classifier does better than predicting the proportion of positive + /// instances as the probability for each instance. /// + /// + /// + /// + /// public double Entropy { get; } internal CalibratedBinaryClassificationMetrics(IHost host, DataViewRow overallResult, IDataView confusionMatrix) diff --git a/src/Microsoft.ML.Data/Evaluators/Metrics/ClusteringMetrics.cs b/src/Microsoft.ML.Data/Evaluators/Metrics/ClusteringMetrics.cs index f10d32fc5a..169ff4f16a 100644 --- a/src/Microsoft.ML.Data/Evaluators/Metrics/ClusteringMetrics.cs +++ b/src/Microsoft.ML.Data/Evaluators/Metrics/ClusteringMetrics.cs @@ -15,14 +15,14 @@ public sealed class ClusteringMetrics /// Normalized Mutual Information is a measure of the mutual dependence of the variables. /// This metric is only calculated if the Label column is provided. /// - /// Its value ranged from 0 to 1, where higher numbers are better. + /// Its value ranges from 0 to 1, where higher numbers are better. /// Normalized variants. public double NormalizedMutualInformation { get; } /// /// Average Score. For the K-Means algorithm, the 'score' is the distance from the centroid to the example. /// The average score is, therefore, a measure of proximity of the examples to cluster centroids. - /// In other words, it's the 'cluster tightness' measure. + /// In other words, it is a measure of 'cluster tightness'. /// Note however, that this metric will only decrease if the number of clusters is increased, /// and in the extreme case (where each distinct example is its own cluster) it will be equal to zero. /// diff --git a/src/Microsoft.ML.Data/Evaluators/Metrics/MulticlassClassificationMetrics.cs b/src/Microsoft.ML.Data/Evaluators/Metrics/MulticlassClassificationMetrics.cs index db100e95b0..68fed130d2 100644 --- a/src/Microsoft.ML.Data/Evaluators/Metrics/MulticlassClassificationMetrics.cs +++ b/src/Microsoft.ML.Data/Evaluators/Metrics/MulticlassClassificationMetrics.cs @@ -14,27 +14,37 @@ namespace Microsoft.ML.Data public sealed class MulticlassClassificationMetrics { /// - /// Gets the average log-loss of the classifier. + /// Gets the average log-loss of the classifier. Log-loss measures the performance of a classifier + /// with respect to how much the predicted probabilities diverge from the true class label. Lower + /// log-loss indicates a better model. A perfect model, which predicts a probability of 1 for the + /// true class, will have a log-loss of 0. /// /// - /// The log-loss metric, is computed as follows: - /// LL = - (1/m) * sum( log(p[i])) - /// where m is the number of instances in the test set. - /// p[i] is the probability returned by the classifier if the instance belongs to class 1, - /// and 1 minus the probability returned by the classifier if the instance belongs to class 0. + /// + /// /// public double LogLoss { get; } /// /// Gets the log-loss reduction (also known as relative log-loss, or reduction in information gain - RIG) - /// of the classifier. + /// of the classifier. It gives a measure of how much a model improves on a model that gives random predictions. + /// Log-loss reduction closer to 1 indicates a better model. /// /// + /// + /// /// public double LogLossReduction { get; private set; } @@ -54,8 +64,9 @@ public sealed class MulticlassClassificationMetrics /// Gets the micro-average accuracy of the model. /// /// - /// The micro-average is the fraction of instances predicted correctly. - /// The micro-average does not take class membership into account. + /// The micro-average is the fraction of instances predicted correctly across all classes. Micro-average can + /// be a more useful metric than macro-average if class imbalance is suspected (i.e. one class has many more + /// instances than the rest). /// public double MicroAccuracy { get; } @@ -71,7 +82,10 @@ public sealed class MulticlassClassificationMetrics public int TopKPredictionCount { get; } /// - /// Gets the log-loss of the classifier for each class. + /// Gets the log-loss of the classifier for each class. Log-loss measures the performance of a classifier + /// with respect to how much the predicted probabilities diverge from the true class label. Lower + /// log-loss indicates a better model. A perfect model, which predicts a probability of 1 for the + /// true class, will have a log-loss of 0. /// /// /// The log-loss metric is computed as $-\frac{1}{m} \sum_{i=1}^m \log(p_i)$, diff --git a/src/Microsoft.ML.Data/Evaluators/Metrics/RankingMetrics.cs b/src/Microsoft.ML.Data/Evaluators/Metrics/RankingMetrics.cs index 26650e6b66..771a3a3cde 100644 --- a/src/Microsoft.ML.Data/Evaluators/Metrics/RankingMetrics.cs +++ b/src/Microsoft.ML.Data/Evaluators/Metrics/RankingMetrics.cs @@ -14,19 +14,36 @@ namespace Microsoft.ML.Data public sealed class RankingMetrics { /// - /// Array of normalized discounted cumulative gains where i-th element represent NDCG@i. - /// + /// + /// /// + /// + /// Normalized Discounted Cumulative Gain + /// public IReadOnlyList NormalizedDiscountedCumulativeGains { get; } /// - /// Array of discounted cumulative gains where i-th element represent DCG@i. - /// Discounted Cumulative gain is the sum of the gains, for all the instances i, - /// normalized by the natural logarithm of the instance + 1. - /// Note that unline the Wikipedia article, ML.Net uses the natural logarithm. - /// + /// + /// /// - /// Discounted Cumulative gain. + /// + /// Discounted Cumulative Gain + /// public IReadOnlyList DiscountedCumulativeGains { get; } private static T Fetch(IExceptionContext ectx, DataViewRow row, string name) diff --git a/src/Microsoft.ML.Data/Evaluators/Metrics/RegressionMetrics.cs b/src/Microsoft.ML.Data/Evaluators/Metrics/RegressionMetrics.cs index c992f1615c..5bc01a1c04 100644 --- a/src/Microsoft.ML.Data/Evaluators/Metrics/RegressionMetrics.cs +++ b/src/Microsoft.ML.Data/Evaluators/Metrics/RegressionMetrics.cs @@ -15,11 +15,16 @@ public sealed class RegressionMetrics /// Gets the absolute loss of the model. /// /// + /// + /// /// public double MeanAbsoluteError { get; } @@ -27,16 +32,21 @@ public sealed class RegressionMetrics /// Gets the squared loss of the model. /// /// + /// + /// /// public double MeanSquaredError { get; } /// - /// Gets the root mean square loss (or RMS) which is the square root of the L2 loss. + /// Gets the root mean square loss (or RMS) which is the square root of the L2 loss . /// public double RootMeanSquaredError { get; } @@ -50,8 +60,9 @@ public sealed class RegressionMetrics public double LossFunction { get; } /// - /// Gets the R squared value of the model, which is also known as - /// the coefficient of determination​. + /// Gets the R-squared value of the model, which is also known as + /// the coefficient of determination​. + /// R-Squared closer to 1 indicates a better fitted model. /// public double RSquared { get; } diff --git a/src/Microsoft.ML.Data/TrainCatalog.cs b/src/Microsoft.ML.Data/TrainCatalog.cs index f89c8d6f51..dd9ad3ac11 100644 --- a/src/Microsoft.ML.Data/TrainCatalog.cs +++ b/src/Microsoft.ML.Data/TrainCatalog.cs @@ -213,7 +213,8 @@ public BinaryClassificationMetrics EvaluateNonCalibrated(IDataView data, string /// /// Run cross-validation over folds of , by fitting , /// and respecting if provided. - /// Then evaluate each sub-model against and return metrics. + /// Then evaluate each sub-model against and return a object, which + /// do not include probability-based metrics, for each sub-model. Each sub-model is evaluated on the cross-validation fold that it did not see during training. /// /// The data to run cross-validation on. /// The estimator to fit. @@ -237,7 +238,8 @@ public IReadOnlyList> CrossVa /// /// Run cross-validation over folds of , by fitting , /// and respecting if provided. - /// Then evaluate each sub-model against and return metrics. + /// Then evaluate each sub-model against and return a object, which + /// includes probability-based metrics, for each sub-model. Each sub-model is evaluated on the cross-validation fold that it did not see during training. /// /// The data to run cross-validation on. /// The estimator to fit. diff --git a/src/Microsoft.ML.Data/Transforms/RowShufflingTransformer.cs b/src/Microsoft.ML.Data/Transforms/RowShufflingTransformer.cs index 72984958ce..afa16e320a 100644 --- a/src/Microsoft.ML.Data/Transforms/RowShufflingTransformer.cs +++ b/src/Microsoft.ML.Data/Transforms/RowShufflingTransformer.cs @@ -229,7 +229,7 @@ public static DataViewRowCursor GetShuffledCursor(IChannelProvider provider, int provider.CheckValue(cursor, nameof(cursor)); // REVIEW: In principle, we could limit this check to only active columns, // if we extend the use of this utility. - provider.CheckParam(CanShuffleAll(cursor.Schema), nameof(cursor), "Cannot shuffle a cursor with some uncachable columns"); + provider.CheckParam(CanShuffleAll(cursor.Schema), nameof(cursor), "Cannot shuffle a cursor with some uncacheable columns"); provider.CheckValue(rand, nameof(rand)); if (poolRows == 1) diff --git a/src/Microsoft.ML.DataView/IDataView.cs b/src/Microsoft.ML.DataView/IDataView.cs index d4a93c5555..49bf90164e 100644 --- a/src/Microsoft.ML.DataView/IDataView.cs +++ b/src/Microsoft.ML.DataView/IDataView.cs @@ -39,11 +39,12 @@ public interface IDataView long? GetRowCount(); /// - /// Get a row cursor. The active column indices are those for which needCol(col) returns true. - /// The schema of the returned cursor will be the same as the schema of the IDataView, but getting - /// a getter for inactive columns will throw. The indicate the columns that are needed - /// to iterate over.If set to an empty no column is requested. + /// Get a row cursor. The indicate the active columns that are needed + /// to iterate over. If set to an empty no column is requested. The schema of the returned + /// cursor will be the same as the schema of the IDataView, but getting a getter for inactive columns will throw. /// + /// The active columns needed. If passed an empty no column is requested. + /// An instance of to seed randomizing the access for a shuffled cursor. DataViewRowCursor GetRowCursor(IEnumerable columnsNeeded, Random rand = null); /// @@ -103,7 +104,7 @@ public abstract class DataViewRow : IDisposable /// This provides a means for reconciling multiple rows that have been produced generally from /// . When getting a set, there is a need /// to, while allowing parallel processing to proceed, always have an aim that the original order should be - /// recoverable. Note, whether or not a user cares about that original order in ones specific application is + /// recoverable. Note, whether or not a user cares about that original order in one's specific application is /// another story altogether (most callers of this as a practical matter do not, otherwise they would not call /// it), but at least in principle it should be possible to reconstruct the original order one would get from an /// identically configured . So: for any cursor