From f47dc18f42f82d44d60966794969393a626e4796 Mon Sep 17 00:00:00 2001 From: Senja Filipi Date: Thu, 9 May 2019 14:42:59 -0700 Subject: [PATCH 1/4] Sample of using LoadFromEnumerable with a SchemaDefinition --- .../DataOperations/LoadFromEnumerable.cs | 83 +++++++++++++++++++ .../DataLoadSave/DataOperationsCatalog.cs | 2 +- 2 files changed, 84 insertions(+), 1 deletion(-) create mode 100644 docs/samples/Microsoft.ML.Samples/Dynamic/DataOperations/LoadFromEnumerable.cs diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/DataOperations/LoadFromEnumerable.cs b/docs/samples/Microsoft.ML.Samples/Dynamic/DataOperations/LoadFromEnumerable.cs new file mode 100644 index 0000000000..4c57be6c0d --- /dev/null +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/DataOperations/LoadFromEnumerable.cs @@ -0,0 +1,83 @@ +using System; +using System.Collections.Generic; +using Microsoft.ML; +using Microsoft.ML.Data; + +namespace Samples.Dynamic +{ + public static class LoadFromEnumerable + { + // A simple case of creating IDataView from IEnumerable. + public static void Example() + { + // Create a new context for ML.NET operations. It can be used for exception tracking and logging, + // as a catalog of available operations and as the source of randomness. + var mlContext = new MLContext(); + + // Get a small dataset as an IEnumerable. + IEnumerable enumerableKnownSize = new DataPointVector[] + { + new DataPointVector{ Features = new float[]{ 1.2f, 3.4f, 4.5f, 3.2f, 7,5f } }, + new DataPointVector{ Features = new float[]{ 4.2f, 3.4f, 14.65f, 3.2f, 3,5f } }, + new DataPointVector{ Features = new float[]{ 1.6f, 3.5f, 4.5f, 6.2f, 3,5f } }, + }; + + // Load dataset into an IDataView. + IDataView data = mlContext.Data.LoadFromEnumerable(enumerableKnownSize); + var featureColumn = data.Schema["Features"].Type as VectorDataViewType; + // Inspecting the schema + Console.WriteLine($"Is the size of the Features column known: {featureColumn.IsKnownSize}.\nSize: {featureColumn.Size}"); + + // Preview + // + // Is the size of the Features column known? True. + // Size: 5. + + // If the size of the vector is unknown at compile time, it can be set at runtime. + IEnumerable enumerableUnknownSize = new DataPoint[] + { + new DataPoint{ Features = new float[]{ 1.2f, 3.4f, 4.5f } }, + new DataPoint{ Features = new float[]{ 4.2f, 3.4f, 1.6f } }, + new DataPoint{ Features = new float[]{ 1.6f, 3.5f, 4.5f } }, + }; + + // The feature dimension retrievable at runtime. + int featureDimension = 3; + var definedSchema = SchemaDefinition.Create(typeof(DataPoint)); + featureColumn = definedSchema["Features"].ColumnType as VectorDataViewType; + Console.WriteLine($"Is the size of the Features column known: {featureColumn.IsKnownSize}.\nSize: {featureColumn.Size}"); + + // Preview + // + // Is the size of the Features column known? False. + //Size: 0. + + // Set the column type to be a known-size vector. + var vectorItemType = ((VectorDataViewType)definedSchema[0].ColumnType).ItemType; + definedSchema[0].ColumnType = new VectorDataViewType(vectorItemType, featureDimension); + + // Read the data into an IDataView with the schema supplied in + IDataView data2 = mlContext.Data.LoadFromEnumerable(enumerableUnknownSize, definedSchema); + + featureColumn = data2.Schema["Features"].Type as VectorDataViewType; + // Inspecting the schema + Console.WriteLine($"Is the size of the Features column known: {featureColumn.IsKnownSize}.\nSize: {featureColumn.Size}"); + + // Preview + // + // Is the size of the Features column known? True. + // Size: 3. + } + } + + public class DataPoint + { + public float[] Features { get; set; } + } + + public class DataPointVector + { + [VectorType(5)] + public float[] Features { get; set; } + } +} diff --git a/src/Microsoft.ML.Data/DataLoadSave/DataOperationsCatalog.cs b/src/Microsoft.ML.Data/DataLoadSave/DataOperationsCatalog.cs index 36e6593675..0c45082330 100644 --- a/src/Microsoft.ML.Data/DataLoadSave/DataOperationsCatalog.cs +++ b/src/Microsoft.ML.Data/DataLoadSave/DataOperationsCatalog.cs @@ -70,7 +70,7 @@ internal DataOperationsCatalog(IHostEnvironment env) /// /// /// /// /// From ae7a8ca0ea70f261c21fb6531e82a29493a751a8 Mon Sep 17 00:00:00 2001 From: Senja Filipi Date: Fri, 10 May 2019 08:36:31 -0700 Subject: [PATCH 2/4] Adding the header --- .../Dynamic/DataOperations/LoadFromEnumerable.cs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/DataOperations/LoadFromEnumerable.cs b/docs/samples/Microsoft.ML.Samples/Dynamic/DataOperations/LoadFromEnumerable.cs index 4c57be6c0d..8172e5a48f 100644 --- a/docs/samples/Microsoft.ML.Samples/Dynamic/DataOperations/LoadFromEnumerable.cs +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/DataOperations/LoadFromEnumerable.cs @@ -1,4 +1,8 @@ -using System; +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; using System.Collections.Generic; using Microsoft.ML; using Microsoft.ML.Data; From eb3b9664637d19325507cd69d2f2277565c26c81 Mon Sep 17 00:00:00 2001 From: Senja Filipi Date: Wed, 15 May 2019 23:10:48 -0700 Subject: [PATCH 3/4] addressing PR comments --- .../DataOperations/LoadFromEnumerable.cs | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/DataOperations/LoadFromEnumerable.cs b/docs/samples/Microsoft.ML.Samples/Dynamic/DataOperations/LoadFromEnumerable.cs index 8172e5a48f..e982d3ede4 100644 --- a/docs/samples/Microsoft.ML.Samples/Dynamic/DataOperations/LoadFromEnumerable.cs +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/DataOperations/LoadFromEnumerable.cs @@ -1,8 +1,4 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. -// See the LICENSE file in the project root for more information. - -using System; +using System; using System.Collections.Generic; using Microsoft.ML; using Microsoft.ML.Data; @@ -11,7 +7,11 @@ namespace Samples.Dynamic { public static class LoadFromEnumerable { - // A simple case of creating IDataView from IEnumerable. + // Creating IDataView from IEnumerable, and setting the size of the vector at runtime. + // When the data model is defined through types, setting the size of the vector is done through the VectorType + // annotation. When the size of the data is not known at compile time, the Schema can be directly modified at runtime + // and the size of the vector set there. + // This is important, because most of the ML.NET trainers require the Features vector to be of known size. public static void Example() { // Create a new context for ML.NET operations. It can be used for exception tracking and logging, @@ -45,7 +45,8 @@ public static void Example() new DataPoint{ Features = new float[]{ 1.6f, 3.5f, 4.5f } }, }; - // The feature dimension retrievable at runtime. + // The feature dimension (typically this will be the Count of the array of the features vector + // known at runtime). int featureDimension = 3; var definedSchema = SchemaDefinition.Create(typeof(DataPoint)); featureColumn = definedSchema["Features"].ColumnType as VectorDataViewType; @@ -54,7 +55,7 @@ public static void Example() // Preview // // Is the size of the Features column known? False. - //Size: 0. + // Size: 0. // Set the column type to be a known-size vector. var vectorItemType = ((VectorDataViewType)definedSchema[0].ColumnType).ItemType; From 7eae29c6d58570ad01d84dd0994eafa06d347afb Mon Sep 17 00:00:00 2001 From: Senja Filipi Date: Fri, 17 May 2019 15:31:35 -0700 Subject: [PATCH 4/4] PR comment --- .../Dynamic/DataOperations/LoadFromEnumerable.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/samples/Microsoft.ML.Samples/Dynamic/DataOperations/LoadFromEnumerable.cs b/docs/samples/Microsoft.ML.Samples/Dynamic/DataOperations/LoadFromEnumerable.cs index e982d3ede4..64bc929850 100644 --- a/docs/samples/Microsoft.ML.Samples/Dynamic/DataOperations/LoadFromEnumerable.cs +++ b/docs/samples/Microsoft.ML.Samples/Dynamic/DataOperations/LoadFromEnumerable.cs @@ -61,7 +61,7 @@ public static void Example() var vectorItemType = ((VectorDataViewType)definedSchema[0].ColumnType).ItemType; definedSchema[0].ColumnType = new VectorDataViewType(vectorItemType, featureDimension); - // Read the data into an IDataView with the schema supplied in + // Read the data into an IDataView with the modified schema supplied in IDataView data2 = mlContext.Data.LoadFromEnumerable(enumerableUnknownSize, definedSchema); featureColumn = data2.Schema["Features"].Type as VectorDataViewType;