-
Notifications
You must be signed in to change notification settings - Fork 1.9k
Added samples for loading text #3793
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Changes from all commits
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,77 @@ | ||
| using System; | ||
| using System.Collections.Generic; | ||
| using System.IO; | ||
| using System.Text; | ||
| using Microsoft.ML; | ||
| using Microsoft.ML.Data; | ||
|
|
||
| namespace Samples.Dynamic.DataOperations | ||
| { | ||
| public static class LoadingText | ||
| { | ||
| // This examples shows all the ways to load data with TextLoader. | ||
| public static void Example() | ||
| { | ||
| // Create 5 data files to illustrate different loading methods. | ||
| var dataFiles = new List<string>(); | ||
| var random = new Random(); | ||
| var dataDirectoryName = "DataDir"; | ||
| Directory.CreateDirectory(dataDirectoryName); | ||
| for (int i = 0; i < 5; i++) | ||
| { | ||
| var fileName = Path.Combine(dataDirectoryName, $"Data_{i}.csv"); | ||
| dataFiles.Add(fileName); | ||
| using (var fs = File.CreateText(fileName)) | ||
| // Write random lines without header | ||
| for (int line = 0; line < 10; line++) | ||
| fs.WriteLine(random.NextDouble().ToString()); | ||
| } | ||
|
|
||
| // Create a TextLoader. | ||
| var mlContext = new MLContext(); | ||
| var loader = mlContext.Data.CreateTextLoader( | ||
| columns: new[] | ||
| { | ||
| new TextLoader.Column("RandomFeature", DataKind.Single, 0) | ||
| }, | ||
| hasHeader: false | ||
| ); | ||
|
|
||
| // Load a single file from path. | ||
| var singleFileData = loader.Load(dataFiles[0]); | ||
| PrintRowCount(singleFileData); | ||
|
|
||
| // Expected Output: | ||
| // 10 | ||
|
|
||
|
|
||
| // Load all 5 files from path. | ||
| var multipleFilesData = loader.Load(dataFiles.ToArray()); | ||
| PrintRowCount(multipleFilesData); | ||
|
|
||
| // Expected Output: | ||
| // 50 | ||
|
|
||
|
|
||
| // Load all files using path wildcard. | ||
| var multipleFilesWildcardData = | ||
| loader.Load(Path.Combine(dataDirectoryName, "*")); | ||
| PrintRowCount(multipleFilesWildcardData); | ||
|
|
||
| // Expected Output: | ||
| // 50 | ||
| } | ||
|
|
||
| private static void PrintRowCount(IDataView idv) | ||
| { | ||
| // IDataView is lazy so we need to iterate through it | ||
| // to get the number of rows. | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Could you also print out the first and the last lines in the loaded file?
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. The information I'm trying to convey here is how many files were loaded using each of the Load() variants (1 or 5 files). Count of rows is best for that. I don't think the user can know what's the first line and what's the last one, since the rows are randomly generated. That info won't be super relevant. In reply to: 289104819 [](ancestors = 289104819) |
||
| long rowCount = 0; | ||
| using (var cursor = idv.GetRowCursor(idv.Schema)) | ||
| while (cursor.MoveNext()) | ||
| rowCount++; | ||
|
|
||
| Console.WriteLine(rowCount); | ||
| } | ||
| } | ||
| } | ||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Could we have another example with header?
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
this is an example for Load() method and all it's variants. header applies to creating the loader itself. I'll add that separately to CreateTextLoader() API.
In reply to: 289105567 [](ancestors = 289105567)