Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
@@ -0,0 +1,77 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Text;
using Microsoft.ML;
using Microsoft.ML.Data;

namespace Samples.Dynamic.DataOperations
{
public static class LoadingText
{
// This examples shows all the ways to load data with TextLoader.
public static void Example()
{
// Create 5 data files to illustrate different loading methods.
var dataFiles = new List<string>();
var random = new Random();
var dataDirectoryName = "DataDir";
Directory.CreateDirectory(dataDirectoryName);
for (int i = 0; i < 5; i++)
{
var fileName = Path.Combine(dataDirectoryName, $"Data_{i}.csv");
dataFiles.Add(fileName);
using (var fs = File.CreateText(fileName))
// Write random lines without header

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Could we have another example with header?

Copy link
Copy Markdown
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

this is an example for Load() method and all it's variants. header applies to creating the loader itself. I'll add that separately to CreateTextLoader() API.


In reply to: 289105567 [](ancestors = 289105567)

for (int line = 0; line < 10; line++)
fs.WriteLine(random.NextDouble().ToString());
}

// Create a TextLoader.
var mlContext = new MLContext();
var loader = mlContext.Data.CreateTextLoader(
columns: new[]
{
new TextLoader.Column("RandomFeature", DataKind.Single, 0)
},
hasHeader: false
);

// Load a single file from path.
var singleFileData = loader.Load(dataFiles[0]);
PrintRowCount(singleFileData);

// Expected Output:
// 10


// Load all 5 files from path.
var multipleFilesData = loader.Load(dataFiles.ToArray());
PrintRowCount(multipleFilesData);

// Expected Output:
// 50


// Load all files using path wildcard.
var multipleFilesWildcardData =
loader.Load(Path.Combine(dataDirectoryName, "*"));
PrintRowCount(multipleFilesWildcardData);

// Expected Output:
// 50
}

private static void PrintRowCount(IDataView idv)
{
// IDataView is lazy so we need to iterate through it
// to get the number of rows.

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Could you also print out the first and the last lines in the loaded file?

Copy link
Copy Markdown
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The information I'm trying to convey here is how many files were loaded using each of the Load() variants (1 or 5 files). Count of rows is best for that. I don't think the user can know what's the first line and what's the last one, since the rows are randomly generated. That info won't be super relevant.


In reply to: 289104819 [](ancestors = 289104819)

long rowCount = 0;
using (var cursor = idv.GetRowCursor(idv.Schema))
while (cursor.MoveNext())
rowCount++;

Console.WriteLine(rowCount);
}
}
}
7 changes: 7 additions & 0 deletions src/Microsoft.ML.Data/DataLoadSave/DataLoaderExtensions.cs
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,13 @@ public static class DataLoaderExtensions
/// </summary>
/// <param name="loader">The loader to use.</param>
/// <param name="path">One or more paths from which to load data.</param>
/// <example>
/// <format type="text/markdown">
/// <![CDATA[
/// [!code-csharp[Load](~/../docs/samples/docs/samples/Microsoft.ML.Samples/Dynamic/DataOperations/LoadingText.cs)]
/// ]]>
/// </format>
/// </example>
public static IDataView Load(this IDataLoader<IMultiStreamSource> loader, params string[] path)
=> loader.Load(new MultiFileSource(path));
}
Expand Down
7 changes: 7 additions & 0 deletions src/Microsoft.ML.Data/DataLoadSave/Text/TextLoader.cs
Original file line number Diff line number Diff line change
Expand Up @@ -1432,6 +1432,13 @@ void ICanSaveModel.Save(ModelSaveContext ctx)
/// Loads data from <paramref name="source"/> into an <see cref="IDataView"/>.
/// </summary>
/// <param name="source">The source from which to load data.</param>
/// <example>
/// <format type="text/markdown">
/// <![CDATA[
/// [!code-csharp[Load](~/../docs/samples/docs/samples/Microsoft.ML.Samples/Dynamic/DataOperations/LoadingText.cs)]
/// ]]>
/// </format>
/// </example>
public IDataView Load(IMultiStreamSource source) => new BoundLoader(this, source);

internal static TextLoader CreateTextLoader<TInput>(IHostEnvironment host,
Expand Down