Skip to content

reimplement binary experiment using AutoMLExperiment #6246

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
LittleLittleCloud merged 7 commits into from
Jul 18, 2022
Merged

reimplement binary experiment using AutoMLExperiment #6246

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
b668bda
reimplement binary experiment using AutoML experiment
LittleLittleCloud Jul 6, 2022
357f09d
Merge branch 'main' into u/xiaoyun/reimplement-binary
LittleLittleCloud Jul 6, 2022
7964bbd
fix build error
LittleLittleCloud Jul 6, 2022
dfc76a9
fix tests
LittleLittleCloud Jul 11, 2022
8446e6e
force canceling when there's complete trial
LittleLittleCloud Jul 13, 2022
c10800f
Update AutoFitTests.cs
LittleLittleCloud Jul 15, 2022
ae4148c
Update AutoFitTests.cs
LittleLittleCloud Jul 15, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion src/Microsoft.ML.AutoML.Interactive/NotebookMonitor.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -55,7 +55,7 @@ public void ReportCompletedTrial(TrialResult result)
ThrottledUpdate();
}

public void ReportFailTrial(TrialResult result)
public void ReportFailTrial(TrialSettings setting, Exception exp = null)
{
// TODO figure out what to do with failed trials.
ThrottledUpdate();
Expand Down
232 changes: 232 additions & 0 deletions src/Microsoft.ML.AutoML/API/BinaryClassificationExperiment.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -147,6 +147,9 @@ public enum BinaryClassificationTrainer
/// </example>
public sealed class BinaryClassificationExperiment : ExperimentBase<BinaryClassificationMetrics, BinaryExperimentSettings>
{
private readonly AutoMLExperiment _experiment;
private const string Features = "__Features__";

@LittleLittleCloud LittleLittleCloud Jul 6, 2022

Copy link
Copy Markdown
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Output column name for Concatenating all features


internal BinaryClassificationExperiment(MLContext context, BinaryExperimentSettings settings)
: base(context,
new BinaryMetricsAgent(context, settings.OptimizingMetric),
Expand All @@ -155,6 +158,161 @@ internal BinaryClassificationExperiment(MLContext context, BinaryExperimentSetti
TaskKind.BinaryClassification,
TrainerExtensionUtil.GetTrainerNames(settings.Trainers))
{
_experiment = context.Auto().CreateExperiment();
}

public override ExperimentResult<BinaryClassificationMetrics> Execute(IDataView trainData, ColumnInformation columnInformation, IEstimator<ITransformer> preFeaturizer = null, IProgress<RunDetail<BinaryClassificationMetrics>> progressHandler = null)
{
var label = columnInformation.LabelColumnName;
_experiment.SetEvaluateMetric(Settings.OptimizingMetric, label);
_experiment.SetTrainingTimeInSeconds(Settings.MaxExperimentTimeInSeconds);

// Cross val threshold for # of dataset rows --
// If dataset has < threshold # of rows, use cross val.
// Else, run experiment using train-validate split.
const int crossValRowCountThreshold = 15000;
var rowCount = DatasetDimensionsUtil.CountRows(trainData, crossValRowCountThreshold);
// TODO
// split cross validation result according to sample key as well.
if (rowCount < crossValRowCountThreshold)
{
const int numCrossValFolds = 10;
_experiment.SetDataset(trainData, numCrossValFolds);
}
else
{
var splitData = Context.Data.TrainTestSplit(trainData);
_experiment.SetDataset(splitData.TrainSet, splitData.TestSet);
}

MultiModelPipeline pipeline = new MultiModelPipeline();
if (preFeaturizer != null)
{
pipeline = pipeline.Append(preFeaturizer);
}

pipeline = pipeline.Append(Context.Auto().Featurizer(trainData, columnInformation, Features))
Comment thread
michaelgsharp marked this conversation as resolved.
.Append(Context.Auto().BinaryClassification(label, Features));
_experiment.SetPipeline(pipeline);

var monitor = new BinaryClassificationTrialResultMonitor();
monitor.OnTrialCompleted += (o, e) =>
{
var detail = ToRunDetail(e);
progressHandler?.Report(detail);
};

_experiment.SetMonitor(monitor);
_experiment.Run();

var runDetails = monitor.RunDetails.Select(e => ToRunDetail(e));
var bestRun = ToRunDetail(monitor.BestRun);
var result = new ExperimentResult<BinaryClassificationMetrics>(runDetails, bestRun);

return result;
}

public override ExperimentResult<BinaryClassificationMetrics> Execute(IDataView trainData, IDataView validationData, ColumnInformation columnInformation, IEstimator<ITransformer> preFeaturizer = null, IProgress<RunDetail<BinaryClassificationMetrics>> progressHandler = null)
Comment thread
michaelgsharp marked this conversation as resolved.
{
var label = columnInformation.LabelColumnName;
_experiment.SetEvaluateMetric(Settings.OptimizingMetric, label);
_experiment.SetTrainingTimeInSeconds(Settings.MaxExperimentTimeInSeconds);
_experiment.SetDataset(trainData, validationData);

MultiModelPipeline pipeline = new MultiModelPipeline();
if (preFeaturizer != null)
{
pipeline = pipeline.Append(preFeaturizer);
}

pipeline = pipeline.Append(Context.Auto().Featurizer(trainData, columnInformation, "__Features__"))
Comment thread
michaelgsharp marked this conversation as resolved.
.Append(Context.Auto().BinaryClassification(label, featureColumnName: Features));

_experiment.SetPipeline(pipeline);
var monitor = new BinaryClassificationTrialResultMonitor();
monitor.OnTrialCompleted += (o, e) =>
{
var detail = ToRunDetail(e);
progressHandler?.Report(detail);
};

_experiment.SetMonitor(monitor);
_experiment.Run();

var runDetails = monitor.RunDetails.Select(e => ToRunDetail(e));
var bestRun = ToRunDetail(monitor.BestRun);
var result = new ExperimentResult<BinaryClassificationMetrics>(runDetails, bestRun);

return result;
}

public override ExperimentResult<BinaryClassificationMetrics> Execute(IDataView trainData, IDataView validationData, string labelColumnName = "Label", IEstimator<ITransformer> preFeaturizer = null, IProgress<RunDetail<BinaryClassificationMetrics>> progressHandler = null)
{
var columnInformation = new ColumnInformation()
{
LabelColumnName = labelColumnName,
};

return this.Execute(trainData, validationData, columnInformation, preFeaturizer, progressHandler);
}

public override ExperimentResult<BinaryClassificationMetrics> Execute(IDataView trainData, string labelColumnName = "Label", string samplingKeyColumn = null, IEstimator<ITransformer> preFeaturizer = null, IProgress<RunDetail<BinaryClassificationMetrics>> progressHandler = null)
{
var columnInformation = new ColumnInformation()
{
LabelColumnName = labelColumnName,
SamplingKeyColumnName = samplingKeyColumn,
};

return this.Execute(trainData, columnInformation, preFeaturizer, progressHandler);
}

public override CrossValidationExperimentResult<BinaryClassificationMetrics> Execute(IDataView trainData, uint numberOfCVFolds, ColumnInformation columnInformation = null, IEstimator<ITransformer> preFeaturizer = null, IProgress<CrossValidationRunDetail<BinaryClassificationMetrics>> progressHandler = null)
{
var label = columnInformation.LabelColumnName;
_experiment.SetEvaluateMetric(Settings.OptimizingMetric, label);
_experiment.SetTrainingTimeInSeconds(Settings.MaxExperimentTimeInSeconds);
_experiment.SetDataset(trainData, (int)numberOfCVFolds);

MultiModelPipeline pipeline = new MultiModelPipeline();
if (preFeaturizer != null)
{
pipeline = pipeline.Append(preFeaturizer);
}

pipeline = pipeline.Append(Context.Auto().Featurizer(trainData, columnInformation, "__Features__"))
.Append(Context.Auto().BinaryClassification(label, featureColumnName: Features));

_experiment.SetPipeline(pipeline);

var monitor = new BinaryClassificationTrialResultMonitor();
monitor.OnTrialCompleted += (o, e) =>
{
var runDetails = ToCrossValidationRunDetail(e);

progressHandler?.Report(runDetails);
};

_experiment.SetMonitor(monitor);
_experiment.Run();

var runDetails = monitor.RunDetails.Select(e => ToCrossValidationRunDetail(e));
var bestResult = ToCrossValidationRunDetail(monitor.BestRun);

var result = new CrossValidationExperimentResult<BinaryClassificationMetrics>(runDetails, bestResult);

return result;
}

public override CrossValidationExperimentResult<BinaryClassificationMetrics> Execute(IDataView trainData, uint numberOfCVFolds, string labelColumnName = "Label", string samplingKeyColumn = null, IEstimator<ITransformer> preFeaturizer = null, IProgress<CrossValidationRunDetail<BinaryClassificationMetrics>> progressHandler = null)
{
var columnInformation = new ColumnInformation()
{
LabelColumnName = labelColumnName,
SamplingKeyColumnName = samplingKeyColumn,
};

return this.Execute(trainData, numberOfCVFolds, columnInformation, preFeaturizer, progressHandler);
}

private protected override RunDetail<BinaryClassificationMetrics> GetBestRun(IEnumerable<RunDetail<BinaryClassificationMetrics>> results)
Expand All @@ -166,5 +324,79 @@ private protected override CrossValidationRunDetail<BinaryClassificationMetrics>
{
return BestResultUtil.GetBestRun(results, MetricsAgent, OptimizingMetricInfo.IsMaximizing);
}

private RunDetail<BinaryClassificationMetrics> ToRunDetail(BinaryClassificationTrialResult result)
{
var pipeline = result.TrialSettings.Pipeline;
var trainerName = pipeline.ToString();
var parameter = result.TrialSettings.Parameter;
var estimator = pipeline.BuildTrainingPipeline(Context, parameter);
var modelContainer = new ModelContainer(Context, result.Model);
return new RunDetail<BinaryClassificationMetrics>(trainerName, estimator, null, modelContainer, result.BinaryClassificationMetrics, result.Exception);
}

private CrossValidationRunDetail<BinaryClassificationMetrics> ToCrossValidationRunDetail(BinaryClassificationTrialResult result)
{
var pipeline = result.TrialSettings.Pipeline;
var trainerName = pipeline.ToString();
var parameter = result.TrialSettings.Parameter;
var estimator = pipeline.BuildTrainingPipeline(Context, parameter);
var crossValidationResult = result.CrossValidationMetrics.Select(m => new TrainResult<BinaryClassificationMetrics>(new ModelContainer(Context, m.Model), m.Metrics, result.Exception));
return new CrossValidationRunDetail<BinaryClassificationMetrics>(trainerName, estimator, null, crossValidationResult);
}
}

internal class BinaryClassificationTrialResultMonitor : IMonitor
{
public BinaryClassificationTrialResultMonitor()
{
this.RunDetails = new List<BinaryClassificationTrialResult>();
}

public event EventHandler<BinaryClassificationTrialResult> OnTrialCompleted;

public List<BinaryClassificationTrialResult> RunDetails { get; }

public BinaryClassificationTrialResult BestRun { get; private set; }

public void ReportBestTrial(TrialResult result)
{
if (result is BinaryClassificationTrialResult binaryClassificationResult)
{
BestRun = binaryClassificationResult;
}
else
{
throw new ArgumentException($"result must be of type {typeof(BinaryClassificationTrialResult)}");
}
}

public void ReportCompletedTrial(TrialResult result)
{
if (result is BinaryClassificationTrialResult binaryClassificationResult)
{
RunDetails.Add(binaryClassificationResult);
OnTrialCompleted?.Invoke(this, binaryClassificationResult);
}
else
{
throw new ArgumentException($"result must be of type {typeof(BinaryClassificationTrialResult)}");
}
}

public void ReportFailTrial(TrialSettings settings, Exception exp)
{
var result = new BinaryClassificationTrialResult
{
TrialSettings = settings,
Exception = exp,
};

RunDetails.Add(result);
}

public void ReportRunningTrial(TrialSettings setting)
Comment thread
michaelgsharp marked this conversation as resolved.
{
}
}
}
12 changes: 6 additions & 6 deletions src/Microsoft.ML.AutoML/API/ExperimentBase.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -64,7 +64,7 @@ internal ExperimentBase(MLContext context,
/// <remarks>
/// Depending on the size of your data, the AutoML experiment could take a long time to execute.
/// </remarks>
public ExperimentResult<TMetrics> Execute(IDataView trainData, string labelColumnName = DefaultColumnNames.Label,
public virtual ExperimentResult<TMetrics> Execute(IDataView trainData, string labelColumnName = DefaultColumnNames.Label,

@LittleLittleCloud LittleLittleCloud Jul 6, 2022

Copy link
Copy Markdown
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

add virtual keyword so it can be overrided

string samplingKeyColumn = null, IEstimator<ITransformer> preFeaturizer = null, IProgress<RunDetail<TMetrics>> progressHandler = null)
{
ColumnInformation columnInformation;
Expand Down Expand Up @@ -106,7 +106,7 @@ public ExperimentResult<TMetrics> Execute(IDataView trainData, string labelColum
/// <remarks>
/// Depending on the size of your data, the AutoML experiment could take a long time to execute.
/// </remarks>
public ExperimentResult<TMetrics> Execute(IDataView trainData, ColumnInformation columnInformation,
public virtual ExperimentResult<TMetrics> Execute(IDataView trainData, ColumnInformation columnInformation,
IEstimator<ITransformer> preFeaturizer = null, IProgress<RunDetail<TMetrics>> progressHandler = null)
{
// Cross val threshold for # of dataset rows --
Expand Down Expand Up @@ -156,7 +156,7 @@ private string GetSamplingKey(string groupIdColumnName, string samplingKeyColumn
/// <remarks>
/// Depending on the size of your data, the AutoML experiment could take a long time to execute.
/// </remarks>
public ExperimentResult<TMetrics> Execute(IDataView trainData, IDataView validationData, string labelColumnName = DefaultColumnNames.Label, IEstimator<ITransformer> preFeaturizer = null, IProgress<RunDetail<TMetrics>> progressHandler = null)
public virtual ExperimentResult<TMetrics> Execute(IDataView trainData, IDataView validationData, string labelColumnName = DefaultColumnNames.Label, IEstimator<ITransformer> preFeaturizer = null, IProgress<RunDetail<TMetrics>> progressHandler = null)
{
var columnInformation = (_task == TaskKind.Ranking) ?
new ColumnInformation() { LabelColumnName = labelColumnName, GroupIdColumnName = DefaultColumnNames.GroupId } :
Expand Down Expand Up @@ -184,7 +184,7 @@ public ExperimentResult<TMetrics> Execute(IDataView trainData, IDataView validat
/// <remarks>
/// Depending on the size of your data, the AutoML experiment could take a long time to execute.
/// </remarks>
public ExperimentResult<TMetrics> Execute(IDataView trainData, IDataView validationData,
public virtual ExperimentResult<TMetrics> Execute(IDataView trainData, IDataView validationData,
ColumnInformation columnInformation, IEstimator<ITransformer> preFeaturizer = null,
IProgress<RunDetail<TMetrics>> progressHandler = null)
{
Expand Down Expand Up @@ -214,7 +214,7 @@ public ExperimentResult<TMetrics> Execute(IDataView trainData, IDataView validat
/// <remarks>
/// Depending on the size of your data, the AutoML experiment could take a long time to execute.
/// </remarks>
public CrossValidationExperimentResult<TMetrics> Execute(IDataView trainData, uint numberOfCVFolds,
public virtual CrossValidationExperimentResult<TMetrics> Execute(IDataView trainData, uint numberOfCVFolds,
ColumnInformation columnInformation = null, IEstimator<ITransformer> preFeaturizer = null,
IProgress<CrossValidationRunDetail<TMetrics>> progressHandler = null)
{
Expand Down Expand Up @@ -244,7 +244,7 @@ public CrossValidationExperimentResult<TMetrics> Execute(IDataView trainData, ui
/// <remarks>
/// Depending on the size of your data, the AutoML experiment could take a long time to execute.
/// </remarks>
public CrossValidationExperimentResult<TMetrics> Execute(IDataView trainData,
public virtual CrossValidationExperimentResult<TMetrics> Execute(IDataView trainData,
uint numberOfCVFolds, string labelColumnName = DefaultColumnNames.Label,
string samplingKeyColumn = null, IEstimator<ITransformer> preFeaturizer = null,
IProgress<CrossValidationRunDetail<TMetrics>> progressHandler = null)
Expand Down
Loading