diff --git a/src/Microsoft.ML.AutoML/API/AutoCatalog.cs b/src/Microsoft.ML.AutoML/API/AutoCatalog.cs index ed8a2a9ef4..280777db70 100644 --- a/src/Microsoft.ML.AutoML/API/AutoCatalog.cs +++ b/src/Microsoft.ML.AutoML/API/AutoCatalog.cs @@ -654,18 +654,18 @@ internal SweepablePipeline ImagePathFeaturizer(string outputColumnName, string i /// them into a single feature column as output. /// /// input data. - /// columns that should be treated as catalog. If not specified, it will automatically infer if a column is catalog or not. + /// columns that should be treated as catalog. If not specified, it will automatically infer if a column is catalog or not. /// columns that should be treated as numeric. If not specified, it will automatically infer if a column is catalog or not. /// columns that should be treated as text. If not specified, it will automatically infer if a column is catalog or not. /// columns that should be treated as image path. If not specified, it will automatically infer if a column is catalog or not. /// output feature column. /// columns that won't be included when featurizing, like label - public SweepablePipeline Featurizer(IDataView data, string outputColumnName = "Features", string[] catalogColumns = null, string[] numericColumns = null, string[] textColumns = null, string[] imagePathColumns = null, string[] excludeColumns = null) + public SweepablePipeline Featurizer(IDataView data, string outputColumnName = "Features", string[] catelogicalColumns = null, string[] numericColumns = null, string[] textColumns = null, string[] imagePathColumns = null, string[] excludeColumns = null) { Contracts.CheckValue(data, nameof(data)); // validate if there's overlapping among catalogColumns, numericColumns, textColumns and excludeColumns - var overallColumns = new string[][] { catalogColumns, numericColumns, textColumns, excludeColumns } + var overallColumns = new string[][] { catelogicalColumns, numericColumns, textColumns, excludeColumns } .Where(c => c != null) .SelectMany(c => c); @@ -684,9 +684,9 @@ public SweepablePipeline Featurizer(IDataView data, string outputColumnName = "F } } - if (catalogColumns != null) + if (catelogicalColumns != null) { - foreach (var catalogColumn in catalogColumns) + foreach (var catalogColumn in catelogicalColumns) { columnInfo.CategoricalColumnNames.Add(catalogColumn); } diff --git a/src/Microsoft.ML.AutoML/API/AutoMLExperimentExtension.cs b/src/Microsoft.ML.AutoML/API/AutoMLExperimentExtension.cs index 65c08bcf91..14aedab4b3 100644 --- a/src/Microsoft.ML.AutoML/API/AutoMLExperimentExtension.cs +++ b/src/Microsoft.ML.AutoML/API/AutoMLExperimentExtension.cs @@ -6,6 +6,7 @@ using System.Collections.Generic; using System.Text; using Microsoft.Extensions.DependencyInjection; +using Microsoft.ML.Runtime; using static Microsoft.ML.DataOperationsCatalog; namespace Microsoft.ML.AutoML @@ -144,6 +145,18 @@ public static AutoMLExperiment SetPipeline(this AutoMLExperiment experiment, Swe return experiment; } + public static AutoMLExperiment SetPerformanceMonitor(this AutoMLExperiment experiment, int checkIntervalInMilliseconds = 1000) + { + experiment.SetPerformanceMonitor((service) => + { + var channel = service.GetService(); + + return new DefaultPerformanceMonitor(channel, checkIntervalInMilliseconds); + }); + + return experiment; + } + private static AutoMLExperiment SetEvaluateMetric(this AutoMLExperiment experiment, TEvaluateMetricManager metricManager) where TEvaluateMetricManager : class, IEvaluateMetricManager { diff --git a/src/Microsoft.ML.AutoML/API/BinaryClassificationExperiment.cs b/src/Microsoft.ML.AutoML/API/BinaryClassificationExperiment.cs index b9e41062a3..e26826966e 100644 --- a/src/Microsoft.ML.AutoML/API/BinaryClassificationExperiment.cs +++ b/src/Microsoft.ML.AutoML/API/BinaryClassificationExperiment.cs @@ -6,6 +6,8 @@ using System.Collections.Generic; using System.Diagnostics; using System.Linq; +using System.Threading; +using System.Threading.Tasks; using Microsoft.Extensions.DependencyInjection; using Microsoft.ML.Data; using Microsoft.ML.Runtime; @@ -144,6 +146,10 @@ internal BinaryClassificationExperiment(MLContext context, BinaryExperimentSetti TrainerExtensionUtil.GetTrainerNames(settings.Trainers)) { _experiment = context.Auto().CreateExperiment(); + if (settings.MaximumMemoryUsageInMegaByte is double d) + { + _experiment.SetMaximumMemoryUsageInMegaByte(d); + } } public override ExperimentResult Execute(IDataView trainData, ColumnInformation columnInformation, IEstimator preFeaturizer = null, IProgress> progressHandler = null) @@ -332,7 +338,7 @@ private SweepablePipeline CreateBinaryClassificationPipeline(IDataView trainData internal class BinaryClassificationRunner : ITrialRunner { - private readonly MLContext _context; + private MLContext _context; private readonly IDatasetManager _datasetManager; private readonly IMetricManager _metricManager; private readonly SweepablePipeline _pipeline; @@ -346,6 +352,12 @@ public BinaryClassificationRunner(MLContext context, IDatasetManager datasetMana _rnd = settings.Seed.HasValue ? new Random(settings.Seed.Value) : new Random(); } + public void Dispose() + { + _context.CancelExecution(); + _context = null; + } + public TrialResult Run(TrialSettings settings) { if (_metricManager is BinaryMetricManager metricManager) @@ -421,5 +433,27 @@ public TrialResult Run(TrialSettings settings) throw new ArgumentException($"The runner metric manager is of type {_metricManager.GetType()} which expected to be of type {typeof(ITrainTestDatasetManager)} or {typeof(ICrossValidateDatasetManager)}"); } + + public Task RunAsync(TrialSettings settings, CancellationToken ct) + { + try + { + using (var ctRegistration = ct.Register(() => + { + _context?.CancelExecution(); + })) + { + return Task.Run(() => Run(settings)); + } + } + catch (Exception ex) when (ct.IsCancellationRequested) + { + throw new OperationCanceledException(ex.Message, ex.InnerException); + } + catch (Exception) + { + throw; + } + } } } diff --git a/src/Microsoft.ML.AutoML/API/ExperimentSettings.cs b/src/Microsoft.ML.AutoML/API/ExperimentSettings.cs index 8b54f697c4..a505eb4895 100644 --- a/src/Microsoft.ML.AutoML/API/ExperimentSettings.cs +++ b/src/Microsoft.ML.AutoML/API/ExperimentSettings.cs @@ -2,6 +2,7 @@ // The .NET Foundation licenses this file to you under the MIT license. // See the LICENSE file in the project root for more information. +using System; using System.Threading; namespace Microsoft.ML.AutoML @@ -56,6 +57,8 @@ public abstract class ExperimentSettings /// The default value is . public CacheBeforeTrainer CacheBeforeTrainer { get; set; } + public double? MaximumMemoryUsageInMegaByte { get; set; } + internal int MaxModels; /// diff --git a/src/Microsoft.ML.AutoML/API/MulticlassClassificationExperiment.cs b/src/Microsoft.ML.AutoML/API/MulticlassClassificationExperiment.cs index 78c503673f..67ead38f0e 100644 --- a/src/Microsoft.ML.AutoML/API/MulticlassClassificationExperiment.cs +++ b/src/Microsoft.ML.AutoML/API/MulticlassClassificationExperiment.cs @@ -6,6 +6,8 @@ using System.Collections.Generic; using System.Diagnostics; using System.Linq; +using System.Threading; +using System.Threading.Tasks; using Microsoft.Extensions.DependencyInjection; using Microsoft.ML.Data; using Microsoft.ML.Runtime; @@ -135,6 +137,11 @@ internal MulticlassClassificationExperiment(MLContext context, MulticlassExperim TrainerExtensionUtil.GetTrainerNames(settings.Trainers)) { _experiment = context.Auto().CreateExperiment(); + + if (settings.MaximumMemoryUsageInMegaByte is double d) + { + _experiment.SetMaximumMemoryUsageInMegaByte(d); + } } public override ExperimentResult Execute(IDataView trainData, ColumnInformation columnInformation, IEstimator preFeaturizer = null, IProgress> progressHandler = null) @@ -189,7 +196,6 @@ public override ExperimentResult Execute(IDataV return result; } - public override ExperimentResult Execute(IDataView trainData, IDataView validationData, ColumnInformation columnInformation, IEstimator preFeaturizer = null, IProgress> progressHandler = null) { var label = columnInformation.LabelColumnName; @@ -333,7 +339,7 @@ private SweepablePipeline CreateMulticlassClassificationPipeline(IDataView train internal class MulticlassClassificationRunner : ITrialRunner { - private readonly MLContext _context; + private MLContext _context; private readonly IDatasetManager _datasetManager; private readonly IMetricManager _metricManager; private readonly SweepablePipeline _pipeline; @@ -424,5 +430,33 @@ public TrialResult Run(TrialSettings settings) throw new ArgumentException($"The runner metric manager is of type {_metricManager.GetType()} which expected to be of type {typeof(ITrainTestDatasetManager)} or {typeof(ICrossValidateDatasetManager)}"); } + + public Task RunAsync(TrialSettings settings, CancellationToken ct) + { + try + { + using (var ctRegistration = ct.Register(() => + { + _context?.CancelExecution(); + })) + { + return Task.Run(() => Run(settings)); + } + } + catch (Exception ex) when (ct.IsCancellationRequested) + { + throw new OperationCanceledException(ex.Message, ex.InnerException); + } + catch (Exception) + { + throw; + } + } + + public void Dispose() + { + _context.CancelExecution(); + _context = null; + } } } diff --git a/src/Microsoft.ML.AutoML/AutoMLExperiment/AutoMLExperiment.cs b/src/Microsoft.ML.AutoML/AutoMLExperiment/AutoMLExperiment.cs index e142579e14..51d2257d8e 100644 --- a/src/Microsoft.ML.AutoML/AutoMLExperiment/AutoMLExperiment.cs +++ b/src/Microsoft.ML.AutoML/AutoMLExperiment/AutoMLExperiment.cs @@ -3,6 +3,7 @@ // See the LICENSE file in the project root for more information. using System; +using System.Text.Json; using System.Threading; using System.Threading.Tasks; using Microsoft.Extensions.DependencyInjection; @@ -22,7 +23,7 @@ public class AutoMLExperiment private double _bestError = double.MaxValue; private TrialResult _bestTrialResult = null; private readonly IServiceCollection _serviceCollection; - private CancellationTokenSource _cts; + private CancellationTokenSource _globalCancellationTokenSource; public AutoMLExperiment(MLContext context, AutoMLExperimentSettings settings) { @@ -40,14 +41,16 @@ public AutoMLExperiment(MLContext context, AutoMLExperimentSettings settings) } _serviceCollection = new ServiceCollection(); + InitializeServiceCollection(); } private void InitializeServiceCollection() { _serviceCollection.TryAddTransient((provider) => { + var channel = provider.GetRequiredService(); var context = new MLContext(_settings.Seed); - _cts.Token.Register(() => + _globalCancellationTokenSource.Token.Register(() => { // only force-canceling running trials when there's completed trials. // otherwise, wait for the current running trial to be completed. @@ -55,18 +58,37 @@ private void InitializeServiceCollection() context.CancelExecution(); }); + context.Log += (o, e) => + { + // Relay logs that are generated by the current MLContext to the Experiment class's + // _logger. + switch (e.Kind) + { + case ChannelMessageKind.Trace: + channel.Trace(e.Message); + break; + case ChannelMessageKind.Info: + channel.Info(e.Message); + break; + case ChannelMessageKind.Warning: + channel.Warning(e.Message); + break; + case ChannelMessageKind.Error: + channel.Error(e.Message); + break; + default: + break; + } + }; + return context; }); + + this.SetPerformanceMonitor(2000); _serviceCollection.TryAddSingleton(_settings); _serviceCollection.TryAddSingleton(((IChannelProvider)_context).Start(nameof(AutoMLExperiment))); } - private void Initialize() - { - _cts = new CancellationTokenSource(); - InitializeServiceCollection(); - } - internal IServiceCollection ServiceCollection { get => _serviceCollection; } public AutoMLExperiment SetTrainingTimeInSeconds(uint trainingTimeInSeconds) @@ -81,6 +103,13 @@ public AutoMLExperiment SetIsMaximize(bool isMaximize) return this; } + public AutoMLExperiment SetMaximumMemoryUsageInMegaByte(double value = double.MaxValue) + { + Contracts.Assert(!double.IsNaN(value) && value > 0, "value can't be nan or non-positive"); + _settings.MaximumMemoryUsageInMegaByte = value; + return this; + } + public AutoMLExperiment AddSearchSpace(string key, SearchSpace.SearchSpace searchSpace) { _settings.SearchSpace[key] = searchSpace; @@ -167,6 +196,22 @@ public AutoMLExperiment SetTuner() return this; } + internal AutoMLExperiment SetPerformanceMonitor() + where TPerformanceMonitor : class, IPerformanceMonitor + { + _serviceCollection.AddTransient(); + + return this; + } + + internal AutoMLExperiment SetPerformanceMonitor(Func factory) + where TPerformanceMonitor : class, IPerformanceMonitor + { + _serviceCollection.AddTransient(factory); + + return this; + } + /// /// Run experiment and return the best trial result synchronizely. /// @@ -185,47 +230,88 @@ public TrialResult Run() public async Task RunAsync(CancellationToken ct = default) { ValidateSettings(); - Initialize(); + _globalCancellationTokenSource = new CancellationTokenSource(); _settings.CancellationToken = ct; - _cts.CancelAfter((int)_settings.MaxExperimentTimeInSeconds * 1000); - _settings.CancellationToken.Register(() => _cts.Cancel()); + // use TimeSpan to avoid overflow. + _globalCancellationTokenSource.CancelAfter(TimeSpan.FromSeconds(_settings.MaxExperimentTimeInSeconds)); + _settings.CancellationToken.Register(() => _globalCancellationTokenSource.Cancel()); var serviceProvider = _serviceCollection.BuildServiceProvider(); var monitor = serviceProvider.GetService(); + var logger = serviceProvider.GetRequiredService(); var trialNum = 0; var tuner = serviceProvider.GetService(); Contracts.Assert(tuner != null, "tuner can't be null"); - while (true) + + while (!_globalCancellationTokenSource.Token.IsCancellationRequested) { - if (_cts.Token.IsCancellationRequested) - { - break; - } var setting = new TrialSettings() { TrialId = trialNum++, Parameter = Parameter.CreateNestedParameter(), }; - var parameter = tuner.Propose(setting); setting.Parameter = parameter; - monitor?.ReportRunningTrial(setting); - var runner = serviceProvider.GetRequiredService(); + monitor?.ReportRunningTrial(setting); try { - var trialResult = runner.Run(setting); - monitor?.ReportCompletedTrial(trialResult); - tuner.Update(trialResult); - - var error = _settings.IsMaximize ? 1 - trialResult.Metric : trialResult.Metric; - if (error < _bestError) + using (var trialCancellationTokenSource = new CancellationTokenSource()) + using (var deregisterCallback = _globalCancellationTokenSource.Token.Register(() => { - _bestTrialResult = trialResult; - _bestError = error; - monitor?.ReportBestTrial(trialResult); + // only force-canceling running trials when there's completed trials. + // otherwise, wait for the current running trial to be completed. + if (_bestTrialResult != null) + trialCancellationTokenSource.Cancel(); + })) + using (var performanceMonitor = serviceProvider.GetService()) + using (var runner = serviceProvider.GetRequiredService()) + { + performanceMonitor.MemoryUsageInMegaByte += (o, m) => + { + if (_settings.MaximumMemoryUsageInMegaByte is double d && m > d && !trialCancellationTokenSource.IsCancellationRequested) + { + logger.Trace($"cancel current trial {setting.TrialId} because it uses {m} mb memory and the maximum memory usage is {d}"); + trialCancellationTokenSource.Cancel(); + + GC.AddMemoryPressure(Convert.ToInt64(m) * 1024 * 1024); + GC.Collect(); + } + }; + + performanceMonitor.Start(); + logger.Trace($"trial setting - {JsonSerializer.Serialize(setting)}"); + var trialResult = await runner.RunAsync(setting, trialCancellationTokenSource.Token); + + var peakCpu = performanceMonitor?.GetPeakCpuUsage(); + var peakMemoryInMB = performanceMonitor?.GetPeakMemoryUsageInMegaByte(); + trialResult.PeakCpu = peakCpu; + trialResult.PeakMemoryInMegaByte = peakMemoryInMB; + + monitor?.ReportCompletedTrial(trialResult); + tuner.Update(trialResult); + + var error = _settings.IsMaximize ? 1 - trialResult.Metric : trialResult.Metric; + if (error < _bestError) + { + _bestTrialResult = trialResult; + _bestError = error; + monitor?.ReportBestTrial(trialResult); + } } } - catch (OperationCanceledException) + catch (OperationCanceledException ex) when (_globalCancellationTokenSource.IsCancellationRequested == false) + { + monitor?.ReportFailTrial(setting, ex); + var result = new TrialResult + { + TrialSettings = setting, + Metric = _settings.IsMaximize ? double.MinValue : double.MaxValue, + }; + + tuner.Update(result); + continue; + } + catch (OperationCanceledException) when (_globalCancellationTokenSource.IsCancellationRequested) { break; } @@ -233,7 +319,7 @@ public async Task RunAsync(CancellationToken ct = default) { monitor?.ReportFailTrial(setting, ex); - if (!_cts.IsCancellationRequested && _bestTrialResult == null) + if (!_globalCancellationTokenSource.IsCancellationRequested && _bestTrialResult == null) { // TODO // it's questionable on whether to abort the entire training process diff --git a/src/Microsoft.ML.AutoML/AutoMLExperiment/IMonitor.cs b/src/Microsoft.ML.AutoML/AutoMLExperiment/IMonitor.cs index fb52ebe686..6a28d80010 100644 --- a/src/Microsoft.ML.AutoML/AutoMLExperiment/IMonitor.cs +++ b/src/Microsoft.ML.AutoML/AutoMLExperiment/IMonitor.cs @@ -44,7 +44,7 @@ public virtual void ReportBestTrial(TrialResult result) public virtual void ReportCompletedTrial(TrialResult result) { - _logger.Info($"Update Completed Trial - Id: {result.TrialSettings.TrialId} - Metric: {result.Metric} - Pipeline: {_pipeline.ToString(result.TrialSettings.Parameter)} - Duration: {result.DurationInMilliseconds}"); + _logger.Info($"Update Completed Trial - Id: {result.TrialSettings.TrialId} - Metric: {result.Metric} - Pipeline: {_pipeline.ToString(result.TrialSettings.Parameter)} - Duration: {result.DurationInMilliseconds} - Peak CPU: {result.PeakCpu?.ToString("p")} - Peak Memory in MB: {result.PeakMemoryInMegaByte?.ToString("F")}"); _completedTrials.Add(result); } diff --git a/src/Microsoft.ML.AutoML/AutoMLExperiment/IPerformanceMonitor.cs b/src/Microsoft.ML.AutoML/AutoMLExperiment/IPerformanceMonitor.cs new file mode 100644 index 0000000000..fa86bb5894 --- /dev/null +++ b/src/Microsoft.ML.AutoML/AutoMLExperiment/IPerformanceMonitor.cs @@ -0,0 +1,119 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Text; +using System.Threading.Tasks; +using System.Timers; +using Microsoft.ML.Runtime; + +namespace Microsoft.ML.AutoML +{ + internal interface IPerformanceMonitor : IDisposable + { + void Start(); + + void Stop(); + + double? GetPeakMemoryUsageInMegaByte(); + + double? GetPeakCpuUsage(); + + public event EventHandler CpuUsage; + + public event EventHandler MemoryUsageInMegaByte; + } + + internal class DefaultPerformanceMonitor : IPerformanceMonitor + { + private readonly IChannel _logger; + private Timer _timer; + private double? _peakCpuUsage; + private double? _peakMemoryUsage; + private readonly int _checkIntervalInMilliseconds; + private TimeSpan _totalCpuProcessorTime; + + public DefaultPerformanceMonitor(IChannel logger, int checkIntervalInMilliseconds) + { + _logger = logger; + _checkIntervalInMilliseconds = checkIntervalInMilliseconds; + } + + + public event EventHandler CpuUsage; + + public event EventHandler MemoryUsageInMegaByte; + + + public void Dispose() + { + Stop(); + } + + public double? GetPeakCpuUsage() + { + return _peakCpuUsage; + } + + public double? GetPeakMemoryUsageInMegaByte() + { + return _peakMemoryUsage; + } + + public void Start() + { + if (_timer == null) + { + _timer = new Timer(_checkIntervalInMilliseconds); + _totalCpuProcessorTime = Process.GetCurrentProcess().TotalProcessorTime; + _timer.Elapsed += OnCheckCpuAndMemoryUsage; + _timer.AutoReset = true; + _timer.Enabled = true; + _logger?.Trace($"{typeof(DefaultPerformanceMonitor)} has been started"); + } + } + + public void Stop() + { + _timer?.Stop(); + _timer?.Dispose(); + _timer = null; + _peakCpuUsage = null; + _peakMemoryUsage = null; + } + + private void OnCheckCpuAndMemoryUsage(object source, ElapsedEventArgs e) + { + SampleCpuAndMemoryUsage(); + } + + private void SampleCpuAndMemoryUsage() + { + // calculate CPU usage in % + // the % of CPU usage is calculating in the following way + // for every _totalCpuProcessorTime + // total CPU time is _totalCpuProcessorTime * ProcessorCount + // total CPU time used by current process is currentCpuProcessorTime + // the % of CPU usage by current process is simply currentCpuProcessorTime / total CPU time. + using (var process = Process.GetCurrentProcess()) + { + var currentCpuProcessorTime = Process.GetCurrentProcess().TotalProcessorTime; + var elapseCpuProcessorTime = currentCpuProcessorTime - _totalCpuProcessorTime; + var cpuUsedMs = elapseCpuProcessorTime.TotalMilliseconds; + var cpuUsageInTotal = cpuUsedMs / (Environment.ProcessorCount * _checkIntervalInMilliseconds); + _totalCpuProcessorTime = currentCpuProcessorTime; + _peakCpuUsage = Math.Max(cpuUsageInTotal, _peakCpuUsage ?? 0); + + // calculate Memory Usage in MB + var memoryUsage = process.PrivateMemorySize64 * 1.0 / (1024 * 1024); + _peakMemoryUsage = Math.Max(memoryUsage, _peakMemoryUsage ?? 0); + _logger?.Trace($"current CPU: {cpuUsageInTotal}, current Memory(mb): {memoryUsage}"); + MemoryUsageInMegaByte?.Invoke(this, memoryUsage); + CpuUsage?.Invoke(this, cpuUsageInTotal); + } + } + } +} diff --git a/src/Microsoft.ML.AutoML/AutoMLExperiment/Runner/ITrialRunner.cs b/src/Microsoft.ML.AutoML/AutoMLExperiment/Runner/ITrialRunner.cs index c19ae220b6..1cb80e6663 100644 --- a/src/Microsoft.ML.AutoML/AutoMLExperiment/Runner/ITrialRunner.cs +++ b/src/Microsoft.ML.AutoML/AutoMLExperiment/Runner/ITrialRunner.cs @@ -5,6 +5,8 @@ using System; using System.Diagnostics; using System.Linq; +using System.Threading; +using System.Threading.Tasks; using Microsoft.ML.Data; namespace Microsoft.ML.AutoML @@ -12,8 +14,10 @@ namespace Microsoft.ML.AutoML /// /// interface for all trial runners. /// - public interface ITrialRunner + public interface ITrialRunner : IDisposable { TrialResult Run(TrialSettings settings); + + Task RunAsync(TrialSettings settings, CancellationToken ct); } } diff --git a/src/Microsoft.ML.AutoML/AutoMLExperiment/Runner/SweepablePipelineRunner.cs b/src/Microsoft.ML.AutoML/AutoMLExperiment/Runner/SweepablePipelineRunner.cs index f869df6ba4..b4913e1238 100644 --- a/src/Microsoft.ML.AutoML/AutoMLExperiment/Runner/SweepablePipelineRunner.cs +++ b/src/Microsoft.ML.AutoML/AutoMLExperiment/Runner/SweepablePipelineRunner.cs @@ -2,28 +2,34 @@ // The .NET Foundation licenses this file to you under the MIT license. // See the LICENSE file in the project root for more information. +#nullable enable using System; using System.Collections.Generic; using System.Diagnostics; using System.Linq; using System.Text; +using System.Threading; +using System.Threading.Tasks; using Microsoft.ML.Data; +using Microsoft.ML.Runtime; namespace Microsoft.ML.AutoML { internal class SweepablePipelineRunner : ITrialRunner { - private readonly MLContext _mLContext; + private MLContext? _mLContext; private readonly IEvaluateMetricManager _metricManager; private readonly IDatasetManager _datasetManager; private readonly SweepablePipeline _pipeline; + private readonly IChannel? _logger; - public SweepablePipelineRunner(MLContext context, SweepablePipeline pipeline, IEvaluateMetricManager metricManager, IDatasetManager datasetManager) + public SweepablePipelineRunner(MLContext context, SweepablePipeline pipeline, IEvaluateMetricManager metricManager, IDatasetManager datasetManager, IChannel? logger = null) { _mLContext = context; _metricManager = metricManager; _pipeline = pipeline; _datasetManager = datasetManager; + _logger = logger; } public TrialResult Run(TrialSettings settings) @@ -32,10 +38,9 @@ public TrialResult Run(TrialSettings settings) stopWatch.Start(); var parameter = settings.Parameter[AutoMLExperiment.PipelineSearchspaceName]; var mlnetPipeline = _pipeline.BuildFromOption(_mLContext, parameter); - if (_datasetManager is ICrossValidateDatasetManager crossValidateDatasetManager) { - var datasetSplit = _mLContext.Data.CrossValidationSplit(crossValidateDatasetManager.Dataset, crossValidateDatasetManager.Fold ?? 5); + var datasetSplit = _mLContext!.Data.CrossValidationSplit(crossValidateDatasetManager.Dataset, crossValidateDatasetManager.Fold ?? 5); var metrics = new List(); var models = new List(); foreach (var split in datasetSplit) @@ -75,5 +80,33 @@ public TrialResult Run(TrialSettings settings) throw new ArgumentException("IDatasetManager must be either ITrainTestDatasetManager or ICrossValidationDatasetManager"); } + + public Task RunAsync(TrialSettings settings, CancellationToken ct) + { + try + { + using (var ctRegistration = ct.Register(() => + { + _mLContext?.CancelExecution(); + })) + { + return Task.Run(() => Run(settings)); + } + } + catch (Exception ex) when (ct.IsCancellationRequested) + { + throw new OperationCanceledException(ex.Message, ex.InnerException); + } + catch (Exception) + { + throw; + } + } + + public void Dispose() + { + _mLContext!.CancelExecution(); + _mLContext = null; + } } } diff --git a/src/Microsoft.ML.AutoML/AutoMLExperiment/TrialResult.cs b/src/Microsoft.ML.AutoML/AutoMLExperiment/TrialResult.cs index f08e499d2f..69a1371a78 100644 --- a/src/Microsoft.ML.AutoML/AutoMLExperiment/TrialResult.cs +++ b/src/Microsoft.ML.AutoML/AutoMLExperiment/TrialResult.cs @@ -19,6 +19,10 @@ public class TrialResult public double Metric { get; set; } public double DurationInMilliseconds { get; set; } + + public double? PeakCpu { get; set; } + + public double? PeakMemoryInMegaByte { get; set; } } /// diff --git a/src/Microsoft.ML.CpuMath/CpuMathUtils.netstandard.cs b/src/Microsoft.ML.CpuMath/CpuMathUtils.netstandard.cs index e9f3f40f0e..2c8fcf15c9 100644 --- a/src/Microsoft.ML.CpuMath/CpuMathUtils.netstandard.cs +++ b/src/Microsoft.ML.CpuMath/CpuMathUtils.netstandard.cs @@ -20,7 +20,9 @@ public static int GetVectorAlignment() => Vector128Alignment; /// - /// Check if 's alignment is suitable to SSE instructions. Returns + /// + /// + /// if 's alignment is suitable to SSE instructions. Returns /// if 's alignment is ok and otherwise. /// /// The vector being checked. diff --git a/src/Microsoft.ML.FastTree/FastTree.cs b/src/Microsoft.ML.FastTree/FastTree.cs index 381b844bc1..c1cf5a5166 100644 --- a/src/Microsoft.ML.FastTree/FastTree.cs +++ b/src/Microsoft.ML.FastTree/FastTree.cs @@ -647,6 +647,8 @@ private protected virtual void Train(IChannel ch) pch.SetHeader(new ProgressHeader("trees"), e => e.SetProgress(0, Ensemble.NumTrees, numTotalTrees)); while (Ensemble.NumTrees < numTotalTrees) { + ch.Trace($"numTotalTrees left: {numTotalTrees}"); + Host.CheckAlive(); using (Timer.Time(TimerEvent.Iteration)) { #if NO_STORE @@ -687,6 +689,7 @@ private protected virtual void Train(IChannel ch) baggingProvider.GetCurrentOutOfBagPartition().Documents); } + Host.CheckAlive(); CustomizedTrainingIteration(tree); using (Timer.Time(TimerEvent.Test)) @@ -738,6 +741,7 @@ private protected virtual void Train(IChannel ch) } } + Host.CheckAlive(); if (earlyStoppingRule != null) { Contracts.Assert(numTotalTrees == 0 || bestIteration > 0); @@ -750,8 +754,13 @@ private protected virtual void Train(IChannel ch) bestIteration = GetBestIteration(ch); } + Host.CheckAlive(); OptimizationAlgorithm.FinalizeLearning(bestIteration); + + Host.CheckAlive(); Ensemble.PopulateRawThresholds(TrainSet); + + Host.CheckAlive(); ParallelTraining.FinalizeTreeLearner(); } diff --git a/src/Microsoft.ML.LightGbm/LightGbmTrainerBase.cs b/src/Microsoft.ML.LightGbm/LightGbmTrainerBase.cs index e8bf78bda9..4f17e19128 100644 --- a/src/Microsoft.ML.LightGbm/LightGbmTrainerBase.cs +++ b/src/Microsoft.ML.LightGbm/LightGbmTrainerBase.cs @@ -619,7 +619,7 @@ private void TrainCore(IChannel ch, IProgressChannel pch, Dataset dtrain, Catego Host.AssertValue(pch); Host.AssertValue(dtrain); Host.AssertValueOrNull(dvalid); - + Host.CheckAlive(); // For multi class, the number of labels is required. ch.Assert(((ITrainer)this).PredictionKind != PredictionKind.MulticlassClassification || GbmOptions.ContainsKey("num_class"), "LightGBM requires the number of classes to be specified in the parameters."); @@ -628,7 +628,7 @@ private void TrainCore(IChannel ch, IProgressChannel pch, Dataset dtrain, Catego lock (LightGbmShared.LockForMultiThreadingInside) { ch.Info("LightGBM objective={0}", GbmOptions["objective"]); - using (Booster bst = WrappedLightGbmTraining.Train(ch, pch, GbmOptions, dtrain, + using (Booster bst = WrappedLightGbmTraining.Train(Host, ch, pch, GbmOptions, dtrain, dvalid: dvalid, numIteration: LightGbmTrainerOptions.NumberOfIterations, verboseEval: LightGbmTrainerOptions.Verbose, earlyStoppingRound: LightGbmTrainerOptions.EarlyStoppingRound)) { diff --git a/src/Microsoft.ML.LightGbm/WrappedLightGbmTraining.cs b/src/Microsoft.ML.LightGbm/WrappedLightGbmTraining.cs index d898018e9a..169467f43a 100644 --- a/src/Microsoft.ML.LightGbm/WrappedLightGbmTraining.cs +++ b/src/Microsoft.ML.LightGbm/WrappedLightGbmTraining.cs @@ -15,7 +15,7 @@ internal static class WrappedLightGbmTraining /// /// Train and return a booster. /// - public static Booster Train(IChannel ch, IProgressChannel pch, + public static Booster Train(IHost host, IChannel ch, IProgressChannel pch, Dictionary parameters, Dataset dtrain, Dataset dvalid = null, int numIteration = 100, bool verboseEval = true, int earlyStoppingRound = 0) { @@ -67,6 +67,7 @@ public static Booster Train(IChannel ch, IProgressChannel pch, }); for (iter = 0; iter < numIteration; ++iter) { + host.CheckAlive(); if (bst.Update()) break; diff --git a/test/Microsoft.ML.AutoML.Tests/AutoMLExperimentTests.cs b/test/Microsoft.ML.AutoML.Tests/AutoMLExperimentTests.cs index 246ab5f66b..e38dace417 100644 --- a/test/Microsoft.ML.AutoML.Tests/AutoMLExperimentTests.cs +++ b/test/Microsoft.ML.AutoML.Tests/AutoMLExperimentTests.cs @@ -12,8 +12,10 @@ using FluentAssertions; using Microsoft.Data.Analysis; using Microsoft.Extensions.DependencyInjection; +using Microsoft.ML.AutoML.CodeGen; using Microsoft.ML.Runtime; using Microsoft.ML.TestFramework; +using Microsoft.ML.TestFramework.Attributes; using Xunit; using Xunit.Abstractions; @@ -55,6 +57,63 @@ public async Task AutoMLExperiment_throw_timeout_exception_when_ct_is_canceled_a await runExperimentAction.Should().ThrowExactlyAsync(); } + [Fact] + public async Task AutoMLExperiment_cancel_trial_when_exceeds_memory_limit_Async() + { + var context = new MLContext(1); + var experiment = context.Auto().CreateExperiment(); + context.Log += (o, e) => + { + if (e.Source.StartsWith("AutoMLExperiment")) + { + this.Output.WriteLine(e.RawMessage); + } + }; + // the following experiment set memory usage limit to 0.01mb + // so all trials should be canceled and there should be no successful trials. + // therefore when experiment finishes, it should throw timeout exception with no model trained message. + experiment.SetTrainingTimeInSeconds(10) + .SetTrialRunner((serviceProvider) => + { + var channel = serviceProvider.GetService(); + var settings = serviceProvider.GetService(); + return new DummyTrialRunner(settings, 5, channel); + }) + .SetTuner() + .SetMaximumMemoryUsageInMegaByte(0.01) + .SetPerformanceMonitor(); + + var runExperimentAction = async () => await experiment.RunAsync(); + await runExperimentAction.Should().ThrowExactlyAsync(); + } + + [LightGBMFact] + public async Task AutoMLExperiment_lgbm_cancel_trial_when_exceeds_memory_limit_Async() + { + // this test is to verify that lightGbm can be cancelled during training booster. + var context = new MLContext(1); + context.Log += (o, e) => + { + if (e.Message.Contains("LightGBM objective")) + { + context.CancelExecution(); + } + }; + var data = DatasetUtil.GetUciAdultDataView(); + var experiment = context.Auto().CreateExperiment(); + var pipeline = context.Auto().Featurizer(data, "_Features_", excludeColumns: new[] { DatasetUtil.UciAdultLabel }) + .Append(context.BinaryClassification.Trainers.LightGbm(DatasetUtil.UciAdultLabel, "_Features_", numberOfIterations: 10000)); + + experiment.SetDataset(context.Data.TrainTestSplit(data)) + .SetBinaryClassificationMetric(BinaryClassificationMetric.AreaUnderRocCurve, DatasetUtil.UciAdultLabel) + .SetPipeline(pipeline) + .SetTrainingTimeInSeconds(10) + .SetMaximumMemoryUsageInMegaByte(10); + + var runExperimentAction = async () => await experiment.RunAsync(); + await runExperimentAction.Should().ThrowExactlyAsync(); + } + [Fact] public async Task AutoMLExperiment_return_current_best_trial_when_ct_is_canceled_with_trial_completed_Async() { @@ -83,7 +142,7 @@ public async Task AutoMLExperiment_return_current_best_trial_when_ct_is_canceled var res = await experiment.RunAsync(cts.Token); stopWatch.Stop(); - stopWatch.ElapsedMilliseconds.Should().BeLessOrEqualTo(2 * 1000); + stopWatch.ElapsedMilliseconds.Should().BeLessOrEqualTo(2 * 1000 + 500); cts.IsCancellationRequested.Should().BeTrue(); res.Metric.Should().BeGreaterThan(0); } @@ -111,6 +170,7 @@ public async Task AutoMLExperiment_finish_training_when_time_is_up_Async() cts.IsCancellationRequested.Should().BeFalse(); } + [Fact] public async Task AutoMLExperiment_UCI_Adult_Train_Test_Split_Test() { @@ -282,6 +342,10 @@ public DummyTrialRunner(AutoMLExperiment.AutoMLExperimentSettings automlSettings _logger = logger; } + public void Dispose() + { + } + public TrialResult Run(TrialSettings settings) { _logger.Info("Update Running Trial"); @@ -295,5 +359,71 @@ public TrialResult Run(TrialSettings settings) Metric = 1.000 + 0.01 * settings.TrialId, }; } + + public async Task RunAsync(TrialSettings settings, CancellationToken ct) + { + _logger.Info("Update Running Trial"); + await Task.Delay(_finishAfterNSeconds * 1000, ct); + _ct.ThrowIfCancellationRequested(); + _logger.Info("Update Completed Trial"); + return new TrialResult + { + TrialSettings = settings, + DurationInMilliseconds = _finishAfterNSeconds * 1000, + Metric = 1.000 + 0.01 * settings.TrialId, + }; + } + } + + class DummyPeformanceMonitor : IPerformanceMonitor + { + private readonly int _checkIntervalInMilliseconds; + private System.Timers.Timer _timer; + + public DummyPeformanceMonitor() + { + _checkIntervalInMilliseconds = 1000; + } + + public event EventHandler CpuUsage; + + public event EventHandler MemoryUsageInMegaByte; + + public void Dispose() + { + } + + public double? GetPeakCpuUsage() + { + return 100; + } + + public double? GetPeakMemoryUsageInMegaByte() + { + return 1000; + } + + public void Start() + { + if (_timer == null) + { + _timer = new System.Timers.Timer(_checkIntervalInMilliseconds); + _timer.Elapsed += (o, e) => + { + CpuUsage?.Invoke(this, 100); + MemoryUsageInMegaByte?.Invoke(this, 1000); + }; + + _timer.AutoReset = true; + _timer.Enabled = true; + } + } + + public void Stop() + { + _timer?.Stop(); + _timer?.Dispose(); + _timer = null; + } } } diff --git a/test/Microsoft.ML.Tests/Microsoft.ML.Tests.csproj b/test/Microsoft.ML.Tests/Microsoft.ML.Tests.csproj index adf8e3b2bd..4da6994b3b 100644 --- a/test/Microsoft.ML.Tests/Microsoft.ML.Tests.csproj +++ b/test/Microsoft.ML.Tests/Microsoft.ML.Tests.csproj @@ -59,6 +59,7 @@ + diff --git a/test/Microsoft.ML.Tests/TrainerEstimators/TreeEstimators.cs b/test/Microsoft.ML.Tests/TrainerEstimators/TreeEstimators.cs index 6188b7e34b..2384545fa0 100644 --- a/test/Microsoft.ML.Tests/TrainerEstimators/TreeEstimators.cs +++ b/test/Microsoft.ML.Tests/TrainerEstimators/TreeEstimators.cs @@ -6,6 +6,7 @@ using System.Collections.Generic; using System.Linq; using System.Threading; +using System.Threading.Tasks; using Microsoft.ML.Calibrators; using Microsoft.ML.Data; using Microsoft.ML.Internal.Utilities; @@ -18,6 +19,7 @@ using Microsoft.ML.Trainers.LightGbm; using Microsoft.ML.Transforms; using Xunit; +using FluentAssertions; namespace Microsoft.ML.Tests.TrainerEstimators { @@ -498,7 +500,7 @@ private void LightGbmHelper(bool useSoftmax, double sigmoid, out string modelStr { var host = (mlContext as IHostEnvironment).Register("Training LightGBM..."); - using (var gbmNative = WrappedLightGbmTraining.Train(ch, pch, gbmParams, gbmDataSet, numIteration: numberOfTrainingIterations)) + using (var gbmNative = WrappedLightGbmTraining.Train(host, ch, pch, gbmParams, gbmDataSet, numIteration: numberOfTrainingIterations)) { int nativeLength = 0; unsafe @@ -992,6 +994,28 @@ public void FastForestBinaryClassificationTestSummary() Done(); } + [Fact] + public void FastTreeMultiClassificationCancellationTest() + { + // verify that FastForest can be canceled after training start. + // we need to create a new context for cancellation. + var context = new MLContext(seed: 1); + var (pipeline, dataView) = GetMulticlassPipeline(); + var estimator = pipeline.Append(context.MulticlassClassification.Trainers.OneVersusAll(context.BinaryClassification.Trainers.FastTree())); + + context.Log += (o, e) => + { + + if (e.Source.StartsWith("FastTreeTraining")) + { + context.CancelExecution(); + } + }; + + var action = () => estimator.Fit(dataView); + action.Should().Throw(); + } + [LightGBMFact] public void LightGbmBinaryClassificationTestSummary() { @@ -1017,5 +1041,61 @@ public void LightGbmBinaryClassificationTestSummary() CheckSummary(modelParameters, trainedTreeEnsemble.Bias, trainedTreeEnsemble.TreeWeights, trainedTreeEnsemble.Trees); Done(); } + + [LightGBMFact] + public void LightGbmBinaryClassificationCancellationTest() + { + var context = new MLContext(seed: 1); + context.Log += (o, e) => + { + if (e.Message.Contains("LightGBM objective")) + { + context.CancelExecution(); + } + }; + + var (pipeline, dataView) = GetOneHotBinaryClassificationPipeline(); + + // Attention: Do not set NumberOfThreads here, left this to use default value to avoid test crash. + // Details can be found here: https://github.com/dotnet/machinelearning/pull/4918 + var trainer = pipeline.Append(context.BinaryClassification.Trainers.LightGbm( + new LightGbmBinaryTrainer.Options + { + NumberOfIterations = 1000, + NumberOfLeaves = 5, + UseCategoricalSplit = true + })); + + var action = () => trainer.Fit(dataView); + action.Should().Throw(); + } + + [LightGBMFact] + public void LightGbmMultiClassificationCancellationTest() + { + var context = new MLContext(seed: 1); + context.Log += (o, e) => + { + if (e.Message.Contains("LightGBM objective")) + { + context.CancelExecution(); + } + }; + + var (pipeline, dataView) = GetMulticlassPipeline(); + + // Attention: Do not set NumberOfThreads here, left this to use default value to avoid test crash. + // Details can be found here: https://github.com/dotnet/machinelearning/pull/4918 + var trainer = pipeline.Append(context.MulticlassClassification.Trainers.LightGbm( + new LightGbmMulticlassTrainer.Options + { + NumberOfIterations = 1000, + NumberOfLeaves = 5, + UseCategoricalSplit = true + })); + + var action = () => trainer.Fit(dataView); + action.Should().Throw(); + } } }