From fc6379de1a3ca4e93e797ee6b0eb2926529cf195 Mon Sep 17 00:00:00 2001 From: XiaoYun Zhang Date: Thu, 25 Aug 2022 12:40:31 -0700 Subject: [PATCH 01/23] add IPerformanceMonitor --- .../AutoMLExperiment/AutoMLExperiment.cs | 115 +++++++++++------- .../AutoMLExperiment/IMonitor.cs | 2 +- .../AutoMLExperiment/IPerformanceMonitor.cs | 56 +++++++++ .../AutoMLExperiment/TrialResult.cs | 4 + 4 files changed, 131 insertions(+), 46 deletions(-) create mode 100644 src/Microsoft.ML.AutoML/AutoMLExperiment/IPerformanceMonitor.cs diff --git a/src/Microsoft.ML.AutoML/AutoMLExperiment/AutoMLExperiment.cs b/src/Microsoft.ML.AutoML/AutoMLExperiment/AutoMLExperiment.cs index e142579e14..48cecc5bef 100644 --- a/src/Microsoft.ML.AutoML/AutoMLExperiment/AutoMLExperiment.cs +++ b/src/Microsoft.ML.AutoML/AutoMLExperiment/AutoMLExperiment.cs @@ -167,6 +167,14 @@ public AutoMLExperiment SetTuner() return this; } + internal AutoMLExperiment SetPerformanceMonitor() + where TPerformanceMonitor : class, IPerformanceMonitor + { + _serviceCollection.AddTransient(); + + return this; + } + /// /// Run experiment and return the best trial result synchronizely. /// @@ -191,67 +199,84 @@ public async Task RunAsync(CancellationToken ct = default) _settings.CancellationToken.Register(() => _cts.Cancel()); var serviceProvider = _serviceCollection.BuildServiceProvider(); var monitor = serviceProvider.GetService(); + var trialNum = 0; var tuner = serviceProvider.GetService(); Contracts.Assert(tuner != null, "tuner can't be null"); - while (true) + + using (var performanceMonitor = serviceProvider.GetService()) { - if (_cts.Token.IsCancellationRequested) - { - break; - } - var setting = new TrialSettings() + while (true) { - TrialId = trialNum++, - Parameter = Parameter.CreateNestedParameter(), - }; + if (_cts.Token.IsCancellationRequested) + { + break; + } + var setting = new TrialSettings() + { + TrialId = trialNum++, + Parameter = Parameter.CreateNestedParameter(), + }; - var parameter = tuner.Propose(setting); - setting.Parameter = parameter; - monitor?.ReportRunningTrial(setting); - var runner = serviceProvider.GetRequiredService(); + var parameter = tuner.Propose(setting); + setting.Parameter = parameter; + monitor?.ReportRunningTrial(setting); + var runner = serviceProvider.GetRequiredService(); - try - { - var trialResult = runner.Run(setting); - monitor?.ReportCompletedTrial(trialResult); - tuner.Update(trialResult); + try + { + performanceMonitor?.Start(); + + var trialResult = runner.Run(setting); + + var peakCpu = performanceMonitor?.GetPeakCpuUsage(); + var peakMemoryInMB = performanceMonitor?.GetPeakMemoryUsageInMegaByte(); + trialResult.PeakCpu = peakCpu; + trialResult.PeakMemoryInMegaByte = peakMemoryInMB; - var error = _settings.IsMaximize ? 1 - trialResult.Metric : trialResult.Metric; - if (error < _bestError) + monitor?.ReportCompletedTrial(trialResult); + tuner.Update(trialResult); + + var error = _settings.IsMaximize ? 1 - trialResult.Metric : trialResult.Metric; + if (error < _bestError) + { + _bestTrialResult = trialResult; + _bestError = error; + monitor?.ReportBestTrial(trialResult); + } + } + catch (OperationCanceledException) + { + break; + } + catch (Exception ex) { - _bestTrialResult = trialResult; - _bestError = error; - monitor?.ReportBestTrial(trialResult); + monitor?.ReportFailTrial(setting, ex); + + if (!_cts.IsCancellationRequested && _bestTrialResult == null) + { + // TODO + // it's questionable on whether to abort the entire training process + // for a single fail trial. We should make it an option and only exit + // when error is fatal (like schema mismatch). + throw; + } + } + finally + { + performanceMonitor?.Stop(); } } - catch (OperationCanceledException) + + if (_bestTrialResult == null) { - break; + throw new TimeoutException("Training time finished without completing a trial run"); } - catch (Exception ex) + else { - monitor?.ReportFailTrial(setting, ex); - - if (!_cts.IsCancellationRequested && _bestTrialResult == null) - { - // TODO - // it's questionable on whether to abort the entire training process - // for a single fail trial. We should make it an option and only exit - // when error is fatal (like schema mismatch). - throw; - } + return await Task.FromResult(_bestTrialResult); } } - - if (_bestTrialResult == null) - { - throw new TimeoutException("Training time finished without completing a trial run"); - } - else - { - return await Task.FromResult(_bestTrialResult); - } } private void ValidateSettings() diff --git a/src/Microsoft.ML.AutoML/AutoMLExperiment/IMonitor.cs b/src/Microsoft.ML.AutoML/AutoMLExperiment/IMonitor.cs index fb52ebe686..af14d0feb6 100644 --- a/src/Microsoft.ML.AutoML/AutoMLExperiment/IMonitor.cs +++ b/src/Microsoft.ML.AutoML/AutoMLExperiment/IMonitor.cs @@ -44,7 +44,7 @@ public virtual void ReportBestTrial(TrialResult result) public virtual void ReportCompletedTrial(TrialResult result) { - _logger.Info($"Update Completed Trial - Id: {result.TrialSettings.TrialId} - Metric: {result.Metric} - Pipeline: {_pipeline.ToString(result.TrialSettings.Parameter)} - Duration: {result.DurationInMilliseconds}"); + _logger.Info($"Update Completed Trial - Id: {result.TrialSettings.TrialId} - Metric: {result.Metric} - Pipeline: {_pipeline.ToString(result.TrialSettings.Parameter)} - Duration: {result.DurationInMilliseconds} - Peak CPU: {result.PeakCpu} - Peak Memory in MB: {result.PeakMemoryInMegaByte}"); _completedTrials.Add(result); } diff --git a/src/Microsoft.ML.AutoML/AutoMLExperiment/IPerformanceMonitor.cs b/src/Microsoft.ML.AutoML/AutoMLExperiment/IPerformanceMonitor.cs new file mode 100644 index 0000000000..af0729d271 --- /dev/null +++ b/src/Microsoft.ML.AutoML/AutoMLExperiment/IPerformanceMonitor.cs @@ -0,0 +1,56 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.Collections.Generic; +using System.Text; + +namespace Microsoft.ML.AutoML +{ + public interface IPerformanceMonitor : IDisposable + { + void Start(); + + void Stop(); + + float GetPeakMemoryUsageInMegaByte(); + + float GetPeakCpuUsage(); + + //event EventHandler CpuUsage; + + //event EventHandler MemoryUsageInMegaByte; + } + + internal class DefaultPerformanceMonitor : IPerformanceMonitor + { + //public event EventHandler CpuUsage; + + //public event EventHandler MemoryUsageInMegaByte; + public void Dispose() + { + throw new NotImplementedException(); + } + + public float GetPeakCpuUsage() + { + throw new NotImplementedException(); + } + + public float GetPeakMemoryUsageInMegaByte() + { + throw new NotImplementedException(); + } + + public void Start() + { + throw new NotImplementedException(); + } + + public void Stop() + { + throw new NotImplementedException(); + } + } +} diff --git a/src/Microsoft.ML.AutoML/AutoMLExperiment/TrialResult.cs b/src/Microsoft.ML.AutoML/AutoMLExperiment/TrialResult.cs index f08e499d2f..74e1624828 100644 --- a/src/Microsoft.ML.AutoML/AutoMLExperiment/TrialResult.cs +++ b/src/Microsoft.ML.AutoML/AutoMLExperiment/TrialResult.cs @@ -19,6 +19,10 @@ public class TrialResult public double Metric { get; set; } public double DurationInMilliseconds { get; set; } + + public float? PeakCpu { get; set; } + + public float? PeakMemoryInMegaByte { get; set; } } /// From 045244ad35c1228e1879a2b17f114ff0041bad48 Mon Sep 17 00:00:00 2001 From: XiaoYun Zhang Date: Thu, 25 Aug 2022 15:16:18 -0700 Subject: [PATCH 02/23] implement DefaultPerformanceMonitor --- .../API/AutoMLExperimentExtension.cs | 13 +++ .../AutoMLExperiment/AutoMLExperiment.cs | 8 ++ .../AutoMLExperiment/IMonitor.cs | 2 +- .../AutoMLExperiment/IPerformanceMonitor.cs | 83 ++++++++++++++++--- .../AutoMLExperiment/TrialResult.cs | 4 +- 5 files changed, 94 insertions(+), 16 deletions(-) diff --git a/src/Microsoft.ML.AutoML/API/AutoMLExperimentExtension.cs b/src/Microsoft.ML.AutoML/API/AutoMLExperimentExtension.cs index 65c08bcf91..14aedab4b3 100644 --- a/src/Microsoft.ML.AutoML/API/AutoMLExperimentExtension.cs +++ b/src/Microsoft.ML.AutoML/API/AutoMLExperimentExtension.cs @@ -6,6 +6,7 @@ using System.Collections.Generic; using System.Text; using Microsoft.Extensions.DependencyInjection; +using Microsoft.ML.Runtime; using static Microsoft.ML.DataOperationsCatalog; namespace Microsoft.ML.AutoML @@ -144,6 +145,18 @@ public static AutoMLExperiment SetPipeline(this AutoMLExperiment experiment, Swe return experiment; } + public static AutoMLExperiment SetPerformanceMonitor(this AutoMLExperiment experiment, int checkIntervalInMilliseconds = 1000) + { + experiment.SetPerformanceMonitor((service) => + { + var channel = service.GetService(); + + return new DefaultPerformanceMonitor(channel, checkIntervalInMilliseconds); + }); + + return experiment; + } + private static AutoMLExperiment SetEvaluateMetric(this AutoMLExperiment experiment, TEvaluateMetricManager metricManager) where TEvaluateMetricManager : class, IEvaluateMetricManager { diff --git a/src/Microsoft.ML.AutoML/AutoMLExperiment/AutoMLExperiment.cs b/src/Microsoft.ML.AutoML/AutoMLExperiment/AutoMLExperiment.cs index 48cecc5bef..0b2a85d0fd 100644 --- a/src/Microsoft.ML.AutoML/AutoMLExperiment/AutoMLExperiment.cs +++ b/src/Microsoft.ML.AutoML/AutoMLExperiment/AutoMLExperiment.cs @@ -175,6 +175,14 @@ internal AutoMLExperiment SetPerformanceMonitor() return this; } + internal AutoMLExperiment SetPerformanceMonitor(Func factory) + where TPerformanceMonitor : class, IPerformanceMonitor + { + _serviceCollection.AddTransient(factory); + + return this; + } + /// /// Run experiment and return the best trial result synchronizely. /// diff --git a/src/Microsoft.ML.AutoML/AutoMLExperiment/IMonitor.cs b/src/Microsoft.ML.AutoML/AutoMLExperiment/IMonitor.cs index af14d0feb6..6a28d80010 100644 --- a/src/Microsoft.ML.AutoML/AutoMLExperiment/IMonitor.cs +++ b/src/Microsoft.ML.AutoML/AutoMLExperiment/IMonitor.cs @@ -44,7 +44,7 @@ public virtual void ReportBestTrial(TrialResult result) public virtual void ReportCompletedTrial(TrialResult result) { - _logger.Info($"Update Completed Trial - Id: {result.TrialSettings.TrialId} - Metric: {result.Metric} - Pipeline: {_pipeline.ToString(result.TrialSettings.Parameter)} - Duration: {result.DurationInMilliseconds} - Peak CPU: {result.PeakCpu} - Peak Memory in MB: {result.PeakMemoryInMegaByte}"); + _logger.Info($"Update Completed Trial - Id: {result.TrialSettings.TrialId} - Metric: {result.Metric} - Pipeline: {_pipeline.ToString(result.TrialSettings.Parameter)} - Duration: {result.DurationInMilliseconds} - Peak CPU: {result.PeakCpu?.ToString("p")} - Peak Memory in MB: {result.PeakMemoryInMegaByte?.ToString("F")}"); _completedTrials.Add(result); } diff --git a/src/Microsoft.ML.AutoML/AutoMLExperiment/IPerformanceMonitor.cs b/src/Microsoft.ML.AutoML/AutoMLExperiment/IPerformanceMonitor.cs index af0729d271..4c7caf5a94 100644 --- a/src/Microsoft.ML.AutoML/AutoMLExperiment/IPerformanceMonitor.cs +++ b/src/Microsoft.ML.AutoML/AutoMLExperiment/IPerformanceMonitor.cs @@ -4,7 +4,11 @@ using System; using System.Collections.Generic; +using System.Diagnostics; using System.Text; +using System.Threading.Tasks; +using System.Timers; +using Microsoft.ML.Runtime; namespace Microsoft.ML.AutoML { @@ -14,43 +18,96 @@ public interface IPerformanceMonitor : IDisposable void Stop(); - float GetPeakMemoryUsageInMegaByte(); + double? GetPeakMemoryUsageInMegaByte(); - float GetPeakCpuUsage(); + double? GetPeakCpuUsage(); - //event EventHandler CpuUsage; + public event EventHandler CpuUsage; - //event EventHandler MemoryUsageInMegaByte; + public event EventHandler MemoryUsageInMegaByte; } internal class DefaultPerformanceMonitor : IPerformanceMonitor { - //public event EventHandler CpuUsage; + private readonly IChannel _logger; + private Timer _timer; + private double? _peakCpuUsage; + private double? _peakMemoryUsage; + private readonly int _checkIntervalInMilliseconds; + private TimeSpan _totalCpuProcessorTime; + + public DefaultPerformanceMonitor(IChannel logger, int checkIntervalInMilliseconds) + { + _logger = logger; + _checkIntervalInMilliseconds = checkIntervalInMilliseconds; + } + + + public event EventHandler CpuUsage; + + public event EventHandler MemoryUsageInMegaByte; + - //public event EventHandler MemoryUsageInMegaByte; public void Dispose() { - throw new NotImplementedException(); + Stop(); } - public float GetPeakCpuUsage() + public double? GetPeakCpuUsage() { - throw new NotImplementedException(); + return _peakCpuUsage; } - public float GetPeakMemoryUsageInMegaByte() + public double? GetPeakMemoryUsageInMegaByte() { - throw new NotImplementedException(); + return _peakMemoryUsage; } public void Start() { - throw new NotImplementedException(); + if (_timer == null) + { + _timer = new Timer(_checkIntervalInMilliseconds); + _totalCpuProcessorTime = Process.GetCurrentProcess().TotalProcessorTime; + _timer.Elapsed += OnCheckCpuAndMemoryUsage; + _timer.AutoReset = true; + _timer.Enabled = true; + _logger?.Info($"{typeof(DefaultPerformanceMonitor)} has been started"); + } } public void Stop() { - throw new NotImplementedException(); + _timer?.Stop(); + _timer?.Dispose(); + _timer = null; + _peakCpuUsage = null; + _peakMemoryUsage = null; + } + + private void OnCheckCpuAndMemoryUsage(object source, ElapsedEventArgs e) + { + SampleCpuAndMemoryUsage(); + } + + private void SampleCpuAndMemoryUsage() + { + // calculate CPU usage in % + using (var process = Process.GetCurrentProcess()) + { + var currentCpuProcessorTime = Process.GetCurrentProcess().TotalProcessorTime; + var elapseCpuProcessorTime = currentCpuProcessorTime - _totalCpuProcessorTime; + var cpuUsedMs = elapseCpuProcessorTime.TotalMilliseconds; + var cpuUsageInTotal = cpuUsedMs / (Environment.ProcessorCount * _checkIntervalInMilliseconds); + _totalCpuProcessorTime = currentCpuProcessorTime; + CpuUsage?.Invoke(this, cpuUsageInTotal); + _peakCpuUsage = Math.Max(cpuUsageInTotal, _peakCpuUsage ?? 0); + + // calculate Memory Usage in MB + var memoryUsage = process.PrivateMemorySize64 * 1.0 / (1024 * 1024); + MemoryUsageInMegaByte?.Invoke(this, memoryUsage); + _peakMemoryUsage = Math.Max(memoryUsage, _peakMemoryUsage ?? 0); + } } } } diff --git a/src/Microsoft.ML.AutoML/AutoMLExperiment/TrialResult.cs b/src/Microsoft.ML.AutoML/AutoMLExperiment/TrialResult.cs index 74e1624828..69a1371a78 100644 --- a/src/Microsoft.ML.AutoML/AutoMLExperiment/TrialResult.cs +++ b/src/Microsoft.ML.AutoML/AutoMLExperiment/TrialResult.cs @@ -20,9 +20,9 @@ public class TrialResult public double DurationInMilliseconds { get; set; } - public float? PeakCpu { get; set; } + public double? PeakCpu { get; set; } - public float? PeakMemoryInMegaByte { get; set; } + public double? PeakMemoryInMegaByte { get; set; } } /// From c611afd2326c7b8c8661dc3a1971d1cb49707ea6 Mon Sep 17 00:00:00 2001 From: XiaoYun Zhang Date: Thu, 25 Aug 2022 18:00:13 -0700 Subject: [PATCH 03/23] implement memory restricution --- .../API/BinaryClassificationExperiment.cs | 12 +++ .../API/MulticlassClassificationExperiment.cs | 38 +++++++- .../AutoMLExperiment/AutoMLExperiment.cs | 95 ++++++++++++------- .../AutoMLExperiment/IPerformanceMonitor.cs | 5 +- .../AutoMLExperiment/Runner/ITrialRunner.cs | 6 +- .../Runner/SweepablePipelineRunner.cs | 92 +++++++++++++++++- .../AutoMLExperimentTests.cs | 16 +++- 7 files changed, 221 insertions(+), 43 deletions(-) diff --git a/src/Microsoft.ML.AutoML/API/BinaryClassificationExperiment.cs b/src/Microsoft.ML.AutoML/API/BinaryClassificationExperiment.cs index b9e41062a3..0790d2edfb 100644 --- a/src/Microsoft.ML.AutoML/API/BinaryClassificationExperiment.cs +++ b/src/Microsoft.ML.AutoML/API/BinaryClassificationExperiment.cs @@ -6,6 +6,8 @@ using System.Collections.Generic; using System.Diagnostics; using System.Linq; +using System.Threading; +using System.Threading.Tasks; using Microsoft.Extensions.DependencyInjection; using Microsoft.ML.Data; using Microsoft.ML.Runtime; @@ -346,6 +348,11 @@ public BinaryClassificationRunner(MLContext context, IDatasetManager datasetMana _rnd = settings.Seed.HasValue ? new Random(settings.Seed.Value) : new Random(); } + public void Dispose() + { + throw new NotImplementedException(); + } + public TrialResult Run(TrialSettings settings) { if (_metricManager is BinaryMetricManager metricManager) @@ -421,5 +428,10 @@ public TrialResult Run(TrialSettings settings) throw new ArgumentException($"The runner metric manager is of type {_metricManager.GetType()} which expected to be of type {typeof(ITrainTestDatasetManager)} or {typeof(ICrossValidateDatasetManager)}"); } + + public Task RunAsync(TrialSettings settings, CancellationToken ct) + { + throw new NotImplementedException(); + } } } diff --git a/src/Microsoft.ML.AutoML/API/MulticlassClassificationExperiment.cs b/src/Microsoft.ML.AutoML/API/MulticlassClassificationExperiment.cs index 78c503673f..30c99a7b19 100644 --- a/src/Microsoft.ML.AutoML/API/MulticlassClassificationExperiment.cs +++ b/src/Microsoft.ML.AutoML/API/MulticlassClassificationExperiment.cs @@ -6,6 +6,8 @@ using System.Collections.Generic; using System.Diagnostics; using System.Linq; +using System.Threading; +using System.Threading.Tasks; using Microsoft.Extensions.DependencyInjection; using Microsoft.ML.Data; using Microsoft.ML.Runtime; @@ -189,7 +191,6 @@ public override ExperimentResult Execute(IDataV return result; } - public override ExperimentResult Execute(IDataView trainData, IDataView validationData, ColumnInformation columnInformation, IEstimator preFeaturizer = null, IProgress> progressHandler = null) { var label = columnInformation.LabelColumnName; @@ -338,6 +339,7 @@ internal class MulticlassClassificationRunner : ITrialRunner private readonly IMetricManager _metricManager; private readonly SweepablePipeline _pipeline; private readonly Random _rnd; + private bool _disposedValue; public MulticlassClassificationRunner(MLContext context, IDatasetManager datasetManager, IMetricManager metricManager, SweepablePipeline pipeline, AutoMLExperiment.AutoMLExperimentSettings settings) { @@ -424,5 +426,39 @@ public TrialResult Run(TrialSettings settings) throw new ArgumentException($"The runner metric manager is of type {_metricManager.GetType()} which expected to be of type {typeof(ITrainTestDatasetManager)} or {typeof(ICrossValidateDatasetManager)}"); } + + public Task RunAsync(TrialSettings settings, CancellationToken ct) + { + throw new NotImplementedException(); + } + + protected virtual void Dispose(bool disposing) + { + if (!_disposedValue) + { + if (disposing) + { + // TODO: dispose managed state (managed objects) + } + + // TODO: free unmanaged resources (unmanaged objects) and override finalizer + // TODO: set large fields to null + _disposedValue = true; + } + } + + // // TODO: override finalizer only if 'Dispose(bool disposing)' has code to free unmanaged resources + // ~MulticlassClassificationRunner() + // { + // // Do not change this code. Put cleanup code in 'Dispose(bool disposing)' method + // Dispose(disposing: false); + // } + + public void Dispose() + { + // Do not change this code. Put cleanup code in 'Dispose(bool disposing)' method + Dispose(disposing: true); + GC.SuppressFinalize(this); + } } } diff --git a/src/Microsoft.ML.AutoML/AutoMLExperiment/AutoMLExperiment.cs b/src/Microsoft.ML.AutoML/AutoMLExperiment/AutoMLExperiment.cs index 0b2a85d0fd..93a3f34aad 100644 --- a/src/Microsoft.ML.AutoML/AutoMLExperiment/AutoMLExperiment.cs +++ b/src/Microsoft.ML.AutoML/AutoMLExperiment/AutoMLExperiment.cs @@ -22,7 +22,7 @@ public class AutoMLExperiment private double _bestError = double.MaxValue; private TrialResult _bestTrialResult = null; private readonly IServiceCollection _serviceCollection; - private CancellationTokenSource _cts; + private CancellationTokenSource _globalCancellationTokenSource; public AutoMLExperiment(MLContext context, AutoMLExperimentSettings settings) { @@ -40,6 +40,7 @@ public AutoMLExperiment(MLContext context, AutoMLExperimentSettings settings) } _serviceCollection = new ServiceCollection(); + InitializeServiceCollection(); } private void InitializeServiceCollection() @@ -47,7 +48,7 @@ private void InitializeServiceCollection() _serviceCollection.TryAddTransient((provider) => { var context = new MLContext(_settings.Seed); - _cts.Token.Register(() => + _globalCancellationTokenSource.Token.Register(() => { // only force-canceling running trials when there's completed trials. // otherwise, wait for the current running trial to be completed. @@ -57,14 +58,15 @@ private void InitializeServiceCollection() return context; }); + this.SetMaximumMemoryUsageInMegaByte(); + this.SetPerformanceMonitor(1000); _serviceCollection.TryAddSingleton(_settings); _serviceCollection.TryAddSingleton(((IChannelProvider)_context).Start(nameof(AutoMLExperiment))); } private void Initialize() { - _cts = new CancellationTokenSource(); - InitializeServiceCollection(); + _globalCancellationTokenSource = new CancellationTokenSource(); } internal IServiceCollection ServiceCollection { get => _serviceCollection; } @@ -81,6 +83,12 @@ public AutoMLExperiment SetIsMaximize(bool isMaximize) return this; } + public AutoMLExperiment SetMaximumMemoryUsageInMegaByte(double value = double.MaxValue) + { + _settings.MaximumMemoryUsageInMegaByte = value; + return this; + } + public AutoMLExperiment AddSearchSpace(string key, SearchSpace.SearchSpace searchSpace) { _settings.SearchSpace[key] = searchSpace; @@ -203,8 +211,8 @@ public async Task RunAsync(CancellationToken ct = default) ValidateSettings(); Initialize(); _settings.CancellationToken = ct; - _cts.CancelAfter((int)_settings.MaxExperimentTimeInSeconds * 1000); - _settings.CancellationToken.Register(() => _cts.Cancel()); + _globalCancellationTokenSource.CancelAfter((int)_settings.MaxExperimentTimeInSeconds * 1000); + _settings.CancellationToken.Register(() => _globalCancellationTokenSource.Cancel()); var serviceProvider = _serviceCollection.BuildServiceProvider(); var monitor = serviceProvider.GetService(); @@ -214,46 +222,65 @@ public async Task RunAsync(CancellationToken ct = default) using (var performanceMonitor = serviceProvider.GetService()) { - while (true) + while (!_globalCancellationTokenSource.Token.IsCancellationRequested) { - if (_cts.Token.IsCancellationRequested) - { - break; - } var setting = new TrialSettings() { TrialId = trialNum++, Parameter = Parameter.CreateNestedParameter(), }; - var parameter = tuner.Propose(setting); setting.Parameter = parameter; - monitor?.ReportRunningTrial(setting); - var runner = serviceProvider.GetRequiredService(); + monitor?.ReportRunningTrial(setting); try { - performanceMonitor?.Start(); - - var trialResult = runner.Run(setting); - - var peakCpu = performanceMonitor?.GetPeakCpuUsage(); - var peakMemoryInMB = performanceMonitor?.GetPeakMemoryUsageInMegaByte(); - trialResult.PeakCpu = peakCpu; - trialResult.PeakMemoryInMegaByte = peakMemoryInMB; - - monitor?.ReportCompletedTrial(trialResult); - tuner.Update(trialResult); - - var error = _settings.IsMaximize ? 1 - trialResult.Metric : trialResult.Metric; - if (error < _bestError) + using (var runner = serviceProvider.GetRequiredService()) { - _bestTrialResult = trialResult; - _bestError = error; - monitor?.ReportBestTrial(trialResult); + var trialCancellationTokenSource = new CancellationTokenSource(); + _globalCancellationTokenSource.Token.Register(() => trialCancellationTokenSource.Cancel()); + performanceMonitor.MemoryUsageInMegaByte += (o, m) => + { + if (m > _settings.MaximumMemoryUsageInMegaByte) + { + trialCancellationTokenSource.Cancel(); + } + }; + + performanceMonitor.Start(); + + var trialResult = await runner.RunAsync(setting, trialCancellationTokenSource.Token); + + var peakCpu = performanceMonitor?.GetPeakCpuUsage(); + var peakMemoryInMB = performanceMonitor?.GetPeakMemoryUsageInMegaByte(); + trialResult.PeakCpu = peakCpu; + trialResult.PeakMemoryInMegaByte = peakMemoryInMB; + + monitor?.ReportCompletedTrial(trialResult); + tuner.Update(trialResult); + + var error = _settings.IsMaximize ? 1 - trialResult.Metric : trialResult.Metric; + if (error < _bestError) + { + _bestTrialResult = trialResult; + _bestError = error; + monitor?.ReportBestTrial(trialResult); + } } } - catch (OperationCanceledException) + catch (OperationCanceledException ex) when (_globalCancellationTokenSource.IsCancellationRequested == false) + { + monitor?.ReportFailTrial(setting, ex); + var result = new TrialResult + { + TrialSettings = setting, + Metric = _settings.IsMaximize ? double.MinValue : double.MaxValue, + }; + + tuner.Update(result); + continue; + } + catch (OperationCanceledException) when (_globalCancellationTokenSource.IsCancellationRequested) { break; } @@ -261,7 +288,7 @@ public async Task RunAsync(CancellationToken ct = default) { monitor?.ReportFailTrial(setting, ex); - if (!_cts.IsCancellationRequested && _bestTrialResult == null) + if (!_globalCancellationTokenSource.IsCancellationRequested && _bestTrialResult == null) { // TODO // it's questionable on whether to abort the entire training process @@ -300,6 +327,8 @@ public class AutoMLExperimentSettings : ExperimentSettings public SearchSpace.SearchSpace SearchSpace { get; set; } public bool IsMaximize { get; set; } + + public double MaximumMemoryUsageInMegaByte { get; set; } } } } diff --git a/src/Microsoft.ML.AutoML/AutoMLExperiment/IPerformanceMonitor.cs b/src/Microsoft.ML.AutoML/AutoMLExperiment/IPerformanceMonitor.cs index 4c7caf5a94..b819cd1bd2 100644 --- a/src/Microsoft.ML.AutoML/AutoMLExperiment/IPerformanceMonitor.cs +++ b/src/Microsoft.ML.AutoML/AutoMLExperiment/IPerformanceMonitor.cs @@ -100,13 +100,14 @@ private void SampleCpuAndMemoryUsage() var cpuUsedMs = elapseCpuProcessorTime.TotalMilliseconds; var cpuUsageInTotal = cpuUsedMs / (Environment.ProcessorCount * _checkIntervalInMilliseconds); _totalCpuProcessorTime = currentCpuProcessorTime; - CpuUsage?.Invoke(this, cpuUsageInTotal); _peakCpuUsage = Math.Max(cpuUsageInTotal, _peakCpuUsage ?? 0); // calculate Memory Usage in MB var memoryUsage = process.PrivateMemorySize64 * 1.0 / (1024 * 1024); - MemoryUsageInMegaByte?.Invoke(this, memoryUsage); _peakMemoryUsage = Math.Max(memoryUsage, _peakMemoryUsage ?? 0); + + MemoryUsageInMegaByte?.Invoke(this, memoryUsage); + CpuUsage?.Invoke(this, cpuUsageInTotal); } } } diff --git a/src/Microsoft.ML.AutoML/AutoMLExperiment/Runner/ITrialRunner.cs b/src/Microsoft.ML.AutoML/AutoMLExperiment/Runner/ITrialRunner.cs index c19ae220b6..1cb80e6663 100644 --- a/src/Microsoft.ML.AutoML/AutoMLExperiment/Runner/ITrialRunner.cs +++ b/src/Microsoft.ML.AutoML/AutoMLExperiment/Runner/ITrialRunner.cs @@ -5,6 +5,8 @@ using System; using System.Diagnostics; using System.Linq; +using System.Threading; +using System.Threading.Tasks; using Microsoft.ML.Data; namespace Microsoft.ML.AutoML @@ -12,8 +14,10 @@ namespace Microsoft.ML.AutoML /// /// interface for all trial runners. /// - public interface ITrialRunner + public interface ITrialRunner : IDisposable { TrialResult Run(TrialSettings settings); + + Task RunAsync(TrialSettings settings, CancellationToken ct); } } diff --git a/src/Microsoft.ML.AutoML/AutoMLExperiment/Runner/SweepablePipelineRunner.cs b/src/Microsoft.ML.AutoML/AutoMLExperiment/Runner/SweepablePipelineRunner.cs index f869df6ba4..df5074c51f 100644 --- a/src/Microsoft.ML.AutoML/AutoMLExperiment/Runner/SweepablePipelineRunner.cs +++ b/src/Microsoft.ML.AutoML/AutoMLExperiment/Runner/SweepablePipelineRunner.cs @@ -2,28 +2,37 @@ // The .NET Foundation licenses this file to you under the MIT license. // See the LICENSE file in the project root for more information. +#nullable enable using System; using System.Collections.Generic; using System.Diagnostics; using System.Linq; using System.Text; +using System.Threading; +using System.Threading.Tasks; using Microsoft.ML.Data; +using Microsoft.ML.Runtime; namespace Microsoft.ML.AutoML { internal class SweepablePipelineRunner : ITrialRunner { - private readonly MLContext _mLContext; + private MLContext? _mLContext; private readonly IEvaluateMetricManager _metricManager; private readonly IDatasetManager _datasetManager; private readonly SweepablePipeline _pipeline; + private readonly IChannel? _logger; + private bool _disposedValue; + private Task? _disposableTrainingTask; + private Task? _disposableCancellationTask; - public SweepablePipelineRunner(MLContext context, SweepablePipeline pipeline, IEvaluateMetricManager metricManager, IDatasetManager datasetManager) + public SweepablePipelineRunner(MLContext context, SweepablePipeline pipeline, IEvaluateMetricManager metricManager, IDatasetManager datasetManager, IChannel? logger = null) { _mLContext = context; _metricManager = metricManager; _pipeline = pipeline; _datasetManager = datasetManager; + _logger = logger; } public TrialResult Run(TrialSettings settings) @@ -32,10 +41,9 @@ public TrialResult Run(TrialSettings settings) stopWatch.Start(); var parameter = settings.Parameter[AutoMLExperiment.PipelineSearchspaceName]; var mlnetPipeline = _pipeline.BuildFromOption(_mLContext, parameter); - if (_datasetManager is ICrossValidateDatasetManager crossValidateDatasetManager) { - var datasetSplit = _mLContext.Data.CrossValidationSplit(crossValidateDatasetManager.Dataset, crossValidateDatasetManager.Fold ?? 5); + var datasetSplit = _mLContext!.Data.CrossValidationSplit(crossValidateDatasetManager.Dataset, crossValidateDatasetManager.Fold ?? 5); var metrics = new List(); var models = new List(); foreach (var split in datasetSplit) @@ -75,5 +83,81 @@ public TrialResult Run(TrialSettings settings) throw new ArgumentException("IDatasetManager must be either ITrainTestDatasetManager or ICrossValidationDatasetManager"); } + + public async Task RunAsync(TrialSettings settings, CancellationToken ct) + { + var cts = new CancellationTokenSource(); + ct.Register(async () => + { + cts.Cancel(); + await Task.Delay(100); + _mLContext?.CancelExecution(); + }); + + _disposableTrainingTask = Task.Run(() => Run(settings)); + _disposableCancellationTask = Task.Run(async () => + { + while (!ct.IsCancellationRequested) + { + await Task.Delay(100); + } + + return new TrialResult(); + }); + var task = await Task.WhenAny(_disposableTrainingTask, _disposableCancellationTask); + var result = await task; + if (!cts.Token.IsCancellationRequested) + { + cts.Cancel(); + await _disposableCancellationTask; + + return result; + } + else + { + try + { + await _disposableTrainingTask; + } + catch (Exception ex) + { + throw new OperationCanceledException(ex.Message, ex); + } + } + } + + protected virtual void Dispose(bool disposing) + { + if (!_disposedValue) + { + if (disposing) + { + _mLContext?.CancelExecution(); + _mLContext = null; + _disposableTrainingTask?.Dispose(); + _disposableTrainingTask = null; + _disposableCancellationTask?.Dispose(); + _disposableCancellationTask = null; + } + + // TODO: free unmanaged resources (unmanaged objects) and override finalizer + // TODO: set large fields to null + _disposedValue = true; + } + } + + // // TODO: override finalizer only if 'Dispose(bool disposing)' has code to free unmanaged resources + // ~SweepablePipelineRunner() + // { + // // Do not change this code. Put cleanup code in 'Dispose(bool disposing)' method + // Dispose(disposing: false); + // } + + public void Dispose() + { + // Do not change this code. Put cleanup code in 'Dispose(bool disposing)' method + Dispose(disposing: true); + GC.SuppressFinalize(this); + } } } diff --git a/test/Microsoft.ML.AutoML.Tests/AutoMLExperimentTests.cs b/test/Microsoft.ML.AutoML.Tests/AutoMLExperimentTests.cs index 246ab5f66b..03e374f3c5 100644 --- a/test/Microsoft.ML.AutoML.Tests/AutoMLExperimentTests.cs +++ b/test/Microsoft.ML.AutoML.Tests/AutoMLExperimentTests.cs @@ -236,7 +236,8 @@ public async Task AutoMLExperiment_Taxi_Fare_Train_Test_Split_Test() experiment.SetDataset(train, test) .SetRegressionMetric(RegressionMetric.RSquared, label) .SetPipeline(pipeline) - .SetTrainingTimeInSeconds(50); + .SetTrainingTimeInSeconds(50) + .SetMaximumMemoryUsageInMegaByte(10); var result = await experiment.RunAsync(); result.Metric.Should().BeGreaterThan(0.5); @@ -262,7 +263,8 @@ public async Task AutoMLExperiment_Taxi_Fare_CV_5_Test() experiment.SetDataset(train, 5) .SetRegressionMetric(RegressionMetric.RSquared, label) .SetPipeline(pipeline) - .SetTrainingTimeInSeconds(50); + .SetTrainingTimeInSeconds(50) + .SetMaximumMemoryUsageInMegaByte(10); var result = await experiment.RunAsync(); result.Metric.Should().BeGreaterThan(0.5); @@ -282,6 +284,10 @@ public DummyTrialRunner(AutoMLExperiment.AutoMLExperimentSettings automlSettings _logger = logger; } + public void Dispose() + { + } + public TrialResult Run(TrialSettings settings) { _logger.Info("Update Running Trial"); @@ -295,5 +301,11 @@ public TrialResult Run(TrialSettings settings) Metric = 1.000 + 0.01 * settings.TrialId, }; } + + public Task RunAsync(TrialSettings settings, CancellationToken ct) + { + ct.Register(() => _ct.ThrowIfCancellationRequested()); + return Task.Run(() => Run(settings)); + } } } From 313d8d44eab3826f20eb84ba13f9f926dd3171c9 Mon Sep 17 00:00:00 2001 From: XiaoYun Zhang Date: Thu, 25 Aug 2022 22:13:19 -0700 Subject: [PATCH 04/23] update --- .../API/BinaryClassificationExperiment.cs | 20 +++++- .../API/ExperimentSettings.cs | 3 + .../API/MulticlassClassificationExperiment.cs | 37 ++++------ .../AutoMLExperiment/AutoMLExperiment.cs | 15 ++-- .../AutoMLExperiment/IPerformanceMonitor.cs | 2 +- .../Runner/SweepablePipelineRunner.cs | 71 +++---------------- .../AutoMLExperimentTests.cs | 46 +++++++++--- 7 files changed, 93 insertions(+), 101 deletions(-) diff --git a/src/Microsoft.ML.AutoML/API/BinaryClassificationExperiment.cs b/src/Microsoft.ML.AutoML/API/BinaryClassificationExperiment.cs index 0790d2edfb..f4396918ef 100644 --- a/src/Microsoft.ML.AutoML/API/BinaryClassificationExperiment.cs +++ b/src/Microsoft.ML.AutoML/API/BinaryClassificationExperiment.cs @@ -350,7 +350,7 @@ public BinaryClassificationRunner(MLContext context, IDatasetManager datasetMana public void Dispose() { - throw new NotImplementedException(); + GC.SuppressFinalize(this); } public TrialResult Run(TrialSettings settings) @@ -431,7 +431,23 @@ public TrialResult Run(TrialSettings settings) public Task RunAsync(TrialSettings settings, CancellationToken ct) { - throw new NotImplementedException(); + ct.Register(() => + { + _context?.CancelExecution(); + }); + + try + { + return Task.FromResult(Run(settings)); + } + catch (Exception ex) when (ct.IsCancellationRequested) + { + throw new OperationCanceledException(ex.Message, ex.InnerException); + } + catch (Exception) + { + throw; + } } } } diff --git a/src/Microsoft.ML.AutoML/API/ExperimentSettings.cs b/src/Microsoft.ML.AutoML/API/ExperimentSettings.cs index 8b54f697c4..a505eb4895 100644 --- a/src/Microsoft.ML.AutoML/API/ExperimentSettings.cs +++ b/src/Microsoft.ML.AutoML/API/ExperimentSettings.cs @@ -2,6 +2,7 @@ // The .NET Foundation licenses this file to you under the MIT license. // See the LICENSE file in the project root for more information. +using System; using System.Threading; namespace Microsoft.ML.AutoML @@ -56,6 +57,8 @@ public abstract class ExperimentSettings /// The default value is . public CacheBeforeTrainer CacheBeforeTrainer { get; set; } + public double? MaximumMemoryUsageInMegaByte { get; set; } + internal int MaxModels; /// diff --git a/src/Microsoft.ML.AutoML/API/MulticlassClassificationExperiment.cs b/src/Microsoft.ML.AutoML/API/MulticlassClassificationExperiment.cs index 30c99a7b19..83622fe624 100644 --- a/src/Microsoft.ML.AutoML/API/MulticlassClassificationExperiment.cs +++ b/src/Microsoft.ML.AutoML/API/MulticlassClassificationExperiment.cs @@ -339,7 +339,6 @@ internal class MulticlassClassificationRunner : ITrialRunner private readonly IMetricManager _metricManager; private readonly SweepablePipeline _pipeline; private readonly Random _rnd; - private bool _disposedValue; public MulticlassClassificationRunner(MLContext context, IDatasetManager datasetManager, IMetricManager metricManager, SweepablePipeline pipeline, AutoMLExperiment.AutoMLExperimentSettings settings) { @@ -429,35 +428,27 @@ public TrialResult Run(TrialSettings settings) public Task RunAsync(TrialSettings settings, CancellationToken ct) { - throw new NotImplementedException(); - } - - protected virtual void Dispose(bool disposing) - { - if (!_disposedValue) + ct.Register(() => { - if (disposing) - { - // TODO: dispose managed state (managed objects) - } + _context?.CancelExecution(); + }); - // TODO: free unmanaged resources (unmanaged objects) and override finalizer - // TODO: set large fields to null - _disposedValue = true; + try + { + return Task.FromResult(Run(settings)); + } + catch (Exception ex) when (ct.IsCancellationRequested) + { + throw new OperationCanceledException(ex.Message, ex.InnerException); + } + catch (Exception) + { + throw; } } - // // TODO: override finalizer only if 'Dispose(bool disposing)' has code to free unmanaged resources - // ~MulticlassClassificationRunner() - // { - // // Do not change this code. Put cleanup code in 'Dispose(bool disposing)' method - // Dispose(disposing: false); - // } - public void Dispose() { - // Do not change this code. Put cleanup code in 'Dispose(bool disposing)' method - Dispose(disposing: true); GC.SuppressFinalize(this); } } diff --git a/src/Microsoft.ML.AutoML/AutoMLExperiment/AutoMLExperiment.cs b/src/Microsoft.ML.AutoML/AutoMLExperiment/AutoMLExperiment.cs index 93a3f34aad..c52936c81a 100644 --- a/src/Microsoft.ML.AutoML/AutoMLExperiment/AutoMLExperiment.cs +++ b/src/Microsoft.ML.AutoML/AutoMLExperiment/AutoMLExperiment.cs @@ -58,7 +58,7 @@ private void InitializeServiceCollection() return context; }); - this.SetMaximumMemoryUsageInMegaByte(); + this.SetPerformanceMonitor(1000); _serviceCollection.TryAddSingleton(_settings); _serviceCollection.TryAddSingleton(((IChannelProvider)_context).Start(nameof(AutoMLExperiment))); @@ -238,10 +238,17 @@ public async Task RunAsync(CancellationToken ct = default) using (var runner = serviceProvider.GetRequiredService()) { var trialCancellationTokenSource = new CancellationTokenSource(); - _globalCancellationTokenSource.Token.Register(() => trialCancellationTokenSource.Cancel()); + _globalCancellationTokenSource.Token.Register(() => + { + // only force-canceling running trials when there's completed trials. + // otherwise, wait for the current running trial to be completed. + if (_bestTrialResult != null) + trialCancellationTokenSource.Cancel(); + }); + performanceMonitor.MemoryUsageInMegaByte += (o, m) => { - if (m > _settings.MaximumMemoryUsageInMegaByte) + if (_settings.MaximumMemoryUsageInMegaByte is double d && m > d) { trialCancellationTokenSource.Cancel(); } @@ -327,8 +334,6 @@ public class AutoMLExperimentSettings : ExperimentSettings public SearchSpace.SearchSpace SearchSpace { get; set; } public bool IsMaximize { get; set; } - - public double MaximumMemoryUsageInMegaByte { get; set; } } } } diff --git a/src/Microsoft.ML.AutoML/AutoMLExperiment/IPerformanceMonitor.cs b/src/Microsoft.ML.AutoML/AutoMLExperiment/IPerformanceMonitor.cs index b819cd1bd2..3e4847b13c 100644 --- a/src/Microsoft.ML.AutoML/AutoMLExperiment/IPerformanceMonitor.cs +++ b/src/Microsoft.ML.AutoML/AutoMLExperiment/IPerformanceMonitor.cs @@ -72,7 +72,7 @@ public void Start() _timer.Elapsed += OnCheckCpuAndMemoryUsage; _timer.AutoReset = true; _timer.Enabled = true; - _logger?.Info($"{typeof(DefaultPerformanceMonitor)} has been started"); + _logger?.Trace($"{typeof(DefaultPerformanceMonitor)} has been started"); } } diff --git a/src/Microsoft.ML.AutoML/AutoMLExperiment/Runner/SweepablePipelineRunner.cs b/src/Microsoft.ML.AutoML/AutoMLExperiment/Runner/SweepablePipelineRunner.cs index df5074c51f..c823d53596 100644 --- a/src/Microsoft.ML.AutoML/AutoMLExperiment/Runner/SweepablePipelineRunner.cs +++ b/src/Microsoft.ML.AutoML/AutoMLExperiment/Runner/SweepablePipelineRunner.cs @@ -17,14 +17,11 @@ namespace Microsoft.ML.AutoML { internal class SweepablePipelineRunner : ITrialRunner { - private MLContext? _mLContext; + private readonly MLContext? _mLContext; private readonly IEvaluateMetricManager _metricManager; private readonly IDatasetManager _datasetManager; private readonly SweepablePipeline _pipeline; private readonly IChannel? _logger; - private bool _disposedValue; - private Task? _disposableTrainingTask; - private Task? _disposableCancellationTask; public SweepablePipelineRunner(MLContext context, SweepablePipeline pipeline, IEvaluateMetricManager metricManager, IDatasetManager datasetManager, IChannel? logger = null) { @@ -84,79 +81,29 @@ public TrialResult Run(TrialSettings settings) throw new ArgumentException("IDatasetManager must be either ITrainTestDatasetManager or ICrossValidationDatasetManager"); } - public async Task RunAsync(TrialSettings settings, CancellationToken ct) + public Task RunAsync(TrialSettings settings, CancellationToken ct) { - var cts = new CancellationTokenSource(); - ct.Register(async () => + ct.Register(() => { - cts.Cancel(); - await Task.Delay(100); _mLContext?.CancelExecution(); }); - _disposableTrainingTask = Task.Run(() => Run(settings)); - _disposableCancellationTask = Task.Run(async () => + try { - while (!ct.IsCancellationRequested) - { - await Task.Delay(100); - } - - return new TrialResult(); - }); - var task = await Task.WhenAny(_disposableTrainingTask, _disposableCancellationTask); - var result = await task; - if (!cts.Token.IsCancellationRequested) - { - cts.Cancel(); - await _disposableCancellationTask; - - return result; + return Task.FromResult(Run(settings)); } - else + catch (Exception ex) when (ct.IsCancellationRequested) { - try - { - await _disposableTrainingTask; - } - catch (Exception ex) - { - throw new OperationCanceledException(ex.Message, ex); - } + throw new OperationCanceledException(ex.Message, ex.InnerException); } - } - - protected virtual void Dispose(bool disposing) - { - if (!_disposedValue) + catch (Exception) { - if (disposing) - { - _mLContext?.CancelExecution(); - _mLContext = null; - _disposableTrainingTask?.Dispose(); - _disposableTrainingTask = null; - _disposableCancellationTask?.Dispose(); - _disposableCancellationTask = null; - } - - // TODO: free unmanaged resources (unmanaged objects) and override finalizer - // TODO: set large fields to null - _disposedValue = true; + throw; } } - // // TODO: override finalizer only if 'Dispose(bool disposing)' has code to free unmanaged resources - // ~SweepablePipelineRunner() - // { - // // Do not change this code. Put cleanup code in 'Dispose(bool disposing)' method - // Dispose(disposing: false); - // } - public void Dispose() { - // Do not change this code. Put cleanup code in 'Dispose(bool disposing)' method - Dispose(disposing: true); GC.SuppressFinalize(this); } } diff --git a/test/Microsoft.ML.AutoML.Tests/AutoMLExperimentTests.cs b/test/Microsoft.ML.AutoML.Tests/AutoMLExperimentTests.cs index 03e374f3c5..516c68d06c 100644 --- a/test/Microsoft.ML.AutoML.Tests/AutoMLExperimentTests.cs +++ b/test/Microsoft.ML.AutoML.Tests/AutoMLExperimentTests.cs @@ -55,6 +55,29 @@ public async Task AutoMLExperiment_throw_timeout_exception_when_ct_is_canceled_a await runExperimentAction.Should().ThrowExactlyAsync(); } + [Fact] + public async Task AutoMLExperiment_cancel_trial_when_exceeds_memory_limit_Async() + { + var context = new MLContext(1); + var experiment = context.Auto().CreateExperiment(); + + // the following experiment set memory usage limit to 0.01mb + // so all trials should be canceled and there should be no successful trials. + // therefore when experiment finishes, it should throw timeout exception with no model trained message. + experiment.SetTrainingTimeInSeconds(10) + .SetTrialRunner((serviceProvider) => + { + var channel = serviceProvider.GetService(); + var settings = serviceProvider.GetService(); + return new DummyTrialRunner(settings, 5, channel); + }) + .SetTuner() + .SetMaximumMemoryUsageInMegaByte(0.01); + + var runExperimentAction = async () => await experiment.RunAsync(); + await runExperimentAction.Should().ThrowExactlyAsync(); + } + [Fact] public async Task AutoMLExperiment_return_current_best_trial_when_ct_is_canceled_with_trial_completed_Async() { @@ -83,7 +106,7 @@ public async Task AutoMLExperiment_return_current_best_trial_when_ct_is_canceled var res = await experiment.RunAsync(cts.Token); stopWatch.Stop(); - stopWatch.ElapsedMilliseconds.Should().BeLessOrEqualTo(2 * 1000); + stopWatch.ElapsedMilliseconds.Should().BeLessOrEqualTo(2 * 1000 + 500); cts.IsCancellationRequested.Should().BeTrue(); res.Metric.Should().BeGreaterThan(0); } @@ -111,6 +134,7 @@ public async Task AutoMLExperiment_finish_training_when_time_is_up_Async() cts.IsCancellationRequested.Should().BeFalse(); } + [Fact] public async Task AutoMLExperiment_UCI_Adult_Train_Test_Split_Test() { @@ -236,8 +260,7 @@ public async Task AutoMLExperiment_Taxi_Fare_Train_Test_Split_Test() experiment.SetDataset(train, test) .SetRegressionMetric(RegressionMetric.RSquared, label) .SetPipeline(pipeline) - .SetTrainingTimeInSeconds(50) - .SetMaximumMemoryUsageInMegaByte(10); + .SetTrainingTimeInSeconds(50); var result = await experiment.RunAsync(); result.Metric.Should().BeGreaterThan(0.5); @@ -263,8 +286,7 @@ public async Task AutoMLExperiment_Taxi_Fare_CV_5_Test() experiment.SetDataset(train, 5) .SetRegressionMetric(RegressionMetric.RSquared, label) .SetPipeline(pipeline) - .SetTrainingTimeInSeconds(50) - .SetMaximumMemoryUsageInMegaByte(10); + .SetTrainingTimeInSeconds(50); var result = await experiment.RunAsync(); result.Metric.Should().BeGreaterThan(0.5); @@ -302,10 +324,18 @@ public TrialResult Run(TrialSettings settings) }; } - public Task RunAsync(TrialSettings settings, CancellationToken ct) + public async Task RunAsync(TrialSettings settings, CancellationToken ct) { - ct.Register(() => _ct.ThrowIfCancellationRequested()); - return Task.Run(() => Run(settings)); + _logger.Info("Update Running Trial"); + await Task.Delay(_finishAfterNSeconds * 1000, ct); + _ct.ThrowIfCancellationRequested(); + _logger.Info("Update Completed Trial"); + return new TrialResult + { + TrialSettings = settings, + DurationInMilliseconds = _finishAfterNSeconds * 1000, + Metric = 1.000 + 0.01 * settings.TrialId, + }; } } } From 87f8734518feea1a0fb4aafd13aefb6b4ab0dd11 Mon Sep 17 00:00:00 2001 From: XiaoYun Zhang Date: Thu, 25 Aug 2022 23:09:38 -0700 Subject: [PATCH 05/23] update --- .../API/BinaryClassificationExperiment.cs | 6 ++++-- .../API/MulticlassClassificationExperiment.cs | 4 +++- .../AutoMLExperiment/AutoMLExperiment.cs | 2 +- 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/src/Microsoft.ML.AutoML/API/BinaryClassificationExperiment.cs b/src/Microsoft.ML.AutoML/API/BinaryClassificationExperiment.cs index f4396918ef..01f031a434 100644 --- a/src/Microsoft.ML.AutoML/API/BinaryClassificationExperiment.cs +++ b/src/Microsoft.ML.AutoML/API/BinaryClassificationExperiment.cs @@ -334,7 +334,7 @@ private SweepablePipeline CreateBinaryClassificationPipeline(IDataView trainData internal class BinaryClassificationRunner : ITrialRunner { - private readonly MLContext _context; + private MLContext _context; private readonly IDatasetManager _datasetManager; private readonly IMetricManager _metricManager; private readonly SweepablePipeline _pipeline; @@ -350,6 +350,8 @@ public BinaryClassificationRunner(MLContext context, IDatasetManager datasetMana public void Dispose() { + _context.CancelExecution(); + _context = null; GC.SuppressFinalize(this); } @@ -433,7 +435,7 @@ public Task RunAsync(TrialSettings settings, CancellationToken ct) { ct.Register(() => { - _context?.CancelExecution(); + _context.CancelExecution(); }); try diff --git a/src/Microsoft.ML.AutoML/API/MulticlassClassificationExperiment.cs b/src/Microsoft.ML.AutoML/API/MulticlassClassificationExperiment.cs index 83622fe624..e04a247165 100644 --- a/src/Microsoft.ML.AutoML/API/MulticlassClassificationExperiment.cs +++ b/src/Microsoft.ML.AutoML/API/MulticlassClassificationExperiment.cs @@ -334,7 +334,7 @@ private SweepablePipeline CreateMulticlassClassificationPipeline(IDataView train internal class MulticlassClassificationRunner : ITrialRunner { - private readonly MLContext _context; + private MLContext _context; private readonly IDatasetManager _datasetManager; private readonly IMetricManager _metricManager; private readonly SweepablePipeline _pipeline; @@ -449,6 +449,8 @@ public Task RunAsync(TrialSettings settings, CancellationToken ct) public void Dispose() { + _context.CancelExecution(); + _context = null; GC.SuppressFinalize(this); } } diff --git a/src/Microsoft.ML.AutoML/AutoMLExperiment/AutoMLExperiment.cs b/src/Microsoft.ML.AutoML/AutoMLExperiment/AutoMLExperiment.cs index c52936c81a..3a6f59317f 100644 --- a/src/Microsoft.ML.AutoML/AutoMLExperiment/AutoMLExperiment.cs +++ b/src/Microsoft.ML.AutoML/AutoMLExperiment/AutoMLExperiment.cs @@ -59,7 +59,7 @@ private void InitializeServiceCollection() return context; }); - this.SetPerformanceMonitor(1000); + this.SetPerformanceMonitor(500); _serviceCollection.TryAddSingleton(_settings); _serviceCollection.TryAddSingleton(((IChannelProvider)_context).Start(nameof(AutoMLExperiment))); } From 82c7075ae5f662eebccff3326f746c39c8ce4ca7 Mon Sep 17 00:00:00 2001 From: XiaoYun Zhang Date: Thu, 25 Aug 2022 23:33:44 -0700 Subject: [PATCH 06/23] update --- .../API/BinaryClassificationExperiment.cs | 4 ++++ .../API/MulticlassClassificationExperiment.cs | 5 +++++ 2 files changed, 9 insertions(+) diff --git a/src/Microsoft.ML.AutoML/API/BinaryClassificationExperiment.cs b/src/Microsoft.ML.AutoML/API/BinaryClassificationExperiment.cs index 01f031a434..99baebfba0 100644 --- a/src/Microsoft.ML.AutoML/API/BinaryClassificationExperiment.cs +++ b/src/Microsoft.ML.AutoML/API/BinaryClassificationExperiment.cs @@ -146,6 +146,10 @@ internal BinaryClassificationExperiment(MLContext context, BinaryExperimentSetti TrainerExtensionUtil.GetTrainerNames(settings.Trainers)) { _experiment = context.Auto().CreateExperiment(); + if (settings.MaximumMemoryUsageInMegaByte is double d) + { + _experiment.SetMaximumMemoryUsageInMegaByte(d); + } } public override ExperimentResult Execute(IDataView trainData, ColumnInformation columnInformation, IEstimator preFeaturizer = null, IProgress> progressHandler = null) diff --git a/src/Microsoft.ML.AutoML/API/MulticlassClassificationExperiment.cs b/src/Microsoft.ML.AutoML/API/MulticlassClassificationExperiment.cs index e04a247165..c29d64ec15 100644 --- a/src/Microsoft.ML.AutoML/API/MulticlassClassificationExperiment.cs +++ b/src/Microsoft.ML.AutoML/API/MulticlassClassificationExperiment.cs @@ -137,6 +137,11 @@ internal MulticlassClassificationExperiment(MLContext context, MulticlassExperim TrainerExtensionUtil.GetTrainerNames(settings.Trainers)) { _experiment = context.Auto().CreateExperiment(); + + if (settings.MaximumMemoryUsageInMegaByte is double d) + { + _experiment.SetMaximumMemoryUsageInMegaByte(d); + } } public override ExperimentResult Execute(IDataView trainData, ColumnInformation columnInformation, IEstimator preFeaturizer = null, IProgress> progressHandler = null) From d091aad9ac58134f6fb1f3eb32c980bcc3d7217c Mon Sep 17 00:00:00 2001 From: XiaoYun Zhang Date: Fri, 26 Aug 2022 10:19:17 -0700 Subject: [PATCH 07/23] update --- .../AutoMLExperiment/AutoMLExperiment.cs | 155 +++++++++--------- .../AutoMLExperiment/IPerformanceMonitor.cs | 2 +- 2 files changed, 76 insertions(+), 81 deletions(-) diff --git a/src/Microsoft.ML.AutoML/AutoMLExperiment/AutoMLExperiment.cs b/src/Microsoft.ML.AutoML/AutoMLExperiment/AutoMLExperiment.cs index 3a6f59317f..556dd379f0 100644 --- a/src/Microsoft.ML.AutoML/AutoMLExperiment/AutoMLExperiment.cs +++ b/src/Microsoft.ML.AutoML/AutoMLExperiment/AutoMLExperiment.cs @@ -59,7 +59,7 @@ private void InitializeServiceCollection() return context; }); - this.SetPerformanceMonitor(500); + this.SetPerformanceMonitor(100); _serviceCollection.TryAddSingleton(_settings); _serviceCollection.TryAddSingleton(((IChannelProvider)_context).Start(nameof(AutoMLExperiment))); } @@ -215,110 +215,105 @@ public async Task RunAsync(CancellationToken ct = default) _settings.CancellationToken.Register(() => _globalCancellationTokenSource.Cancel()); var serviceProvider = _serviceCollection.BuildServiceProvider(); var monitor = serviceProvider.GetService(); - + var logger = serviceProvider.GetRequiredService(); var trialNum = 0; var tuner = serviceProvider.GetService(); Contracts.Assert(tuner != null, "tuner can't be null"); - using (var performanceMonitor = serviceProvider.GetService()) + while (!_globalCancellationTokenSource.Token.IsCancellationRequested) { - while (!_globalCancellationTokenSource.Token.IsCancellationRequested) + var setting = new TrialSettings() { - var setting = new TrialSettings() - { - TrialId = trialNum++, - Parameter = Parameter.CreateNestedParameter(), - }; - var parameter = tuner.Propose(setting); - setting.Parameter = parameter; - - monitor?.ReportRunningTrial(setting); - try + TrialId = trialNum++, + Parameter = Parameter.CreateNestedParameter(), + }; + var parameter = tuner.Propose(setting); + setting.Parameter = parameter; + + monitor?.ReportRunningTrial(setting); + try + { + using (var performanceMonitor = serviceProvider.GetService()) + using (var runner = serviceProvider.GetRequiredService()) { - using (var runner = serviceProvider.GetRequiredService()) + var trialCancellationTokenSource = new CancellationTokenSource(); + _globalCancellationTokenSource.Token.Register(() => { - var trialCancellationTokenSource = new CancellationTokenSource(); - _globalCancellationTokenSource.Token.Register(() => - { - // only force-canceling running trials when there's completed trials. - // otherwise, wait for the current running trial to be completed. - if (_bestTrialResult != null) - trialCancellationTokenSource.Cancel(); - }); + // only force-canceling running trials when there's completed trials. + // otherwise, wait for the current running trial to be completed. + if (_bestTrialResult != null) + trialCancellationTokenSource.Cancel(); + }); - performanceMonitor.MemoryUsageInMegaByte += (o, m) => + performanceMonitor.MemoryUsageInMegaByte += (o, m) => + { + if (_settings.MaximumMemoryUsageInMegaByte is double d && m > d) { - if (_settings.MaximumMemoryUsageInMegaByte is double d && m > d) - { - trialCancellationTokenSource.Cancel(); - } - }; + logger.Trace($"cancel current trial {setting.TrialId} because it uses {m} mb memory and the maximum memory usage is {d}"); + trialCancellationTokenSource.Cancel(); + } + }; - performanceMonitor.Start(); + performanceMonitor.Start(); - var trialResult = await runner.RunAsync(setting, trialCancellationTokenSource.Token); + var trialResult = await runner.RunAsync(setting, trialCancellationTokenSource.Token); - var peakCpu = performanceMonitor?.GetPeakCpuUsage(); - var peakMemoryInMB = performanceMonitor?.GetPeakMemoryUsageInMegaByte(); - trialResult.PeakCpu = peakCpu; - trialResult.PeakMemoryInMegaByte = peakMemoryInMB; + var peakCpu = performanceMonitor?.GetPeakCpuUsage(); + var peakMemoryInMB = performanceMonitor?.GetPeakMemoryUsageInMegaByte(); + trialResult.PeakCpu = peakCpu; + trialResult.PeakMemoryInMegaByte = peakMemoryInMB; - monitor?.ReportCompletedTrial(trialResult); - tuner.Update(trialResult); + monitor?.ReportCompletedTrial(trialResult); + tuner.Update(trialResult); - var error = _settings.IsMaximize ? 1 - trialResult.Metric : trialResult.Metric; - if (error < _bestError) - { - _bestTrialResult = trialResult; - _bestError = error; - monitor?.ReportBestTrial(trialResult); - } - } - } - catch (OperationCanceledException ex) when (_globalCancellationTokenSource.IsCancellationRequested == false) - { - monitor?.ReportFailTrial(setting, ex); - var result = new TrialResult + var error = _settings.IsMaximize ? 1 - trialResult.Metric : trialResult.Metric; + if (error < _bestError) { - TrialSettings = setting, - Metric = _settings.IsMaximize ? double.MinValue : double.MaxValue, - }; - - tuner.Update(result); - continue; - } - catch (OperationCanceledException) when (_globalCancellationTokenSource.IsCancellationRequested) - { - break; - } - catch (Exception ex) - { - monitor?.ReportFailTrial(setting, ex); - - if (!_globalCancellationTokenSource.IsCancellationRequested && _bestTrialResult == null) - { - // TODO - // it's questionable on whether to abort the entire training process - // for a single fail trial. We should make it an option and only exit - // when error is fatal (like schema mismatch). - throw; + _bestTrialResult = trialResult; + _bestError = error; + monitor?.ReportBestTrial(trialResult); } } - finally - { - performanceMonitor?.Stop(); - } } + catch (OperationCanceledException ex) when (_globalCancellationTokenSource.IsCancellationRequested == false) + { + monitor?.ReportFailTrial(setting, ex); + var result = new TrialResult + { + TrialSettings = setting, + Metric = _settings.IsMaximize ? double.MinValue : double.MaxValue, + }; - if (_bestTrialResult == null) + tuner.Update(result); + continue; + } + catch (OperationCanceledException) when (_globalCancellationTokenSource.IsCancellationRequested) { - throw new TimeoutException("Training time finished without completing a trial run"); + break; } - else + catch (Exception ex) { - return await Task.FromResult(_bestTrialResult); + monitor?.ReportFailTrial(setting, ex); + + if (!_globalCancellationTokenSource.IsCancellationRequested && _bestTrialResult == null) + { + // TODO + // it's questionable on whether to abort the entire training process + // for a single fail trial. We should make it an option and only exit + // when error is fatal (like schema mismatch). + throw; + } } } + + if (_bestTrialResult == null) + { + throw new TimeoutException("Training time finished without completing a trial run"); + } + else + { + return await Task.FromResult(_bestTrialResult); + } } private void ValidateSettings() diff --git a/src/Microsoft.ML.AutoML/AutoMLExperiment/IPerformanceMonitor.cs b/src/Microsoft.ML.AutoML/AutoMLExperiment/IPerformanceMonitor.cs index 3e4847b13c..33306b1fb7 100644 --- a/src/Microsoft.ML.AutoML/AutoMLExperiment/IPerformanceMonitor.cs +++ b/src/Microsoft.ML.AutoML/AutoMLExperiment/IPerformanceMonitor.cs @@ -105,7 +105,7 @@ private void SampleCpuAndMemoryUsage() // calculate Memory Usage in MB var memoryUsage = process.PrivateMemorySize64 * 1.0 / (1024 * 1024); _peakMemoryUsage = Math.Max(memoryUsage, _peakMemoryUsage ?? 0); - + _logger?.Trace($"current CPU: {cpuUsageInTotal}, current Memory(mb): {memoryUsage}"); MemoryUsageInMegaByte?.Invoke(this, memoryUsage); CpuUsage?.Invoke(this, cpuUsageInTotal); } From e49d9b911c90aa78ced05a552e521ddb4c31d72c Mon Sep 17 00:00:00 2001 From: XiaoYun Zhang Date: Fri, 26 Aug 2022 11:51:10 -0700 Subject: [PATCH 08/23] add trace --- src/Microsoft.ML.AutoML/AutoMLExperiment/AutoMLExperiment.cs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/Microsoft.ML.AutoML/AutoMLExperiment/AutoMLExperiment.cs b/src/Microsoft.ML.AutoML/AutoMLExperiment/AutoMLExperiment.cs index 556dd379f0..49e1ec7213 100644 --- a/src/Microsoft.ML.AutoML/AutoMLExperiment/AutoMLExperiment.cs +++ b/src/Microsoft.ML.AutoML/AutoMLExperiment/AutoMLExperiment.cs @@ -3,6 +3,7 @@ // See the LICENSE file in the project root for more information. using System; +using System.Text.Json; using System.Threading; using System.Threading.Tasks; using Microsoft.Extensions.DependencyInjection; @@ -59,7 +60,7 @@ private void InitializeServiceCollection() return context; }); - this.SetPerformanceMonitor(100); + this.SetPerformanceMonitor(300); _serviceCollection.TryAddSingleton(_settings); _serviceCollection.TryAddSingleton(((IChannelProvider)_context).Start(nameof(AutoMLExperiment))); } @@ -255,7 +256,7 @@ public async Task RunAsync(CancellationToken ct = default) }; performanceMonitor.Start(); - + logger.Trace($"trial setting - {JsonSerializer.Serialize(setting)}"); var trialResult = await runner.RunAsync(setting, trialCancellationTokenSource.Token); var peakCpu = performanceMonitor?.GetPeakCpuUsage(); From b4f2007fb8003e0136a92a01ae0cd6bac889f193 Mon Sep 17 00:00:00 2001 From: XiaoYun Zhang Date: Fri, 26 Aug 2022 12:32:22 -0700 Subject: [PATCH 09/23] add checkalive --- src/Microsoft.ML.FastTree/FastTree.cs | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/Microsoft.ML.FastTree/FastTree.cs b/src/Microsoft.ML.FastTree/FastTree.cs index 381b844bc1..12ee382ef7 100644 --- a/src/Microsoft.ML.FastTree/FastTree.cs +++ b/src/Microsoft.ML.FastTree/FastTree.cs @@ -647,6 +647,7 @@ private protected virtual void Train(IChannel ch) pch.SetHeader(new ProgressHeader("trees"), e => e.SetProgress(0, Ensemble.NumTrees, numTotalTrees)); while (Ensemble.NumTrees < numTotalTrees) { + Host.CheckAlive(); using (Timer.Time(TimerEvent.Iteration)) { #if NO_STORE @@ -687,6 +688,7 @@ private protected virtual void Train(IChannel ch) baggingProvider.GetCurrentOutOfBagPartition().Documents); } + Host.CheckAlive(); CustomizedTrainingIteration(tree); using (Timer.Time(TimerEvent.Test)) @@ -738,6 +740,7 @@ private protected virtual void Train(IChannel ch) } } + Host.CheckAlive(); if (earlyStoppingRule != null) { Contracts.Assert(numTotalTrees == 0 || bestIteration > 0); @@ -750,8 +753,13 @@ private protected virtual void Train(IChannel ch) bestIteration = GetBestIteration(ch); } + Host.CheckAlive(); OptimizationAlgorithm.FinalizeLearning(bestIteration); + + Host.CheckAlive(); Ensemble.PopulateRawThresholds(TrainSet); + + Host.CheckAlive(); ParallelTraining.FinalizeTreeLearner(); } From 9fc36b02b466959fe72530bf1dbc1e88cdb0d681 Mon Sep 17 00:00:00 2001 From: XiaoYun Zhang Date: Fri, 26 Aug 2022 13:11:43 -0700 Subject: [PATCH 10/23] add test for fast tree cancellation --- .../CpuMathUtils.netstandard.cs | 4 ++- .../Microsoft.ML.Tests.csproj | 1 + .../TrainerEstimators/TreeEstimators.cs | 25 +++++++++++++++++++ 3 files changed, 29 insertions(+), 1 deletion(-) diff --git a/src/Microsoft.ML.CpuMath/CpuMathUtils.netstandard.cs b/src/Microsoft.ML.CpuMath/CpuMathUtils.netstandard.cs index e9f3f40f0e..2c8fcf15c9 100644 --- a/src/Microsoft.ML.CpuMath/CpuMathUtils.netstandard.cs +++ b/src/Microsoft.ML.CpuMath/CpuMathUtils.netstandard.cs @@ -20,7 +20,9 @@ public static int GetVectorAlignment() => Vector128Alignment; /// - /// Check if 's alignment is suitable to SSE instructions. Returns + /// + /// + /// if 's alignment is suitable to SSE instructions. Returns /// if 's alignment is ok and otherwise. /// /// The vector being checked. diff --git a/test/Microsoft.ML.Tests/Microsoft.ML.Tests.csproj b/test/Microsoft.ML.Tests/Microsoft.ML.Tests.csproj index adf8e3b2bd..4da6994b3b 100644 --- a/test/Microsoft.ML.Tests/Microsoft.ML.Tests.csproj +++ b/test/Microsoft.ML.Tests/Microsoft.ML.Tests.csproj @@ -59,6 +59,7 @@ + diff --git a/test/Microsoft.ML.Tests/TrainerEstimators/TreeEstimators.cs b/test/Microsoft.ML.Tests/TrainerEstimators/TreeEstimators.cs index 6188b7e34b..c409388227 100644 --- a/test/Microsoft.ML.Tests/TrainerEstimators/TreeEstimators.cs +++ b/test/Microsoft.ML.Tests/TrainerEstimators/TreeEstimators.cs @@ -6,6 +6,7 @@ using System.Collections.Generic; using System.Linq; using System.Threading; +using System.Threading.Tasks; using Microsoft.ML.Calibrators; using Microsoft.ML.Data; using Microsoft.ML.Internal.Utilities; @@ -18,6 +19,7 @@ using Microsoft.ML.Trainers.LightGbm; using Microsoft.ML.Transforms; using Xunit; +using FluentAssertions; namespace Microsoft.ML.Tests.TrainerEstimators { @@ -992,6 +994,29 @@ public void FastForestBinaryClassificationTestSummary() Done(); } + [Fact] + public void FastForestBinaryClassificationCancellationTest() + { + // verify that FastForest can be canceled after training start. + // we need to create a new context for cancellation. + var context = new MLContext(seed: 1); + var (pipeline, dataView) = GetOneHotBinaryClassificationPipeline(); + var estimator = pipeline.Append(context.BinaryClassification.Trainers.FastForest()); + + context.Log += (o, e) => + { + Output.WriteLine(e.Message); + if (e.Message.Contains("Starting to train ...")) + { + context.CancelExecution(); + } + }; + + var action = () => estimator.Fit(dataView); + action.Should().Throw(); + Done(); + } + [LightGBMFact] public void LightGbmBinaryClassificationTestSummary() { From 3f4f29d9c3e4bc79ebb326e53e2f6473d915987d Mon Sep 17 00:00:00 2001 From: XiaoYun Zhang Date: Fri, 26 Aug 2022 14:02:52 -0700 Subject: [PATCH 11/23] add log --- src/Microsoft.ML.FastTree/FastTree.cs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Microsoft.ML.FastTree/FastTree.cs b/src/Microsoft.ML.FastTree/FastTree.cs index 12ee382ef7..c1cf5a5166 100644 --- a/src/Microsoft.ML.FastTree/FastTree.cs +++ b/src/Microsoft.ML.FastTree/FastTree.cs @@ -647,6 +647,7 @@ private protected virtual void Train(IChannel ch) pch.SetHeader(new ProgressHeader("trees"), e => e.SetProgress(0, Ensemble.NumTrees, numTotalTrees)); while (Ensemble.NumTrees < numTotalTrees) { + ch.Trace($"numTotalTrees left: {numTotalTrees}"); Host.CheckAlive(); using (Timer.Time(TimerEvent.Iteration)) { From 039fb8f0ab1756ac8b046ade8b15b537ff457181 Mon Sep 17 00:00:00 2001 From: XiaoYun Zhang Date: Fri, 26 Aug 2022 14:36:02 -0700 Subject: [PATCH 12/23] add log to automl --- .../AutoMLExperiment/AutoMLExperiment.cs | 24 +++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/src/Microsoft.ML.AutoML/AutoMLExperiment/AutoMLExperiment.cs b/src/Microsoft.ML.AutoML/AutoMLExperiment/AutoMLExperiment.cs index 49e1ec7213..f85710488c 100644 --- a/src/Microsoft.ML.AutoML/AutoMLExperiment/AutoMLExperiment.cs +++ b/src/Microsoft.ML.AutoML/AutoMLExperiment/AutoMLExperiment.cs @@ -48,6 +48,7 @@ private void InitializeServiceCollection() { _serviceCollection.TryAddTransient((provider) => { + var channel = provider.GetRequiredService(); var context = new MLContext(_settings.Seed); _globalCancellationTokenSource.Token.Register(() => { @@ -57,6 +58,29 @@ private void InitializeServiceCollection() context.CancelExecution(); }); + context.Log += (o, e) => + { + // Relay logs that are generated by the current MLContext to the Experiment class's + // _logger. + switch (e.Kind) + { + case ChannelMessageKind.Trace: + channel.Trace(e.Message); + break; + case ChannelMessageKind.Info: + channel.Info(e.Message); + break; + case ChannelMessageKind.Warning: + channel.Warning(e.Message); + break; + case ChannelMessageKind.Error: + channel.Error(e.Message); + break; + default: + break; + } + }; + return context; }); From 26e6642c4e9673e9284486bdfb57e2a824877fc5 Mon Sep 17 00:00:00 2001 From: XiaoYun Zhang Date: Fri, 26 Aug 2022 14:37:47 -0700 Subject: [PATCH 13/23] lower refreshing speed --- src/Microsoft.ML.AutoML/AutoMLExperiment/AutoMLExperiment.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Microsoft.ML.AutoML/AutoMLExperiment/AutoMLExperiment.cs b/src/Microsoft.ML.AutoML/AutoMLExperiment/AutoMLExperiment.cs index f85710488c..4fbf50f164 100644 --- a/src/Microsoft.ML.AutoML/AutoMLExperiment/AutoMLExperiment.cs +++ b/src/Microsoft.ML.AutoML/AutoMLExperiment/AutoMLExperiment.cs @@ -84,7 +84,7 @@ private void InitializeServiceCollection() return context; }); - this.SetPerformanceMonitor(300); + this.SetPerformanceMonitor(1000); _serviceCollection.TryAddSingleton(_settings); _serviceCollection.TryAddSingleton(((IChannelProvider)_context).Start(nameof(AutoMLExperiment))); } From 90f1f8d211e51119cfa373f8dab07c2683a47319 Mon Sep 17 00:00:00 2001 From: XiaoYun Zhang Date: Fri, 26 Aug 2022 18:38:33 -0700 Subject: [PATCH 14/23] add cancellation check to lgbm and add test --- .../LightGbmTrainerBase.cs | 4 +- .../WrappedLightGbmTraining.cs | 3 +- .../AutoMLExperimentTests.cs | 29 ++++++++ .../TrainerEstimators/TreeEstimators.cs | 70 +++++++++++++++++-- 4 files changed, 96 insertions(+), 10 deletions(-) diff --git a/src/Microsoft.ML.LightGbm/LightGbmTrainerBase.cs b/src/Microsoft.ML.LightGbm/LightGbmTrainerBase.cs index e8bf78bda9..4f17e19128 100644 --- a/src/Microsoft.ML.LightGbm/LightGbmTrainerBase.cs +++ b/src/Microsoft.ML.LightGbm/LightGbmTrainerBase.cs @@ -619,7 +619,7 @@ private void TrainCore(IChannel ch, IProgressChannel pch, Dataset dtrain, Catego Host.AssertValue(pch); Host.AssertValue(dtrain); Host.AssertValueOrNull(dvalid); - + Host.CheckAlive(); // For multi class, the number of labels is required. ch.Assert(((ITrainer)this).PredictionKind != PredictionKind.MulticlassClassification || GbmOptions.ContainsKey("num_class"), "LightGBM requires the number of classes to be specified in the parameters."); @@ -628,7 +628,7 @@ private void TrainCore(IChannel ch, IProgressChannel pch, Dataset dtrain, Catego lock (LightGbmShared.LockForMultiThreadingInside) { ch.Info("LightGBM objective={0}", GbmOptions["objective"]); - using (Booster bst = WrappedLightGbmTraining.Train(ch, pch, GbmOptions, dtrain, + using (Booster bst = WrappedLightGbmTraining.Train(Host, ch, pch, GbmOptions, dtrain, dvalid: dvalid, numIteration: LightGbmTrainerOptions.NumberOfIterations, verboseEval: LightGbmTrainerOptions.Verbose, earlyStoppingRound: LightGbmTrainerOptions.EarlyStoppingRound)) { diff --git a/src/Microsoft.ML.LightGbm/WrappedLightGbmTraining.cs b/src/Microsoft.ML.LightGbm/WrappedLightGbmTraining.cs index d898018e9a..169467f43a 100644 --- a/src/Microsoft.ML.LightGbm/WrappedLightGbmTraining.cs +++ b/src/Microsoft.ML.LightGbm/WrappedLightGbmTraining.cs @@ -15,7 +15,7 @@ internal static class WrappedLightGbmTraining /// /// Train and return a booster. /// - public static Booster Train(IChannel ch, IProgressChannel pch, + public static Booster Train(IHost host, IChannel ch, IProgressChannel pch, Dictionary parameters, Dataset dtrain, Dataset dvalid = null, int numIteration = 100, bool verboseEval = true, int earlyStoppingRound = 0) { @@ -67,6 +67,7 @@ public static Booster Train(IChannel ch, IProgressChannel pch, }); for (iter = 0; iter < numIteration; ++iter) { + host.CheckAlive(); if (bst.Update()) break; diff --git a/test/Microsoft.ML.AutoML.Tests/AutoMLExperimentTests.cs b/test/Microsoft.ML.AutoML.Tests/AutoMLExperimentTests.cs index 516c68d06c..aaf183fbff 100644 --- a/test/Microsoft.ML.AutoML.Tests/AutoMLExperimentTests.cs +++ b/test/Microsoft.ML.AutoML.Tests/AutoMLExperimentTests.cs @@ -12,8 +12,10 @@ using FluentAssertions; using Microsoft.Data.Analysis; using Microsoft.Extensions.DependencyInjection; +using Microsoft.ML.AutoML.CodeGen; using Microsoft.ML.Runtime; using Microsoft.ML.TestFramework; +using Microsoft.ML.TestFramework.Attributes; using Xunit; using Xunit.Abstractions; @@ -78,6 +80,33 @@ public async Task AutoMLExperiment_cancel_trial_when_exceeds_memory_limit_Async( await runExperimentAction.Should().ThrowExactlyAsync(); } + [LightGBMFact] + public async Task AutoMLExperiment_lgbm_cancel_trial_when_exceeds_memory_limit_Async() + { + // this test is to verify that lightGbm can be cancelled during training booster. + var context = new MLContext(1); + context.Log += (o, e) => + { + if (e.Message.Contains("LightGBM objective")) + { + context.CancelExecution(); + } + }; + var data = DatasetUtil.GetUciAdultDataView(); + var experiment = context.Auto().CreateExperiment(); + var pipeline = context.Auto().Featurizer(data, "_Features_", excludeColumns: new[] { DatasetUtil.UciAdultLabel }) + .Append(context.BinaryClassification.Trainers.LightGbm(DatasetUtil.UciAdultLabel, "_Features_", numberOfIterations: 10000)); + + experiment.SetDataset(context.Data.TrainTestSplit(data)) + .SetBinaryClassificationMetric(BinaryClassificationMetric.AreaUnderRocCurve, DatasetUtil.UciAdultLabel) + .SetPipeline(pipeline) + .SetTrainingTimeInSeconds(10) + .SetMaximumMemoryUsageInMegaByte(10); + + var runExperimentAction = async () => await experiment.RunAsync(); + await runExperimentAction.Should().ThrowExactlyAsync(); + } + [Fact] public async Task AutoMLExperiment_return_current_best_trial_when_ct_is_canceled_with_trial_completed_Async() { diff --git a/test/Microsoft.ML.Tests/TrainerEstimators/TreeEstimators.cs b/test/Microsoft.ML.Tests/TrainerEstimators/TreeEstimators.cs index c409388227..a2f6c39d9c 100644 --- a/test/Microsoft.ML.Tests/TrainerEstimators/TreeEstimators.cs +++ b/test/Microsoft.ML.Tests/TrainerEstimators/TreeEstimators.cs @@ -500,7 +500,7 @@ private void LightGbmHelper(bool useSoftmax, double sigmoid, out string modelStr { var host = (mlContext as IHostEnvironment).Register("Training LightGBM..."); - using (var gbmNative = WrappedLightGbmTraining.Train(ch, pch, gbmParams, gbmDataSet, numIteration: numberOfTrainingIterations)) + using (var gbmNative = WrappedLightGbmTraining.Train(host, ch, pch, gbmParams, gbmDataSet, numIteration: numberOfTrainingIterations)) { int nativeLength = 0; unsafe @@ -995,20 +995,20 @@ public void FastForestBinaryClassificationTestSummary() } [Fact] - public void FastForestBinaryClassificationCancellationTest() + public void FastTreeMultiClassificationCancellationTest() { // verify that FastForest can be canceled after training start. // we need to create a new context for cancellation. var context = new MLContext(seed: 1); - var (pipeline, dataView) = GetOneHotBinaryClassificationPipeline(); - var estimator = pipeline.Append(context.BinaryClassification.Trainers.FastForest()); + var (pipeline, dataView) = GetMulticlassPipeline(); + var estimator = pipeline.Append(context.MulticlassClassification.Trainers.OneVersusAll(context.BinaryClassification.Trainers.FastTree())); context.Log += (o, e) => { - Output.WriteLine(e.Message); - if (e.Message.Contains("Starting to train ...")) + + if (e.Source.StartsWith("FastTreeTraining")) { - context.CancelExecution(); + Output.WriteLine(e.Message); } }; @@ -1042,5 +1042,61 @@ public void LightGbmBinaryClassificationTestSummary() CheckSummary(modelParameters, trainedTreeEnsemble.Bias, trainedTreeEnsemble.TreeWeights, trainedTreeEnsemble.Trees); Done(); } + + [LightGBMFact] + public void LightGbmBinaryClassificationCancellationTest() + { + var context = new MLContext(seed: 1); + context.Log += (o, e) => + { + if (e.Message.Contains("LightGBM objective")) + { + context.CancelExecution(); + } + }; + + var (pipeline, dataView) = GetOneHotBinaryClassificationPipeline(); + + // Attention: Do not set NumberOfThreads here, left this to use default value to avoid test crash. + // Details can be found here: https://github.com/dotnet/machinelearning/pull/4918 + var trainer = pipeline.Append(context.BinaryClassification.Trainers.LightGbm( + new LightGbmBinaryTrainer.Options + { + NumberOfIterations = 1000, + NumberOfLeaves = 5, + UseCategoricalSplit = true + })); + + var action = () => trainer.Fit(dataView); + action.Should().Throw(); + } + + [LightGBMFact] + public void LightGbmMultiClassificationCancellationTest() + { + var context = new MLContext(seed: 1); + context.Log += (o, e) => + { + if (e.Message.Contains("LightGBM objective")) + { + context.CancelExecution(); + } + }; + + var (pipeline, dataView) = GetMulticlassPipeline(); + + // Attention: Do not set NumberOfThreads here, left this to use default value to avoid test crash. + // Details can be found here: https://github.com/dotnet/machinelearning/pull/4918 + var trainer = pipeline.Append(context.MulticlassClassification.Trainers.LightGbm( + new LightGbmMulticlassTrainer.Options + { + NumberOfIterations = 1000, + NumberOfLeaves = 5, + UseCategoricalSplit = true + })); + + var action = () => trainer.Fit(dataView); + action.Should().Throw(); + } } } From cc1ca6888fc73f3aa93a74aa8f33c76fd0e0305b Mon Sep 17 00:00:00 2001 From: XiaoYun Zhang Date: Sun, 28 Aug 2022 15:41:15 -0700 Subject: [PATCH 15/23] dispose ml context --- .../AutoMLExperiment/Runner/SweepablePipelineRunner.cs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/Microsoft.ML.AutoML/AutoMLExperiment/Runner/SweepablePipelineRunner.cs b/src/Microsoft.ML.AutoML/AutoMLExperiment/Runner/SweepablePipelineRunner.cs index c823d53596..32dbfb9fdc 100644 --- a/src/Microsoft.ML.AutoML/AutoMLExperiment/Runner/SweepablePipelineRunner.cs +++ b/src/Microsoft.ML.AutoML/AutoMLExperiment/Runner/SweepablePipelineRunner.cs @@ -17,7 +17,7 @@ namespace Microsoft.ML.AutoML { internal class SweepablePipelineRunner : ITrialRunner { - private readonly MLContext? _mLContext; + private MLContext? _mLContext; private readonly IEvaluateMetricManager _metricManager; private readonly IDatasetManager _datasetManager; private readonly SweepablePipeline _pipeline; @@ -104,6 +104,8 @@ public Task RunAsync(TrialSettings settings, CancellationToken ct) public void Dispose() { + _mLContext!.CancelExecution(); + _mLContext = null; GC.SuppressFinalize(this); } } From 3c96c4e607e8390ec0be43b4594482d44432b93a Mon Sep 17 00:00:00 2001 From: XiaoYun Zhang Date: Sun, 28 Aug 2022 15:53:27 -0700 Subject: [PATCH 16/23] update --- .../API/BinaryClassificationExperiment.cs | 13 ++++++------ .../API/MulticlassClassificationExperiment.cs | 13 ++++++------ .../AutoMLExperiment/AutoMLExperiment.cs | 21 +++++++++---------- .../Runner/SweepablePipelineRunner.cs | 13 ++++++------ 4 files changed, 31 insertions(+), 29 deletions(-) diff --git a/src/Microsoft.ML.AutoML/API/BinaryClassificationExperiment.cs b/src/Microsoft.ML.AutoML/API/BinaryClassificationExperiment.cs index 99baebfba0..36814405ce 100644 --- a/src/Microsoft.ML.AutoML/API/BinaryClassificationExperiment.cs +++ b/src/Microsoft.ML.AutoML/API/BinaryClassificationExperiment.cs @@ -437,14 +437,15 @@ public TrialResult Run(TrialSettings settings) public Task RunAsync(TrialSettings settings, CancellationToken ct) { - ct.Register(() => - { - _context.CancelExecution(); - }); - try { - return Task.FromResult(Run(settings)); + using (var ctRegistration = ct.Register(() => + { + _context?.CancelExecution(); + })) + { + return Task.FromResult(Run(settings)); + } } catch (Exception ex) when (ct.IsCancellationRequested) { diff --git a/src/Microsoft.ML.AutoML/API/MulticlassClassificationExperiment.cs b/src/Microsoft.ML.AutoML/API/MulticlassClassificationExperiment.cs index c29d64ec15..44d20a4ee4 100644 --- a/src/Microsoft.ML.AutoML/API/MulticlassClassificationExperiment.cs +++ b/src/Microsoft.ML.AutoML/API/MulticlassClassificationExperiment.cs @@ -433,14 +433,15 @@ public TrialResult Run(TrialSettings settings) public Task RunAsync(TrialSettings settings, CancellationToken ct) { - ct.Register(() => - { - _context?.CancelExecution(); - }); - try { - return Task.FromResult(Run(settings)); + using (var ctRegistration = ct.Register(() => + { + _context?.CancelExecution(); + })) + { + return Task.FromResult(Run(settings)); + } } catch (Exception ex) when (ct.IsCancellationRequested) { diff --git a/src/Microsoft.ML.AutoML/AutoMLExperiment/AutoMLExperiment.cs b/src/Microsoft.ML.AutoML/AutoMLExperiment/AutoMLExperiment.cs index 4fbf50f164..ddf94b5308 100644 --- a/src/Microsoft.ML.AutoML/AutoMLExperiment/AutoMLExperiment.cs +++ b/src/Microsoft.ML.AutoML/AutoMLExperiment/AutoMLExperiment.cs @@ -84,7 +84,7 @@ private void InitializeServiceCollection() return context; }); - this.SetPerformanceMonitor(1000); + this.SetPerformanceMonitor(2000); _serviceCollection.TryAddSingleton(_settings); _serviceCollection.TryAddSingleton(((IChannelProvider)_context).Start(nameof(AutoMLExperiment))); } @@ -258,21 +258,20 @@ public async Task RunAsync(CancellationToken ct = default) monitor?.ReportRunningTrial(setting); try { + using (var trialCancellationTokenSource = new CancellationTokenSource()) + using (var deregisterCallback = _globalCancellationTokenSource.Token.Register(() => + { + // only force-canceling running trials when there's completed trials. + // otherwise, wait for the current running trial to be completed. + if (_bestTrialResult != null) + trialCancellationTokenSource.Cancel(); + })) using (var performanceMonitor = serviceProvider.GetService()) using (var runner = serviceProvider.GetRequiredService()) { - var trialCancellationTokenSource = new CancellationTokenSource(); - _globalCancellationTokenSource.Token.Register(() => - { - // only force-canceling running trials when there's completed trials. - // otherwise, wait for the current running trial to be completed. - if (_bestTrialResult != null) - trialCancellationTokenSource.Cancel(); - }); - performanceMonitor.MemoryUsageInMegaByte += (o, m) => { - if (_settings.MaximumMemoryUsageInMegaByte is double d && m > d) + if (_settings.MaximumMemoryUsageInMegaByte is double d && m > d && !trialCancellationTokenSource.IsCancellationRequested) { logger.Trace($"cancel current trial {setting.TrialId} because it uses {m} mb memory and the maximum memory usage is {d}"); trialCancellationTokenSource.Cancel(); diff --git a/src/Microsoft.ML.AutoML/AutoMLExperiment/Runner/SweepablePipelineRunner.cs b/src/Microsoft.ML.AutoML/AutoMLExperiment/Runner/SweepablePipelineRunner.cs index 32dbfb9fdc..5f6d61945e 100644 --- a/src/Microsoft.ML.AutoML/AutoMLExperiment/Runner/SweepablePipelineRunner.cs +++ b/src/Microsoft.ML.AutoML/AutoMLExperiment/Runner/SweepablePipelineRunner.cs @@ -83,14 +83,15 @@ public TrialResult Run(TrialSettings settings) public Task RunAsync(TrialSettings settings, CancellationToken ct) { - ct.Register(() => - { - _mLContext?.CancelExecution(); - }); - try { - return Task.FromResult(Run(settings)); + using (var ctRegistration = ct.Register(() => + { + _mLContext?.CancelExecution(); + })) + { + return Task.FromResult(Run(settings)); + } } catch (Exception ex) when (ct.IsCancellationRequested) { From 937aa8a889d6bce3d32b741b8bb973a2733c532d Mon Sep 17 00:00:00 2001 From: XiaoYun Zhang Date: Mon, 29 Aug 2022 10:35:11 -0700 Subject: [PATCH 17/23] fix tests --- test/Microsoft.ML.Tests/TrainerEstimators/TreeEstimators.cs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/test/Microsoft.ML.Tests/TrainerEstimators/TreeEstimators.cs b/test/Microsoft.ML.Tests/TrainerEstimators/TreeEstimators.cs index a2f6c39d9c..2384545fa0 100644 --- a/test/Microsoft.ML.Tests/TrainerEstimators/TreeEstimators.cs +++ b/test/Microsoft.ML.Tests/TrainerEstimators/TreeEstimators.cs @@ -1008,13 +1008,12 @@ public void FastTreeMultiClassificationCancellationTest() if (e.Source.StartsWith("FastTreeTraining")) { - Output.WriteLine(e.Message); + context.CancelExecution(); } }; var action = () => estimator.Fit(dataView); action.Should().Throw(); - Done(); } [LightGBMFact] From 16eeeae3fa81e3558e431807b809d053e8ac78e8 Mon Sep 17 00:00:00 2001 From: XiaoYun Zhang Date: Wed, 31 Aug 2022 14:26:52 -0700 Subject: [PATCH 18/23] fix based on comment --- src/Microsoft.ML.AutoML/API/BinaryClassificationExperiment.cs | 3 +-- .../API/MulticlassClassificationExperiment.cs | 3 +-- src/Microsoft.ML.AutoML/AutoMLExperiment/AutoMLExperiment.cs | 3 +++ .../AutoMLExperiment/Runner/SweepablePipelineRunner.cs | 3 +-- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/Microsoft.ML.AutoML/API/BinaryClassificationExperiment.cs b/src/Microsoft.ML.AutoML/API/BinaryClassificationExperiment.cs index 36814405ce..e26826966e 100644 --- a/src/Microsoft.ML.AutoML/API/BinaryClassificationExperiment.cs +++ b/src/Microsoft.ML.AutoML/API/BinaryClassificationExperiment.cs @@ -356,7 +356,6 @@ public void Dispose() { _context.CancelExecution(); _context = null; - GC.SuppressFinalize(this); } public TrialResult Run(TrialSettings settings) @@ -444,7 +443,7 @@ public Task RunAsync(TrialSettings settings, CancellationToken ct) _context?.CancelExecution(); })) { - return Task.FromResult(Run(settings)); + return Task.Run(() => Run(settings)); } } catch (Exception ex) when (ct.IsCancellationRequested) diff --git a/src/Microsoft.ML.AutoML/API/MulticlassClassificationExperiment.cs b/src/Microsoft.ML.AutoML/API/MulticlassClassificationExperiment.cs index 44d20a4ee4..67ead38f0e 100644 --- a/src/Microsoft.ML.AutoML/API/MulticlassClassificationExperiment.cs +++ b/src/Microsoft.ML.AutoML/API/MulticlassClassificationExperiment.cs @@ -440,7 +440,7 @@ public Task RunAsync(TrialSettings settings, CancellationToken ct) _context?.CancelExecution(); })) { - return Task.FromResult(Run(settings)); + return Task.Run(() => Run(settings)); } } catch (Exception ex) when (ct.IsCancellationRequested) @@ -457,7 +457,6 @@ public void Dispose() { _context.CancelExecution(); _context = null; - GC.SuppressFinalize(this); } } } diff --git a/src/Microsoft.ML.AutoML/AutoMLExperiment/AutoMLExperiment.cs b/src/Microsoft.ML.AutoML/AutoMLExperiment/AutoMLExperiment.cs index ddf94b5308..91a02955a3 100644 --- a/src/Microsoft.ML.AutoML/AutoMLExperiment/AutoMLExperiment.cs +++ b/src/Microsoft.ML.AutoML/AutoMLExperiment/AutoMLExperiment.cs @@ -275,6 +275,9 @@ public async Task RunAsync(CancellationToken ct = default) { logger.Trace($"cancel current trial {setting.TrialId} because it uses {m} mb memory and the maximum memory usage is {d}"); trialCancellationTokenSource.Cancel(); + + GC.AddMemoryPressure(Convert.ToInt64(m) * 1024 * 1024); + GC.Collect(); } }; diff --git a/src/Microsoft.ML.AutoML/AutoMLExperiment/Runner/SweepablePipelineRunner.cs b/src/Microsoft.ML.AutoML/AutoMLExperiment/Runner/SweepablePipelineRunner.cs index 5f6d61945e..b4913e1238 100644 --- a/src/Microsoft.ML.AutoML/AutoMLExperiment/Runner/SweepablePipelineRunner.cs +++ b/src/Microsoft.ML.AutoML/AutoMLExperiment/Runner/SweepablePipelineRunner.cs @@ -90,7 +90,7 @@ public Task RunAsync(TrialSettings settings, CancellationToken ct) _mLContext?.CancelExecution(); })) { - return Task.FromResult(Run(settings)); + return Task.Run(() => Run(settings)); } } catch (Exception ex) when (ct.IsCancellationRequested) @@ -107,7 +107,6 @@ public void Dispose() { _mLContext!.CancelExecution(); _mLContext = null; - GC.SuppressFinalize(this); } } } From 6bf43608f33796bd30e773fdf2d8b3ed541f730a Mon Sep 17 00:00:00 2001 From: XiaoYun Zhang Date: Thu, 1 Sep 2022 13:54:23 -0700 Subject: [PATCH 19/23] fix comments --- .../AutoMLExperiment/AutoMLExperiment.cs | 11 ++++------- .../AutoMLExperiment/IPerformanceMonitor.cs | 7 ++++++- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/src/Microsoft.ML.AutoML/AutoMLExperiment/AutoMLExperiment.cs b/src/Microsoft.ML.AutoML/AutoMLExperiment/AutoMLExperiment.cs index 91a02955a3..aeed0de390 100644 --- a/src/Microsoft.ML.AutoML/AutoMLExperiment/AutoMLExperiment.cs +++ b/src/Microsoft.ML.AutoML/AutoMLExperiment/AutoMLExperiment.cs @@ -89,11 +89,6 @@ private void InitializeServiceCollection() _serviceCollection.TryAddSingleton(((IChannelProvider)_context).Start(nameof(AutoMLExperiment))); } - private void Initialize() - { - _globalCancellationTokenSource = new CancellationTokenSource(); - } - internal IServiceCollection ServiceCollection { get => _serviceCollection; } public AutoMLExperiment SetTrainingTimeInSeconds(uint trainingTimeInSeconds) @@ -110,6 +105,7 @@ public AutoMLExperiment SetIsMaximize(bool isMaximize) public AutoMLExperiment SetMaximumMemoryUsageInMegaByte(double value = double.MaxValue) { + Contracts.Assert(!double.IsNaN(value) && value > 0, "value can't be nan or non-positive"); _settings.MaximumMemoryUsageInMegaByte = value; return this; } @@ -234,9 +230,10 @@ public TrialResult Run() public async Task RunAsync(CancellationToken ct = default) { ValidateSettings(); - Initialize(); + _globalCancellationTokenSource = new CancellationTokenSource(); _settings.CancellationToken = ct; - _globalCancellationTokenSource.CancelAfter((int)_settings.MaxExperimentTimeInSeconds * 1000); + // use TimeSpan to avoid overflow. + _globalCancellationTokenSource.CancelAfter(TimeSpan.FromSeconds(_settings.MaxExperimentTimeInSeconds * 1000)); _settings.CancellationToken.Register(() => _globalCancellationTokenSource.Cancel()); var serviceProvider = _serviceCollection.BuildServiceProvider(); var monitor = serviceProvider.GetService(); diff --git a/src/Microsoft.ML.AutoML/AutoMLExperiment/IPerformanceMonitor.cs b/src/Microsoft.ML.AutoML/AutoMLExperiment/IPerformanceMonitor.cs index 33306b1fb7..fa86bb5894 100644 --- a/src/Microsoft.ML.AutoML/AutoMLExperiment/IPerformanceMonitor.cs +++ b/src/Microsoft.ML.AutoML/AutoMLExperiment/IPerformanceMonitor.cs @@ -12,7 +12,7 @@ namespace Microsoft.ML.AutoML { - public interface IPerformanceMonitor : IDisposable + internal interface IPerformanceMonitor : IDisposable { void Start(); @@ -93,6 +93,11 @@ private void OnCheckCpuAndMemoryUsage(object source, ElapsedEventArgs e) private void SampleCpuAndMemoryUsage() { // calculate CPU usage in % + // the % of CPU usage is calculating in the following way + // for every _totalCpuProcessorTime + // total CPU time is _totalCpuProcessorTime * ProcessorCount + // total CPU time used by current process is currentCpuProcessorTime + // the % of CPU usage by current process is simply currentCpuProcessorTime / total CPU time. using (var process = Process.GetCurrentProcess()) { var currentCpuProcessorTime = Process.GetCurrentProcess().TotalProcessorTime; From b1d50c0cccbde9ea9ea8b49e49c59f40cfb4db7b Mon Sep 17 00:00:00 2001 From: XiaoYun Zhang Date: Thu, 1 Sep 2022 17:05:04 -0700 Subject: [PATCH 20/23] fix time convertion error --- src/Microsoft.ML.AutoML/AutoMLExperiment/AutoMLExperiment.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Microsoft.ML.AutoML/AutoMLExperiment/AutoMLExperiment.cs b/src/Microsoft.ML.AutoML/AutoMLExperiment/AutoMLExperiment.cs index aeed0de390..51d2257d8e 100644 --- a/src/Microsoft.ML.AutoML/AutoMLExperiment/AutoMLExperiment.cs +++ b/src/Microsoft.ML.AutoML/AutoMLExperiment/AutoMLExperiment.cs @@ -233,7 +233,7 @@ public async Task RunAsync(CancellationToken ct = default) _globalCancellationTokenSource = new CancellationTokenSource(); _settings.CancellationToken = ct; // use TimeSpan to avoid overflow. - _globalCancellationTokenSource.CancelAfter(TimeSpan.FromSeconds(_settings.MaxExperimentTimeInSeconds * 1000)); + _globalCancellationTokenSource.CancelAfter(TimeSpan.FromSeconds(_settings.MaxExperimentTimeInSeconds)); _settings.CancellationToken.Register(() => _globalCancellationTokenSource.Cancel()); var serviceProvider = _serviceCollection.BuildServiceProvider(); var monitor = serviceProvider.GetService(); From b517c9f2ad74d9cc10c3043aa6b0f989c26d7272 Mon Sep 17 00:00:00 2001 From: XiaoYun Zhang Date: Fri, 2 Sep 2022 11:12:15 -0700 Subject: [PATCH 21/23] add log to AutoMLExperiment_cancel_trial_when_exceeds_memory_limit_Async --- test/Microsoft.ML.AutoML.Tests/AutoMLExperimentTests.cs | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/test/Microsoft.ML.AutoML.Tests/AutoMLExperimentTests.cs b/test/Microsoft.ML.AutoML.Tests/AutoMLExperimentTests.cs index aaf183fbff..5d192666ff 100644 --- a/test/Microsoft.ML.AutoML.Tests/AutoMLExperimentTests.cs +++ b/test/Microsoft.ML.AutoML.Tests/AutoMLExperimentTests.cs @@ -62,7 +62,13 @@ public async Task AutoMLExperiment_cancel_trial_when_exceeds_memory_limit_Async( { var context = new MLContext(1); var experiment = context.Auto().CreateExperiment(); - + context.Log += (o, e) => + { + if (e.Source.StartsWith("AutoMLExperiment")) + { + this.Output.WriteLine(e.RawMessage); + } + }; // the following experiment set memory usage limit to 0.01mb // so all trials should be canceled and there should be no successful trials. // therefore when experiment finishes, it should throw timeout exception with no model trained message. From 29bbc266a13312ed1858708a6eea7622e4a290e1 Mon Sep 17 00:00:00 2001 From: XiaoYun Zhang Date: Fri, 2 Sep 2022 11:14:11 -0700 Subject: [PATCH 22/23] rename catelogical column --- src/Microsoft.ML.AutoML/API/AutoCatalog.cs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/Microsoft.ML.AutoML/API/AutoCatalog.cs b/src/Microsoft.ML.AutoML/API/AutoCatalog.cs index ed8a2a9ef4..280777db70 100644 --- a/src/Microsoft.ML.AutoML/API/AutoCatalog.cs +++ b/src/Microsoft.ML.AutoML/API/AutoCatalog.cs @@ -654,18 +654,18 @@ internal SweepablePipeline ImagePathFeaturizer(string outputColumnName, string i /// them into a single feature column as output. /// /// input data. - /// columns that should be treated as catalog. If not specified, it will automatically infer if a column is catalog or not. + /// columns that should be treated as catalog. If not specified, it will automatically infer if a column is catalog or not. /// columns that should be treated as numeric. If not specified, it will automatically infer if a column is catalog or not. /// columns that should be treated as text. If not specified, it will automatically infer if a column is catalog or not. /// columns that should be treated as image path. If not specified, it will automatically infer if a column is catalog or not. /// output feature column. /// columns that won't be included when featurizing, like label - public SweepablePipeline Featurizer(IDataView data, string outputColumnName = "Features", string[] catalogColumns = null, string[] numericColumns = null, string[] textColumns = null, string[] imagePathColumns = null, string[] excludeColumns = null) + public SweepablePipeline Featurizer(IDataView data, string outputColumnName = "Features", string[] catelogicalColumns = null, string[] numericColumns = null, string[] textColumns = null, string[] imagePathColumns = null, string[] excludeColumns = null) { Contracts.CheckValue(data, nameof(data)); // validate if there's overlapping among catalogColumns, numericColumns, textColumns and excludeColumns - var overallColumns = new string[][] { catalogColumns, numericColumns, textColumns, excludeColumns } + var overallColumns = new string[][] { catelogicalColumns, numericColumns, textColumns, excludeColumns } .Where(c => c != null) .SelectMany(c => c); @@ -684,9 +684,9 @@ public SweepablePipeline Featurizer(IDataView data, string outputColumnName = "F } } - if (catalogColumns != null) + if (catelogicalColumns != null) { - foreach (var catalogColumn in catalogColumns) + foreach (var catalogColumn in catelogicalColumns) { columnInfo.CategoricalColumnNames.Add(catalogColumn); } From 54964031a4e7627091e75bc090ba0b06d55f98c3 Mon Sep 17 00:00:00 2001 From: XiaoYun Zhang Date: Tue, 6 Sep 2022 15:51:46 -0700 Subject: [PATCH 23/23] use dummy performance monitor --- .../AutoMLExperimentTests.cs | 55 ++++++++++++++++++- 1 file changed, 54 insertions(+), 1 deletion(-) diff --git a/test/Microsoft.ML.AutoML.Tests/AutoMLExperimentTests.cs b/test/Microsoft.ML.AutoML.Tests/AutoMLExperimentTests.cs index 5d192666ff..e38dace417 100644 --- a/test/Microsoft.ML.AutoML.Tests/AutoMLExperimentTests.cs +++ b/test/Microsoft.ML.AutoML.Tests/AutoMLExperimentTests.cs @@ -80,7 +80,8 @@ public async Task AutoMLExperiment_cancel_trial_when_exceeds_memory_limit_Async( return new DummyTrialRunner(settings, 5, channel); }) .SetTuner() - .SetMaximumMemoryUsageInMegaByte(0.01); + .SetMaximumMemoryUsageInMegaByte(0.01) + .SetPerformanceMonitor(); var runExperimentAction = async () => await experiment.RunAsync(); await runExperimentAction.Should().ThrowExactlyAsync(); @@ -373,4 +374,56 @@ public async Task RunAsync(TrialSettings settings, CancellationToke }; } } + + class DummyPeformanceMonitor : IPerformanceMonitor + { + private readonly int _checkIntervalInMilliseconds; + private System.Timers.Timer _timer; + + public DummyPeformanceMonitor() + { + _checkIntervalInMilliseconds = 1000; + } + + public event EventHandler CpuUsage; + + public event EventHandler MemoryUsageInMegaByte; + + public void Dispose() + { + } + + public double? GetPeakCpuUsage() + { + return 100; + } + + public double? GetPeakMemoryUsageInMegaByte() + { + return 1000; + } + + public void Start() + { + if (_timer == null) + { + _timer = new System.Timers.Timer(_checkIntervalInMilliseconds); + _timer.Elapsed += (o, e) => + { + CpuUsage?.Invoke(this, 100); + MemoryUsageInMegaByte?.Invoke(this, 1000); + }; + + _timer.AutoReset = true; + _timer.Enabled = true; + } + } + + public void Stop() + { + _timer?.Stop(); + _timer?.Dispose(); + _timer = null; + } + } }