diff --git a/Microsoft.ML.sln b/Microsoft.ML.sln index 87fe950e9c..93d58d6721 100644 --- a/Microsoft.ML.sln +++ b/Microsoft.ML.sln @@ -1,4 +1,4 @@ -Microsoft Visual Studio Solution File, Format Version 12.00 +Microsoft Visual Studio Solution File, Format Version 12.00 # Visual Studio Version 17 VisualStudioVersion = 17.1.32120.378 MinimumVisualStudioVersion = 10.0.40219.1 @@ -159,6 +159,8 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.ML.Tokenizers", " EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.ML.Tokenizers.Tests", "test\Microsoft.ML.Tokenizers.Tests\Microsoft.ML.Tokenizers.Tests.csproj", "{C3D82402-F207-4F19-8C57-5AF0FBAF9682}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Microsoft.ML.XGBoost", "src\Microsoft.ML.XGBoost\Microsoft.ML.XGBoost.csproj", "{9CF22D6B-3094-4F42-9CBF-1B07087CF1EE}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -407,6 +409,14 @@ Global {A7222F41-1CF0-47D9-B80C-B4D77B027A61}.Release|Any CPU.Build.0 = Release|Any CPU {A7222F41-1CF0-47D9-B80C-B4D77B027A61}.Release|x64.ActiveCfg = Release|Any CPU {A7222F41-1CF0-47D9-B80C-B4D77B027A61}.Release|x64.Build.0 = Release|Any CPU + {A7222F94-2AF1-10C9-A21C-C4D22B137A69}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {A7222F94-2AF1-10C9-A21C-C4D22B137A69}.Debug|Any CPU.Build.0 = Debug|Any CPU + {A7222F94-2AF1-10C9-A21C-C4D22B137A69}.Debug|x64.ActiveCfg = Debug|Any CPU + {A7222F94-2AF1-10C9-A21C-C4D22B137A69}.Debug|x64.Build.0 = Debug|Any CPU + {A7222F94-2AF1-10C9-A21C-C4D22B137A69}.Release|Any CPU.ActiveCfg = Release|Any CPU + {A7222F94-2AF1-10C9-A21C-C4D22B137A69}.Release|Any CPU.Build.0 = Release|Any CPU + {A7222F94-2AF1-10C9-A21C-C4D22B137A69}.Release|x64.ActiveCfg = Release|Any CPU + {A7222F94-2AF1-10C9-A21C-C4D22B137A69}.Release|x64.Build.0 = Release|Any CPU {570A0B8A-5463-44D2-8521-54C0CA4CACA9}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {570A0B8A-5463-44D2-8521-54C0CA4CACA9}.Debug|Any CPU.Build.0 = Debug|Any CPU {570A0B8A-5463-44D2-8521-54C0CA4CACA9}.Debug|x64.ActiveCfg = Debug|Any CPU @@ -747,6 +757,14 @@ Global {C3D82402-F207-4F19-8C57-5AF0FBAF9682}.Release|Any CPU.Build.0 = Release|Any CPU {C3D82402-F207-4F19-8C57-5AF0FBAF9682}.Release|x64.ActiveCfg = Release|Any CPU {C3D82402-F207-4F19-8C57-5AF0FBAF9682}.Release|x64.Build.0 = Release|Any CPU + {9CF22D6B-3094-4F42-9CBF-1B07087CF1EE}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {9CF22D6B-3094-4F42-9CBF-1B07087CF1EE}.Debug|Any CPU.Build.0 = Debug|Any CPU + {9CF22D6B-3094-4F42-9CBF-1B07087CF1EE}.Debug|x64.ActiveCfg = Debug|Any CPU + {9CF22D6B-3094-4F42-9CBF-1B07087CF1EE}.Debug|x64.Build.0 = Debug|Any CPU + {9CF22D6B-3094-4F42-9CBF-1B07087CF1EE}.Release|Any CPU.ActiveCfg = Release|Any CPU + {9CF22D6B-3094-4F42-9CBF-1B07087CF1EE}.Release|Any CPU.Build.0 = Release|Any CPU + {9CF22D6B-3094-4F42-9CBF-1B07087CF1EE}.Release|x64.ActiveCfg = Release|Any CPU + {9CF22D6B-3094-4F42-9CBF-1B07087CF1EE}.Release|x64.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -825,6 +843,7 @@ Global {FF0BD187-4451-4A3B-934B-2AE3454896E2} = {09EADF06-BE25-4228-AB53-95AE3E15B530} {BBC3A950-BD68-45AC-9DBD-A8F4D8847745} = {09EADF06-BE25-4228-AB53-95AE3E15B530} {C3D82402-F207-4F19-8C57-5AF0FBAF9682} = {AED9C836-31E3-4F3F-8ABC-929555D3F3C4} + {9CF22D6B-3094-4F42-9CBF-1B07087CF1EE} = {09EADF06-BE25-4228-AB53-95AE3E15B530} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {41165AF1-35BB-4832-A189-73060F82B01D} diff --git a/src/Microsoft.ML.Core/Properties/AssemblyInfo.cs b/src/Microsoft.ML.Core/Properties/AssemblyInfo.cs index bbfd94eee5..93bc7fd305 100644 --- a/src/Microsoft.ML.Core/Properties/AssemblyInfo.cs +++ b/src/Microsoft.ML.Core/Properties/AssemblyInfo.cs @@ -27,6 +27,7 @@ [assembly: InternalsVisibleTo(assemblyName: "Microsoft.ML.Mkl.Components" + PublicKey.Value)] [assembly: InternalsVisibleTo(assemblyName: "Microsoft.ML.KMeansClustering" + PublicKey.Value)] [assembly: InternalsVisibleTo(assemblyName: "Microsoft.ML.LightGbm" + PublicKey.Value)] +[assembly: InternalsVisibleTo(assemblyName: "Microsoft.ML.XGBoost" + PublicKey.Value)] [assembly: InternalsVisibleTo(assemblyName: "Microsoft.ML.OnnxConverter" + PublicKey.Value)] [assembly: InternalsVisibleTo(assemblyName: "Microsoft.ML.OnnxTransformer" + PublicKey.Value)] [assembly: InternalsVisibleTo(assemblyName: "Microsoft.ML.Parquet" + PublicKey.Value)] diff --git a/src/Microsoft.ML.Data/Properties/AssemblyInfo.cs b/src/Microsoft.ML.Data/Properties/AssemblyInfo.cs index d8e6944256..efed04f2af 100644 --- a/src/Microsoft.ML.Data/Properties/AssemblyInfo.cs +++ b/src/Microsoft.ML.Data/Properties/AssemblyInfo.cs @@ -26,6 +26,7 @@ [assembly: InternalsVisibleTo(assemblyName: "Microsoft.ML.Mkl.Components" + PublicKey.Value)] [assembly: InternalsVisibleTo(assemblyName: "Microsoft.ML.KMeansClustering" + PublicKey.Value)] [assembly: InternalsVisibleTo(assemblyName: "Microsoft.ML.LightGbm" + PublicKey.Value)] +[assembly: InternalsVisibleTo(assemblyName: "Microsoft.ML.XGBoost" + PublicKey.Value)] [assembly: InternalsVisibleTo(assemblyName: "Microsoft.ML.OnnxConverter" + PublicKey.Value)] [assembly: InternalsVisibleTo(assemblyName: "Microsoft.ML.OnnxTransformer" + PublicKey.Value)] [assembly: InternalsVisibleTo(assemblyName: "Microsoft.ML.Parquet" + PublicKey.Value)] diff --git a/src/Microsoft.ML.FastTree/Properties/AssemblyInfo.cs b/src/Microsoft.ML.FastTree/Properties/AssemblyInfo.cs index 4a1575a5aa..4ee0690c06 100644 --- a/src/Microsoft.ML.FastTree/Properties/AssemblyInfo.cs +++ b/src/Microsoft.ML.FastTree/Properties/AssemblyInfo.cs @@ -9,6 +9,7 @@ [assembly: InternalsVisibleTo(assemblyName: "Microsoft.ML.Predictor.Tests" + PublicKey.Value)] [assembly: InternalsVisibleTo(assemblyName: "Microsoft.ML.LightGbm" + PublicKey.Value)] +[assembly: InternalsVisibleTo(assemblyName: "Microsoft.ML.XGBoost" + PublicKey.Value)] [assembly: InternalsVisibleTo(assemblyName: "Microsoft.ML.Sweeper" + PublicKey.Value)] [assembly: InternalsVisibleTo(assemblyName: "Microsoft.ML.Internal.FastTree" + InternalPublicKey.Value)] diff --git a/src/Microsoft.ML.XGBoost/Microsoft.ML.XGBoost.csproj b/src/Microsoft.ML.XGBoost/Microsoft.ML.XGBoost.csproj new file mode 100644 index 0000000000..3a955d4ac7 --- /dev/null +++ b/src/Microsoft.ML.XGBoost/Microsoft.ML.XGBoost.csproj @@ -0,0 +1,34 @@ + + + + + netstandard2.0 + Microsoft.ML.XGBoost + true + ML.NET component for XGBoost + + + + + + + + all + + + all + + + all + + + + + + + + + + diff --git a/src/Microsoft.ML.XGBoost/Properties/AssemblyInfo.cs b/src/Microsoft.ML.XGBoost/Properties/AssemblyInfo.cs new file mode 100644 index 0000000000..9d1bb36754 --- /dev/null +++ b/src/Microsoft.ML.XGBoost/Properties/AssemblyInfo.cs @@ -0,0 +1,13 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System.Runtime.CompilerServices; +using Microsoft.ML; + +[assembly: InternalsVisibleTo(assemblyName: "Microsoft.ML.Tests" + PublicKey.TestValue)] +[assembly: InternalsVisibleTo(assemblyName: "Microsoft.ML.Predictor.Tests" + PublicKey.Value)] + +[assembly: InternalsVisibleTo(assemblyName: "RunTests" + InternalPublicKey.Value)] + +[assembly: WantsToBeBestFriends] diff --git a/src/Microsoft.ML.XGBoost/WrappedXGBoostInterface.cs b/src/Microsoft.ML.XGBoost/WrappedXGBoostInterface.cs new file mode 100644 index 0000000000..141e0ec60c --- /dev/null +++ b/src/Microsoft.ML.XGBoost/WrappedXGBoostInterface.cs @@ -0,0 +1,166 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.Collections.Generic; +using System.Globalization; +using System.Runtime.InteropServices; +using System.Runtime; +using System.Text; + +namespace Microsoft.ML.Trainers.XGBoost +{ + /// + /// Wrapper of the c interfaces of XGBoost + /// Refer to https://xgboost.readthedocs.io/en/stable/tutorials/c_api_tutorial.html to get the details. + /// + + internal static class WrappedXGBoostInterface + { + + private const string DllName = "xgboost"; + + [DllImport(DllName)] + public static extern void XGBoostVersion(out int major, out int minor, out int patch); + + #region Error API + + [DllImport(DllName)] + public static extern string XGBGetLastError(); + + #endregion + + #region DMatrix API + + [DllImport(DllName)] + public static extern int XGDMatrixCreateFromMat(float[] data, ulong nrow, ulong ncol, + float missing, out IntPtr handle); + + [DllImport(DllName)] + public static extern int XGDMatrixFree(IntPtr handle); + + [DllImport(DllName)] + public static extern int XGDMatrixNumRow(IntPtr handle, out ulong nrows); + + [DllImport(DllName)] + public static extern int XGDMatrixNumCol(IntPtr handle, out ulong ncols); + + [DllImport(DllName)] + public static extern int XGDMatrixGetFloatInfo(IntPtr handle, string field, + out ulong len, out IntPtr result); + + [DllImport(DllName)] + public static extern int XGDMatrixSetFloatInfo(IntPtr handle, string field, + IntPtr array, ulong len); + #endregion + + + #region API Booster + + [DllImport(DllName)] + public static extern int XGBoosterCreate(IntPtr[] dmats, + ulong len, out IntPtr handle); + + [DllImport(DllName)] + public static extern int XGBoosterFree(IntPtr handle); + + [DllImport(DllName)] + public static extern int XGBoosterSetParam(IntPtr handle, string name, string val); + + #endregion + + + #region API train + [DllImport(DllName)] + public static extern int XGBoosterUpdateOneIter(IntPtr bHandle, int iter, + IntPtr dHandle); + + [DllImport(DllName)] + public static extern int XGBoosterEvalOneIter(); + #endregion + + #region API predict + [DllImport(DllName)] + public static extern int XGBoosterPredict(IntPtr bHandle, IntPtr dHandle, + int optionMask, int ntreeLimit, int training, + out ulong predsLen, out IntPtr predsPtr); + #endregion + + #region API serialization +#pragma warning disable MSML_ParameterLocalVarName + [DllImport(DllName)] + public static extern int XGBoosterDumpModel(IntPtr handle, string fmap, int with_stats, out int out_len, out IntPtr dumpStr); + + [DllImport(DllName)] + public static extern int XGBoosterDumpModelEx(IntPtr handle, string fmap, int with_stats, string format, out int out_len, out IntPtr dumpStr); +#pragma warning restore MSML_ParameterLocalVarName + #endregion + + } + + internal static class XGBoostInterfaceUtils + { + /// + /// Checks if XGBoost has a pending error message. Raises an exception in that case. + /// + public static void Check(int res) + { + if (res != 0) + { + string mes = WrappedXGBoostInterface.XGBGetLastError(); + throw new Exception($"XGBoost Error, code is {res}, error message is '{mes}'."); + } + } + + public static float[] GetPredictionsArray(IntPtr predsPtr, ulong predsLen) + { + var length = unchecked((int)predsLen); + var preds = new float[length]; + for (var i = 0; i < length; i++) + { + var floatBytes = new byte[4]; + for (var b = 0; b < 4; b++) + { + floatBytes[b] = Marshal.ReadByte(predsPtr, 4 * i + b); + } + preds[i] = BitConverter.ToSingle(floatBytes, 0); + } + return preds; + } + + /// + /// Helper function used for generating the LightGbm argument name. + /// When given a name, this will convert the name to lower-case with underscores. + /// The underscore will be placed when an upper-case letter is encountered. + /// + public static string GetOptionName(string name) + { + // Otherwise convert the name to the light gbm argument + StringBuilder strBuf = new StringBuilder(); + bool first = true; + foreach (char c in name) + { + if (char.IsUpper(c)) + { + if (first) + first = false; + else + strBuf.Append('_'); + strBuf.Append(char.ToLower(c)); + } + else + strBuf.Append(c); + } + return strBuf.ToString(); + } + + /// + /// Convert the pointer of c string to c# string. + /// + public static string GetString(IntPtr src) + { + return Marshal.PtrToStringAnsi(src); + } + } +} diff --git a/src/Microsoft.ML.XGBoost/WrappedXGBooster.cs b/src/Microsoft.ML.XGBoost/WrappedXGBooster.cs new file mode 100644 index 0000000000..8bc45aba95 --- /dev/null +++ b/src/Microsoft.ML.XGBoost/WrappedXGBooster.cs @@ -0,0 +1,312 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.Collections.Generic; +using System.Globalization; +using System.Linq; +using System.Text.RegularExpressions; +using System.Text.Json; +using System.Runtime.InteropServices; +using System.Text; +using Microsoft.ML.Runtime; +//#if false +using Microsoft.ML.Trainers.FastTree; +//#else +using Microsoft.ML; +using Microsoft.ML.CommandLine; +using Microsoft.ML.Data; +using Microsoft.ML.EntryPoints; +using Microsoft.ML.Internal.CpuMath; +using Microsoft.ML.Internal.Utilities; +//using Microsoft.ML.Model.OnnxConverter; +using Microsoft.ML.Numeric; +using Microsoft.ML.Transforms; +//#endif + +namespace Microsoft.ML.Trainers.XGBoost +{ + /// + /// Wrapper of Booster object of XGBoost. + /// + internal class Booster : IDisposable + { +#pragma warning disable MSML_PrivateFieldName // Private field name not in: _camelCase format + private bool disposed; + private readonly IntPtr _handle; + private const int normalPrediction = 0; // Value for the optionMask in prediction + private int numClass = 1; +#pragma warning restore MSML_PrivateFieldName + + public IntPtr Handle => _handle; + + public Booster(Dictionary parameters, DMatrix trainDMatrix) + { + var dmats = new[] { trainDMatrix.Handle }; + var len = unchecked((ulong)dmats.Length); + var errp = WrappedXGBoostInterface.XGBoosterCreate(dmats, len, out _handle); + if (errp == -1) + { + string reason = WrappedXGBoostInterface.XGBGetLastError(); + throw new XGBoostDLLException(reason); + } + SetParameters(parameters); + } + + public Booster(DMatrix trainDMatrix) + { + var dmats = new[] { trainDMatrix.Handle }; + var len = unchecked((ulong)dmats.Length); + var errp = WrappedXGBoostInterface.XGBoosterCreate(dmats, len, out _handle); + if (errp == -1) + { + string reason = WrappedXGBoostInterface.XGBGetLastError(); + throw new XGBoostDLLException(reason); + } + } + + public void Update(DMatrix train, int iter) + { + var errp = WrappedXGBoostInterface.XGBoosterUpdateOneIter(_handle, iter, train.Handle); + if (errp == -1) + { + string reason = WrappedXGBoostInterface.XGBGetLastError(); + throw new XGBoostDLLException(reason); + } + } + + public float[] Predict(DMatrix test) + { + ulong predsLen; + IntPtr predsPtr; + /* + allowed values of optionmask: + 0:normal prediction + 1:output margin instead of transformed value + 2:output leaf index of trees instead of leaf value, note leaf index is unique per tree + 4:output feature contributions to individual predictions + + // using `0` for ntreeLimit means use all the trees + */ + + var errp = WrappedXGBoostInterface.XGBoosterPredict(_handle, test.Handle, 0, 0, 0, out predsLen, out predsPtr); + if (errp == -1) + { + string reason = WrappedXGBoostInterface.XGBGetLastError(); + throw new XGBoostDLLException(reason); + } + return XGBoostInterfaceUtils.GetPredictionsArray(predsPtr, predsLen); + } + +#if false + // Should enable XGBoosterSaveModelToBuffer + [BestFriend] + internal unsafe string GetModelString() + { + int bufLen = 2 << 15; + byte[] buffer = new byte[bufLen]; + int size = 0; + fixed (byte* ptr = buffer) + LightGbmInterfaceUtils.Check(WrappedLightGbmInterface.BoosterSaveModelToString(Handle, 0, BestIteration, bufLen, ref size, ptr)); + // If buffer size is not enough, reallocate buffer and get again. + if (size > bufLen) + { + bufLen = size; + buffer = new byte[bufLen]; + fixed (byte* ptr = buffer) + LightGbmInterfaceUtils.Check(WrappedLightGbmInterface.BoosterSaveModelToString(Handle, 0, BestIteration, bufLen, ref size, ptr)); + } + byte[] content = new byte[size]; + Array.Copy(buffer, content, size); + fixed (byte* ptr = content) + return LightGbmInterfaceUtils.GetString((IntPtr)ptr); + } +#endif + + public void SetParameters(Dictionary parameters) + { + // support internationalisation i.e. support floats with commas (e.g. 0,5F) + var nfi = new NumberFormatInfo { NumberDecimalSeparator = "." }; + + SetParameter("max_depth", ((int)parameters["max_depth"]).ToString()); + SetParameter("learning_rate", ((float)parameters["learning_rate"]).ToString(nfi)); + SetParameter("n_estimators", ((int)parameters["n_estimators"]).ToString()); + SetParameter("silent", ((bool)parameters["silent"]).ToString()); + SetParameter("objective", (string)parameters["objective"]); + SetParameter("booster", (string)parameters["booster"]); + SetParameter("tree_method", (string)parameters["tree_method"]); + + SetParameter("nthread", ((int)parameters["nthread"]).ToString()); + SetParameter("gamma", ((float)parameters["gamma"]).ToString(nfi)); + SetParameter("min_child_weight", ((int)parameters["min_child_weight"]).ToString()); + SetParameter("max_delta_step", ((int)parameters["max_delta_step"]).ToString()); + SetParameter("subsample", ((float)parameters["subsample"]).ToString(nfi)); + SetParameter("colsample_bytree", ((float)parameters["colsample_bytree"]).ToString(nfi)); + SetParameter("colsample_bylevel", ((float)parameters["colsample_bylevel"]).ToString(nfi)); + SetParameter("reg_alpha", ((float)parameters["reg_alpha"]).ToString(nfi)); + SetParameter("reg_lambda", ((float)parameters["reg_lambda"]).ToString(nfi)); + SetParameter("scale_pos_weight", ((float)parameters["scale_pos_weight"]).ToString(nfi)); + + SetParameter("base_score", ((float)parameters["base_score"]).ToString(nfi)); + SetParameter("seed", ((int)parameters["seed"]).ToString()); + SetParameter("missing", ((float)parameters["missing"]).ToString(nfi)); + + SetParameter("sample_type", (string)parameters["sample_type"]); + SetParameter("normalize_type ", (string)parameters["normalize_type"]); + SetParameter("rate_drop", ((float)parameters["rate_drop"]).ToString(nfi)); + SetParameter("one_drop", ((int)parameters["one_drop"]).ToString()); + SetParameter("skip_drop", ((float)parameters["skip_drop"]).ToString(nfi)); + + if (parameters.TryGetValue("num_class", out var value)) + { + numClass = (int)value; + SetParameter("num_class", numClass.ToString()); + } + + } + + public void SetParameter(string name, string val) + { + var errp = WrappedXGBoostInterface.XGBoosterSetParam(_handle, name, val); + + if (errp == -1) + { + string reason = WrappedXGBoostInterface.XGBGetLastError(); + throw new XGBoostDLLException(reason); + } + } + + #region Create Models +#pragma warning disable MSML_ParameterLocalVarName + public string[] DumpModelEx(string fmap, int with_stats, string format) +#pragma warning restore MSML_ParameterLocalVarName + { + int length; + IntPtr treePtr; + var intptrSize = IntPtr.Size; + + WrappedXGBoostInterface.XGBoosterDumpModelEx(_handle, fmap, with_stats, format, out length, out treePtr); + + var trees = new string[length]; + int readSize = 0; + var handle2 = GCHandle.Alloc(treePtr, GCHandleType.Pinned); + + //iterate through the length of the tree ensemble and pull the strings out from the returned pointer's array of pointers. prepend python's api convention of adding booster[i] to the beginning of the tree + for (var i = 0; i < length; i++) + { + var ipt1 = Marshal.ReadIntPtr(Marshal.ReadIntPtr(handle2.AddrOfPinnedObject()), intptrSize * i); + string s = Marshal.PtrToStringAnsi(ipt1); + trees[i] = string.Format("booster[{0}]\n{1}", i, s); + var bytesToRead = (s.Length * 2) + IntPtr.Size; + readSize += bytesToRead; + } + handle2.Free(); + return trees; + } + + public void GetModel() + { + var ss = DumpModelEx("", with_stats: 0, format: "json"); + var boosterPattern = @"^booster\[\d+\]"; +#if false + List ensemble = new List(); // should probably return this +#endif + + for (int i = 0; i < ss.Length; i++) + { + var m = Regex.Matches(ss[i], boosterPattern, RegexOptions.IgnoreCase); + if ((m.Count >= 1) && (m[0].Groups.Count >= 1)) + { + // every booster representation should match + var structString = ss[i].Substring(m[0].Groups[0].Value.Length); + var doc = JsonDocument.Parse(structString); +#if false + TreeNode t = TreeNode.Create(doc.RootElement); + ensemble.Add(t); +#else + //var table = new TablePopulator(doc); +#endif + } + } + } + + private class TablePopulator + { +#pragma warning disable MSML_GeneralName + public Dictionary> dict = new(); + public Dictionary nodes = new(); + public Dictionary lte = new(); + public Dictionary gt = new(); +#pragma warning restore MSML_GeneralName + + public TablePopulator(JsonElement elm) + { + PopulateTable(elm, 0); + } + + public void PopulateTable(JsonElement elm, int level) + { + string nodeId = ""; + if (elm.TryGetProperty("nodeid", out JsonElement nodeidElm)) + { + // If this test fails, should probably bail, as the syntax of the booster is incorrect + nodeId = nodeidElm.ToString(); + nodes.Add(nodeId, level); + } + + if (!dict.ContainsKey(level)) + { + dict.Add(level, new List()); + } + + if (elm.TryGetProperty("leaf", out JsonElement leafJsonNode)) + { + dict[level].Add(elm); + } + else if (elm.TryGetProperty("children", out JsonElement internalJsonNode)) + { + dict[level].Add(elm); + if (elm.TryGetProperty("yes", out JsonElement yesNodeId)) + { + lte.Add(nodeId, yesNodeId.ToString()); + } + + if (elm.TryGetProperty("no", out JsonElement noNodeId)) + { + gt.Add(nodeId, noNodeId.ToString()); + } + + foreach (var e in internalJsonNode.EnumerateArray()) + { + PopulateTable(e, level + 1); + } + } + else + { + throw new Exception("Invalid booster content"); + } + } + } + #endregion + + #region IDisposable Support + public void Dispose() + { + Dispose(true); + GC.SuppressFinalize(this); + } + + protected virtual void Dispose(bool disposing) + { + if (disposed) + { + return; + } + WrappedXGBoostInterface.XGBoosterFree(_handle); + disposed = true; + } + #endregion + } +} + diff --git a/src/Microsoft.ML.XGBoost/WrappedXGDMatrix.cs b/src/Microsoft.ML.XGBoost/WrappedXGDMatrix.cs new file mode 100644 index 0000000000..cd2d797525 --- /dev/null +++ b/src/Microsoft.ML.XGBoost/WrappedXGDMatrix.cs @@ -0,0 +1,110 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.Runtime.InteropServices; + +namespace Microsoft.ML.Trainers.XGBoost +{ + /// + /// Wrapper of DMatrix object of XGBoost + /// + internal sealed class DMatrix : IDisposable + { +#pragma warning disable MSML_PrivateFieldName + private bool disposed = false; +#pragma warning restore MSML_PrivateFieldName +#pragma warning disable IDE0044 + private IntPtr _handle; +#pragma warning restore IDE0044 + public IntPtr Handle => _handle; + private const float Missing = 0f; + + /// + /// Create a for storing training and prediction data under XGBoost framework. + /// +#nullable enable + public unsafe DMatrix(float[] data, uint nrows, uint ncols, float[]? labels = null) + { + int errp = WrappedXGBoostInterface.XGDMatrixCreateFromMat(data, nrows, ncols, Missing, out _handle); + if (errp == -1) + { + string reason = WrappedXGBoostInterface.XGBGetLastError(); + throw new XGBoostDLLException(reason); + } + + if (labels != null) + { + SetLabel(labels); + } + + } +#nullable disable + + public ulong GetNumRows() + { + ulong numRows; + int errp = WrappedXGBoostInterface.XGDMatrixNumRow(_handle, out numRows); + if (errp == -1) + { + string reason = WrappedXGBoostInterface.XGBGetLastError(); + throw new XGBoostDLLException(reason); + } + return numRows; + } + + public ulong GetNumCols() + { + ulong numCols; + int errp = WrappedXGBoostInterface.XGDMatrixNumCol(_handle, out numCols); + if (errp == -1) + { + string reason = WrappedXGBoostInterface.XGBGetLastError(); + throw new XGBoostDLLException(reason); + } + return numCols; + } + + public unsafe void SetLabel(float[] labels) + { +#if false + Contracts.AssertValue(labels); + Contracts.Assert(labels.Length == GetNumRows()); +#endif + fixed (float* ptr = labels) + { + int errp = WrappedXGBoostInterface.XGDMatrixSetFloatInfo(_handle, "label", (IntPtr)ptr, (ulong)labels.Length); + if (errp == -1) + { + string reason = WrappedXGBoostInterface.XGBGetLastError(); + throw new XGBoostDLLException(reason); + } + } + } + + public void Dispose() + { + Dispose(true); + GC.SuppressFinalize(this); + } + + void Dispose(bool disposing) + { + if (disposed) + { + return; + } + + int errp = WrappedXGBoostInterface.XGDMatrixFree(_handle); + if (errp == -1) + { + string reason = WrappedXGBoostInterface.XGBGetLastError(); + throw new XGBoostDLLException(reason); + } + disposed = true; + + } + } +} + diff --git a/src/Microsoft.ML.XGBoost/XGBoostArguments.cs b/src/Microsoft.ML.XGBoost/XGBoostArguments.cs new file mode 100644 index 0000000000..8f39c2cc58 --- /dev/null +++ b/src/Microsoft.ML.XGBoost/XGBoostArguments.cs @@ -0,0 +1,307 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System.Collections.Generic; +using System.Reflection; +using Microsoft.ML; +using Microsoft.ML.CommandLine; +using Microsoft.ML.EntryPoints; +using Microsoft.ML.Internal.Internallearn; +using Microsoft.ML.Runtime; +using Microsoft.ML.Trainers.XGBoost; + +[assembly: LoadableClass(typeof(GradientBooster), typeof(GradientBooster.Options), + typeof(SignatureXGBoostBooster), GradientBooster.FriendlyName, GradientBooster.Name)] +[assembly: LoadableClass(typeof(DartBooster), typeof(DartBooster.Options), + typeof(SignatureXGBoostBooster), DartBooster.FriendlyName, DartBooster.Name)] + +[assembly: EntryPointModule(typeof(GradientBooster.Options))] +[assembly: EntryPointModule(typeof(DartBooster.Options))] + +namespace Microsoft.ML.Trainers.XGBoost +{ +#if true + internal delegate void SignatureXGBoostBooster(); + + [TlcModule.ComponentKind("BoosterParameterFunction")] + internal interface IBoosterParameterFactory : IComponentFactory + { + new BoosterParameterBase CreateComponent(IHostEnvironment env); + } +#endif + + public abstract class BoosterParameterBase + { + + private protected static Dictionary NameMapping = new Dictionary() + { + {nameof(OptionsBase.MinimumSplitGain), "min_split_gain" }, +#if false + {nameof(OptionsBase.MaximumTreeDepth), "max_depth"}, + {nameof(OptionsBase.MinimumChildWeight), "min_child_weight"}, + {nameof(OptionsBase.SubsampleFraction), "subsample"}, + {nameof(OptionsBase.SubsampleFrequency), "subsample_freq"}, + {nameof(OptionsBase.L1Regularization), "reg_alpha"}, + {nameof(OptionsBase.L2Regularization), "reg_lambda"}, +#endif + }; + + + public BoosterParameterBase(OptionsBase options) + { + Contracts.CheckUserArg(options.MinimumSplitGain >= 0, nameof(OptionsBase.MinimumSplitGain), "must be >= 0."); +#if false + Contracts.CheckUserArg(options.MinimumChildWeight >= 0, nameof(OptionsBase.MinimumChildWeight), "must be >= 0."); + Contracts.CheckUserArg(options.SubsampleFraction > 0 && options.SubsampleFraction <= 1, nameof(OptionsBase.SubsampleFraction), "must be in (0,1]."); + Contracts.CheckUserArg(options.FeatureFraction > 0 && options.FeatureFraction <= 1, nameof(OptionsBase.FeatureFraction), "must be in (0,1]."); + Contracts.CheckUserArg(options.L2Regularization >= 0, nameof(OptionsBase.L2Regularization), "must be >= 0."); + Contracts.CheckUserArg(options.L1Regularization >= 0, nameof(OptionsBase.L1Regularization), "must be >= 0."); +#endif + BoosterOptions = options; + } + + public abstract class OptionsBase : IBoosterParameterFactory + { + internal BoosterParameterBase GetBooster() { return null; } + /// + /// The minimum loss reduction required to make a further partition on a leaf node of the tree. + /// + /// + /// Larger values make the algorithm more conservative. + /// + [Argument(ArgumentType.AtMostOnce, + HelpText = "Minimum loss reduction required to make a further partition on a leaf node of the tree. the larger, " + + "the more conservative the algorithm will be.")] + [TlcModule.Range(Min = 0.0)] + public double MinimumSplitGain = 0; + +#if false + /// + /// The maximum depth of a tree. + /// + /// + /// 0 means no limit. + /// + [Argument(ArgumentType.AtMostOnce, + HelpText = "Maximum depth of a tree. 0 means no limit. However, tree still grows by best-first.")] + [TlcModule.Range(Min = 0, Max = int.MaxValue)] + public int MaximumTreeDepth = 0; + + /// + /// The minimum sum of instance weight needed to form a new node. + /// + /// + /// If the tree partition step results in a leaf node with the sum of instance weight less than , + /// the building process will give up further partitioning. In linear regression mode, this simply corresponds to minimum number + /// of instances needed to be in each node. The larger, the more conservative the algorithm will be. + /// + [Argument(ArgumentType.AtMostOnce, + HelpText = "Minimum sum of instance weight(hessian) needed in a child. If the tree partition step results in a leaf " + + "node with the sum of instance weight less than min_child_weight, then the building process will give up further partitioning. In linear regression mode, " + + "this simply corresponds to minimum number of instances needed to be in each node. The larger, the more conservative the algorithm will be.")] + [TlcModule.Range(Min = 0.0)] + public double MinimumChildWeight = 0.1; + + /// + /// The frequency of performing subsampling (bagging). + /// + /// + /// 0 means disable bagging. N means perform bagging at every N iterations. + /// To enable bagging, should also be set to a value less than 1.0. + /// + [Argument(ArgumentType.AtMostOnce, + HelpText = "Subsample frequency for bagging. 0 means no subsample. " + + "Specifies the frequency at which the bagging occurs, where if this is set to N, the subsampling will happen at every N iterations." + + "This must be set with Subsample as this specifies the amount to subsample.")] + [TlcModule.Range(Min = 0, Max = int.MaxValue)] + public int SubsampleFrequency = 0; + + /// + /// The fraction of training data used for creating trees. + /// + /// + /// Setting it to 0.5 means that LightGBM randomly picks half of the data points to grow trees. + /// This can be used to speed up training and to reduce over-fitting. Valid range is (0,1]. + /// + [Argument(ArgumentType.AtMostOnce, + HelpText = "Subsample ratio of the training instance. Setting it to 0.5 means that LightGBM randomly collected " + + "half of the data instances to grow trees and this will prevent overfitting. Range: (0,1].")] + [TlcModule.Range(Inf = 0.0, Max = 1.0)] + public double SubsampleFraction = 1; + + /// + /// The fraction of features used when creating trees. + /// + /// + /// If is smaller than 1.0, LightGBM will randomly select fraction of features to train each tree. + /// For example, if you set it to 0.8, LightGBM will select 80% of features before training each tree. + /// This can be used to speed up training and to reduce over-fitting. Valid range is (0,1]. + /// + [Argument(ArgumentType.AtMostOnce, + HelpText = "Subsample ratio of columns when constructing each tree. Range: (0,1].", + ShortName = "ff")] + [TlcModule.Range(Inf = 0.0, Max = 1.0)] + public double FeatureFraction = 1; + + /// + /// The L2 regularization term on weights. + /// + /// + /// Increasing this value could help reduce over-fitting. + /// + [Argument(ArgumentType.AtMostOnce, + HelpText = "L2 regularization term on weights, increasing this value will make model more conservative.", + ShortName = "l2")] + [TlcModule.Range(Min = 0.0)] + [TGUI(Label = "Lambda(L2)", SuggestedSweeps = "0,0.5,1")] + [TlcModule.SweepableDiscreteParam("RegLambda", new object[] { 0f, 0.5f, 1f })] + public double L2Regularization = 0.01; + + /// + /// The L1 regularization term on weights. + /// + /// + /// Increasing this value could help reduce over-fitting. + /// + [Argument(ArgumentType.AtMostOnce, + HelpText = "L1 regularization term on weights, increase this value will make model more conservative.", + ShortName = "l1")] + [TlcModule.Range(Min = 0.0)] + [TGUI(Label = "Alpha(L1)", SuggestedSweeps = "0,0.5,1")] + [TlcModule.SweepableDiscreteParam("RegAlpha", new object[] { 0f, 0.5f, 1f })] + public double L1Regularization = 0; +#endif + + BoosterParameterBase IComponentFactory.CreateComponent(IHostEnvironment env) + => BuildOptions(); + + BoosterParameterBase IBoosterParameterFactory.CreateComponent(IHostEnvironment env) + => BuildOptions(); + + internal abstract BoosterParameterBase BuildOptions(); + } + + internal void UpdateParameters(Dictionary res) + { + FieldInfo[] fields = BoosterOptions.GetType().GetFields(BindingFlags.Public | BindingFlags.NonPublic | BindingFlags.Instance); + foreach (var field in fields) + { + var attribute = field.GetCustomAttribute(false); + + if (attribute == null) + continue; + + var name = NameMapping.ContainsKey(field.Name) ? NameMapping[field.Name] : XGBoostInterfaceUtils.GetOptionName(field.Name); + res[name] = field.GetValue(BoosterOptions); + } + } + + /// + /// Create for supporting legacy infra built upon . + /// + internal abstract IBoosterParameterFactory BuildFactory(); + internal abstract string BoosterName { get; } + + private protected OptionsBase BoosterOptions; + } + + /// + /// Gradient boosting decision tree. + /// + /// + /// For details, please see gradient tree boosting. + /// + public sealed class GradientBooster : BoosterParameterBase + { + internal const string Name = "gbdt"; + internal const string FriendlyName = "Tree Booster"; + + /// + /// The options for , used for setting . + /// + [TlcModule.Component(Name = Name, FriendlyName = FriendlyName, Desc = "Traditional Gradient Boosting Decision Tree.")] + public sealed class Options : OptionsBase + { + internal override BoosterParameterBase BuildOptions() => new GradientBooster(this); + } + + internal GradientBooster(Options options) + : base(options) + { + } + + internal override IBoosterParameterFactory BuildFactory() => BoosterOptions; + + internal override string BoosterName => Name; + + } + + + /// + /// DART booster (Dropouts meet Multiple Additive Regression Trees) + /// + /// + /// For details, please see here. + /// + public sealed class DartBooster : BoosterParameterBase + { + internal const string Name = "dart"; + internal const string FriendlyName = "Tree Dropout Tree Booster"; + + + /// + /// The options for , used for setting . + /// + [TlcModule.Component(Name = Name, FriendlyName = FriendlyName, Desc = "Dropouts meet Multiple Additive Regresion Trees. See https://arxiv.org/abs/1505.01866")] + public sealed class Options : OptionsBase + { + static Options() + { + // Add additional name mappings + NameMapping.Add(nameof(TreeDropFraction), "rate_drop"); + NameMapping.Add(nameof(SkipDropFraction), "skip_drop"); + } + + /// + /// The dropout rate, i.e. the fraction of previous trees to drop during the dropout. + /// + /// + /// Valid range is [0,1]. + /// + [Argument(ArgumentType.AtMostOnce, HelpText = "The drop ratio for trees. Range:(0,1).")] + [TlcModule.Range(Inf = 0.0, Max = 1.0)] + public double TreeDropFraction = 0.1; + + /// + /// The probability of skipping the dropout procedure during a boosting iteration. + /// + [Argument(ArgumentType.AtMostOnce, HelpText = "Probability for not dropping in a boosting round.")] + [TlcModule.Range(Inf = 0.0, Max = 1.0)] + public double SkipDropFraction = 0.5; + +#if false + /// + /// Whether to enable uniform drop. + /// Allowed values: "uniform" or "weighted" + /// + [Argument(ArgumentType.AtMostOnce, HelpText = "True will enable uniform drop.")] + public bool UniformDrop = false; +#endif + + internal override BoosterParameterBase BuildOptions() => new DartBooster(this); + } + + internal DartBooster(Options options) + : base(options) + { + Contracts.CheckUserArg(options.TreeDropFraction > 0 && options.TreeDropFraction < 1, nameof(options.TreeDropFraction), "must be in (0,1)."); + Contracts.CheckUserArg(options.SkipDropFraction >= 0 && options.SkipDropFraction < 1, nameof(options.SkipDropFraction), "must be in [0,1)."); + BoosterOptions = options; + } + + internal override IBoosterParameterFactory BuildFactory() => BoosterOptions; + internal override string BoosterName => Name; + } + +} diff --git a/src/Microsoft.ML.XGBoost/XGBoostCatalog.cs b/src/Microsoft.ML.XGBoost/XGBoostCatalog.cs new file mode 100644 index 0000000000..45e5e3b3e7 --- /dev/null +++ b/src/Microsoft.ML.XGBoost/XGBoostCatalog.cs @@ -0,0 +1,54 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.Collections.Generic; +using System.Text; + +using Microsoft.ML.Data; +using Microsoft.ML.Runtime; +using Microsoft.ML.Trainers.XGBoost; + +namespace Microsoft.ML +{ + /// + /// Collection of extension methods for the and + /// catalogs. + /// + public static class XGBoostExtensions + { + /// + /// Create , which predicts a target using a gradient boosting decision tree regression model. + /// + /// The . + /// The name of the label column. The column data must be . + /// The name of the feature column. The column data must be a known-sized vector of . + /// The name of the example weight column (optional). + /// The maximum number of leaves in one tree. + /// The minimal number of data points required to form a new tree leaf. + /// The learning rate. + /// The number of boosting iterations. A new tree is created in each iteration, so this is equivalent to the number of trees. + public static XGBoostRegressionTrainer LightGbm(this RegressionCatalog.RegressionTrainers catalog, + string labelColumnName = DefaultColumnNames.Label, + string featureColumnName = DefaultColumnNames.Features, + string exampleWeightColumnName = null, + int? numberOfLeaves = null, + int? minimumExampleCountPerLeaf = null, + double? learningRate = null, + int numberOfIterations = Defaults.NumberOfIterations) + { + Contracts.CheckValue(catalog, nameof(catalog)); + var env = CatalogUtils.GetEnvironment(catalog); + return new XGBoostRegressionTrainer(env, labelColumnName, featureColumnName, exampleWeightColumnName, numberOfLeaves, minimumExampleCountPerLeaf, learningRate, numberOfIterations); + } + + public static XGBoostRegressionTrainer XGBoost(this RegressionCatalog.RegressionTrainers catalog, + XGBoostRegressionTrainer.Options options) + { + Contracts.CheckValue(catalog, nameof(catalog)); + var env = CatalogUtils.GetEnvironment(catalog); + return new XGBoostRegressionTrainer(env, options); + } + } +} diff --git a/src/Microsoft.ML.XGBoost/XGBoostInterface.cs b/src/Microsoft.ML.XGBoost/XGBoostInterface.cs new file mode 100644 index 0000000000..56d8cd3ad5 --- /dev/null +++ b/src/Microsoft.ML.XGBoost/XGBoostInterface.cs @@ -0,0 +1,54 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.Collections.Generic; +using System.Globalization; +using System.Linq; +using Microsoft.ML.Runtime; + +namespace Microsoft.ML.Trainers.XGBoost +{ + +#pragma warning disable MSML_GeneralName + public class XGBoostDLLException : Exception +#pragma warning restore MSML_GeneralName + { + public XGBoostDLLException() + { + /* empty */ + } + + public XGBoostDLLException(string message) + : base(message) + { + /* empty */ + } + + } + + public static class XGBoost + { + public struct XGBoostVersion + { + public int Major; + public int Minor; + public int Patch; + } + + public static XGBoostVersion Version() + { + int major; + int minor; + int patch; + WrappedXGBoostInterface.XGBoostVersion(out major, out minor, out patch); + return new XGBoostVersion + { + Major = major, + Minor = minor, + Patch = patch + }; + } + } +} diff --git a/src/Microsoft.ML.XGBoost/XGBoostRegressionTrainer.cs b/src/Microsoft.ML.XGBoost/XGBoostRegressionTrainer.cs new file mode 100644 index 0000000000..785efd9a7e --- /dev/null +++ b/src/Microsoft.ML.XGBoost/XGBoostRegressionTrainer.cs @@ -0,0 +1,231 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System.Collections.Generic; +using Microsoft.ML; +using Microsoft.ML.CommandLine; +using Microsoft.ML.Data; +using Microsoft.ML.EntryPoints; +using Microsoft.ML.Runtime; +using Microsoft.ML.Trainers.FastTree; +using Microsoft.ML.Trainers.XGBoost; + +[assembly: LoadableClass(XGBoostRegressionTrainer.Summary, typeof(XGBoostRegressionTrainer), typeof(XGBoostRegressionTrainer.Options), + new[] { typeof(SignatureRegressorTrainer), typeof(SignatureTrainer), typeof(SignatureTreeEnsembleTrainer) }, + XGBoostRegressionTrainer.UserNameValue, XGBoostRegressionTrainer.LoadNameValue, XGBoostRegressionTrainer.ShortName)] + +[assembly: LoadableClass(typeof(XGBoostRegressionModelParameters), null, typeof(SignatureLoadModel), + "XGBoost Regression Executor", + XGBoostRegressionModelParameters.LoaderSignature)] +namespace Microsoft.ML.Trainers.XGBoost +{ + /// + /// Model parameters for . + /// + public sealed class XGBoostRegressionModelParameters : TreeEnsembleModelParametersBasedOnRegressionTree + { + internal const string LoaderSignature = "XGBoostRegressionExec"; + internal const string RegistrationName = "XGBoostRegressionPredictor"; + + private static VersionInfo GetVersionInfo() + { + // REVIEW: can we decouple the version from FastTree predictor version ? + return new VersionInfo( + modelSignature: "XGBSIREG", + verWrittenCur: 0x00010001, // Initial (not sure if "Categorical splits") are supported + // verWrittenCur: 0x00010002, // _numFeatures serialized + // verWrittenCur: 0x00010003, // Ini content out of predictor + // verWrittenCur: 0x00010004, // Add _defaultValueForMissing + verReadableCur: 0x00010004, + verWeCanReadBack: 0x00010001, + loaderSignature: LoaderSignature, + loaderAssemblyName: typeof(XGBoostRegressionModelParameters).Assembly.FullName); + } + private protected override uint VerNumFeaturesSerialized => 0x00010002; + private protected override uint VerDefaultValueSerialized => 0x00010004; + private protected override uint VerCategoricalSplitSerialized => 0x00010005; + private protected override PredictionKind PredictionKind => PredictionKind.Regression; + + internal XGBoostRegressionModelParameters(IHostEnvironment env, InternalTreeEnsemble trainedEnsemble, int featureCount, string innerArgs) + : base(env, RegistrationName, trainedEnsemble, featureCount, innerArgs) + { + } + + private XGBoostRegressionModelParameters(IHostEnvironment env, ModelLoadContext ctx) + : base(env, RegistrationName, ctx, GetVersionInfo()) + { + } + + private protected override void SaveCore(ModelSaveContext ctx) + { + base.SaveCore(ctx); + ctx.SetVersionInfo(GetVersionInfo()); + } + + internal static XGBoostRegressionModelParameters Create(IHostEnvironment env, ModelLoadContext ctx) + { + Contracts.CheckValue(env, nameof(env)); + env.CheckValue(ctx, nameof(ctx)); + ctx.CheckAtModel(GetVersionInfo()); + return new XGBoostRegressionModelParameters(env, ctx); + } + } + + /// + /// The for training a boosted decision tree regression model using XGBoost. + /// + public sealed class XGBoostRegressionTrainer : + XGBoostTrainerBase, XGBoostRegressionModelParameters> + { + internal new const string Summary = "XGBoost Regression"; + internal new const string LoadNameValue = "XGBoostRegression"; + internal const string ShortName = "XGBoostR"; + internal new const string UserNameValue = "XGBoost Regressor"; + + public XGBoostRegressionTrainer(IHost host, SchemaShape.Column feature, SchemaShape.Column label, SchemaShape.Column weight = default, SchemaShape.Column groupId = default) : base(host, feature, label, weight, groupId) + { + } + + public override TrainerInfo Info => throw new System.NotImplementedException(); + + private protected override PredictionKind PredictionKind => PredictionKind.Regression; + + private protected override SchemaShape.Column[] GetOutputColumnsCore(SchemaShape inputSchema) + { + return new[] + { + new SchemaShape.Column(DefaultColumnNames.Score, SchemaShape.Column.VectorKind.Scalar, NumberDataViewType.Single, false, new SchemaShape(AnnotationUtils.GetTrainerOutputAnnotation())) + }; + } + + private protected override RegressionPredictionTransformer MakeTransformer(XGBoostRegressionModelParameters model, DataViewSchema trainSchema) + => new RegressionPredictionTransformer(Host, model, trainSchema, FeatureColumn.Name); + + /// + /// Options for the as used in + /// [XGBoost(Options)](xref:Microsoft.ML.XGBoostExtensions.XGBoost(Microsoft.ML.RegressionCatalog.RegressionTrainers,Microsoft.ML.Trainers.XGBoost.XGBoostRegressionTrainer.Options)). + /// + public sealed class Options : OptionsBase + { + public enum EvaluateMetricType + { + None, + Default, + MeanAbsoluteError, + RootMeanSquaredError, + MeanSquaredError + }; + + /// + /// Determines what evaluation metric to use. + /// + [Argument(ArgumentType.AtMostOnce, + HelpText = "Evaluation metrics.", + ShortName = "em")] + public EvaluateMetricType EvaluationMetric = EvaluateMetricType.RootMeanSquaredError; + +#if false + static Options() + { + NameMapping.Add(nameof(EvaluateMetricType), "metric"); + NameMapping.Add(nameof(EvaluateMetricType.None), "None"); + NameMapping.Add(nameof(EvaluateMetricType.Default), ""); + NameMapping.Add(nameof(EvaluateMetricType.MeanAbsoluteError), "mae"); + NameMapping.Add(nameof(EvaluateMetricType.RootMeanSquaredError), "rmse"); + NameMapping.Add(nameof(EvaluateMetricType.MeanSquaredError), "mse"); + } + + internal override Dictionary ToDictionary(IHost host) + { + var res = base.ToDictionary(host); + res[GetOptionName(nameof(EvaluateMetricType))] = GetOptionName(EvaluationMetric.ToString()); + + return res; + } +#endif + } + + /// + /// Initializes a new instance of + /// + /// The private instance of . + /// The name of the label column. + /// The name of the feature column. + /// The name of the example weight column (optional). + /// The number of leaves to use. + /// The minimal number of data points allowed in a leaf of the tree, out of the subsampled data. + /// The learning rate. + /// Number of iterations. + internal XGBoostRegressionTrainer(IHostEnvironment env, + string labelColumnName = DefaultColumnNames.Label, + string featureColumnName = DefaultColumnNames.Features, + string exampleWeightColumnName = null, + int? numberOfLeaves = null, + int? minimumExampleCountPerLeaf = null, + double? learningRate = null, + int numberOfIterations = Defaults.NumberOfIterations + ) + : this(env, new Options() + { + LabelColumnName = labelColumnName, + FeatureColumnName = featureColumnName, + ExampleWeightColumnName = exampleWeightColumnName, +#if false + NumberOfLeaves = numberOfLeaves, + MinimumExampleCountPerLeaf = minimumExampleCountPerLeaf, + LearningRate = learningRate, + NumberOfIterations = numberOfIterations +#endif + }) + { + } + + internal XGBoostRegressionTrainer(IHostEnvironment env, Options options) + : base(env, LoadNameValue, options, TrainerUtils.MakeR4ScalarColumn(options.LabelColumnName)) + { + } + + private protected override XGBoostRegressionModelParameters CreatePredictor() + { + Host.Check(TrainedEnsemble != null, + "The predictor cannot be created before training is complete"); +#if true + return null; +#else + var innerArgs = LightGbmInterfaceUtils.JoinParameters(GbmOptions); + return new LightGbmRegressionModelParameters(Host, TrainedEnsemble, FeatureCount, innerArgs); +#endif + } + +#if false + private protected override void CheckDataValid(IChannel ch, RoleMappedData data) + { + Host.AssertValue(ch); + base.CheckDataValid(ch, data); + var labelType = data.Schema.Label.Value.Type; + if (!(labelType is BooleanDataViewType || labelType is KeyDataViewType || labelType == NumberDataViewType.Single)) + { + throw ch.ExceptParam(nameof(data), + $"Label column '{data.Schema.Label.Value.Name}' is of type '{labelType.RawType}', but must be an unsigned int, boolean or float."); + } + } + + private protected override void CheckAndUpdateParametersBeforeTraining(IChannel ch, RoleMappedData data, float[] labels, int[] groups) + { + GbmOptions["objective"] = "regression"; + } + + private protected override SchemaShape.Column[] GetOutputColumnsCore(SchemaShape inputSchema) + { + return new[] + { + new SchemaShape.Column(DefaultColumnNames.Score, SchemaShape.Column.VectorKind.Scalar, NumberDataViewType.Single, false, new SchemaShape(AnnotationUtils.GetTrainerOutputAnnotation())) + }; + } + + private protected override RegressionPredictionTransformer MakeTransformer(LightGbmRegressionModelParameters model, DataViewSchema trainSchema) + => new RegressionPredictionTransformer(Host, model, trainSchema, FeatureColumn.Name); +#endif + } +} diff --git a/src/Microsoft.ML.XGBoost/XGBoostTrainerBase.cs b/src/Microsoft.ML.XGBoost/XGBoostTrainerBase.cs new file mode 100644 index 0000000000..80d49e9c7c --- /dev/null +++ b/src/Microsoft.ML.XGBoost/XGBoostTrainerBase.cs @@ -0,0 +1,224 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using Microsoft.ML.CommandLine; +using Microsoft.ML.Data; +using Microsoft.ML.EntryPoints; +using Microsoft.ML.Internal.Internallearn; +using Microsoft.ML.Internal.Utilities; +using Microsoft.ML.Runtime; +using Microsoft.ML.Trainers; +using Microsoft.ML.Trainers.FastTree; + +namespace Microsoft.ML.Trainers.XGBoost +{ + [BestFriend] + internal static class Defaults + { + public const int NumberOfIterations = 100; + } + + public abstract class XGBoostTrainerBase : TrainerEstimatorBaseWithGroupId + where TTransformer : ISingleFeaturePredictionTransformer + where TModel : class // IPredictorProducing + where TOptions : XGBoostTrainerBase.OptionsBase, new() + { + internal const string LoadNameValue = "XGBoostPredictor"; + internal const string UserNameValue = "XGBoost Predictor"; + internal const string Summary = "The base logic for all XGBoost-based trainers."; + + private protected int FeatureCount; + private protected InternalTreeEnsemble TrainedEnsemble; + +#if false + /// + /// The shrinkage rate for trees, used to prevent over-fitting. + /// Also aliased to "eta" + /// + /// + /// Valid range is (0,1]. + /// + [Argument(ArgumentType.AtMostOnce, + HelpText = "Shrinkage rate for trees, used to prevent over-fitting. Range: (0,1].", + SortOrder = 2, ShortName = "lr", NullName = "")] + public double? LearningRate; + + /// + /// The maximum number of leaves in one tree. + /// + [Argument(ArgumentType.AtMostOnce, HelpText = "Maximum leaves for trees.", + SortOrder = 2, ShortName = "nl", NullName = "")] + public int? NumberOfLeaves; + + /// + /// Minimum loss reduction required to make a further partition on a leaf node of + /// the tree. The larger gamma is, the more conservative the algorithm will be. + /// aka: gamma + /// range: [0,\infnty] + /// + public int? MinSplitLoss; +#endif + + /// + /// Maximum depth of a tree. Increasing this value will make the model more complex and + /// more likely to overfit. 0 indicates no limit on depth. Beware that XGBoost aggressively + /// consumes memory when training a deep tree. exact tree method requires non-zero value. + /// range: [0,\infnty], default=6 + /// + public int? MaxDepth; + + /// + /// Minimum sum of instance weight (hessian) needed in a child. If the tree partition step + /// results in a leaf node with the sum of instance weight less than min_child_weight, then + /// the building process will give up further partitioning. In linear regression task, this + /// simply corresponds to minimum number of instances needed to be in each node. The larger + /// MinChildWeight is, the more conservative the algorithm will be. + /// range: [0,\infnty] + /// + public float? MinChildWeight; + + private protected XGBoostTrainerBase(IHost host, + SchemaShape.Column feature, + SchemaShape.Column label, SchemaShape.Column weight = default, SchemaShape.Column groupId = default) : base(host, feature, label, weight, groupId) + { + } + +#if false + /// + /// L2 regularization term on weights. Increasing this value will make model more conservative + /// + public float? L2Regularization; + + /// + /// L1 regularization term on weights. Increasing this value will make model more conservative. + /// + public float? L1Regularization; +#endif + + public class OptionsBase : TrainerInputBaseWithGroupId + { + + // Static override name map that maps friendly names to XGBMArguments arguments. + // If an argument is not here, then its name is identical to a lightGBM argument + // and does not require a mapping, for example, Subsample. + // For a complete list, see https://xgboost.readthedocs.io/en/latest/parameter.html + private protected static Dictionary NameMapping = new Dictionary() + { +#if false + {nameof(MinSplitLoss), "min_split_loss"}, + {nameof(NumberOfLeaves), "num_leaves"}, +#endif + {nameof(MaxDepth), "max_depth" }, + {nameof(MinChildWeight), "min_child_weight" }, +#if false + {nameof(L2Regularization), "lambda" }, + {nameof(L1Regularization), "alpha" } +#endif + }; + + private BoosterParameterBase.OptionsBase _boosterParameter; + +#if true + /// + /// Determines which booster to use. + /// + /// + /// Available boosters are , and . + /// + [Argument(ArgumentType.Multiple, + HelpText = "Which booster to use, can be gbtree, gblinear or dart. gbtree and dart use tree based model while gblinear uses linear function.", + Name = "Booster", + SortOrder = 3)] + internal IBoosterParameterFactory BoosterFactory = new GradientBooster.Options(); +#endif + + /// + /// Booster parameter to use + /// + public BoosterParameterBase.OptionsBase Booster + { + get => _boosterParameter; + + set + { + _boosterParameter = value; + BoosterFactory = _boosterParameter; + } + + } + + private protected string GetOptionName(string name) + { + if (NameMapping.ContainsKey(name)) + return NameMapping[name]; + return XGBoostInterfaceUtils.GetOptionName(name); + } + } + + private protected override TModel TrainModelCore(TrainContext context) + { +#if true + return null; +#else + InitializeBeforeTraining(); + + Host.CheckValue(context, nameof(context)); + + Dataset dtrain = null; + Dataset dvalid = null; + CategoricalMetaData catMetaData; + try + { + using (var ch = Host.Start("Loading data for XGBoost")) + { + using (var pch = Host.StartProgressChannel("Loading data for XGBoost")) + { + dtrain = LoadTrainingData(ch, context.TrainingSet, out catMetaData); + if (context.ValidationSet != null) + dvalid = LoadValidationData(ch, dtrain, context.ValidationSet, catMetaData); + } + } + using (var ch = Host.Start("Training with XGBoost")) + { + using (var pch = Host.StartProgressChannel("Training with XGBoost")) + TrainCore(ch, pch, dtrain, catMetaData, dvalid); + } + } + finally + { + dtrain?.Dispose(); + dvalid?.Dispose(); + DisposeParallelTraining(); + } + return CreatePredictor(); +#endif + } + + private protected XGBoostTrainerBase(IHostEnvironment env, string name, TOptions options, SchemaShape.Column label) + : base(Contracts.CheckRef(env, nameof(env)).Register(name), TrainerUtils.MakeR4VecFeature(options.FeatureColumnName), label, + TrainerUtils.MakeR4ScalarWeightColumn(options.ExampleWeightColumnName), TrainerUtils.MakeU4ScalarColumn(options.RowGroupColumnName)) + { + Host.CheckValue(options, nameof(options)); +#if false + Contracts.CheckUserArg(options.NumberOfIterations >= 0, nameof(options.NumberOfIterations), "must be >= 0."); + Contracts.CheckUserArg(options.MaximumBinCountPerFeature > 0, nameof(options.MaximumBinCountPerFeature), "must be > 0."); + Contracts.CheckUserArg(options.MinimumExampleCountPerGroup > 0, nameof(options.MinimumExampleCountPerGroup), "must be > 0."); + Contracts.CheckUserArg(options.MaximumCategoricalSplitPointCount > 0, nameof(options.MaximumCategoricalSplitPointCount), "must be > 0."); + Contracts.CheckUserArg(options.CategoricalSmoothing >= 0, nameof(options.CategoricalSmoothing), "must be >= 0."); + Contracts.CheckUserArg(options.L2CategoricalRegularization >= 0.0, nameof(options.L2CategoricalRegularization), "must be >= 0."); +#endif + +#if false + XGBoostTrainerOptions = options; + GbmOptions = XGBoostTrainerOption.ToDictionary(Host); +#endif + } + + private protected abstract TModel CreatePredictor(); + } +}