From 50e2a1056bbd61fcba05e020f210b48189615235 Mon Sep 17 00:00:00 2001 From: Aleksei Smirnov Date: Sun, 1 Oct 2023 16:37:32 +0300 Subject: [PATCH 1/7] Fixes incorrect work of DataFrame with VBufferColumn when number of elements is greater than Int.MaxValue --- .../VBufferDataFrameColumn.cs | 25 ++++++++----------- .../DataFrameTests.cs | 21 +++++++++++++--- 2 files changed, 28 insertions(+), 18 deletions(-) diff --git a/src/Microsoft.Data.Analysis/VBufferDataFrameColumn.cs b/src/Microsoft.Data.Analysis/VBufferDataFrameColumn.cs index ee860cfa33..dddf94e8e5 100644 --- a/src/Microsoft.Data.Analysis/VBufferDataFrameColumn.cs +++ b/src/Microsoft.Data.Analysis/VBufferDataFrameColumn.cs @@ -6,11 +6,6 @@ using System.Collections; using System.Collections.Generic; using System.Diagnostics; -using System.Linq; -using System.Runtime.InteropServices; -using System.Text; -using Apache.Arrow; -using Apache.Arrow.Types; using Microsoft.ML; using Microsoft.ML.Data; @@ -83,7 +78,7 @@ public void Append(VBuffer value) Length++; } - private int GetBufferIndexContainingRowIndex(ref long rowIndex) + private int GetBufferIndexContainingRowIndex(long rowIndex) { if (rowIndex >= Length) { @@ -95,22 +90,23 @@ private int GetBufferIndexContainingRowIndex(ref long rowIndex) protected override object GetValue(long rowIndex) { - int bufferIndex = GetBufferIndexContainingRowIndex(ref rowIndex); - return _vBuffers[bufferIndex][(int)rowIndex]; + int bufferIndex = GetBufferIndexContainingRowIndex(rowIndex); + return _vBuffers[bufferIndex][(int)(rowIndex % int.MaxValue)]; } protected override IReadOnlyList GetValues(long startIndex, int length) { var ret = new List(); - int bufferIndex = GetBufferIndexContainingRowIndex(ref startIndex); + int bufferIndex = GetBufferIndexContainingRowIndex(startIndex); + int bufferOffset = (int)(startIndex % int.MaxValue); while (ret.Count < length && bufferIndex < _vBuffers.Count) { - for (int i = (int)startIndex; ret.Count < length && i < _vBuffers[bufferIndex].Count; i++) + for (int i = bufferOffset; ret.Count < length && i < _vBuffers[bufferIndex].Count; i++) { ret.Add(_vBuffers[bufferIndex][i]); } bufferIndex++; - startIndex = 0; + bufferOffset = 0; } return ret; } @@ -119,9 +115,10 @@ protected override void SetValue(long rowIndex, object value) { if (value == null || value is VBuffer) { - int bufferIndex = GetBufferIndexContainingRowIndex(ref rowIndex); - var oldValue = this[rowIndex]; - _vBuffers[bufferIndex][(int)rowIndex] = (VBuffer)value; + int bufferIndex = GetBufferIndexContainingRowIndex(rowIndex); + int bufferOffset = (int)(rowIndex % int.MaxValue); + var oldValue = _vBuffers[bufferIndex][bufferOffset]; + _vBuffers[bufferIndex][bufferOffset] = (VBuffer)value; if (!oldValue.Equals((VBuffer)value)) { if (value == null) diff --git a/test/Microsoft.Data.Analysis.Tests/DataFrameTests.cs b/test/Microsoft.Data.Analysis.Tests/DataFrameTests.cs index 85e4ccd79c..cf17565ae8 100644 --- a/test/Microsoft.Data.Analysis.Tests/DataFrameTests.cs +++ b/test/Microsoft.Data.Analysis.Tests/DataFrameTests.cs @@ -75,7 +75,7 @@ public static ArrowStringDataFrameColumn CreateArrowStringColumn(int length, boo return new ArrowStringDataFrameColumn("ArrowString", dataMemory, offsetMemory, nullMemory, length, nullCount); } - public static VBufferDataFrameColumn CreateVBufferDataFrame(int length) + public static VBufferDataFrameColumn CreateVBufferDataFrameColumn(int length) { var buffers = Enumerable.Repeat(new VBuffer(5, new[] { 0, 1, 2, 3, 4 }), length).ToArray(); return new VBufferDataFrameColumn("VBuffer", buffers); @@ -85,7 +85,7 @@ public static DataFrame MakeDataFrameWithAllColumnTypes(int length, bool withNul { DataFrame df = MakeDataFrameWithAllMutableAndArrowColumnTypes(length, withNulls); - var vBufferColumn = CreateVBufferDataFrame(length); + var vBufferColumn = CreateVBufferDataFrameColumn(length); df.Columns.Insert(df.Columns.Count, vBufferColumn); return df; @@ -230,15 +230,28 @@ public DataFrame SplitTrainTest(DataFrame input, float testRatio, out DataFrame } [Fact] - public void TestVBufferColumn() + public void TestVBufferColumn_Creation() { - var vBufferColumn = CreateVBufferDataFrame(10); + var vBufferColumn = CreateVBufferDataFrameColumn(10); Assert.Equal(10, vBufferColumn.Length); Assert.Equal(5, vBufferColumn[0].GetValues().Length); Assert.Equal(0, vBufferColumn[0].GetValues()[0]); } + [Fact] + public void TestVBufferColumn_Indexer() + { + var buffer = new VBuffer(5, new[] { 4, 3, 2, 1, 0 }); + + var vBufferColumn = new VBufferDataFrameColumn("VBuffer", 1); + vBufferColumn[0] = buffer; + + Assert.Equal(1, vBufferColumn.Length); + Assert.Equal(5, vBufferColumn[0].GetValues().Length); + Assert.Equal(0, vBufferColumn[0].GetValues()[4]); + } + [Fact] public void TestIndexer() { From f53e56d5fc666f9b9839062286c6a928921fba14 Mon Sep 17 00:00:00 2001 From: Aleksei Smirnov Date: Sun, 1 Oct 2023 17:26:46 +0300 Subject: [PATCH 2/7] Fix calculation of max capacity and amount of required buffers --- .../VBufferDataFrameColumn.cs | 28 +++++++++++-------- .../DataFrameTests.cs | 16 +++++++++++ 2 files changed, 33 insertions(+), 11 deletions(-) diff --git a/src/Microsoft.Data.Analysis/VBufferDataFrameColumn.cs b/src/Microsoft.Data.Analysis/VBufferDataFrameColumn.cs index dddf94e8e5..e87fa94351 100644 --- a/src/Microsoft.Data.Analysis/VBufferDataFrameColumn.cs +++ b/src/Microsoft.Data.Analysis/VBufferDataFrameColumn.cs @@ -5,7 +5,9 @@ using System; using System.Collections; using System.Collections.Generic; +using System.Data; using System.Diagnostics; +using System.Runtime.CompilerServices; using Microsoft.ML; using Microsoft.ML.Data; @@ -16,6 +18,10 @@ namespace Microsoft.Data.Analysis /// public partial class VBufferDataFrameColumn : DataFrameColumn, IEnumerable> { + //The maximum size in any single dimension for byte array is 0x7FFFFFc7 - 2147483591 + //See https://learn.microsoft.com/en-us/dotnet/framework/configure-apps/file-schema/runtime/gcallowverylargeobjects-element + public static int MaxCapacity = 2147483591 / Unsafe.SizeOf>(); + private readonly List>> _vBuffers = new List>>(); // To store more than intMax number of vbuffers /// @@ -25,11 +31,11 @@ public partial class VBufferDataFrameColumn : DataFrameColumn, IEnumerableLength of values public VBufferDataFrameColumn(string name, long length = 0) : base(name, length, typeof(VBuffer)) { - int numberOfBuffersRequired = Math.Max((int)(length / int.MaxValue), 1); + int numberOfBuffersRequired = (int)(length / MaxCapacity + 1); for (int i = 0; i < numberOfBuffersRequired; i++) { - long bufferLen = length - _vBuffers.Count * int.MaxValue; - List> buffer = new List>((int)Math.Min(int.MaxValue, bufferLen)); + int bufferLen = (int)Math.Min(MaxCapacity, length - _vBuffers.Count * MaxCapacity); + List> buffer = new List>(bufferLen); _vBuffers.Add(buffer); for (int j = 0; j < bufferLen; j++) { @@ -69,7 +75,7 @@ protected internal override void Resize(long length) public void Append(VBuffer value) { List> lastBuffer = _vBuffers[_vBuffers.Count - 1]; - if (lastBuffer.Count == int.MaxValue) + if (lastBuffer.Count == MaxCapacity) { lastBuffer = new List>(); _vBuffers.Add(lastBuffer); @@ -85,20 +91,20 @@ private int GetBufferIndexContainingRowIndex(long rowIndex) throw new ArgumentOutOfRangeException(Strings.ColumnIndexOutOfRange, nameof(rowIndex)); } - return (int)(rowIndex / int.MaxValue); + return (int)(rowIndex / MaxCapacity); } protected override object GetValue(long rowIndex) { int bufferIndex = GetBufferIndexContainingRowIndex(rowIndex); - return _vBuffers[bufferIndex][(int)(rowIndex % int.MaxValue)]; + return _vBuffers[bufferIndex][(int)(rowIndex % MaxCapacity)]; } protected override IReadOnlyList GetValues(long startIndex, int length) { var ret = new List(); int bufferIndex = GetBufferIndexContainingRowIndex(startIndex); - int bufferOffset = (int)(startIndex % int.MaxValue); + int bufferOffset = (int)(startIndex % MaxCapacity); while (ret.Count < length && bufferIndex < _vBuffers.Count) { for (int i = bufferOffset; ret.Count < length && i < _vBuffers[bufferIndex].Count; i++) @@ -116,7 +122,7 @@ protected override void SetValue(long rowIndex, object value) if (value == null || value is VBuffer) { int bufferIndex = GetBufferIndexContainingRowIndex(rowIndex); - int bufferOffset = (int)(rowIndex % int.MaxValue); + int bufferOffset = (int)(rowIndex % MaxCapacity); var oldValue = _vBuffers[bufferIndex][bufferOffset]; _vBuffers[bufferIndex][bufferOffset] = (VBuffer)value; if (!oldValue.Equals((VBuffer)value)) @@ -247,11 +253,11 @@ private VBufferDataFrameColumn CloneImplementation(PrimitiveDataFrameColum List> setBuffer = ret._vBuffers[0]; long setBufferMinRange = 0; - long setBufferMaxRange = int.MaxValue; + long setBufferMaxRange = MaxCapacity; List> getBuffer = _vBuffers[0]; long getBufferMinRange = 0; - long getBufferMaxRange = int.MaxValue; - long maxCapacity = int.MaxValue; + long getBufferMaxRange = MaxCapacity; + long maxCapacity = MaxCapacity; if (mapIndices.DataType == typeof(long)) { PrimitiveDataFrameColumn longMapIndices = mapIndices as PrimitiveDataFrameColumn; diff --git a/test/Microsoft.Data.Analysis.Tests/DataFrameTests.cs b/test/Microsoft.Data.Analysis.Tests/DataFrameTests.cs index cf17565ae8..273654cb72 100644 --- a/test/Microsoft.Data.Analysis.Tests/DataFrameTests.cs +++ b/test/Microsoft.Data.Analysis.Tests/DataFrameTests.cs @@ -252,6 +252,22 @@ public void TestVBufferColumn_Indexer() Assert.Equal(0, vBufferColumn[0].GetValues()[4]); } + [Fact] + public void TestVBufferColumn_Indexer_MoreThanMaxInt() + { + var buffer = new VBuffer(5, new[] { 4, 3, 2, 1, 0 }); + + var vBufferColumn = new VBufferDataFrameColumn("VBuffer", VBufferDataFrameColumn.MaxCapacity + 3); + + long index = (long)int.MaxValue + 1; + + vBufferColumn[index] = buffer; + + Assert.Equal(1, vBufferColumn.Length); + Assert.Equal(5, vBufferColumn[index].GetValues().Length); + Assert.Equal(0, vBufferColumn[index].GetValues()[4]); + } + [Fact] public void TestIndexer() { From 0f01084846a45e37748b8655ace6cf1f853016d4 Mon Sep 17 00:00:00 2001 From: Aleksei Smirnov Date: Sun, 1 Oct 2023 19:07:11 +0300 Subject: [PATCH 3/7] Fix unit test --- .../VBufferDataFrameColumn.cs | 4 ++-- .../DataFrameTests.cs | 21 ++++++++++++------- 2 files changed, 16 insertions(+), 9 deletions(-) diff --git a/src/Microsoft.Data.Analysis/VBufferDataFrameColumn.cs b/src/Microsoft.Data.Analysis/VBufferDataFrameColumn.cs index e87fa94351..4fb1851a7c 100644 --- a/src/Microsoft.Data.Analysis/VBufferDataFrameColumn.cs +++ b/src/Microsoft.Data.Analysis/VBufferDataFrameColumn.cs @@ -18,9 +18,9 @@ namespace Microsoft.Data.Analysis /// public partial class VBufferDataFrameColumn : DataFrameColumn, IEnumerable> { - //The maximum size in any single dimension for byte array is 0x7FFFFFc7 - 2147483591 + //The maximum size in any single dimension for array containing other types than byte or single byte structure is 0X7FEFFFFF - 2146435071 //See https://learn.microsoft.com/en-us/dotnet/framework/configure-apps/file-schema/runtime/gcallowverylargeobjects-element - public static int MaxCapacity = 2147483591 / Unsafe.SizeOf>(); + public static int MaxCapacity = 2146435071 / Unsafe.SizeOf>(); private readonly List>> _vBuffers = new List>>(); // To store more than intMax number of vbuffers diff --git a/test/Microsoft.Data.Analysis.Tests/DataFrameTests.cs b/test/Microsoft.Data.Analysis.Tests/DataFrameTests.cs index 273654cb72..04e6188827 100644 --- a/test/Microsoft.Data.Analysis.Tests/DataFrameTests.cs +++ b/test/Microsoft.Data.Analysis.Tests/DataFrameTests.cs @@ -255,17 +255,24 @@ public void TestVBufferColumn_Indexer() [Fact] public void TestVBufferColumn_Indexer_MoreThanMaxInt() { - var buffer = new VBuffer(5, new[] { 4, 3, 2, 1, 0 }); + var originalValues = new[] { 4, 3, 2, 1, 0 }; - var vBufferColumn = new VBufferDataFrameColumn("VBuffer", VBufferDataFrameColumn.MaxCapacity + 3); + var length = VBufferDataFrameColumn.MaxCapacity + 3; - long index = (long)int.MaxValue + 1; + var vBufferColumn = new VBufferDataFrameColumn("VBuffer", length); + long index = length - 2; - vBufferColumn[index] = buffer; + vBufferColumn[index] = new VBuffer(5, originalValues); - Assert.Equal(1, vBufferColumn.Length); - Assert.Equal(5, vBufferColumn[index].GetValues().Length); - Assert.Equal(0, vBufferColumn[index].GetValues()[4]); + var values = vBufferColumn[index].GetValues(); + + Assert.Equal(length, vBufferColumn.Length); + Assert.Equal(5, values.Length); + + for (int i = 0; i < values.Length; i++) + { + Assert.Equal(originalValues[i], values[i]); + } } [Fact] From e2d11a8035272b847b2c4abfa3743d682af5127b Mon Sep 17 00:00:00 2001 From: Aleksei Smirnov Date: Sun, 1 Oct 2023 20:42:48 +0300 Subject: [PATCH 4/7] Run test allocating more than 2 Gb of memory on 64bit env only --- test/Microsoft.Data.Analysis.Tests/DataFrameTests.cs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/Microsoft.Data.Analysis.Tests/DataFrameTests.cs b/test/Microsoft.Data.Analysis.Tests/DataFrameTests.cs index 04e6188827..61d9361ade 100644 --- a/test/Microsoft.Data.Analysis.Tests/DataFrameTests.cs +++ b/test/Microsoft.Data.Analysis.Tests/DataFrameTests.cs @@ -9,6 +9,7 @@ using Apache.Arrow; using Microsoft.ML; using Microsoft.ML.Data; +using Microsoft.ML.TestFramework.Attributes; using Xunit; namespace Microsoft.Data.Analysis.Tests @@ -252,7 +253,7 @@ public void TestVBufferColumn_Indexer() Assert.Equal(0, vBufferColumn[0].GetValues()[4]); } - [Fact] + [X64Fact("32-bit doesn't allow to allocate more than 2 Gb")] public void TestVBufferColumn_Indexer_MoreThanMaxInt() { var originalValues = new[] { 4, 3, 2, 1, 0 }; From f1fc194f4e90ee276bf250671293e10de0ec1d4c Mon Sep 17 00:00:00 2001 From: Aleksei Smirnov Date: Mon, 2 Oct 2023 16:29:52 +0300 Subject: [PATCH 5/7] Fix StringDataFrameColumn same way as VBufferDataFrameColumn --- .../StringDataFrameColumn.cs | 48 +++++++++++-------- 1 file changed, 28 insertions(+), 20 deletions(-) diff --git a/src/Microsoft.Data.Analysis/StringDataFrameColumn.cs b/src/Microsoft.Data.Analysis/StringDataFrameColumn.cs index 1a46ff74a9..9eb93931c1 100644 --- a/src/Microsoft.Data.Analysis/StringDataFrameColumn.cs +++ b/src/Microsoft.Data.Analysis/StringDataFrameColumn.cs @@ -6,6 +6,7 @@ using System.Collections; using System.Collections.Generic; using System.Diagnostics; +using System.Runtime.CompilerServices; using Microsoft.ML; using Microsoft.ML.Data; @@ -17,15 +18,19 @@ namespace Microsoft.Data.Analysis /// Is NOT Arrow compatible public partial class StringDataFrameColumn : DataFrameColumn, IEnumerable { + //The maximum size in any single dimension for array containing other types than byte or single byte structure is 0X7FEFFFFF - 2146435071 + //See https://learn.microsoft.com/en-us/dotnet/framework/configure-apps/file-schema/runtime/gcallowverylargeobjects-element + public static int MaxCapacity = 2146435071 / Unsafe.SizeOf(); // Size of pointer + private readonly List> _stringBuffers = new List>(); // To store more than intMax number of strings public StringDataFrameColumn(string name, long length = 0) : base(name, length, typeof(string)) { - int numberOfBuffersRequired = Math.Max((int)(length / int.MaxValue), 1); + int numberOfBuffersRequired = Math.Max((int)(length / MaxCapacity), 1); for (int i = 0; i < numberOfBuffersRequired; i++) { - long bufferLen = length - _stringBuffers.Count * int.MaxValue; - List buffer = new List((int)Math.Min(int.MaxValue, bufferLen)); + long bufferLen = length - _stringBuffers.Count * MaxCapacity; + List buffer = new List((int)Math.Min(MaxCapacity, bufferLen)); _stringBuffers.Add(buffer); for (int j = 0; j < bufferLen; j++) { @@ -64,7 +69,7 @@ protected internal override void Resize(long length) public void Append(string value) { List lastBuffer = _stringBuffers[_stringBuffers.Count - 1]; - if (lastBuffer.Count == int.MaxValue) + if (lastBuffer.Count == MaxCapacity) { lastBuffer = new List(); _stringBuffers.Add(lastBuffer); @@ -75,33 +80,34 @@ public void Append(string value) Length++; } - private int GetBufferIndexContainingRowIndex(ref long rowIndex) + private int GetBufferIndexContainingRowIndex(long rowIndex) { if (rowIndex >= Length) { throw new ArgumentOutOfRangeException(Strings.ColumnIndexOutOfRange, nameof(rowIndex)); } - return (int)(rowIndex / int.MaxValue); + return (int)(rowIndex / MaxCapacity); } protected override object GetValue(long rowIndex) { - int bufferIndex = GetBufferIndexContainingRowIndex(ref rowIndex); - return _stringBuffers[bufferIndex][(int)rowIndex]; + int bufferIndex = GetBufferIndexContainingRowIndex(rowIndex); + return _stringBuffers[bufferIndex][(int)(rowIndex % MaxCapacity)]; } protected override IReadOnlyList GetValues(long startIndex, int length) { var ret = new List(); - int bufferIndex = GetBufferIndexContainingRowIndex(ref startIndex); + int bufferIndex = GetBufferIndexContainingRowIndex(startIndex); + int bufferOffset = (int)(startIndex % MaxCapacity); while (ret.Count < length && bufferIndex < _stringBuffers.Count) { - for (int i = (int)startIndex; ret.Count < length && i < _stringBuffers[bufferIndex].Count; i++) + for (int i = bufferOffset; ret.Count < length && i < _stringBuffers[bufferIndex].Count; i++) { ret.Add(_stringBuffers[bufferIndex][i]); } bufferIndex++; - startIndex = 0; + bufferOffset = 0; } return ret; } @@ -110,9 +116,10 @@ protected override void SetValue(long rowIndex, object value) { if (value == null || value is string) { - int bufferIndex = GetBufferIndexContainingRowIndex(ref rowIndex); + int bufferIndex = GetBufferIndexContainingRowIndex(rowIndex); + int bufferOffset = (int)(rowIndex % MaxCapacity); var oldValue = this[rowIndex]; - _stringBuffers[bufferIndex][(int)rowIndex] = (string)value; + _stringBuffers[bufferIndex][bufferOffset] = (string)value; if (oldValue != (string)value) { if (value == null) @@ -138,15 +145,16 @@ protected override void SetValue(long rowIndex, object value) get { var ret = new List(); - int bufferIndex = GetBufferIndexContainingRowIndex(ref startIndex); + int bufferIndex = GetBufferIndexContainingRowIndex(startIndex); + int bufferOffset = (int)(startIndex % MaxCapacity); while (ret.Count < length && bufferIndex < _stringBuffers.Count) { - for (int i = (int)startIndex; ret.Count < length && i < _stringBuffers[bufferIndex].Count; i++) + for (int i = bufferOffset; ret.Count < length && i < _stringBuffers[bufferIndex].Count; i++) { ret.Add(_stringBuffers[bufferIndex][i]); } bufferIndex++; - startIndex = 0; + bufferOffset = 0; } return ret; } @@ -194,7 +202,7 @@ private PrimitiveDataFrameColumn GetSortIndices(Comparer comparer, sortIndices[i] = i; if (buffer[i] == null) { - columnNullIndices[nullIndicesSlot] = i + bufferSortIndices.Count * int.MaxValue; + columnNullIndices[nullIndicesSlot] = i + bufferSortIndices.Count * MaxCapacity; nullIndicesSlot++; } } @@ -295,11 +303,11 @@ private StringDataFrameColumn CloneImplementation(PrimitiveDataFrameColumn List setBuffer = ret._stringBuffers[0]; long setBufferMinRange = 0; - long setBufferMaxRange = int.MaxValue; + long setBufferMaxRange = MaxCapacity; List getBuffer = _stringBuffers[0]; long getBufferMinRange = 0; - long getBufferMaxRange = int.MaxValue; - long maxCapacity = int.MaxValue; + long getBufferMaxRange = MaxCapacity; + long maxCapacity = MaxCapacity; if (mapIndices.DataType == typeof(long)) { PrimitiveDataFrameColumn longMapIndices = mapIndices as PrimitiveDataFrameColumn; From 251d955c42261a708fe67f369f338060add34979 Mon Sep 17 00:00:00 2001 From: Aleksei Smirnov Date: Mon, 2 Oct 2023 20:13:43 +0300 Subject: [PATCH 6/7] Fix wrong amount of buffers created in constructor of StringDataFrameColumn --- src/Microsoft.Data.Analysis/StringDataFrameColumn.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Microsoft.Data.Analysis/StringDataFrameColumn.cs b/src/Microsoft.Data.Analysis/StringDataFrameColumn.cs index 9eb93931c1..700b42cbad 100644 --- a/src/Microsoft.Data.Analysis/StringDataFrameColumn.cs +++ b/src/Microsoft.Data.Analysis/StringDataFrameColumn.cs @@ -26,7 +26,7 @@ public partial class StringDataFrameColumn : DataFrameColumn, IEnumerable Date: Wed, 4 Oct 2023 00:11:05 +0300 Subject: [PATCH 7/7] Fix code review findings --- src/Microsoft.Data.Analysis/ArrayUtility.cs | 18 ++++++++++++++++++ src/Microsoft.Data.Analysis/DataFrameBuffer.cs | 2 +- .../ReadOnlyDataFrameBuffer.cs | 6 +----- .../StringDataFrameColumn.cs | 4 +--- .../VBufferDataFrameColumn.cs | 5 ++--- .../BufferTests.cs | 2 +- 6 files changed, 24 insertions(+), 13 deletions(-) create mode 100644 src/Microsoft.Data.Analysis/ArrayUtility.cs diff --git a/src/Microsoft.Data.Analysis/ArrayUtility.cs b/src/Microsoft.Data.Analysis/ArrayUtility.cs new file mode 100644 index 0000000000..daffef9f8d --- /dev/null +++ b/src/Microsoft.Data.Analysis/ArrayUtility.cs @@ -0,0 +1,18 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.Collections.Generic; +using System.Text; + +namespace Microsoft.Data.Analysis +{ + internal static class ArrayUtility + { + // Maximum size of one-dimensional array. + // See: https://msdn.microsoft.com/en-us/library/hh285054(v=vs.110).aspx + // Polyfilling Array.MaxLength API for netstandard2.0 + public const int ArrayMaxSize = 0X7FEFFFFF; + } +} diff --git a/src/Microsoft.Data.Analysis/DataFrameBuffer.cs b/src/Microsoft.Data.Analysis/DataFrameBuffer.cs index fd7f23964f..b4a3fc15a8 100644 --- a/src/Microsoft.Data.Analysis/DataFrameBuffer.cs +++ b/src/Microsoft.Data.Analysis/DataFrameBuffer.cs @@ -80,7 +80,7 @@ public void EnsureCapacity(int numberOfValues) if (newLength > Capacity) { //Double buffer size, but not higher than MaxByteCapacity - var doubledSize = (int)Math.Min((long)ReadOnlyBuffer.Length * 2, MaxCapacityInBytes); + var doubledSize = (int)Math.Min((long)ReadOnlyBuffer.Length * 2, ArrayUtility.ArrayMaxSize); var newCapacity = Math.Max(newLength * Size, doubledSize); var memory = new Memory(new byte[newCapacity]); diff --git a/src/Microsoft.Data.Analysis/ReadOnlyDataFrameBuffer.cs b/src/Microsoft.Data.Analysis/ReadOnlyDataFrameBuffer.cs index 069aa3e94a..f08b2120b1 100644 --- a/src/Microsoft.Data.Analysis/ReadOnlyDataFrameBuffer.cs +++ b/src/Microsoft.Data.Analysis/ReadOnlyDataFrameBuffer.cs @@ -36,11 +36,7 @@ public ReadOnlyMemory RawReadOnlyMemory protected int Capacity => ReadOnlyBuffer.Length / Size; - //The maximum size in any single dimension for byte array is 0x7FFFFFc7 - 2147483591 - //See https://learn.microsoft.com/en-us/dotnet/framework/configure-apps/file-schema/runtime/gcallowverylargeobjects-element - public const int MaxCapacityInBytes = 2147483591; - - public static int MaxCapacity => MaxCapacityInBytes / Size; + public static int MaxCapacity => ArrayUtility.ArrayMaxSize / Size; public ReadOnlySpan ReadOnlySpan { diff --git a/src/Microsoft.Data.Analysis/StringDataFrameColumn.cs b/src/Microsoft.Data.Analysis/StringDataFrameColumn.cs index 700b42cbad..03d112d027 100644 --- a/src/Microsoft.Data.Analysis/StringDataFrameColumn.cs +++ b/src/Microsoft.Data.Analysis/StringDataFrameColumn.cs @@ -18,9 +18,7 @@ namespace Microsoft.Data.Analysis /// Is NOT Arrow compatible public partial class StringDataFrameColumn : DataFrameColumn, IEnumerable { - //The maximum size in any single dimension for array containing other types than byte or single byte structure is 0X7FEFFFFF - 2146435071 - //See https://learn.microsoft.com/en-us/dotnet/framework/configure-apps/file-schema/runtime/gcallowverylargeobjects-element - public static int MaxCapacity = 2146435071 / Unsafe.SizeOf(); // Size of pointer + public static int MaxCapacity = ArrayUtility.ArrayMaxSize / Unsafe.SizeOf(); // Max Size in bytes / size of pointer (8 bytes on x64) private readonly List> _stringBuffers = new List>(); // To store more than intMax number of strings diff --git a/src/Microsoft.Data.Analysis/VBufferDataFrameColumn.cs b/src/Microsoft.Data.Analysis/VBufferDataFrameColumn.cs index 4fb1851a7c..fc15c873cb 100644 --- a/src/Microsoft.Data.Analysis/VBufferDataFrameColumn.cs +++ b/src/Microsoft.Data.Analysis/VBufferDataFrameColumn.cs @@ -18,9 +18,8 @@ namespace Microsoft.Data.Analysis /// public partial class VBufferDataFrameColumn : DataFrameColumn, IEnumerable> { - //The maximum size in any single dimension for array containing other types than byte or single byte structure is 0X7FEFFFFF - 2146435071 - //See https://learn.microsoft.com/en-us/dotnet/framework/configure-apps/file-schema/runtime/gcallowverylargeobjects-element - public static int MaxCapacity = 2146435071 / Unsafe.SizeOf>(); + + public static int MaxCapacity = ArrayUtility.ArrayMaxSize / Unsafe.SizeOf>(); private readonly List>> _vBuffers = new List>>(); // To store more than intMax number of vbuffers diff --git a/test/Microsoft.Data.Analysis.Tests/BufferTests.cs b/test/Microsoft.Data.Analysis.Tests/BufferTests.cs index 16672818db..ce797ceda3 100644 --- a/test/Microsoft.Data.Analysis.Tests/BufferTests.cs +++ b/test/Microsoft.Data.Analysis.Tests/BufferTests.cs @@ -457,7 +457,7 @@ public void TestAppend_SizeMoreThanMaxBufferCapacity() [X64Fact("32-bit dosn't allow to allocate more than 2 Gb")] public void TestAppendMany_SizeMoreThanMaxBufferCapacity() { - const int MaxCapacityInBytes = 2147483591; + const int MaxCapacityInBytes = 0X7FEFFFFF; //Check appending values with extending column size over MaxCapacity of ReadOnlyDataFrameBuffer PrimitiveDataFrameColumn intColumn = new PrimitiveDataFrameColumn("Byte1", MaxCapacityInBytes - 5);