From cfc64ea13d9b8a38feaa160a95522157a5e53cb2 Mon Sep 17 00:00:00 2001 From: neon-sunset <20912188+neon-sunset@users.noreply.github.com> Date: Sat, 14 Jan 2023 00:30:15 +0200 Subject: [PATCH 1/6] Vectorize Enumerable.Range(...).ToArray() --- .../src/System/Linq/Range.SpeedOpt.cs | 71 +++++++++++++++++-- 1 file changed, 64 insertions(+), 7 deletions(-) diff --git a/src/libraries/System.Linq/src/System/Linq/Range.SpeedOpt.cs b/src/libraries/System.Linq/src/System/Linq/Range.SpeedOpt.cs index 6704eef5dedc29..f8553e3ceff595 100644 --- a/src/libraries/System.Linq/src/System/Linq/Range.SpeedOpt.cs +++ b/src/libraries/System.Linq/src/System/Linq/Range.SpeedOpt.cs @@ -2,6 +2,10 @@ // The .NET Foundation licenses this file to you under the MIT license. using System.Collections.Generic; +using System.Diagnostics; +using System.Numerics; +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; namespace System.Linq { @@ -16,14 +20,9 @@ public override IEnumerable Select(Func selector public int[] ToArray() { - int[] array = new int[_end - _start]; - int cur = _start; - for (int i = 0; i < array.Length; ++i) - { - array[i] = cur; - ++cur; - } + int[] array = GC.AllocateUninitializedArray(_end - _start); + InitializeSpan(array); return array; } @@ -84,6 +83,64 @@ public int TryGetLast(out bool found) found = true; return _end - 1; } + + // Destination *must* be non-empty and exactly match the range length + private void InitializeSpan(Span destination) + { + Debug.Assert((_end - _start) == destination.Length); + + if (destination.Length < Vector.Count * 2) + { + int end = _start + _end; + ref int pos = ref MemoryMarshal.GetReference(destination); + for (int num = _start; num < end; num++) + { + pos = num; + pos = ref Unsafe.Add(ref pos, 1); + } + } + else + { + InitializeSpanCore(destination); + } + } + + private void InitializeSpanCore(Span destination) + { + int width = Vector.Count; + int stride = Vector.Count * 2; + int remainder = destination.Length % stride; + + // Up to 16 elements which corresponds to AVX512 + Vector initMask = Unsafe.ReadUnaligned>( + ref Unsafe.As(ref MemoryMarshal.GetReference( + (ReadOnlySpan)new int[] { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 }))); + + Vector mask = new Vector(stride); + Vector value = new Vector(_start) + initMask; + Vector value2 = value + new Vector(width); + + ref int pos = ref MemoryMarshal.GetReference(destination); + ref int limit = ref Unsafe.Add(ref pos, destination.Length - remainder); + while (!Unsafe.AreSame(ref pos, ref limit)) + { + Unsafe.WriteUnaligned(ref Unsafe.As(ref pos), value); + Unsafe.WriteUnaligned(ref Unsafe.As(ref Unsafe.Add(ref pos, width)), value2); + + value += mask; + value2 += mask; + pos = ref Unsafe.Add(ref pos, stride); + } + + int cur = _start + (destination.Length - remainder); + int end = _end; + while (cur < end) + { + pos = cur; + pos = ref Unsafe.Add(ref pos, 1); + cur++; + } + } } } } From eda1c69fcb5b8d01ece7993acdadcbc016634e34 Mon Sep 17 00:00:00 2001 From: neon-sunset <20912188+neon-sunset@users.noreply.github.com> Date: Tue, 17 Jan 2023 11:28:46 +0200 Subject: [PATCH 2/6] Fix impl. and address feedback --- .../System.Linq/src/System/Linq/Range.SpeedOpt.cs | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/src/libraries/System.Linq/src/System/Linq/Range.SpeedOpt.cs b/src/libraries/System.Linq/src/System/Linq/Range.SpeedOpt.cs index f8553e3ceff595..08c3ba356e1c4a 100644 --- a/src/libraries/System.Linq/src/System/Linq/Range.SpeedOpt.cs +++ b/src/libraries/System.Linq/src/System/Linq/Range.SpeedOpt.cs @@ -87,15 +87,13 @@ public int TryGetLast(out bool found) // Destination *must* be non-empty and exactly match the range length private void InitializeSpan(Span destination) { - Debug.Assert((_end - _start) == destination.Length); - if (destination.Length < Vector.Count * 2) { - int end = _start + _end; + int end = _end; ref int pos = ref MemoryMarshal.GetReference(destination); - for (int num = _start; num < end; num++) + for (int cur = _start; cur < end; cur++) { - pos = num; + pos = cur; pos = ref Unsafe.Add(ref pos, 1); } } From 5d728e4599bbc168453b4d053ad5eb8c8cf995f4 Mon Sep 17 00:00:00 2001 From: neon-sunset <20912188+neon-sunset@users.noreply.github.com> Date: Wed, 18 Jan 2023 01:30:15 +0200 Subject: [PATCH 3/6] Reduce cost for short ranges --- src/libraries/System.Linq/src/System/Linq/Range.SpeedOpt.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/libraries/System.Linq/src/System/Linq/Range.SpeedOpt.cs b/src/libraries/System.Linq/src/System/Linq/Range.SpeedOpt.cs index 08c3ba356e1c4a..08937708ce7f8c 100644 --- a/src/libraries/System.Linq/src/System/Linq/Range.SpeedOpt.cs +++ b/src/libraries/System.Linq/src/System/Linq/Range.SpeedOpt.cs @@ -20,8 +20,7 @@ public override IEnumerable Select(Func selector public int[] ToArray() { - int[] array = GC.AllocateUninitializedArray(_end - _start); - + int[] array = new int[_end - _start]; InitializeSpan(array); return array; } @@ -85,6 +84,7 @@ public int TryGetLast(out bool found) } // Destination *must* be non-empty and exactly match the range length + [MethodImpl(MethodImplOptions.AggressiveInlining)] private void InitializeSpan(Span destination) { if (destination.Length < Vector.Count * 2) From f7b6f3f2c050e2126c4f1be09b62351638c31617 Mon Sep 17 00:00:00 2001 From: neon-sunset <20912188+neon-sunset@users.noreply.github.com> Date: Wed, 18 Jan 2023 02:04:22 +0200 Subject: [PATCH 4/6] Remove unused reference --- src/libraries/System.Linq/src/System/Linq/Range.SpeedOpt.cs | 1 - 1 file changed, 1 deletion(-) diff --git a/src/libraries/System.Linq/src/System/Linq/Range.SpeedOpt.cs b/src/libraries/System.Linq/src/System/Linq/Range.SpeedOpt.cs index 08937708ce7f8c..ccd7d54434624a 100644 --- a/src/libraries/System.Linq/src/System/Linq/Range.SpeedOpt.cs +++ b/src/libraries/System.Linq/src/System/Linq/Range.SpeedOpt.cs @@ -2,7 +2,6 @@ // The .NET Foundation licenses this file to you under the MIT license. using System.Collections.Generic; -using System.Diagnostics; using System.Numerics; using System.Runtime.CompilerServices; using System.Runtime.InteropServices; From 4ca8590658d0c3132cd2608d39baffe0aa17a0b2 Mon Sep 17 00:00:00 2001 From: neon-sunset <20912188+neon-sunset@users.noreply.github.com> Date: Thu, 19 Jan 2023 00:38:10 +0200 Subject: [PATCH 5/6] Try to fix tests --- src/libraries/System.Linq/src/System/Linq/Range.SpeedOpt.cs | 1 - 1 file changed, 1 deletion(-) diff --git a/src/libraries/System.Linq/src/System/Linq/Range.SpeedOpt.cs b/src/libraries/System.Linq/src/System/Linq/Range.SpeedOpt.cs index ccd7d54434624a..92252180f4d563 100644 --- a/src/libraries/System.Linq/src/System/Linq/Range.SpeedOpt.cs +++ b/src/libraries/System.Linq/src/System/Linq/Range.SpeedOpt.cs @@ -83,7 +83,6 @@ public int TryGetLast(out bool found) } // Destination *must* be non-empty and exactly match the range length - [MethodImpl(MethodImplOptions.AggressiveInlining)] private void InitializeSpan(Span destination) { if (destination.Length < Vector.Count * 2) From 5d7f00365b0b0ba500d98b40f5994fed8ab3332b Mon Sep 17 00:00:00 2001 From: neon-sunset <20912188+neon-sunset@users.noreply.github.com> Date: Fri, 24 Feb 2023 04:46:29 +0200 Subject: [PATCH 6/6] Address feedback --- .../System.Linq/src/System/Linq/Range.SpeedOpt.cs | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/src/libraries/System.Linq/src/System/Linq/Range.SpeedOpt.cs b/src/libraries/System.Linq/src/System/Linq/Range.SpeedOpt.cs index 92252180f4d563..2907e277229a44 100644 --- a/src/libraries/System.Linq/src/System/Linq/Range.SpeedOpt.cs +++ b/src/libraries/System.Linq/src/System/Linq/Range.SpeedOpt.cs @@ -82,17 +82,16 @@ public int TryGetLast(out bool found) return _end - 1; } - // Destination *must* be non-empty and exactly match the range length + // Destination *must* be non-empty and match the range length private void InitializeSpan(Span destination) { if (destination.Length < Vector.Count * 2) { - int end = _end; - ref int pos = ref MemoryMarshal.GetReference(destination); - for (int cur = _start; cur < end; cur++) + int cur = _start; + for (int i = 0; i < destination.Length; i++) { - pos = cur; - pos = ref Unsafe.Add(ref pos, 1); + destination[i] = cur; + cur++; } } else