From 10430e6e13d271a5146c1d1e55cb170291b2a109 Mon Sep 17 00:00:00 2001 From: Miha Zupan Date: Sat, 6 Jan 2024 19:03:45 +0100 Subject: [PATCH 1/8] Add packed IgnoreCase IndexOfAny variants --- .../System.Memory/tests/Span/SearchValues.cs | 4 + .../System.Private.CoreLib.Shared.projitems | 15 +- .../src/System/Globalization/Ordinal.cs | 2 +- .../Any1CharPackedIgnoreCaseSearchValues.cs | 49 +++++ .../Any1CharPackedSearchValues.cs | 42 +++++ .../System/SearchValues/Any1SearchValues.cs | 52 ++++++ .../SearchValues/Any2ByteSearchValues.cs | 41 ----- .../Any2CharPackedIgnoreCaseSearchValues.cs | 63 +++++++ .../Any2CharPackedSearchValues.cs | 42 +++++ .../SearchValues/Any2CharSearchValues.cs | 51 ------ .../System/SearchValues/Any2SearchValues.cs | 53 ++++++ .../SearchValues/Any3ByteSearchValues.cs | 41 ----- .../Any3CharPackedIgnoreCaseSearchValues.cs | 64 +++++++ .../Any3CharPackedSearchValues.cs | 42 +++++ .../SearchValues/Any3CharSearchValues.cs | 53 ------ .../System/SearchValues/Any3SearchValues.cs | 54 ++++++ .../src/System/SearchValues/SearchValues.cs | 65 +++++-- .../SearchValues/SingleByteSearchValues.cs | 41 ----- .../SearchValues/SingleCharSearchValues.cs | 49 ----- .../src/System/SpanHelpers.Packed.cs | 173 +++++++++++++----- .../src/System/SpanHelpers.T.cs | 9 + .../src/System/String.Searching.cs | 6 +- 22 files changed, 666 insertions(+), 345 deletions(-) create mode 100644 src/libraries/System.Private.CoreLib/src/System/SearchValues/Any1CharPackedIgnoreCaseSearchValues.cs create mode 100644 src/libraries/System.Private.CoreLib/src/System/SearchValues/Any1CharPackedSearchValues.cs create mode 100644 src/libraries/System.Private.CoreLib/src/System/SearchValues/Any1SearchValues.cs delete mode 100644 src/libraries/System.Private.CoreLib/src/System/SearchValues/Any2ByteSearchValues.cs create mode 100644 src/libraries/System.Private.CoreLib/src/System/SearchValues/Any2CharPackedIgnoreCaseSearchValues.cs create mode 100644 src/libraries/System.Private.CoreLib/src/System/SearchValues/Any2CharPackedSearchValues.cs delete mode 100644 src/libraries/System.Private.CoreLib/src/System/SearchValues/Any2CharSearchValues.cs create mode 100644 src/libraries/System.Private.CoreLib/src/System/SearchValues/Any2SearchValues.cs delete mode 100644 src/libraries/System.Private.CoreLib/src/System/SearchValues/Any3ByteSearchValues.cs create mode 100644 src/libraries/System.Private.CoreLib/src/System/SearchValues/Any3CharPackedIgnoreCaseSearchValues.cs create mode 100644 src/libraries/System.Private.CoreLib/src/System/SearchValues/Any3CharPackedSearchValues.cs delete mode 100644 src/libraries/System.Private.CoreLib/src/System/SearchValues/Any3CharSearchValues.cs create mode 100644 src/libraries/System.Private.CoreLib/src/System/SearchValues/Any3SearchValues.cs delete mode 100644 src/libraries/System.Private.CoreLib/src/System/SearchValues/SingleByteSearchValues.cs delete mode 100644 src/libraries/System.Private.CoreLib/src/System/SearchValues/SingleCharSearchValues.cs diff --git a/src/libraries/System.Memory/tests/Span/SearchValues.cs b/src/libraries/System.Memory/tests/Span/SearchValues.cs index 9ef91acec80dab..f2020279cdf4b9 100644 --- a/src/libraries/System.Memory/tests/Span/SearchValues.cs +++ b/src/libraries/System.Memory/tests/Span/SearchValues.cs @@ -44,6 +44,10 @@ public static IEnumerable Values_MemberData() "aaa", "aaaa", "aaaaa", + "Aa", + "AaBb", + "AaBbCc", + "[]{}", "\uFFF0", "\uFFF0\uFFF2", "\uFFF0\uFFF2\uFFF4", diff --git a/src/libraries/System.Private.CoreLib/src/System.Private.CoreLib.Shared.projitems b/src/libraries/System.Private.CoreLib/src/System.Private.CoreLib.Shared.projitems index c9e500d22af6b6..27b69331cdfa20 100644 --- a/src/libraries/System.Private.CoreLib/src/System.Private.CoreLib.Shared.projitems +++ b/src/libraries/System.Private.CoreLib/src/System.Private.CoreLib.Shared.projitems @@ -423,14 +423,17 @@ + + + + + + + + + - - - - - - diff --git a/src/libraries/System.Private.CoreLib/src/System/Globalization/Ordinal.cs b/src/libraries/System.Private.CoreLib/src/System/Globalization/Ordinal.cs index b7de19aab570d7..5b18e8d2b6fcb3 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Globalization/Ordinal.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Globalization/Ordinal.cs @@ -397,7 +397,7 @@ internal static int IndexOfOrdinalIgnoreCase(ReadOnlySpan source, ReadOnly // Do a quick search for the first element of "value". int relativeIndex = isLetter ? PackedSpanHelpers.PackedIndexOfIsSupported - ? PackedSpanHelpers.IndexOfAny(ref Unsafe.Add(ref searchSpace, offset), valueCharU, valueCharL, searchSpaceMinusValueTailLength) + ? PackedSpanHelpers.IndexOfAnyIgnoreCase(ref Unsafe.Add(ref searchSpace, offset), valueCharL, searchSpaceMinusValueTailLength) : SpanHelpers.IndexOfAnyChar(ref Unsafe.Add(ref searchSpace, offset), valueCharU, valueCharL, searchSpaceMinusValueTailLength) : SpanHelpers.IndexOfChar(ref Unsafe.Add(ref searchSpace, offset), valueChar, searchSpaceMinusValueTailLength); if (relativeIndex < 0) diff --git a/src/libraries/System.Private.CoreLib/src/System/SearchValues/Any1CharPackedIgnoreCaseSearchValues.cs b/src/libraries/System.Private.CoreLib/src/System/SearchValues/Any1CharPackedIgnoreCaseSearchValues.cs new file mode 100644 index 00000000000000..84f1b857419f43 --- /dev/null +++ b/src/libraries/System.Private.CoreLib/src/System/SearchValues/Any1CharPackedIgnoreCaseSearchValues.cs @@ -0,0 +1,49 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Diagnostics; +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; +using System.Runtime.Intrinsics.X86; + +namespace System.Buffers +{ + internal sealed class Any1CharPackedIgnoreCaseSearchValues : SearchValues + { + private readonly char _lowerCase, _upperCase; + + public Any1CharPackedIgnoreCaseSearchValues(char value) + { + Debug.Assert(value != 0); + Debug.Assert((value | 0x20) == value); + + _lowerCase = value; + _upperCase = (char)(value & ~0x20); + } + + internal override char[] GetValues() => + [_upperCase, _lowerCase]; + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal override bool ContainsCore(char value) => + value == _lowerCase || value == _upperCase; + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + [CompExactlyDependsOn(typeof(Sse2))] + internal override int IndexOfAny(ReadOnlySpan span) => + PackedSpanHelpers.IndexOfAnyIgnoreCase(ref MemoryMarshal.GetReference(span), _lowerCase, span.Length); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + [CompExactlyDependsOn(typeof(Sse2))] + internal override int IndexOfAnyExcept(ReadOnlySpan span) => + PackedSpanHelpers.IndexOfAnyExceptIgnoreCase(ref MemoryMarshal.GetReference(span), _lowerCase, span.Length); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal override int LastIndexOfAny(ReadOnlySpan span) => + span.LastIndexOfAny(_lowerCase, _upperCase); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal override int LastIndexOfAnyExcept(ReadOnlySpan span) => + span.LastIndexOfAnyExcept(_lowerCase, _upperCase); + } +} diff --git a/src/libraries/System.Private.CoreLib/src/System/SearchValues/Any1CharPackedSearchValues.cs b/src/libraries/System.Private.CoreLib/src/System/SearchValues/Any1CharPackedSearchValues.cs new file mode 100644 index 00000000000000..e6aa0c870486d8 --- /dev/null +++ b/src/libraries/System.Private.CoreLib/src/System/SearchValues/Any1CharPackedSearchValues.cs @@ -0,0 +1,42 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; +using System.Runtime.Intrinsics.X86; + +namespace System.Buffers +{ + internal sealed class Any1CharPackedSearchValues : SearchValues + { + private readonly char _e0; + + public Any1CharPackedSearchValues(char value) => + _e0 = value; + + internal override char[] GetValues() => + [_e0]; + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal override bool ContainsCore(char value) => + value == _e0; + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + [CompExactlyDependsOn(typeof(Sse2))] + internal override int IndexOfAny(ReadOnlySpan span) => + PackedSpanHelpers.IndexOf(ref MemoryMarshal.GetReference(span), _e0, span.Length); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + [CompExactlyDependsOn(typeof(Sse2))] + internal override int IndexOfAnyExcept(ReadOnlySpan span) => + PackedSpanHelpers.IndexOfAnyExcept(ref MemoryMarshal.GetReference(span), _e0, span.Length); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal override int LastIndexOfAny(ReadOnlySpan span) => + span.LastIndexOf(_e0); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal override int LastIndexOfAnyExcept(ReadOnlySpan span) => + span.LastIndexOfAnyExcept(_e0); + } +} diff --git a/src/libraries/System.Private.CoreLib/src/System/SearchValues/Any1SearchValues.cs b/src/libraries/System.Private.CoreLib/src/System/SearchValues/Any1SearchValues.cs new file mode 100644 index 00000000000000..9a52511b987509 --- /dev/null +++ b/src/libraries/System.Private.CoreLib/src/System/SearchValues/Any1SearchValues.cs @@ -0,0 +1,52 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Diagnostics; +using System.Numerics; +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; + +#pragma warning disable 8500 // address of managed types + +namespace System.Buffers +{ + internal sealed class Any1SearchValues : SearchValues + where T : struct, IEquatable + where TImpl : struct, INumber + { + private readonly TImpl _e0; + + public Any1SearchValues(ReadOnlySpan values) + { + Debug.Assert(Unsafe.SizeOf() == Unsafe.SizeOf()); + Debug.Assert(values.Length == 1); + _e0 = values[0]; + } + + internal override unsafe T[] GetValues() + { + TImpl e0 = _e0; + return new[] { *(T*)&e0 }; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal override unsafe bool ContainsCore(T value) => + *(TImpl*)&value == _e0; + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal override int IndexOfAny(ReadOnlySpan span) => + SpanHelpers.NonPackedIndexOfValueType>(ref Unsafe.As(ref MemoryMarshal.GetReference(span)), _e0, span.Length); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal override int IndexOfAnyExcept(ReadOnlySpan span) => + SpanHelpers.NonPackedIndexOfValueType>(ref Unsafe.As(ref MemoryMarshal.GetReference(span)), _e0, span.Length); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal override int LastIndexOfAny(ReadOnlySpan span) => + SpanHelpers.LastIndexOfValueType(ref Unsafe.As(ref MemoryMarshal.GetReference(span)), _e0, span.Length); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal override int LastIndexOfAnyExcept(ReadOnlySpan span) => + SpanHelpers.LastIndexOfAnyExceptValueType(ref Unsafe.As(ref MemoryMarshal.GetReference(span)), _e0, span.Length); + } +} diff --git a/src/libraries/System.Private.CoreLib/src/System/SearchValues/Any2ByteSearchValues.cs b/src/libraries/System.Private.CoreLib/src/System/SearchValues/Any2ByteSearchValues.cs deleted file mode 100644 index 42f4acbfebf8b0..00000000000000 --- a/src/libraries/System.Private.CoreLib/src/System/SearchValues/Any2ByteSearchValues.cs +++ /dev/null @@ -1,41 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -using System.Diagnostics; -using System.Runtime.CompilerServices; - -namespace System.Buffers -{ - internal sealed class Any2ByteSearchValues : SearchValues - { - private readonly byte _e0, _e1; - - public Any2ByteSearchValues(ReadOnlySpan values) - { - Debug.Assert(values.Length == 2); - (_e0, _e1) = (values[0], values[1]); - } - - internal override byte[] GetValues() => new[] { _e0, _e1 }; - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal override bool ContainsCore(byte value) => - value == _e0 || value == _e1; - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal override int IndexOfAny(ReadOnlySpan span) => - span.IndexOfAny(_e0, _e1); - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal override int IndexOfAnyExcept(ReadOnlySpan span) => - span.IndexOfAnyExcept(_e0, _e1); - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal override int LastIndexOfAny(ReadOnlySpan span) => - span.LastIndexOfAny(_e0, _e1); - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal override int LastIndexOfAnyExcept(ReadOnlySpan span) => - span.LastIndexOfAnyExcept(_e0, _e1); - } -} diff --git a/src/libraries/System.Private.CoreLib/src/System/SearchValues/Any2CharPackedIgnoreCaseSearchValues.cs b/src/libraries/System.Private.CoreLib/src/System/SearchValues/Any2CharPackedIgnoreCaseSearchValues.cs new file mode 100644 index 00000000000000..b76e4c974e03e8 --- /dev/null +++ b/src/libraries/System.Private.CoreLib/src/System/SearchValues/Any2CharPackedIgnoreCaseSearchValues.cs @@ -0,0 +1,63 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Diagnostics; +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; +using System.Runtime.Intrinsics.Arm; +using System.Runtime.Intrinsics.Wasm; +using System.Runtime.Intrinsics.X86; + +namespace System.Buffers +{ + internal sealed class Any2CharPackedIgnoreCaseSearchValues : SearchValues + { + private readonly char _e0, _e1; + private IndexOfAnyAsciiSearcher.AsciiState _state; + + public Any2CharPackedIgnoreCaseSearchValues(char value0, char value1) + { + Debug.Assert(value0 != 0 && (value0 | 0x20) == value0 && char.IsAscii(value0)); + Debug.Assert(value1 != 0 && (value1 | 0x20) == value1 && char.IsAscii(value1)); + + (_e0, _e1) = (value0, value1); + IndexOfAnyAsciiSearcher.ComputeAsciiState([(char)(_e0 & ~0x20), _e0, (char)(_e1 & ~0x20), _e1], out _state); + } + + internal override char[] GetValues() => + [(char)(_e0 & ~0x20), _e0, (char)(_e1 & ~0x20), _e1]; + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal override bool ContainsCore(char value) + { + value = (char)(value | 0x20); + return value == _e0 || value == _e1; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + [CompExactlyDependsOn(typeof(Sse2))] + internal override int IndexOfAny(ReadOnlySpan span) => + PackedSpanHelpers.IndexOfAnyIgnoreCase(ref MemoryMarshal.GetReference(span), _e0, _e1, span.Length); + + [CompExactlyDependsOn(typeof(Sse2))] + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal override int IndexOfAnyExcept(ReadOnlySpan span) => + PackedSpanHelpers.IndexOfAnyExceptIgnoreCase(ref MemoryMarshal.GetReference(span), _e0, _e1, span.Length); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + [CompExactlyDependsOn(typeof(Ssse3))] + [CompExactlyDependsOn(typeof(AdvSimd))] + [CompExactlyDependsOn(typeof(PackedSimd))] + internal override int LastIndexOfAny(ReadOnlySpan span) => + IndexOfAnyAsciiSearcher.LastIndexOfAnyVectorized( + ref Unsafe.As(ref MemoryMarshal.GetReference(span)), span.Length, ref _state); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + [CompExactlyDependsOn(typeof(Ssse3))] + [CompExactlyDependsOn(typeof(AdvSimd))] + [CompExactlyDependsOn(typeof(PackedSimd))] + internal override int LastIndexOfAnyExcept(ReadOnlySpan span) => + IndexOfAnyAsciiSearcher.LastIndexOfAnyVectorized( + ref Unsafe.As(ref MemoryMarshal.GetReference(span)), span.Length, ref _state); + } +} diff --git a/src/libraries/System.Private.CoreLib/src/System/SearchValues/Any2CharPackedSearchValues.cs b/src/libraries/System.Private.CoreLib/src/System/SearchValues/Any2CharPackedSearchValues.cs new file mode 100644 index 00000000000000..d951b8e2375e0f --- /dev/null +++ b/src/libraries/System.Private.CoreLib/src/System/SearchValues/Any2CharPackedSearchValues.cs @@ -0,0 +1,42 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; +using System.Runtime.Intrinsics.X86; + +namespace System.Buffers +{ + internal sealed class Any2CharPackedSearchValues : SearchValues + { + private readonly char _e0, _e1; + + public Any2CharPackedSearchValues(char value0, char value1) => + (_e0, _e1) = (value0, value1); + + internal override char[] GetValues() => + [_e0, _e1]; + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal override bool ContainsCore(char value) => + value == _e0 || value == _e1; + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + [CompExactlyDependsOn(typeof(Sse2))] + internal override int IndexOfAny(ReadOnlySpan span) => + PackedSpanHelpers.IndexOfAny(ref MemoryMarshal.GetReference(span), _e0, _e1, span.Length); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + [CompExactlyDependsOn(typeof(Sse2))] + internal override int IndexOfAnyExcept(ReadOnlySpan span) => + PackedSpanHelpers.IndexOfAnyExcept(ref MemoryMarshal.GetReference(span), _e0, _e1, span.Length); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal override int LastIndexOfAny(ReadOnlySpan span) => + span.LastIndexOfAny(_e0, _e1); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal override int LastIndexOfAnyExcept(ReadOnlySpan span) => + span.LastIndexOfAnyExcept(_e0, _e1); + } +} diff --git a/src/libraries/System.Private.CoreLib/src/System/SearchValues/Any2CharSearchValues.cs b/src/libraries/System.Private.CoreLib/src/System/SearchValues/Any2CharSearchValues.cs deleted file mode 100644 index a2bfde7f6fab94..00000000000000 --- a/src/libraries/System.Private.CoreLib/src/System/SearchValues/Any2CharSearchValues.cs +++ /dev/null @@ -1,51 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -using System.Runtime.CompilerServices; -using System.Runtime.InteropServices; - -namespace System.Buffers -{ - internal sealed class Any2CharSearchValues : SearchValues - where TShouldUsePacked : struct, SearchValues.IRuntimeConst - { - private char _e0, _e1; - - public Any2CharSearchValues(char value0, char value1) => - (_e0, _e1) = (value0, value1); - - internal override char[] GetValues() => new[] { _e0, _e1 }; - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal override bool ContainsCore(char value) => - value == _e0 || value == _e1; - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal override int IndexOfAny(ReadOnlySpan span) => - (PackedSpanHelpers.PackedIndexOfIsSupported && TShouldUsePacked.Value) - ? PackedSpanHelpers.IndexOfAny(ref MemoryMarshal.GetReference(span), _e0, _e1, span.Length) - : SpanHelpers.NonPackedIndexOfAnyValueType>( - ref Unsafe.As(ref MemoryMarshal.GetReference(span)), - Unsafe.As(ref _e0), - Unsafe.As(ref _e1), - span.Length); - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal override int IndexOfAnyExcept(ReadOnlySpan span) => - (PackedSpanHelpers.PackedIndexOfIsSupported && TShouldUsePacked.Value) - ? PackedSpanHelpers.IndexOfAnyExcept(ref MemoryMarshal.GetReference(span), _e0, _e1, span.Length) - : SpanHelpers.NonPackedIndexOfAnyValueType>( - ref Unsafe.As(ref MemoryMarshal.GetReference(span)), - Unsafe.As(ref _e0), - Unsafe.As(ref _e1), - span.Length); - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal override int LastIndexOfAny(ReadOnlySpan span) => - span.LastIndexOfAny(_e0, _e1); - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal override int LastIndexOfAnyExcept(ReadOnlySpan span) => - span.LastIndexOfAnyExcept(_e0, _e1); - } -} diff --git a/src/libraries/System.Private.CoreLib/src/System/SearchValues/Any2SearchValues.cs b/src/libraries/System.Private.CoreLib/src/System/SearchValues/Any2SearchValues.cs new file mode 100644 index 00000000000000..6bc47027cbdb2f --- /dev/null +++ b/src/libraries/System.Private.CoreLib/src/System/SearchValues/Any2SearchValues.cs @@ -0,0 +1,53 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Diagnostics; +using System.Numerics; +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; + +#pragma warning disable 8500 // address of managed types + +namespace System.Buffers +{ + internal sealed class Any2SearchValues : SearchValues + where T : struct, IEquatable + where TImpl : struct, INumber + { + private readonly TImpl _e0, _e1; + + public Any2SearchValues(ReadOnlySpan values) + { + Debug.Assert(Unsafe.SizeOf() == Unsafe.SizeOf()); + Debug.Assert(values.Length == 2); + (_e0, _e1) = (values[0], values[1]); + } + + internal override unsafe T[] GetValues() + { + TImpl e0 = _e0, e1 = _e1; + return new[] { *(T*)&e0, *(T*)&e1 }; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal override unsafe bool ContainsCore(T value) => + *(TImpl*)&value == _e0 || + *(TImpl*)&value == _e1; + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal override int IndexOfAny(ReadOnlySpan span) => + SpanHelpers.NonPackedIndexOfAnyValueType>(ref Unsafe.As(ref MemoryMarshal.GetReference(span)), _e0, _e1, span.Length); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal override int IndexOfAnyExcept(ReadOnlySpan span) => + SpanHelpers.NonPackedIndexOfAnyValueType>(ref Unsafe.As(ref MemoryMarshal.GetReference(span)), _e0, _e1, span.Length); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal override int LastIndexOfAny(ReadOnlySpan span) => + SpanHelpers.LastIndexOfAnyValueType(ref Unsafe.As(ref MemoryMarshal.GetReference(span)), _e0, _e1, span.Length); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal override int LastIndexOfAnyExcept(ReadOnlySpan span) => + SpanHelpers.LastIndexOfAnyExceptValueType(ref Unsafe.As(ref MemoryMarshal.GetReference(span)), _e0, _e1, span.Length); + } +} diff --git a/src/libraries/System.Private.CoreLib/src/System/SearchValues/Any3ByteSearchValues.cs b/src/libraries/System.Private.CoreLib/src/System/SearchValues/Any3ByteSearchValues.cs deleted file mode 100644 index d7208dea435067..00000000000000 --- a/src/libraries/System.Private.CoreLib/src/System/SearchValues/Any3ByteSearchValues.cs +++ /dev/null @@ -1,41 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -using System.Diagnostics; -using System.Runtime.CompilerServices; - -namespace System.Buffers -{ - internal sealed class Any3ByteSearchValues : SearchValues - { - private readonly byte _e0, _e1, _e2; - - public Any3ByteSearchValues(ReadOnlySpan values) - { - Debug.Assert(values.Length == 3); - (_e0, _e1, _e2) = (values[0], values[1], values[2]); - } - - internal override byte[] GetValues() => new[] { _e0, _e1, _e2 }; - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal override bool ContainsCore(byte value) => - value == _e0 || value == _e1 || value == _e2; - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal override int IndexOfAny(ReadOnlySpan span) => - span.IndexOfAny(_e0, _e1, _e2); - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal override int IndexOfAnyExcept(ReadOnlySpan span) => - span.IndexOfAnyExcept(_e0, _e1, _e2); - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal override int LastIndexOfAny(ReadOnlySpan span) => - span.LastIndexOfAny(_e0, _e1, _e2); - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal override int LastIndexOfAnyExcept(ReadOnlySpan span) => - span.LastIndexOfAnyExcept(_e0, _e1, _e2); - } -} diff --git a/src/libraries/System.Private.CoreLib/src/System/SearchValues/Any3CharPackedIgnoreCaseSearchValues.cs b/src/libraries/System.Private.CoreLib/src/System/SearchValues/Any3CharPackedIgnoreCaseSearchValues.cs new file mode 100644 index 00000000000000..40844c3d25be7b --- /dev/null +++ b/src/libraries/System.Private.CoreLib/src/System/SearchValues/Any3CharPackedIgnoreCaseSearchValues.cs @@ -0,0 +1,64 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Diagnostics; +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; +using System.Runtime.Intrinsics.Arm; +using System.Runtime.Intrinsics.Wasm; +using System.Runtime.Intrinsics.X86; + +namespace System.Buffers +{ + internal sealed class Any3CharPackedIgnoreCaseSearchValues : SearchValues + { + private readonly char _e0, _e1, _e2; + private IndexOfAnyAsciiSearcher.AsciiState _state; + + public Any3CharPackedIgnoreCaseSearchValues(char value0, char value1, char value2) + { + Debug.Assert(value0 != 0 && (value0 | 0x20) == value0 && char.IsAscii(value0)); + Debug.Assert(value1 != 0 && (value1 | 0x20) == value1 && char.IsAscii(value1)); + Debug.Assert(value2 != 0 && (value2 | 0x20) == value2 && char.IsAscii(value2)); + + (_e0, _e1, _e2) = (value0, value1, value2); + IndexOfAnyAsciiSearcher.ComputeAsciiState([(char)(_e0 & ~0x20), _e0, (char)(_e1 & ~0x20), _e1, (char)(_e2 & ~0x20), _e2], out _state); + } + + internal override char[] GetValues() => + [(char)(_e0 & ~0x20), _e0, (char)(_e1 & ~0x20), _e1, (char)(_e2 & ~0x20), _e2]; + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal override bool ContainsCore(char value) + { + value = (char)(value | 0x20); + return value == _e0 || value == _e1 || value == _e2; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + [CompExactlyDependsOn(typeof(Sse2))] + internal override int IndexOfAny(ReadOnlySpan span) => + PackedSpanHelpers.IndexOfAnyIgnoreCase(ref MemoryMarshal.GetReference(span), _e0, _e1, _e2, span.Length); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + [CompExactlyDependsOn(typeof(Sse2))] + internal override int IndexOfAnyExcept(ReadOnlySpan span) => + PackedSpanHelpers.IndexOfAnyExceptIgnoreCase(ref MemoryMarshal.GetReference(span), _e0, _e1, _e2, span.Length); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + [CompExactlyDependsOn(typeof(Ssse3))] + [CompExactlyDependsOn(typeof(AdvSimd))] + [CompExactlyDependsOn(typeof(PackedSimd))] + internal override int LastIndexOfAny(ReadOnlySpan span) => + IndexOfAnyAsciiSearcher.LastIndexOfAnyVectorized( + ref Unsafe.As(ref MemoryMarshal.GetReference(span)), span.Length, ref _state); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + [CompExactlyDependsOn(typeof(Ssse3))] + [CompExactlyDependsOn(typeof(AdvSimd))] + [CompExactlyDependsOn(typeof(PackedSimd))] + internal override int LastIndexOfAnyExcept(ReadOnlySpan span) => + IndexOfAnyAsciiSearcher.LastIndexOfAnyVectorized( + ref Unsafe.As(ref MemoryMarshal.GetReference(span)), span.Length, ref _state); + } +} diff --git a/src/libraries/System.Private.CoreLib/src/System/SearchValues/Any3CharPackedSearchValues.cs b/src/libraries/System.Private.CoreLib/src/System/SearchValues/Any3CharPackedSearchValues.cs new file mode 100644 index 00000000000000..cebb695aa3fb79 --- /dev/null +++ b/src/libraries/System.Private.CoreLib/src/System/SearchValues/Any3CharPackedSearchValues.cs @@ -0,0 +1,42 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; +using System.Runtime.Intrinsics.X86; + +namespace System.Buffers +{ + internal sealed class Any3CharPackedSearchValues : SearchValues + { + private readonly char _e0, _e1, _e2; + + public Any3CharPackedSearchValues(char value0, char value1, char value2) => + (_e0, _e1, _e2) = (value0, value1, value2); + + internal override char[] GetValues() => + [_e0, _e1, _e2]; + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal override bool ContainsCore(char value) => + value == _e0 || value == _e1 || value == _e2; + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + [CompExactlyDependsOn(typeof(Sse2))] + internal override int IndexOfAny(ReadOnlySpan span) => + PackedSpanHelpers.IndexOfAny(ref MemoryMarshal.GetReference(span), _e0, _e1, _e2, span.Length); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + [CompExactlyDependsOn(typeof(Sse2))] + internal override int IndexOfAnyExcept(ReadOnlySpan span) => + PackedSpanHelpers.IndexOfAnyExcept(ref MemoryMarshal.GetReference(span), _e0, _e1, _e2, span.Length); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal override int LastIndexOfAny(ReadOnlySpan span) => + span.LastIndexOfAny(_e0, _e1, _e2); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal override int LastIndexOfAnyExcept(ReadOnlySpan span) => + span.LastIndexOfAnyExcept(_e0, _e1, _e2); + } +} diff --git a/src/libraries/System.Private.CoreLib/src/System/SearchValues/Any3CharSearchValues.cs b/src/libraries/System.Private.CoreLib/src/System/SearchValues/Any3CharSearchValues.cs deleted file mode 100644 index c69dbff8d0a4c7..00000000000000 --- a/src/libraries/System.Private.CoreLib/src/System/SearchValues/Any3CharSearchValues.cs +++ /dev/null @@ -1,53 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -using System.Runtime.CompilerServices; -using System.Runtime.InteropServices; - -namespace System.Buffers -{ - internal sealed class Any3CharSearchValues : SearchValues - where TShouldUsePacked : struct, SearchValues.IRuntimeConst - { - private char _e0, _e1, _e2; - - public Any3CharSearchValues(char value0, char value1, char value2) => - (_e0, _e1, _e2) = (value0, value1, value2); - - internal override char[] GetValues() => new[] { _e0, _e1, _e2 }; - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal override bool ContainsCore(char value) => - value == _e0 || value == _e1 || value == _e2; - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal override int IndexOfAny(ReadOnlySpan span) => - (PackedSpanHelpers.PackedIndexOfIsSupported && TShouldUsePacked.Value) - ? PackedSpanHelpers.IndexOfAny(ref MemoryMarshal.GetReference(span), _e0, _e1, _e2, span.Length) - : SpanHelpers.NonPackedIndexOfAnyValueType>( - ref Unsafe.As(ref MemoryMarshal.GetReference(span)), - Unsafe.As(ref _e0), - Unsafe.As(ref _e1), - Unsafe.As(ref _e2), - span.Length); - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal override int IndexOfAnyExcept(ReadOnlySpan span) => - (PackedSpanHelpers.PackedIndexOfIsSupported && TShouldUsePacked.Value) - ? PackedSpanHelpers.IndexOfAnyExcept(ref MemoryMarshal.GetReference(span), _e0, _e1, _e2, span.Length) - : SpanHelpers.NonPackedIndexOfAnyValueType>( - ref Unsafe.As(ref MemoryMarshal.GetReference(span)), - Unsafe.As(ref _e0), - Unsafe.As(ref _e1), - Unsafe.As(ref _e2), - span.Length); - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal override int LastIndexOfAny(ReadOnlySpan span) => - span.LastIndexOfAny(_e0, _e1, _e2); - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal override int LastIndexOfAnyExcept(ReadOnlySpan span) => - span.LastIndexOfAnyExcept(_e0, _e1, _e2); - } -} diff --git a/src/libraries/System.Private.CoreLib/src/System/SearchValues/Any3SearchValues.cs b/src/libraries/System.Private.CoreLib/src/System/SearchValues/Any3SearchValues.cs new file mode 100644 index 00000000000000..95662c995a7f5c --- /dev/null +++ b/src/libraries/System.Private.CoreLib/src/System/SearchValues/Any3SearchValues.cs @@ -0,0 +1,54 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Diagnostics; +using System.Numerics; +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; + +#pragma warning disable 8500 // address of managed types + +namespace System.Buffers +{ + internal sealed class Any3SearchValues : SearchValues + where T : struct, IEquatable + where TImpl : struct, INumber + { + private readonly TImpl _e0, _e1, _e2; + + public Any3SearchValues(ReadOnlySpan values) + { + Debug.Assert(Unsafe.SizeOf() == Unsafe.SizeOf()); + Debug.Assert(values.Length == 3); + (_e0, _e1, _e2) = (values[0], values[1], values[2]); + } + + internal override unsafe T[] GetValues() + { + TImpl e0 = _e0, e1 = _e1, e2 = _e2; + return new[] { *(T*)&e0, *(T*)&e1, *(T*)&e2 }; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal override unsafe bool ContainsCore(T value) => + *(TImpl*)&value == _e0 || + *(TImpl*)&value == _e1 || + *(TImpl*)&value == _e2; + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal override int IndexOfAny(ReadOnlySpan span) => + SpanHelpers.NonPackedIndexOfAnyValueType>(ref Unsafe.As(ref MemoryMarshal.GetReference(span)), _e0, _e1, _e2, span.Length); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal override int IndexOfAnyExcept(ReadOnlySpan span) => + SpanHelpers.NonPackedIndexOfAnyValueType>(ref Unsafe.As(ref MemoryMarshal.GetReference(span)), _e0, _e1, _e2, span.Length); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal override int LastIndexOfAny(ReadOnlySpan span) => + SpanHelpers.LastIndexOfAnyValueType(ref Unsafe.As(ref MemoryMarshal.GetReference(span)), _e0, _e1, _e2, span.Length); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal override int LastIndexOfAnyExcept(ReadOnlySpan span) => + SpanHelpers.LastIndexOfAnyExceptValueType(ref Unsafe.As(ref MemoryMarshal.GetReference(span)), _e0, _e1, _e2, span.Length); + } +} diff --git a/src/libraries/System.Private.CoreLib/src/System/SearchValues/SearchValues.cs b/src/libraries/System.Private.CoreLib/src/System/SearchValues/SearchValues.cs index 13c6779a7d9851..1307801afc971c 100644 --- a/src/libraries/System.Private.CoreLib/src/System/SearchValues/SearchValues.cs +++ b/src/libraries/System.Private.CoreLib/src/System/SearchValues/SearchValues.cs @@ -34,7 +34,7 @@ public static SearchValues Create(ReadOnlySpan values) if (values.Length == 1) { - return new SingleByteSearchValues(values); + return new Any1SearchValues(values); } // RangeByteSearchValues is slower than SingleByteSearchValues, but faster than Any2ByteSearchValues @@ -48,8 +48,8 @@ public static SearchValues Create(ReadOnlySpan values) Debug.Assert(values.Length is 2 or 3 or 4 or 5); return values.Length switch { - 2 => new Any2ByteSearchValues(values), - 3 => new Any3ByteSearchValues(values), + 2 => new Any2SearchValues(values), + 3 => new Any3SearchValues(values), 4 => new Any4SearchValues(values), _ => new Any5SearchValues(values), }; @@ -75,12 +75,18 @@ public static SearchValues Create(ReadOnlySpan values) return new EmptySearchValues(); } + // Vector128 isn't valid. Treat the values as shorts instead. + ReadOnlySpan shortValues = MemoryMarshal.CreateReadOnlySpan( + ref Unsafe.As(ref MemoryMarshal.GetReference(values)), + values.Length); + if (values.Length == 1) { char value = values[0]; + return PackedSpanHelpers.PackedIndexOfIsSupported && PackedSpanHelpers.CanUsePackedIndexOf(value) - ? new SingleCharSearchValues(value) - : new SingleCharSearchValues(value); + ? new Any1CharPackedSearchValues(value) + : new Any1SearchValues(shortValues); } // RangeCharSearchValues is slower than SingleCharSearchValues, but faster than Any2CharSearchValues @@ -95,9 +101,15 @@ public static SearchValues Create(ReadOnlySpan values) { char value0 = values[0]; char value1 = values[1]; - return PackedSpanHelpers.PackedIndexOfIsSupported && PackedSpanHelpers.CanUsePackedIndexOf(value0) && PackedSpanHelpers.CanUsePackedIndexOf(value1) - ? new Any2CharSearchValues(value0, value1) - : new Any2CharSearchValues(value0, value1); + + if (PackedSpanHelpers.PackedIndexOfIsSupported && PackedSpanHelpers.CanUsePackedIndexOf(value0) && PackedSpanHelpers.CanUsePackedIndexOf(value1)) + { + return (value0 ^ value1) == 0x20 + ? new Any1CharPackedIgnoreCaseSearchValues((char)Math.Max(value0, value1)) + : new Any2CharPackedSearchValues(value0, value1); + } + + return new Any2SearchValues(shortValues); } if (values.Length == 3) @@ -105,24 +117,47 @@ public static SearchValues Create(ReadOnlySpan values) char value0 = values[0]; char value1 = values[1]; char value2 = values[2]; + return PackedSpanHelpers.PackedIndexOfIsSupported && PackedSpanHelpers.CanUsePackedIndexOf(value0) && PackedSpanHelpers.CanUsePackedIndexOf(value1) && PackedSpanHelpers.CanUsePackedIndexOf(value2) - ? new Any3CharSearchValues(value0, value1, value2) - : new Any3CharSearchValues(value0, value1, value2); + ? new Any3CharPackedSearchValues(value0, value1, value2) + : new Any3SearchValues(shortValues); } // IndexOfAnyAsciiSearcher for chars is slower than Any3CharSearchValues, but faster than Any4SearchValues if (IndexOfAnyAsciiSearcher.IsVectorizationSupported && maxInclusive < 128) { + // If the values are sets of 2 or 3 ASCII letters with both cases, we can use an approach that + // reduces the number of comparisons by masking off the bit that differs between lower and upper case (0x20). + // While this most commonly applies to ASCII letters, it also works for other values that differ by 0x20 (e.g. "[]{}" => "{}"). + if (PackedSpanHelpers.PackedIndexOfIsSupported && values.Length is 4 or 6 && minInclusive > 0) + { + Span copy = stackalloc char[values.Length]; + values.CopyTo(copy); + copy.Sort(); + + if (copy.Length == 4 && + (copy[0] ^ copy[2]) == 0x20 && + (copy[1] ^ copy[3]) == 0x20) + { + // "AaBb" => 'a', 'b' + return new Any2CharPackedIgnoreCaseSearchValues(copy[2], copy[3]); + } + + if (copy.Length == 6 && + (copy[0] ^ copy[3]) == 0x20 && + (copy[1] ^ copy[4]) == 0x20 && + (copy[2] ^ copy[5]) == 0x20) + { + // "AaBbCc" => 'a', 'b', 'c' + return new Any3CharPackedIgnoreCaseSearchValues(copy[3], copy[4], copy[5]); + } + } + return (Ssse3.IsSupported || PackedSimd.IsSupported) && minInclusive == 0 ? new AsciiCharSearchValues(values) : new AsciiCharSearchValues(values); } - // Vector128 isn't valid. Treat the values as shorts instead. - ReadOnlySpan shortValues = MemoryMarshal.CreateReadOnlySpan( - ref Unsafe.As(ref MemoryMarshal.GetReference(values)), - values.Length); - if (values.Length == 4) { return new Any4SearchValues(shortValues); diff --git a/src/libraries/System.Private.CoreLib/src/System/SearchValues/SingleByteSearchValues.cs b/src/libraries/System.Private.CoreLib/src/System/SearchValues/SingleByteSearchValues.cs deleted file mode 100644 index b768c2541562f7..00000000000000 --- a/src/libraries/System.Private.CoreLib/src/System/SearchValues/SingleByteSearchValues.cs +++ /dev/null @@ -1,41 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -using System.Diagnostics; -using System.Runtime.CompilerServices; - -namespace System.Buffers -{ - internal sealed class SingleByteSearchValues : SearchValues - { - private readonly byte _e0; - - public SingleByteSearchValues(ReadOnlySpan values) - { - Debug.Assert(values.Length == 1); - _e0 = values[0]; - } - - internal override byte[] GetValues() => new[] { _e0 }; - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal override bool ContainsCore(byte value) => - value == _e0; - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal override int IndexOfAny(ReadOnlySpan span) => - span.IndexOf(_e0); - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal override int IndexOfAnyExcept(ReadOnlySpan span) => - span.IndexOfAnyExcept(_e0); - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal override int LastIndexOfAny(ReadOnlySpan span) => - span.LastIndexOf(_e0); - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal override int LastIndexOfAnyExcept(ReadOnlySpan span) => - span.LastIndexOfAnyExcept(_e0); - } -} diff --git a/src/libraries/System.Private.CoreLib/src/System/SearchValues/SingleCharSearchValues.cs b/src/libraries/System.Private.CoreLib/src/System/SearchValues/SingleCharSearchValues.cs deleted file mode 100644 index b1348e3859b073..00000000000000 --- a/src/libraries/System.Private.CoreLib/src/System/SearchValues/SingleCharSearchValues.cs +++ /dev/null @@ -1,49 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -using System.Runtime.CompilerServices; -using System.Runtime.InteropServices; - -namespace System.Buffers -{ - internal sealed class SingleCharSearchValues : SearchValues - where TShouldUsePacked : struct, SearchValues.IRuntimeConst - { - private char _e0; - - public SingleCharSearchValues(char value) => - _e0 = value; - - internal override char[] GetValues() => new[] { _e0 }; - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal override bool ContainsCore(char value) => - value == _e0; - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal override int IndexOfAny(ReadOnlySpan span) => - (PackedSpanHelpers.PackedIndexOfIsSupported && TShouldUsePacked.Value) - ? PackedSpanHelpers.IndexOf(ref MemoryMarshal.GetReference(span), _e0, span.Length) - : SpanHelpers.NonPackedIndexOfValueType>( - ref Unsafe.As(ref MemoryMarshal.GetReference(span)), - Unsafe.As(ref _e0), - span.Length); - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal override int IndexOfAnyExcept(ReadOnlySpan span) => - (PackedSpanHelpers.PackedIndexOfIsSupported && TShouldUsePacked.Value) - ? PackedSpanHelpers.IndexOfAnyExcept(ref MemoryMarshal.GetReference(span), _e0, span.Length) - : SpanHelpers.NonPackedIndexOfValueType>( - ref Unsafe.As(ref MemoryMarshal.GetReference(span)), - Unsafe.As(ref _e0), - span.Length); - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal override int LastIndexOfAny(ReadOnlySpan span) => - span.LastIndexOf(_e0); - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal override int LastIndexOfAnyExcept(ReadOnlySpan span) => - span.LastIndexOfAnyExcept(_e0); - } -} diff --git a/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.Packed.cs b/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.Packed.cs index fca176fe438126..93dece79f54bd4 100644 --- a/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.Packed.cs +++ b/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.Packed.cs @@ -1,10 +1,8 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. -using System.Buffers.Binary; using System.Diagnostics; using System.Numerics; -using System.Runtime; using System.Runtime.CompilerServices; using System.Runtime.Intrinsics; using System.Runtime.Intrinsics.X86; @@ -37,32 +35,92 @@ public static unsafe bool CanUsePackedIndexOf(T value) [MethodImpl(MethodImplOptions.AggressiveInlining)] [CompExactlyDependsOn(typeof(Sse2))] public static int IndexOf(ref char searchSpace, char value, int length) => - IndexOf>(ref Unsafe.As(ref searchSpace), (short)value, length); + IndexOf, NopTransform>(ref Unsafe.As(ref searchSpace), (short)value, length); [MethodImpl(MethodImplOptions.AggressiveInlining)] [CompExactlyDependsOn(typeof(Sse2))] public static int IndexOfAnyExcept(ref char searchSpace, char value, int length) => - IndexOf>(ref Unsafe.As(ref searchSpace), (short)value, length); + IndexOf, NopTransform>(ref Unsafe.As(ref searchSpace), (short)value, length); [MethodImpl(MethodImplOptions.AggressiveInlining)] [CompExactlyDependsOn(typeof(Sse2))] public static int IndexOfAny(ref char searchSpace, char value0, char value1, int length) => - IndexOfAny>(ref Unsafe.As(ref searchSpace), (short)value0, (short)value1, length); + IndexOfAny, NopTransform>(ref Unsafe.As(ref searchSpace), (short)value0, (short)value1, length); [MethodImpl(MethodImplOptions.AggressiveInlining)] [CompExactlyDependsOn(typeof(Sse2))] public static int IndexOfAnyExcept(ref char searchSpace, char value0, char value1, int length) => - IndexOfAny>(ref Unsafe.As(ref searchSpace), (short)value0, (short)value1, length); + IndexOfAny, NopTransform>(ref Unsafe.As(ref searchSpace), (short)value0, (short)value1, length); [MethodImpl(MethodImplOptions.AggressiveInlining)] [CompExactlyDependsOn(typeof(Sse2))] public static int IndexOfAny(ref char searchSpace, char value0, char value1, char value2, int length) => - IndexOfAny>(ref Unsafe.As(ref searchSpace), (short)value0, (short)value1, (short)value2, length); + IndexOfAny, NopTransform>(ref Unsafe.As(ref searchSpace), (short)value0, (short)value1, (short)value2, length); [MethodImpl(MethodImplOptions.AggressiveInlining)] [CompExactlyDependsOn(typeof(Sse2))] public static int IndexOfAnyExcept(ref char searchSpace, char value0, char value1, char value2, int length) => - IndexOfAny>(ref Unsafe.As(ref searchSpace), (short)value0, (short)value1, (short)value2, length); + IndexOfAny, NopTransform>(ref Unsafe.As(ref searchSpace), (short)value0, (short)value1, (short)value2, length); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + [CompExactlyDependsOn(typeof(Sse2))] + public static int IndexOfAnyIgnoreCase(ref char searchSpace, char value, int length) + { + Debug.Assert((value | 0x20) == value); + + return IndexOf, Or20Transform>(ref Unsafe.As(ref searchSpace), (short)value, length); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + [CompExactlyDependsOn(typeof(Sse2))] + public static int IndexOfAnyExceptIgnoreCase(ref char searchSpace, char value, int length) + { + Debug.Assert((value | 0x20) == value); + + return IndexOf, Or20Transform>(ref Unsafe.As(ref searchSpace), (short)value, length); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + [CompExactlyDependsOn(typeof(Sse2))] + public static int IndexOfAnyIgnoreCase(ref char searchSpace, char value0, char value1, int length) + { + Debug.Assert((value0 | 0x20) == value0); + Debug.Assert((value1 | 0x20) == value1); + + return IndexOfAny, Or20Transform>(ref Unsafe.As(ref searchSpace), (short)value0, (short)value1, length); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + [CompExactlyDependsOn(typeof(Sse2))] + public static int IndexOfAnyExceptIgnoreCase(ref char searchSpace, char value0, char value1, int length) + { + Debug.Assert((value0 | 0x20) == value0); + Debug.Assert((value1 | 0x20) == value1); + + return IndexOfAny, Or20Transform>(ref Unsafe.As(ref searchSpace), (short)value0, (short)value1, length); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + [CompExactlyDependsOn(typeof(Sse2))] + public static int IndexOfAnyIgnoreCase(ref char searchSpace, char value0, char value1, char value2, int length) + { + Debug.Assert((value0 | 0x20) == value0); + Debug.Assert((value1 | 0x20) == value1); + Debug.Assert((value2 | 0x20) == value2); + + return IndexOfAny, Or20Transform>(ref Unsafe.As(ref searchSpace), (short)value0, (short)value1, (short)value2, length); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + [CompExactlyDependsOn(typeof(Sse2))] + public static int IndexOfAnyExceptIgnoreCase(ref char searchSpace, char value0, char value1, char value2, int length) + { + Debug.Assert((value0 | 0x20) == value0); + Debug.Assert((value1 | 0x20) == value1); + Debug.Assert((value2 | 0x20) == value2); + + return IndexOfAny, Or20Transform>(ref Unsafe.As(ref searchSpace), (short)value0, (short)value1, (short)value2, length); + } [MethodImpl(MethodImplOptions.AggressiveInlining)] [CompExactlyDependsOn(typeof(Sse2))] @@ -277,8 +335,9 @@ public static bool Contains(ref short searchSpace, short value, int length) } [CompExactlyDependsOn(typeof(Sse2))] - private static int IndexOf(ref short searchSpace, short value, int length) + private static int IndexOf(ref short searchSpace, short value, int length) where TNegator : struct, SpanHelpers.INegator + where TTransform : struct, ITransform { Debug.Assert(CanUsePackedIndexOf(value)); @@ -290,10 +349,10 @@ private static int IndexOf(ref short searchSpace, short value, int len { length -= 4; - if (TNegator.NegateIfNeeded(searchSpace == value)) return 0; - if (TNegator.NegateIfNeeded(Unsafe.Add(ref searchSpace, 1) == value)) return 1; - if (TNegator.NegateIfNeeded(Unsafe.Add(ref searchSpace, 2) == value)) return 2; - if (TNegator.NegateIfNeeded(Unsafe.Add(ref searchSpace, 3) == value)) return 3; + if (TNegator.NegateIfNeeded(TTransform.TransformInput(searchSpace) == value)) return 0; + if (TNegator.NegateIfNeeded(TTransform.TransformInput(Unsafe.Add(ref searchSpace, 1)) == value)) return 1; + if (TNegator.NegateIfNeeded(TTransform.TransformInput(Unsafe.Add(ref searchSpace, 2)) == value)) return 2; + if (TNegator.NegateIfNeeded(TTransform.TransformInput(Unsafe.Add(ref searchSpace, 3)) == value)) return 3; offset = 4; } @@ -302,7 +361,7 @@ private static int IndexOf(ref short searchSpace, short value, int len { length -= 1; - if (TNegator.NegateIfNeeded(Unsafe.Add(ref searchSpace, offset) == value)) return (int)offset; + if (TNegator.NegateIfNeeded(TTransform.TransformInput(Unsafe.Add(ref searchSpace, offset)) == value)) return (int)offset; offset += 1; } @@ -329,7 +388,7 @@ private static int IndexOf(ref short searchSpace, short value, int len { Vector512 source0 = Vector512.LoadUnsafe(ref currentSearchSpace); Vector512 source1 = Vector512.LoadUnsafe(ref currentSearchSpace, (nuint)Vector512.Count); - Vector512 packedSource = PackSources(source0, source1); + Vector512 packedSource = TTransform.TransformInput(PackSources(source0, source1)); if (HasMatch(packedValue, packedSource)) { @@ -352,7 +411,7 @@ private static int IndexOf(ref short searchSpace, short value, int len Vector512 source0 = Vector512.LoadUnsafe(ref firstVector); Vector512 source1 = Vector512.LoadUnsafe(ref oneVectorAwayFromEnd); - Vector512 packedSource = PackSources(source0, source1); + Vector512 packedSource = TTransform.TransformInput(PackSources(source0, source1)); if (HasMatch(packedValue, packedSource)) { @@ -378,7 +437,7 @@ private static int IndexOf(ref short searchSpace, short value, int len { Vector256 source0 = Vector256.LoadUnsafe(ref currentSearchSpace); Vector256 source1 = Vector256.LoadUnsafe(ref currentSearchSpace, (nuint)Vector256.Count); - Vector256 packedSource = PackSources(source0, source1); + Vector256 packedSource = TTransform.TransformInput(PackSources(source0, source1)); Vector256 result = Vector256.Equals(packedValue, packedSource); result = NegateIfNeeded(result); @@ -403,7 +462,7 @@ private static int IndexOf(ref short searchSpace, short value, int len Vector256 source0 = Vector256.LoadUnsafe(ref firstVector); Vector256 source1 = Vector256.LoadUnsafe(ref oneVectorAwayFromEnd); - Vector256 packedSource = PackSources(source0, source1); + Vector256 packedSource = TTransform.TransformInput(PackSources(source0, source1)); Vector256 result = Vector256.Equals(packedValue, packedSource); result = NegateIfNeeded(result); @@ -438,7 +497,7 @@ private static int IndexOf(ref short searchSpace, short value, int len { Vector128 source0 = Vector128.LoadUnsafe(ref currentSearchSpace); Vector128 source1 = Vector128.LoadUnsafe(ref currentSearchSpace, (nuint)Vector128.Count); - Vector128 packedSource = PackSources(source0, source1); + Vector128 packedSource = TTransform.TransformInput(PackSources(source0, source1)); Vector128 result = Vector128.Equals(packedValue, packedSource); result = NegateIfNeeded(result); @@ -463,7 +522,7 @@ private static int IndexOf(ref short searchSpace, short value, int len Vector128 source0 = Vector128.LoadUnsafe(ref firstVector); Vector128 source1 = Vector128.LoadUnsafe(ref oneVectorAwayFromEnd); - Vector128 packedSource = PackSources(source0, source1); + Vector128 packedSource = TTransform.TransformInput(PackSources(source0, source1)); Vector128 result = Vector128.Equals(packedValue, packedSource); result = NegateIfNeeded(result); @@ -479,8 +538,9 @@ private static int IndexOf(ref short searchSpace, short value, int len } [CompExactlyDependsOn(typeof(Sse2))] - private static int IndexOfAny(ref short searchSpace, short value0, short value1, int length) + private static int IndexOfAny(ref short searchSpace, short value0, short value1, int length) where TNegator : struct, SpanHelpers.INegator + where TTransform : struct, ITransform { Debug.Assert(CanUsePackedIndexOf(value0)); Debug.Assert(CanUsePackedIndexOf(value1)); @@ -494,13 +554,13 @@ private static int IndexOfAny(ref short searchSpace, short value0, sho { length -= 4; - lookUp = searchSpace; + lookUp = TTransform.TransformInput(searchSpace); if (TNegator.NegateIfNeeded(lookUp == value0 || lookUp == value1)) return 0; - lookUp = Unsafe.Add(ref searchSpace, 1); + lookUp = TTransform.TransformInput(Unsafe.Add(ref searchSpace, 1)); if (TNegator.NegateIfNeeded(lookUp == value0 || lookUp == value1)) return 1; - lookUp = Unsafe.Add(ref searchSpace, 2); + lookUp = TTransform.TransformInput(Unsafe.Add(ref searchSpace, 2)); if (TNegator.NegateIfNeeded(lookUp == value0 || lookUp == value1)) return 2; - lookUp = Unsafe.Add(ref searchSpace, 3); + lookUp = TTransform.TransformInput(Unsafe.Add(ref searchSpace, 3)); if (TNegator.NegateIfNeeded(lookUp == value0 || lookUp == value1)) return 3; offset = 4; @@ -510,7 +570,7 @@ private static int IndexOfAny(ref short searchSpace, short value0, sho { length -= 1; - lookUp = Unsafe.Add(ref searchSpace, offset); + lookUp = TTransform.TransformInput(Unsafe.Add(ref searchSpace, offset)); if (TNegator.NegateIfNeeded(lookUp == value0 || lookUp == value1)) return (int)offset; offset += 1; @@ -538,7 +598,7 @@ private static int IndexOfAny(ref short searchSpace, short value0, sho { Vector512 source0 = Vector512.LoadUnsafe(ref currentSearchSpace); Vector512 source1 = Vector512.LoadUnsafe(ref currentSearchSpace, (nuint)Vector512.Count); - Vector512 packedSource = PackSources(source0, source1); + Vector512 packedSource = TTransform.TransformInput(PackSources(source0, source1)); Vector512 result = NegateIfNeeded(Vector512.Equals(packedValue0, packedSource) | Vector512.Equals(packedValue1, packedSource)); if (result != Vector512.Zero) @@ -562,7 +622,7 @@ private static int IndexOfAny(ref short searchSpace, short value0, sho Vector512 source0 = Vector512.LoadUnsafe(ref firstVector); Vector512 source1 = Vector512.LoadUnsafe(ref oneVectorAwayFromEnd); - Vector512 packedSource = PackSources(source0, source1); + Vector512 packedSource = TTransform.TransformInput(PackSources(source0, source1)); Vector512 result = NegateIfNeeded(Vector512.Equals(packedValue0, packedSource) | Vector512.Equals(packedValue1, packedSource)); if (result != Vector512.Zero) @@ -590,7 +650,7 @@ private static int IndexOfAny(ref short searchSpace, short value0, sho { Vector256 source0 = Vector256.LoadUnsafe(ref currentSearchSpace); Vector256 source1 = Vector256.LoadUnsafe(ref currentSearchSpace, (nuint)Vector256.Count); - Vector256 packedSource = PackSources(source0, source1); + Vector256 packedSource = TTransform.TransformInput(PackSources(source0, source1)); Vector256 result = Vector256.Equals(packedValue0, packedSource) | Vector256.Equals(packedValue1, packedSource); result = NegateIfNeeded(result); @@ -615,7 +675,7 @@ private static int IndexOfAny(ref short searchSpace, short value0, sho Vector256 source0 = Vector256.LoadUnsafe(ref firstVector); Vector256 source1 = Vector256.LoadUnsafe(ref oneVectorAwayFromEnd); - Vector256 packedSource = PackSources(source0, source1); + Vector256 packedSource = TTransform.TransformInput(PackSources(source0, source1)); Vector256 result = Vector256.Equals(packedValue0, packedSource) | Vector256.Equals(packedValue1, packedSource); result = NegateIfNeeded(result); @@ -651,7 +711,7 @@ private static int IndexOfAny(ref short searchSpace, short value0, sho { Vector128 source0 = Vector128.LoadUnsafe(ref currentSearchSpace); Vector128 source1 = Vector128.LoadUnsafe(ref currentSearchSpace, (nuint)Vector128.Count); - Vector128 packedSource = PackSources(source0, source1); + Vector128 packedSource = TTransform.TransformInput(PackSources(source0, source1)); Vector128 result = Vector128.Equals(packedValue0, packedSource) | Vector128.Equals(packedValue1, packedSource); result = NegateIfNeeded(result); @@ -676,7 +736,7 @@ private static int IndexOfAny(ref short searchSpace, short value0, sho Vector128 source0 = Vector128.LoadUnsafe(ref firstVector); Vector128 source1 = Vector128.LoadUnsafe(ref oneVectorAwayFromEnd); - Vector128 packedSource = PackSources(source0, source1); + Vector128 packedSource = TTransform.TransformInput(PackSources(source0, source1)); Vector128 result = Vector128.Equals(packedValue0, packedSource) | Vector128.Equals(packedValue1, packedSource); result = NegateIfNeeded(result); @@ -692,8 +752,9 @@ private static int IndexOfAny(ref short searchSpace, short value0, sho } [CompExactlyDependsOn(typeof(Sse2))] - private static int IndexOfAny(ref short searchSpace, short value0, short value1, short value2, int length) + private static int IndexOfAny(ref short searchSpace, short value0, short value1, short value2, int length) where TNegator : struct, SpanHelpers.INegator + where TTransform : struct, ITransform { Debug.Assert(CanUsePackedIndexOf(value0)); Debug.Assert(CanUsePackedIndexOf(value1)); @@ -708,13 +769,13 @@ private static int IndexOfAny(ref short searchSpace, short value0, sho { length -= 4; - lookUp = searchSpace; + lookUp = TTransform.TransformInput(searchSpace); if (TNegator.NegateIfNeeded(lookUp == value0 || lookUp == value1 || lookUp == value2)) return 0; - lookUp = Unsafe.Add(ref searchSpace, 1); + lookUp = TTransform.TransformInput(Unsafe.Add(ref searchSpace, 1)); if (TNegator.NegateIfNeeded(lookUp == value0 || lookUp == value1 || lookUp == value2)) return 1; - lookUp = Unsafe.Add(ref searchSpace, 2); + lookUp = TTransform.TransformInput(Unsafe.Add(ref searchSpace, 2)); if (TNegator.NegateIfNeeded(lookUp == value0 || lookUp == value1 || lookUp == value2)) return 2; - lookUp = Unsafe.Add(ref searchSpace, 3); + lookUp = TTransform.TransformInput(Unsafe.Add(ref searchSpace, 3)); if (TNegator.NegateIfNeeded(lookUp == value0 || lookUp == value1 || lookUp == value2)) return 3; offset = 4; @@ -724,7 +785,7 @@ private static int IndexOfAny(ref short searchSpace, short value0, sho { length -= 1; - lookUp = Unsafe.Add(ref searchSpace, offset); + lookUp = TTransform.TransformInput(Unsafe.Add(ref searchSpace, offset)); if (TNegator.NegateIfNeeded(lookUp == value0 || lookUp == value1 || lookUp == value2)) return (int)offset; offset += 1; @@ -754,7 +815,7 @@ private static int IndexOfAny(ref short searchSpace, short value0, sho { Vector512 source0 = Vector512.LoadUnsafe(ref currentSearchSpace); Vector512 source1 = Vector512.LoadUnsafe(ref currentSearchSpace, (nuint)Vector512.Count); - Vector512 packedSource = PackSources(source0, source1); + Vector512 packedSource = TTransform.TransformInput(PackSources(source0, source1)); Vector512 result = NegateIfNeeded(Vector512.Equals(packedValue0, packedSource) | Vector512.Equals(packedValue1, packedSource) | Vector512.Equals(packedValue2, packedSource)); if (result != Vector512.Zero) @@ -778,7 +839,7 @@ private static int IndexOfAny(ref short searchSpace, short value0, sho Vector512 source0 = Vector512.LoadUnsafe(ref firstVector); Vector512 source1 = Vector512.LoadUnsafe(ref oneVectorAwayFromEnd); - Vector512 packedSource = PackSources(source0, source1); + Vector512 packedSource = TTransform.TransformInput(PackSources(source0, source1)); Vector512 result = NegateIfNeeded(Vector512.Equals(packedValue0, packedSource) | Vector512.Equals(packedValue1, packedSource) | Vector512.Equals(packedValue2, packedSource)); if (result != Vector512.Zero) @@ -807,7 +868,7 @@ private static int IndexOfAny(ref short searchSpace, short value0, sho { Vector256 source0 = Vector256.LoadUnsafe(ref currentSearchSpace); Vector256 source1 = Vector256.LoadUnsafe(ref currentSearchSpace, (nuint)Vector256.Count); - Vector256 packedSource = PackSources(source0, source1); + Vector256 packedSource = TTransform.TransformInput(PackSources(source0, source1)); Vector256 result = Vector256.Equals(packedValue0, packedSource) | Vector256.Equals(packedValue1, packedSource) | Vector256.Equals(packedValue2, packedSource); result = NegateIfNeeded(result); @@ -832,7 +893,7 @@ private static int IndexOfAny(ref short searchSpace, short value0, sho Vector256 source0 = Vector256.LoadUnsafe(ref firstVector); Vector256 source1 = Vector256.LoadUnsafe(ref oneVectorAwayFromEnd); - Vector256 packedSource = PackSources(source0, source1); + Vector256 packedSource = TTransform.TransformInput(PackSources(source0, source1)); Vector256 result = Vector256.Equals(packedValue0, packedSource) | Vector256.Equals(packedValue1, packedSource) | Vector256.Equals(packedValue2, packedSource); result = NegateIfNeeded(result); @@ -869,7 +930,7 @@ private static int IndexOfAny(ref short searchSpace, short value0, sho { Vector128 source0 = Vector128.LoadUnsafe(ref currentSearchSpace); Vector128 source1 = Vector128.LoadUnsafe(ref currentSearchSpace, (nuint)Vector128.Count); - Vector128 packedSource = PackSources(source0, source1); + Vector128 packedSource = TTransform.TransformInput(PackSources(source0, source1)); Vector128 result = Vector128.Equals(packedValue0, packedSource) | Vector128.Equals(packedValue1, packedSource) | Vector128.Equals(packedValue2, packedSource); result = NegateIfNeeded(result); @@ -894,7 +955,7 @@ private static int IndexOfAny(ref short searchSpace, short value0, sho Vector128 source0 = Vector128.LoadUnsafe(ref firstVector); Vector128 source1 = Vector128.LoadUnsafe(ref oneVectorAwayFromEnd); - Vector128 packedSource = PackSources(source0, source1); + Vector128 packedSource = TTransform.TransformInput(PackSources(source0, source1)); Vector128 result = Vector128.Equals(packedValue0, packedSource) | Vector128.Equals(packedValue1, packedSource) | Vector128.Equals(packedValue2, packedSource); result = NegateIfNeeded(result); @@ -1283,5 +1344,29 @@ internal static Vector512 FixUpPackedVector512Result(Vector512 resul // We want to preserve the order of the two input vectors, so we deinterleave the packed value. return Avx512F.PermuteVar8x64(result.AsInt64(), Vector512.Create(0, 2, 4, 6, 1, 3, 5, 7)).AsByte(); } + + private interface ITransform + { + static abstract short TransformInput(short input); + static abstract Vector128 TransformInput(Vector128 input); + static abstract Vector256 TransformInput(Vector256 input); + static abstract Vector512 TransformInput(Vector512 input); + } + + private readonly struct NopTransform : ITransform + { + public static short TransformInput(short input) => input; + public static Vector128 TransformInput(Vector128 input) => input; + public static Vector256 TransformInput(Vector256 input) => input; + public static Vector512 TransformInput(Vector512 input) => input; + } + + private readonly struct Or20Transform : ITransform + { + public static short TransformInput(short input) => (short)(input | 0x20); + public static Vector128 TransformInput(Vector128 input) => input | Vector128.Create((byte)0x20); + public static Vector256 TransformInput(Vector256 input) => input | Vector256.Create((byte)0x20); + public static Vector512 TransformInput(Vector512 input) => input | Vector512.Create((byte)0x20); + } } } diff --git a/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.T.cs b/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.T.cs index 2cc6dc4405645d..da77d42320a98a 100644 --- a/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.T.cs +++ b/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.T.cs @@ -1672,6 +1672,15 @@ private static unsafe int IndexOfAnyValueType(ref TValue searc { if (PackedSpanHelpers.PackedIndexOfIsSupported && typeof(TValue) == typeof(short) && PackedSpanHelpers.CanUsePackedIndexOf(value0) && PackedSpanHelpers.CanUsePackedIndexOf(value1)) { + if ((*(char*)&value0 ^ *(char*)&value1) == 0x20) + { + char lowerCase = (char)Math.Max(*(char*)&value0, *(char*)&value1); + + return typeof(TNegator) == typeof(DontNegate) + ? PackedSpanHelpers.IndexOfAnyIgnoreCase(ref Unsafe.As(ref searchSpace), lowerCase, length) + : PackedSpanHelpers.IndexOfAnyExceptIgnoreCase(ref Unsafe.As(ref searchSpace), lowerCase, length); + } + return typeof(TNegator) == typeof(DontNegate) ? PackedSpanHelpers.IndexOfAny(ref Unsafe.As(ref searchSpace), *(char*)&value0, *(char*)&value1, length) : PackedSpanHelpers.IndexOfAnyExcept(ref Unsafe.As(ref searchSpace), *(char*)&value0, *(char*)&value1, length); diff --git a/src/libraries/System.Private.CoreLib/src/System/String.Searching.cs b/src/libraries/System.Private.CoreLib/src/System/String.Searching.cs index 2affdb008575a3..56d471168632d9 100644 --- a/src/libraries/System.Private.CoreLib/src/System/String.Searching.cs +++ b/src/libraries/System.Private.CoreLib/src/System/String.Searching.cs @@ -84,10 +84,10 @@ private int IndexOfCharOrdinalIgnoreCase(char value) if (char.IsAsciiLetter(value)) { - char valueUc = (char)(value | 0x20); - char valueLc = (char)(value & ~0x20); + char valueLc = (char)(value | 0x20); + char valueUc = (char)(value & ~0x20); return PackedSpanHelpers.PackedIndexOfIsSupported - ? PackedSpanHelpers.IndexOfAny(ref _firstChar, valueLc, valueUc, Length) + ? PackedSpanHelpers.IndexOfAnyIgnoreCase(ref _firstChar, valueLc, Length) : SpanHelpers.IndexOfAnyChar(ref _firstChar, valueLc, valueUc, Length); } From 1425886267bfe795d5a4dc6ebc0997e7da109bd4 Mon Sep 17 00:00:00 2001 From: Miha Zupan Date: Sat, 6 Jan 2024 19:04:39 +0100 Subject: [PATCH 2/8] Emit SearchValues for ASCII sets of 4/5 values in RegexCompliler --- .../Text/RegularExpressions/RegexCompiler.cs | 76 ++++++++++--------- 1 file changed, 39 insertions(+), 37 deletions(-) diff --git a/src/libraries/System.Text.RegularExpressions/src/System/Text/RegularExpressions/RegexCompiler.cs b/src/libraries/System.Text.RegularExpressions/src/System/Text/RegularExpressions/RegexCompiler.cs index 58abbb8b181eac..bac950c6db2f97 100644 --- a/src/libraries/System.Text.RegularExpressions/src/System/Text/RegularExpressions/RegexCompiler.cs +++ b/src/libraries/System.Text.RegularExpressions/src/System/Text/RegularExpressions/RegexCompiler.cs @@ -920,20 +920,10 @@ void EmitFixedSet_LeftToRight() Call(primarySet.Negated ? s_spanIndexOfAnyExceptCharCharChar : s_spanIndexOfAnyCharCharChar); break; - case 4 or 5: - // tmp = ...IndexOfAny("abcd"); - // Note that this case differs slightly from the source generator, where it might choose to use - // SearchValues instead of a literal, but there's extra cost to doing so for RegexCompiler so - // it just always uses IndexOfAny(span). - Ldstr(new string(primarySet.Chars)); - Call(s_stringAsSpanMethod); - Call(primarySet.Negated ? s_spanIndexOfAnyExceptSpan : s_spanIndexOfAnySpan); - break; - default: + // tmp = ...IndexOfAny(setChars); // tmp = ...IndexOfAny(s_searchValues); - LoadSearchValues(primarySet.Chars); - Call(primarySet.Negated ? s_spanIndexOfAnyExceptSearchValues : s_spanIndexOfAnySearchValues); + EmitIndexOfAnyWithSearchValuesOrLiteral(new string(primarySet.Chars), except: primarySet.Negated); break; } } @@ -3619,9 +3609,7 @@ literal.SetChars is not null || case (true, _): // startingPos = slice.IndexOfAny(literal.SetChars); - Ldstr(literal.SetChars); - Call(s_stringAsSpanMethod); - Call(s_spanIndexOfAnySpan); + EmitIndexOfAnyWithSearchValuesOrLiteral(literal.SetChars); break; case (false, 2): @@ -3634,9 +3622,7 @@ literal.SetChars is not null || case (false, _): // startingPos = slice.IndexOfAny($"{node.Ch}{literal.SetChars}"); - Ldstr($"{node.Ch}{literal.SetChars}"); - Call(s_stringAsSpanMethod); - Call(s_spanIndexOfAnySpan); + EmitIndexOfAnyWithSearchValuesOrLiteral($"{node.Ch}{literal.SetChars}"); break; } } @@ -3650,8 +3636,7 @@ literal.SetChars is not null || Array.Resize(ref asciiChars, asciiChars.Length + 1); asciiChars[^1] = node.Ch; } - LoadSearchValues(asciiChars); - Call(s_spanIndexOfAnySearchValues); + EmitIndexOfAnyWithSearchValuesOrLiteral(new string(asciiChars)); } else if (literal.Range.LowInclusive == literal.Range.HighInclusive) // single char from a RegexNode.One { @@ -5274,15 +5259,7 @@ void EmitIndexOf(RegexNode node, bool useLast, bool negate) return; default: - Ldstr(setChars.ToString()); - Call(s_stringAsSpanMethod); - Call((useLast, negated) switch - { - (false, false) => s_spanIndexOfAnySpan, - (false, true) => s_spanIndexOfAnyExceptSpan, - (true, false) => s_spanLastIndexOfAnySpan, - (true, true) => s_spanLastIndexOfAnyExceptSpan, - }); + EmitIndexOfAnyWithSearchValuesOrLiteral(setChars.ToString(), last: useLast, except: negated); return; } } @@ -5290,14 +5267,7 @@ void EmitIndexOf(RegexNode node, bool useLast, bool negate) // IndexOfAny{Except}(SearchValues) if (RegexCharClass.TryGetAsciiSetChars(node.Str, out char[]? asciiChars)) { - LoadSearchValues(asciiChars); - Call((useLast, negated) switch - { - (false, false) => s_spanIndexOfAnySearchValues, - (false, true) => s_spanIndexOfAnyExceptSearchValues, - (true, false) => s_spanLastIndexOfAnySearchValues, - (true, true) => s_spanLastIndexOfAnyExceptSearchValues, - }); + EmitIndexOfAnyWithSearchValuesOrLiteral(new string(asciiChars), last: useLast, except: negated); return; } } @@ -6192,6 +6162,38 @@ private void EmitTimeoutCheckIfNeeded() } } + /// Emits a call to either IndexOfAny("abcd") or IndexOfAny(SearchValues) depending on the . + private void EmitIndexOfAnyWithSearchValuesOrLiteral(string chars, bool last = false, bool except = false) + { + Debug.Assert(chars.Length > 3); + + // SearchValues is faster than a regular IndexOfAny("abcd") for sets of 4/5 values iff they are ASCII. + // Only emit SearchValues instances when we know they'll be faster to avoid increasing the startup cost too much. + if (chars.Length is 4 or 5 && !RegexCharClass.IsAscii(chars)) + { + Ldstr(chars); + Call(s_stringAsSpanMethod); + Call((last, except) switch + { + (false, false) => s_spanIndexOfAnySpan, + (false, true) => s_spanIndexOfAnyExceptSpan, + (true, false) => s_spanLastIndexOfAnySpan, + (true, true) => s_spanLastIndexOfAnyExceptSpan, + }); + } + else + { + LoadSearchValues(chars.ToCharArray()); + Call((last, except) switch + { + (false, false) => s_spanIndexOfAnySearchValues, + (false, true) => s_spanIndexOfAnyExceptSearchValues, + (true, false) => s_spanLastIndexOfAnySearchValues, + (true, true) => s_spanLastIndexOfAnyExceptSearchValues, + }); + } + } + /// /// Adds an entry in for the given and emits a load of that initialized value. /// From c63391e5036ab12cf626934c01e9e1d423baa3e1 Mon Sep 17 00:00:00 2001 From: Miha Zupan Date: Sun, 7 Jan 2024 00:39:41 +0100 Subject: [PATCH 3/8] Remove Any3CharPackedIgnoreCase --- .../System.Private.CoreLib.Shared.projitems | 1 - .../Any3CharPackedIgnoreCaseSearchValues.cs | 64 ------------------- .../src/System/SearchValues/SearchValues.cs | 18 ++---- .../src/System/SpanHelpers.Packed.cs | 51 ++++----------- 4 files changed, 18 insertions(+), 116 deletions(-) delete mode 100644 src/libraries/System.Private.CoreLib/src/System/SearchValues/Any3CharPackedIgnoreCaseSearchValues.cs diff --git a/src/libraries/System.Private.CoreLib/src/System.Private.CoreLib.Shared.projitems b/src/libraries/System.Private.CoreLib/src/System.Private.CoreLib.Shared.projitems index 27b69331cdfa20..a73e8247a58e72 100644 --- a/src/libraries/System.Private.CoreLib/src/System.Private.CoreLib.Shared.projitems +++ b/src/libraries/System.Private.CoreLib/src/System.Private.CoreLib.Shared.projitems @@ -425,7 +425,6 @@ - diff --git a/src/libraries/System.Private.CoreLib/src/System/SearchValues/Any3CharPackedIgnoreCaseSearchValues.cs b/src/libraries/System.Private.CoreLib/src/System/SearchValues/Any3CharPackedIgnoreCaseSearchValues.cs deleted file mode 100644 index 40844c3d25be7b..00000000000000 --- a/src/libraries/System.Private.CoreLib/src/System/SearchValues/Any3CharPackedIgnoreCaseSearchValues.cs +++ /dev/null @@ -1,64 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -using System.Diagnostics; -using System.Runtime.CompilerServices; -using System.Runtime.InteropServices; -using System.Runtime.Intrinsics.Arm; -using System.Runtime.Intrinsics.Wasm; -using System.Runtime.Intrinsics.X86; - -namespace System.Buffers -{ - internal sealed class Any3CharPackedIgnoreCaseSearchValues : SearchValues - { - private readonly char _e0, _e1, _e2; - private IndexOfAnyAsciiSearcher.AsciiState _state; - - public Any3CharPackedIgnoreCaseSearchValues(char value0, char value1, char value2) - { - Debug.Assert(value0 != 0 && (value0 | 0x20) == value0 && char.IsAscii(value0)); - Debug.Assert(value1 != 0 && (value1 | 0x20) == value1 && char.IsAscii(value1)); - Debug.Assert(value2 != 0 && (value2 | 0x20) == value2 && char.IsAscii(value2)); - - (_e0, _e1, _e2) = (value0, value1, value2); - IndexOfAnyAsciiSearcher.ComputeAsciiState([(char)(_e0 & ~0x20), _e0, (char)(_e1 & ~0x20), _e1, (char)(_e2 & ~0x20), _e2], out _state); - } - - internal override char[] GetValues() => - [(char)(_e0 & ~0x20), _e0, (char)(_e1 & ~0x20), _e1, (char)(_e2 & ~0x20), _e2]; - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal override bool ContainsCore(char value) - { - value = (char)(value | 0x20); - return value == _e0 || value == _e1 || value == _e2; - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - [CompExactlyDependsOn(typeof(Sse2))] - internal override int IndexOfAny(ReadOnlySpan span) => - PackedSpanHelpers.IndexOfAnyIgnoreCase(ref MemoryMarshal.GetReference(span), _e0, _e1, _e2, span.Length); - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - [CompExactlyDependsOn(typeof(Sse2))] - internal override int IndexOfAnyExcept(ReadOnlySpan span) => - PackedSpanHelpers.IndexOfAnyExceptIgnoreCase(ref MemoryMarshal.GetReference(span), _e0, _e1, _e2, span.Length); - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - [CompExactlyDependsOn(typeof(Ssse3))] - [CompExactlyDependsOn(typeof(AdvSimd))] - [CompExactlyDependsOn(typeof(PackedSimd))] - internal override int LastIndexOfAny(ReadOnlySpan span) => - IndexOfAnyAsciiSearcher.LastIndexOfAnyVectorized( - ref Unsafe.As(ref MemoryMarshal.GetReference(span)), span.Length, ref _state); - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - [CompExactlyDependsOn(typeof(Ssse3))] - [CompExactlyDependsOn(typeof(AdvSimd))] - [CompExactlyDependsOn(typeof(PackedSimd))] - internal override int LastIndexOfAnyExcept(ReadOnlySpan span) => - IndexOfAnyAsciiSearcher.LastIndexOfAnyVectorized( - ref Unsafe.As(ref MemoryMarshal.GetReference(span)), span.Length, ref _state); - } -} diff --git a/src/libraries/System.Private.CoreLib/src/System/SearchValues/SearchValues.cs b/src/libraries/System.Private.CoreLib/src/System/SearchValues/SearchValues.cs index 1307801afc971c..d3242209712ea0 100644 --- a/src/libraries/System.Private.CoreLib/src/System/SearchValues/SearchValues.cs +++ b/src/libraries/System.Private.CoreLib/src/System/SearchValues/SearchValues.cs @@ -126,31 +126,21 @@ ref Unsafe.As(ref MemoryMarshal.GetReference(values)), // IndexOfAnyAsciiSearcher for chars is slower than Any3CharSearchValues, but faster than Any4SearchValues if (IndexOfAnyAsciiSearcher.IsVectorizationSupported && maxInclusive < 128) { - // If the values are sets of 2 or 3 ASCII letters with both cases, we can use an approach that + // If the values are sets of 2 ASCII letters with both cases, we can use an approach that // reduces the number of comparisons by masking off the bit that differs between lower and upper case (0x20). // While this most commonly applies to ASCII letters, it also works for other values that differ by 0x20 (e.g. "[]{}" => "{}"). - if (PackedSpanHelpers.PackedIndexOfIsSupported && values.Length is 4 or 6 && minInclusive > 0) + if (PackedSpanHelpers.PackedIndexOfIsSupported && values.Length == 4 && minInclusive > 0) { - Span copy = stackalloc char[values.Length]; + Span copy = stackalloc char[4]; values.CopyTo(copy); copy.Sort(); - if (copy.Length == 4 && - (copy[0] ^ copy[2]) == 0x20 && + if ((copy[0] ^ copy[2]) == 0x20 && (copy[1] ^ copy[3]) == 0x20) { // "AaBb" => 'a', 'b' return new Any2CharPackedIgnoreCaseSearchValues(copy[2], copy[3]); } - - if (copy.Length == 6 && - (copy[0] ^ copy[3]) == 0x20 && - (copy[1] ^ copy[4]) == 0x20 && - (copy[2] ^ copy[5]) == 0x20) - { - // "AaBbCc" => 'a', 'b', 'c' - return new Any3CharPackedIgnoreCaseSearchValues(copy[3], copy[4], copy[5]); - } } return (Ssse3.IsSupported || PackedSimd.IsSupported) && minInclusive == 0 diff --git a/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.Packed.cs b/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.Packed.cs index 93dece79f54bd4..69ce8c1f7fada9 100644 --- a/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.Packed.cs +++ b/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.Packed.cs @@ -55,12 +55,12 @@ public static int IndexOfAnyExcept(ref char searchSpace, char value0, char value [MethodImpl(MethodImplOptions.AggressiveInlining)] [CompExactlyDependsOn(typeof(Sse2))] public static int IndexOfAny(ref char searchSpace, char value0, char value1, char value2, int length) => - IndexOfAny, NopTransform>(ref Unsafe.As(ref searchSpace), (short)value0, (short)value1, (short)value2, length); + IndexOfAny>(ref Unsafe.As(ref searchSpace), (short)value0, (short)value1, (short)value2, length); [MethodImpl(MethodImplOptions.AggressiveInlining)] [CompExactlyDependsOn(typeof(Sse2))] public static int IndexOfAnyExcept(ref char searchSpace, char value0, char value1, char value2, int length) => - IndexOfAny, NopTransform>(ref Unsafe.As(ref searchSpace), (short)value0, (short)value1, (short)value2, length); + IndexOfAny>(ref Unsafe.As(ref searchSpace), (short)value0, (short)value1, (short)value2, length); [MethodImpl(MethodImplOptions.AggressiveInlining)] [CompExactlyDependsOn(typeof(Sse2))] @@ -100,28 +100,6 @@ public static int IndexOfAnyExceptIgnoreCase(ref char searchSpace, char value0, return IndexOfAny, Or20Transform>(ref Unsafe.As(ref searchSpace), (short)value0, (short)value1, length); } - [MethodImpl(MethodImplOptions.AggressiveInlining)] - [CompExactlyDependsOn(typeof(Sse2))] - public static int IndexOfAnyIgnoreCase(ref char searchSpace, char value0, char value1, char value2, int length) - { - Debug.Assert((value0 | 0x20) == value0); - Debug.Assert((value1 | 0x20) == value1); - Debug.Assert((value2 | 0x20) == value2); - - return IndexOfAny, Or20Transform>(ref Unsafe.As(ref searchSpace), (short)value0, (short)value1, (short)value2, length); - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - [CompExactlyDependsOn(typeof(Sse2))] - public static int IndexOfAnyExceptIgnoreCase(ref char searchSpace, char value0, char value1, char value2, int length) - { - Debug.Assert((value0 | 0x20) == value0); - Debug.Assert((value1 | 0x20) == value1); - Debug.Assert((value2 | 0x20) == value2); - - return IndexOfAny, Or20Transform>(ref Unsafe.As(ref searchSpace), (short)value0, (short)value1, (short)value2, length); - } - [MethodImpl(MethodImplOptions.AggressiveInlining)] [CompExactlyDependsOn(typeof(Sse2))] public static int IndexOfAnyInRange(ref char searchSpace, char lowInclusive, char rangeInclusive, int length) => @@ -752,9 +730,8 @@ private static int IndexOfAny(ref short searchSpace, short } [CompExactlyDependsOn(typeof(Sse2))] - private static int IndexOfAny(ref short searchSpace, short value0, short value1, short value2, int length) + private static int IndexOfAny(ref short searchSpace, short value0, short value1, short value2, int length) where TNegator : struct, SpanHelpers.INegator - where TTransform : struct, ITransform { Debug.Assert(CanUsePackedIndexOf(value0)); Debug.Assert(CanUsePackedIndexOf(value1)); @@ -769,13 +746,13 @@ private static int IndexOfAny(ref short searchSpace, short { length -= 4; - lookUp = TTransform.TransformInput(searchSpace); + lookUp = searchSpace; if (TNegator.NegateIfNeeded(lookUp == value0 || lookUp == value1 || lookUp == value2)) return 0; - lookUp = TTransform.TransformInput(Unsafe.Add(ref searchSpace, 1)); + lookUp = Unsafe.Add(ref searchSpace, 1); if (TNegator.NegateIfNeeded(lookUp == value0 || lookUp == value1 || lookUp == value2)) return 1; - lookUp = TTransform.TransformInput(Unsafe.Add(ref searchSpace, 2)); + lookUp = Unsafe.Add(ref searchSpace, 2); if (TNegator.NegateIfNeeded(lookUp == value0 || lookUp == value1 || lookUp == value2)) return 2; - lookUp = TTransform.TransformInput(Unsafe.Add(ref searchSpace, 3)); + lookUp = Unsafe.Add(ref searchSpace, 3); if (TNegator.NegateIfNeeded(lookUp == value0 || lookUp == value1 || lookUp == value2)) return 3; offset = 4; @@ -785,7 +762,7 @@ private static int IndexOfAny(ref short searchSpace, short { length -= 1; - lookUp = TTransform.TransformInput(Unsafe.Add(ref searchSpace, offset)); + lookUp = Unsafe.Add(ref searchSpace, offset); if (TNegator.NegateIfNeeded(lookUp == value0 || lookUp == value1 || lookUp == value2)) return (int)offset; offset += 1; @@ -815,7 +792,7 @@ private static int IndexOfAny(ref short searchSpace, short { Vector512 source0 = Vector512.LoadUnsafe(ref currentSearchSpace); Vector512 source1 = Vector512.LoadUnsafe(ref currentSearchSpace, (nuint)Vector512.Count); - Vector512 packedSource = TTransform.TransformInput(PackSources(source0, source1)); + Vector512 packedSource = PackSources(source0, source1); Vector512 result = NegateIfNeeded(Vector512.Equals(packedValue0, packedSource) | Vector512.Equals(packedValue1, packedSource) | Vector512.Equals(packedValue2, packedSource)); if (result != Vector512.Zero) @@ -839,7 +816,7 @@ private static int IndexOfAny(ref short searchSpace, short Vector512 source0 = Vector512.LoadUnsafe(ref firstVector); Vector512 source1 = Vector512.LoadUnsafe(ref oneVectorAwayFromEnd); - Vector512 packedSource = TTransform.TransformInput(PackSources(source0, source1)); + Vector512 packedSource = PackSources(source0, source1); Vector512 result = NegateIfNeeded(Vector512.Equals(packedValue0, packedSource) | Vector512.Equals(packedValue1, packedSource) | Vector512.Equals(packedValue2, packedSource)); if (result != Vector512.Zero) @@ -868,7 +845,7 @@ private static int IndexOfAny(ref short searchSpace, short { Vector256 source0 = Vector256.LoadUnsafe(ref currentSearchSpace); Vector256 source1 = Vector256.LoadUnsafe(ref currentSearchSpace, (nuint)Vector256.Count); - Vector256 packedSource = TTransform.TransformInput(PackSources(source0, source1)); + Vector256 packedSource = PackSources(source0, source1); Vector256 result = Vector256.Equals(packedValue0, packedSource) | Vector256.Equals(packedValue1, packedSource) | Vector256.Equals(packedValue2, packedSource); result = NegateIfNeeded(result); @@ -893,7 +870,7 @@ private static int IndexOfAny(ref short searchSpace, short Vector256 source0 = Vector256.LoadUnsafe(ref firstVector); Vector256 source1 = Vector256.LoadUnsafe(ref oneVectorAwayFromEnd); - Vector256 packedSource = TTransform.TransformInput(PackSources(source0, source1)); + Vector256 packedSource = PackSources(source0, source1); Vector256 result = Vector256.Equals(packedValue0, packedSource) | Vector256.Equals(packedValue1, packedSource) | Vector256.Equals(packedValue2, packedSource); result = NegateIfNeeded(result); @@ -930,7 +907,7 @@ private static int IndexOfAny(ref short searchSpace, short { Vector128 source0 = Vector128.LoadUnsafe(ref currentSearchSpace); Vector128 source1 = Vector128.LoadUnsafe(ref currentSearchSpace, (nuint)Vector128.Count); - Vector128 packedSource = TTransform.TransformInput(PackSources(source0, source1)); + Vector128 packedSource = PackSources(source0, source1); Vector128 result = Vector128.Equals(packedValue0, packedSource) | Vector128.Equals(packedValue1, packedSource) | Vector128.Equals(packedValue2, packedSource); result = NegateIfNeeded(result); @@ -955,7 +932,7 @@ private static int IndexOfAny(ref short searchSpace, short Vector128 source0 = Vector128.LoadUnsafe(ref firstVector); Vector128 source1 = Vector128.LoadUnsafe(ref oneVectorAwayFromEnd); - Vector128 packedSource = TTransform.TransformInput(PackSources(source0, source1)); + Vector128 packedSource = PackSources(source0, source1); Vector128 result = Vector128.Equals(packedValue0, packedSource) | Vector128.Equals(packedValue1, packedSource) | Vector128.Equals(packedValue2, packedSource); result = NegateIfNeeded(result); From 066a29dbc9d40303ad5ef5beebd2ce90218e5e48 Mon Sep 17 00:00:00 2001 From: Miha Zupan Date: Sun, 7 Jan 2024 00:42:04 +0100 Subject: [PATCH 4/8] Update asserts --- .../SearchValues/Any1CharPackedIgnoreCaseSearchValues.cs | 1 - .../SearchValues/Any2CharPackedIgnoreCaseSearchValues.cs | 4 ++-- .../src/System/SearchValues/Any2SearchValues.cs | 3 +-- .../src/System/SearchValues/Any3SearchValues.cs | 4 +--- 4 files changed, 4 insertions(+), 8 deletions(-) diff --git a/src/libraries/System.Private.CoreLib/src/System/SearchValues/Any1CharPackedIgnoreCaseSearchValues.cs b/src/libraries/System.Private.CoreLib/src/System/SearchValues/Any1CharPackedIgnoreCaseSearchValues.cs index 84f1b857419f43..adea8486de8d6d 100644 --- a/src/libraries/System.Private.CoreLib/src/System/SearchValues/Any1CharPackedIgnoreCaseSearchValues.cs +++ b/src/libraries/System.Private.CoreLib/src/System/SearchValues/Any1CharPackedIgnoreCaseSearchValues.cs @@ -14,7 +14,6 @@ internal sealed class Any1CharPackedIgnoreCaseSearchValues : SearchValues public Any1CharPackedIgnoreCaseSearchValues(char value) { - Debug.Assert(value != 0); Debug.Assert((value | 0x20) == value); _lowerCase = value; diff --git a/src/libraries/System.Private.CoreLib/src/System/SearchValues/Any2CharPackedIgnoreCaseSearchValues.cs b/src/libraries/System.Private.CoreLib/src/System/SearchValues/Any2CharPackedIgnoreCaseSearchValues.cs index b76e4c974e03e8..ef9aca5332f537 100644 --- a/src/libraries/System.Private.CoreLib/src/System/SearchValues/Any2CharPackedIgnoreCaseSearchValues.cs +++ b/src/libraries/System.Private.CoreLib/src/System/SearchValues/Any2CharPackedIgnoreCaseSearchValues.cs @@ -17,8 +17,8 @@ internal sealed class Any2CharPackedIgnoreCaseSearchValues : SearchValues public Any2CharPackedIgnoreCaseSearchValues(char value0, char value1) { - Debug.Assert(value0 != 0 && (value0 | 0x20) == value0 && char.IsAscii(value0)); - Debug.Assert(value1 != 0 && (value1 | 0x20) == value1 && char.IsAscii(value1)); + Debug.Assert((value0 | 0x20) == value0 && char.IsAscii(value0)); + Debug.Assert((value1 | 0x20) == value1 && char.IsAscii(value1)); (_e0, _e1) = (value0, value1); IndexOfAnyAsciiSearcher.ComputeAsciiState([(char)(_e0 & ~0x20), _e0, (char)(_e1 & ~0x20), _e1], out _state); diff --git a/src/libraries/System.Private.CoreLib/src/System/SearchValues/Any2SearchValues.cs b/src/libraries/System.Private.CoreLib/src/System/SearchValues/Any2SearchValues.cs index 6bc47027cbdb2f..640a15f05c8f2b 100644 --- a/src/libraries/System.Private.CoreLib/src/System/SearchValues/Any2SearchValues.cs +++ b/src/libraries/System.Private.CoreLib/src/System/SearchValues/Any2SearchValues.cs @@ -31,8 +31,7 @@ internal override unsafe T[] GetValues() [MethodImpl(MethodImplOptions.AggressiveInlining)] internal override unsafe bool ContainsCore(T value) => - *(TImpl*)&value == _e0 || - *(TImpl*)&value == _e1; + *(TImpl*)&value == _e0 || *(TImpl*)&value == _e1; [MethodImpl(MethodImplOptions.AggressiveInlining)] internal override int IndexOfAny(ReadOnlySpan span) => diff --git a/src/libraries/System.Private.CoreLib/src/System/SearchValues/Any3SearchValues.cs b/src/libraries/System.Private.CoreLib/src/System/SearchValues/Any3SearchValues.cs index 95662c995a7f5c..f166aef42703cd 100644 --- a/src/libraries/System.Private.CoreLib/src/System/SearchValues/Any3SearchValues.cs +++ b/src/libraries/System.Private.CoreLib/src/System/SearchValues/Any3SearchValues.cs @@ -31,9 +31,7 @@ internal override unsafe T[] GetValues() [MethodImpl(MethodImplOptions.AggressiveInlining)] internal override unsafe bool ContainsCore(T value) => - *(TImpl*)&value == _e0 || - *(TImpl*)&value == _e1 || - *(TImpl*)&value == _e2; + *(TImpl*)&value == _e0 || *(TImpl*)&value == _e1 || *(TImpl*)&value == _e2; [MethodImpl(MethodImplOptions.AggressiveInlining)] internal override int IndexOfAny(ReadOnlySpan span) => From b467a131d7549f242b13268e73520e5f44b804bb Mon Sep 17 00:00:00 2001 From: Miha Zupan Date: Sun, 7 Jan 2024 05:24:30 +0100 Subject: [PATCH 5/8] Attribute order --- .../System/SearchValues/Any2CharPackedIgnoreCaseSearchValues.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libraries/System.Private.CoreLib/src/System/SearchValues/Any2CharPackedIgnoreCaseSearchValues.cs b/src/libraries/System.Private.CoreLib/src/System/SearchValues/Any2CharPackedIgnoreCaseSearchValues.cs index ef9aca5332f537..e17bfaf6df8cb7 100644 --- a/src/libraries/System.Private.CoreLib/src/System/SearchValues/Any2CharPackedIgnoreCaseSearchValues.cs +++ b/src/libraries/System.Private.CoreLib/src/System/SearchValues/Any2CharPackedIgnoreCaseSearchValues.cs @@ -39,8 +39,8 @@ internal override bool ContainsCore(char value) internal override int IndexOfAny(ReadOnlySpan span) => PackedSpanHelpers.IndexOfAnyIgnoreCase(ref MemoryMarshal.GetReference(span), _e0, _e1, span.Length); - [CompExactlyDependsOn(typeof(Sse2))] [MethodImpl(MethodImplOptions.AggressiveInlining)] + [CompExactlyDependsOn(typeof(Sse2))] internal override int IndexOfAnyExcept(ReadOnlySpan span) => PackedSpanHelpers.IndexOfAnyExceptIgnoreCase(ref MemoryMarshal.GetReference(span), _e0, _e1, span.Length); From 5d29af7b3ad0a03b54e0a2bb9893f3b0ca5ca50a Mon Sep 17 00:00:00 2001 From: Miha Zupan Date: Sat, 17 Feb 2024 20:49:01 +0100 Subject: [PATCH 6/8] Fix build --- .../SearchValues/Any2CharPackedIgnoreCaseSearchValues.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/libraries/System.Private.CoreLib/src/System/SearchValues/Any2CharPackedIgnoreCaseSearchValues.cs b/src/libraries/System.Private.CoreLib/src/System/SearchValues/Any2CharPackedIgnoreCaseSearchValues.cs index e17bfaf6df8cb7..9c9c6a55ebf34d 100644 --- a/src/libraries/System.Private.CoreLib/src/System/SearchValues/Any2CharPackedIgnoreCaseSearchValues.cs +++ b/src/libraries/System.Private.CoreLib/src/System/SearchValues/Any2CharPackedIgnoreCaseSearchValues.cs @@ -49,7 +49,7 @@ internal override int IndexOfAnyExcept(ReadOnlySpan span) => [CompExactlyDependsOn(typeof(AdvSimd))] [CompExactlyDependsOn(typeof(PackedSimd))] internal override int LastIndexOfAny(ReadOnlySpan span) => - IndexOfAnyAsciiSearcher.LastIndexOfAnyVectorized( + IndexOfAnyAsciiSearcher.LastIndexOfAny( ref Unsafe.As(ref MemoryMarshal.GetReference(span)), span.Length, ref _state); [MethodImpl(MethodImplOptions.AggressiveInlining)] @@ -57,7 +57,7 @@ internal override int LastIndexOfAny(ReadOnlySpan span) => [CompExactlyDependsOn(typeof(AdvSimd))] [CompExactlyDependsOn(typeof(PackedSimd))] internal override int LastIndexOfAnyExcept(ReadOnlySpan span) => - IndexOfAnyAsciiSearcher.LastIndexOfAnyVectorized( + IndexOfAnyAsciiSearcher.LastIndexOfAny( ref Unsafe.As(ref MemoryMarshal.GetReference(span)), span.Length, ref _state); } } From a919e16779d99096f994c998eea7c470bbb80548 Mon Sep 17 00:00:00 2001 From: Miha Zupan Date: Sat, 17 Feb 2024 21:06:06 +0100 Subject: [PATCH 7/8] More comments --- .../SearchValues/Any1CharPackedIgnoreCaseSearchValues.cs | 2 ++ .../SearchValues/Any2CharPackedIgnoreCaseSearchValues.cs | 2 ++ .../src/System/SearchValues/SearchValues.cs | 5 ++++- 3 files changed, 8 insertions(+), 1 deletion(-) diff --git a/src/libraries/System.Private.CoreLib/src/System/SearchValues/Any1CharPackedIgnoreCaseSearchValues.cs b/src/libraries/System.Private.CoreLib/src/System/SearchValues/Any1CharPackedIgnoreCaseSearchValues.cs index adea8486de8d6d..c31303c71b037f 100644 --- a/src/libraries/System.Private.CoreLib/src/System/SearchValues/Any1CharPackedIgnoreCaseSearchValues.cs +++ b/src/libraries/System.Private.CoreLib/src/System/SearchValues/Any1CharPackedIgnoreCaseSearchValues.cs @@ -10,6 +10,8 @@ namespace System.Buffers { internal sealed class Any1CharPackedIgnoreCaseSearchValues : SearchValues { + // While this most commonly applies to ASCII letters, it also works for other values that differ by 0x20 (e.g. "[{" => "{"). + // _lowerCase is therefore not necessarily a lower case ASCII letter, but just the higher value (the one with the 0x20 bit set). private readonly char _lowerCase, _upperCase; public Any1CharPackedIgnoreCaseSearchValues(char value) diff --git a/src/libraries/System.Private.CoreLib/src/System/SearchValues/Any2CharPackedIgnoreCaseSearchValues.cs b/src/libraries/System.Private.CoreLib/src/System/SearchValues/Any2CharPackedIgnoreCaseSearchValues.cs index 9c9c6a55ebf34d..4c11557a4079b8 100644 --- a/src/libraries/System.Private.CoreLib/src/System/SearchValues/Any2CharPackedIgnoreCaseSearchValues.cs +++ b/src/libraries/System.Private.CoreLib/src/System/SearchValues/Any2CharPackedIgnoreCaseSearchValues.cs @@ -12,6 +12,8 @@ namespace System.Buffers { internal sealed class Any2CharPackedIgnoreCaseSearchValues : SearchValues { + // While this most commonly applies to ASCII letters, it also works for other values that differ by 0x20 (e.g. "[]{}" => "{}"). + // _e0 and _e1 are therefore not necessarily lower case ASCII letters, but just the higher values (the ones with the 0x20 bit set). private readonly char _e0, _e1; private IndexOfAnyAsciiSearcher.AsciiState _state; diff --git a/src/libraries/System.Private.CoreLib/src/System/SearchValues/SearchValues.cs b/src/libraries/System.Private.CoreLib/src/System/SearchValues/SearchValues.cs index d3242209712ea0..7428c912c6cfda 100644 --- a/src/libraries/System.Private.CoreLib/src/System/SearchValues/SearchValues.cs +++ b/src/libraries/System.Private.CoreLib/src/System/SearchValues/SearchValues.cs @@ -104,6 +104,9 @@ ref Unsafe.As(ref MemoryMarshal.GetReference(values)), if (PackedSpanHelpers.PackedIndexOfIsSupported && PackedSpanHelpers.CanUsePackedIndexOf(value0) && PackedSpanHelpers.CanUsePackedIndexOf(value1)) { + // If the two values are the same ASCII letter with both cases, we can use an approach that + // reduces the number of comparisons by masking off the bit that differs between lower and upper case (0x20). + // While this most commonly applies to ASCII letters, it also works for other values that differ by 0x20 (e.g. "[{" => "{"). return (value0 ^ value1) == 0x20 ? new Any1CharPackedIgnoreCaseSearchValues((char)Math.Max(value0, value1)) : new Any2CharPackedSearchValues(value0, value1); @@ -138,7 +141,7 @@ ref Unsafe.As(ref MemoryMarshal.GetReference(values)), if ((copy[0] ^ copy[2]) == 0x20 && (copy[1] ^ copy[3]) == 0x20) { - // "AaBb" => 'a', 'b' + // We pick the higher two values (with the 0x20 bit set). "AaBb" => 'a', 'b' return new Any2CharPackedIgnoreCaseSearchValues(copy[2], copy[3]); } } From 5a2568b8b24c7b1d68acf4256219a39ea78e66a1 Mon Sep 17 00:00:00 2001 From: Miha Zupan Date: Sat, 17 Feb 2024 21:17:26 +0100 Subject: [PATCH 8/8] Tweak ContainsCore --- .../SearchValues/Any1CharPackedIgnoreCaseSearchValues.cs | 4 +++- .../SearchValues/Any2CharPackedIgnoreCaseSearchValues.cs | 6 ++++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/src/libraries/System.Private.CoreLib/src/System/SearchValues/Any1CharPackedIgnoreCaseSearchValues.cs b/src/libraries/System.Private.CoreLib/src/System/SearchValues/Any1CharPackedIgnoreCaseSearchValues.cs index c31303c71b037f..dfe6b3631f709e 100644 --- a/src/libraries/System.Private.CoreLib/src/System/SearchValues/Any1CharPackedIgnoreCaseSearchValues.cs +++ b/src/libraries/System.Private.CoreLib/src/System/SearchValues/Any1CharPackedIgnoreCaseSearchValues.cs @@ -13,6 +13,7 @@ internal sealed class Any1CharPackedIgnoreCaseSearchValues : SearchValues // While this most commonly applies to ASCII letters, it also works for other values that differ by 0x20 (e.g. "[{" => "{"). // _lowerCase is therefore not necessarily a lower case ASCII letter, but just the higher value (the one with the 0x20 bit set). private readonly char _lowerCase, _upperCase; + private readonly uint _lowerCaseUint; public Any1CharPackedIgnoreCaseSearchValues(char value) { @@ -20,6 +21,7 @@ public Any1CharPackedIgnoreCaseSearchValues(char value) _lowerCase = value; _upperCase = (char)(value & ~0x20); + _lowerCaseUint = value; } internal override char[] GetValues() => @@ -27,7 +29,7 @@ internal override char[] GetValues() => [MethodImpl(MethodImplOptions.AggressiveInlining)] internal override bool ContainsCore(char value) => - value == _lowerCase || value == _upperCase; + (uint)(value | 0x20) == _lowerCaseUint; [MethodImpl(MethodImplOptions.AggressiveInlining)] [CompExactlyDependsOn(typeof(Sse2))] diff --git a/src/libraries/System.Private.CoreLib/src/System/SearchValues/Any2CharPackedIgnoreCaseSearchValues.cs b/src/libraries/System.Private.CoreLib/src/System/SearchValues/Any2CharPackedIgnoreCaseSearchValues.cs index 4c11557a4079b8..1073fcf3c81858 100644 --- a/src/libraries/System.Private.CoreLib/src/System/SearchValues/Any2CharPackedIgnoreCaseSearchValues.cs +++ b/src/libraries/System.Private.CoreLib/src/System/SearchValues/Any2CharPackedIgnoreCaseSearchValues.cs @@ -15,6 +15,7 @@ internal sealed class Any2CharPackedIgnoreCaseSearchValues : SearchValues // While this most commonly applies to ASCII letters, it also works for other values that differ by 0x20 (e.g. "[]{}" => "{}"). // _e0 and _e1 are therefore not necessarily lower case ASCII letters, but just the higher values (the ones with the 0x20 bit set). private readonly char _e0, _e1; + private readonly uint _uint0, _uint1; private IndexOfAnyAsciiSearcher.AsciiState _state; public Any2CharPackedIgnoreCaseSearchValues(char value0, char value1) @@ -23,6 +24,7 @@ public Any2CharPackedIgnoreCaseSearchValues(char value0, char value1) Debug.Assert((value1 | 0x20) == value1 && char.IsAscii(value1)); (_e0, _e1) = (value0, value1); + (_uint0, _uint1) = (value0, value1); IndexOfAnyAsciiSearcher.ComputeAsciiState([(char)(_e0 & ~0x20), _e0, (char)(_e1 & ~0x20), _e1], out _state); } @@ -32,8 +34,8 @@ internal override char[] GetValues() => [MethodImpl(MethodImplOptions.AggressiveInlining)] internal override bool ContainsCore(char value) { - value = (char)(value | 0x20); - return value == _e0 || value == _e1; + uint lowerCase = (uint)(value | 0x20); + return lowerCase == _uint0 || lowerCase == _uint1; } [MethodImpl(MethodImplOptions.AggressiveInlining)]