diff --git a/src/libraries/System.Memory/tests/Span/SearchValues.cs b/src/libraries/System.Memory/tests/Span/SearchValues.cs index 9ef91acec80dab..f2020279cdf4b9 100644 --- a/src/libraries/System.Memory/tests/Span/SearchValues.cs +++ b/src/libraries/System.Memory/tests/Span/SearchValues.cs @@ -44,6 +44,10 @@ public static IEnumerable Values_MemberData() "aaa", "aaaa", "aaaaa", + "Aa", + "AaBb", + "AaBbCc", + "[]{}", "\uFFF0", "\uFFF0\uFFF2", "\uFFF0\uFFF2\uFFF4", diff --git a/src/libraries/System.Private.CoreLib/src/System.Private.CoreLib.Shared.projitems b/src/libraries/System.Private.CoreLib/src/System.Private.CoreLib.Shared.projitems index c9e500d22af6b6..a73e8247a58e72 100644 --- a/src/libraries/System.Private.CoreLib/src/System.Private.CoreLib.Shared.projitems +++ b/src/libraries/System.Private.CoreLib/src/System.Private.CoreLib.Shared.projitems @@ -423,14 +423,16 @@ + + + + + + + + - - - - - - diff --git a/src/libraries/System.Private.CoreLib/src/System/Globalization/Ordinal.cs b/src/libraries/System.Private.CoreLib/src/System/Globalization/Ordinal.cs index b7de19aab570d7..5b18e8d2b6fcb3 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Globalization/Ordinal.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Globalization/Ordinal.cs @@ -397,7 +397,7 @@ internal static int IndexOfOrdinalIgnoreCase(ReadOnlySpan source, ReadOnly // Do a quick search for the first element of "value". int relativeIndex = isLetter ? PackedSpanHelpers.PackedIndexOfIsSupported - ? PackedSpanHelpers.IndexOfAny(ref Unsafe.Add(ref searchSpace, offset), valueCharU, valueCharL, searchSpaceMinusValueTailLength) + ? PackedSpanHelpers.IndexOfAnyIgnoreCase(ref Unsafe.Add(ref searchSpace, offset), valueCharL, searchSpaceMinusValueTailLength) : SpanHelpers.IndexOfAnyChar(ref Unsafe.Add(ref searchSpace, offset), valueCharU, valueCharL, searchSpaceMinusValueTailLength) : SpanHelpers.IndexOfChar(ref Unsafe.Add(ref searchSpace, offset), valueChar, searchSpaceMinusValueTailLength); if (relativeIndex < 0) diff --git a/src/libraries/System.Private.CoreLib/src/System/SearchValues/Any1CharPackedIgnoreCaseSearchValues.cs b/src/libraries/System.Private.CoreLib/src/System/SearchValues/Any1CharPackedIgnoreCaseSearchValues.cs new file mode 100644 index 00000000000000..dfe6b3631f709e --- /dev/null +++ b/src/libraries/System.Private.CoreLib/src/System/SearchValues/Any1CharPackedIgnoreCaseSearchValues.cs @@ -0,0 +1,52 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Diagnostics; +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; +using System.Runtime.Intrinsics.X86; + +namespace System.Buffers +{ + internal sealed class Any1CharPackedIgnoreCaseSearchValues : SearchValues + { + // While this most commonly applies to ASCII letters, it also works for other values that differ by 0x20 (e.g. "[{" => "{"). + // _lowerCase is therefore not necessarily a lower case ASCII letter, but just the higher value (the one with the 0x20 bit set). + private readonly char _lowerCase, _upperCase; + private readonly uint _lowerCaseUint; + + public Any1CharPackedIgnoreCaseSearchValues(char value) + { + Debug.Assert((value | 0x20) == value); + + _lowerCase = value; + _upperCase = (char)(value & ~0x20); + _lowerCaseUint = value; + } + + internal override char[] GetValues() => + [_upperCase, _lowerCase]; + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal override bool ContainsCore(char value) => + (uint)(value | 0x20) == _lowerCaseUint; + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + [CompExactlyDependsOn(typeof(Sse2))] + internal override int IndexOfAny(ReadOnlySpan span) => + PackedSpanHelpers.IndexOfAnyIgnoreCase(ref MemoryMarshal.GetReference(span), _lowerCase, span.Length); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + [CompExactlyDependsOn(typeof(Sse2))] + internal override int IndexOfAnyExcept(ReadOnlySpan span) => + PackedSpanHelpers.IndexOfAnyExceptIgnoreCase(ref MemoryMarshal.GetReference(span), _lowerCase, span.Length); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal override int LastIndexOfAny(ReadOnlySpan span) => + span.LastIndexOfAny(_lowerCase, _upperCase); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal override int LastIndexOfAnyExcept(ReadOnlySpan span) => + span.LastIndexOfAnyExcept(_lowerCase, _upperCase); + } +} diff --git a/src/libraries/System.Private.CoreLib/src/System/SearchValues/Any1CharPackedSearchValues.cs b/src/libraries/System.Private.CoreLib/src/System/SearchValues/Any1CharPackedSearchValues.cs new file mode 100644 index 00000000000000..e6aa0c870486d8 --- /dev/null +++ b/src/libraries/System.Private.CoreLib/src/System/SearchValues/Any1CharPackedSearchValues.cs @@ -0,0 +1,42 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; +using System.Runtime.Intrinsics.X86; + +namespace System.Buffers +{ + internal sealed class Any1CharPackedSearchValues : SearchValues + { + private readonly char _e0; + + public Any1CharPackedSearchValues(char value) => + _e0 = value; + + internal override char[] GetValues() => + [_e0]; + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal override bool ContainsCore(char value) => + value == _e0; + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + [CompExactlyDependsOn(typeof(Sse2))] + internal override int IndexOfAny(ReadOnlySpan span) => + PackedSpanHelpers.IndexOf(ref MemoryMarshal.GetReference(span), _e0, span.Length); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + [CompExactlyDependsOn(typeof(Sse2))] + internal override int IndexOfAnyExcept(ReadOnlySpan span) => + PackedSpanHelpers.IndexOfAnyExcept(ref MemoryMarshal.GetReference(span), _e0, span.Length); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal override int LastIndexOfAny(ReadOnlySpan span) => + span.LastIndexOf(_e0); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal override int LastIndexOfAnyExcept(ReadOnlySpan span) => + span.LastIndexOfAnyExcept(_e0); + } +} diff --git a/src/libraries/System.Private.CoreLib/src/System/SearchValues/Any1SearchValues.cs b/src/libraries/System.Private.CoreLib/src/System/SearchValues/Any1SearchValues.cs new file mode 100644 index 00000000000000..9a52511b987509 --- /dev/null +++ b/src/libraries/System.Private.CoreLib/src/System/SearchValues/Any1SearchValues.cs @@ -0,0 +1,52 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Diagnostics; +using System.Numerics; +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; + +#pragma warning disable 8500 // address of managed types + +namespace System.Buffers +{ + internal sealed class Any1SearchValues : SearchValues + where T : struct, IEquatable + where TImpl : struct, INumber + { + private readonly TImpl _e0; + + public Any1SearchValues(ReadOnlySpan values) + { + Debug.Assert(Unsafe.SizeOf() == Unsafe.SizeOf()); + Debug.Assert(values.Length == 1); + _e0 = values[0]; + } + + internal override unsafe T[] GetValues() + { + TImpl e0 = _e0; + return new[] { *(T*)&e0 }; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal override unsafe bool ContainsCore(T value) => + *(TImpl*)&value == _e0; + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal override int IndexOfAny(ReadOnlySpan span) => + SpanHelpers.NonPackedIndexOfValueType>(ref Unsafe.As(ref MemoryMarshal.GetReference(span)), _e0, span.Length); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal override int IndexOfAnyExcept(ReadOnlySpan span) => + SpanHelpers.NonPackedIndexOfValueType>(ref Unsafe.As(ref MemoryMarshal.GetReference(span)), _e0, span.Length); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal override int LastIndexOfAny(ReadOnlySpan span) => + SpanHelpers.LastIndexOfValueType(ref Unsafe.As(ref MemoryMarshal.GetReference(span)), _e0, span.Length); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal override int LastIndexOfAnyExcept(ReadOnlySpan span) => + SpanHelpers.LastIndexOfAnyExceptValueType(ref Unsafe.As(ref MemoryMarshal.GetReference(span)), _e0, span.Length); + } +} diff --git a/src/libraries/System.Private.CoreLib/src/System/SearchValues/Any2ByteSearchValues.cs b/src/libraries/System.Private.CoreLib/src/System/SearchValues/Any2ByteSearchValues.cs deleted file mode 100644 index 42f4acbfebf8b0..00000000000000 --- a/src/libraries/System.Private.CoreLib/src/System/SearchValues/Any2ByteSearchValues.cs +++ /dev/null @@ -1,41 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -using System.Diagnostics; -using System.Runtime.CompilerServices; - -namespace System.Buffers -{ - internal sealed class Any2ByteSearchValues : SearchValues - { - private readonly byte _e0, _e1; - - public Any2ByteSearchValues(ReadOnlySpan values) - { - Debug.Assert(values.Length == 2); - (_e0, _e1) = (values[0], values[1]); - } - - internal override byte[] GetValues() => new[] { _e0, _e1 }; - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal override bool ContainsCore(byte value) => - value == _e0 || value == _e1; - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal override int IndexOfAny(ReadOnlySpan span) => - span.IndexOfAny(_e0, _e1); - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal override int IndexOfAnyExcept(ReadOnlySpan span) => - span.IndexOfAnyExcept(_e0, _e1); - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal override int LastIndexOfAny(ReadOnlySpan span) => - span.LastIndexOfAny(_e0, _e1); - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal override int LastIndexOfAnyExcept(ReadOnlySpan span) => - span.LastIndexOfAnyExcept(_e0, _e1); - } -} diff --git a/src/libraries/System.Private.CoreLib/src/System/SearchValues/Any2CharPackedIgnoreCaseSearchValues.cs b/src/libraries/System.Private.CoreLib/src/System/SearchValues/Any2CharPackedIgnoreCaseSearchValues.cs new file mode 100644 index 00000000000000..1073fcf3c81858 --- /dev/null +++ b/src/libraries/System.Private.CoreLib/src/System/SearchValues/Any2CharPackedIgnoreCaseSearchValues.cs @@ -0,0 +1,67 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Diagnostics; +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; +using System.Runtime.Intrinsics.Arm; +using System.Runtime.Intrinsics.Wasm; +using System.Runtime.Intrinsics.X86; + +namespace System.Buffers +{ + internal sealed class Any2CharPackedIgnoreCaseSearchValues : SearchValues + { + // While this most commonly applies to ASCII letters, it also works for other values that differ by 0x20 (e.g. "[]{}" => "{}"). + // _e0 and _e1 are therefore not necessarily lower case ASCII letters, but just the higher values (the ones with the 0x20 bit set). + private readonly char _e0, _e1; + private readonly uint _uint0, _uint1; + private IndexOfAnyAsciiSearcher.AsciiState _state; + + public Any2CharPackedIgnoreCaseSearchValues(char value0, char value1) + { + Debug.Assert((value0 | 0x20) == value0 && char.IsAscii(value0)); + Debug.Assert((value1 | 0x20) == value1 && char.IsAscii(value1)); + + (_e0, _e1) = (value0, value1); + (_uint0, _uint1) = (value0, value1); + IndexOfAnyAsciiSearcher.ComputeAsciiState([(char)(_e0 & ~0x20), _e0, (char)(_e1 & ~0x20), _e1], out _state); + } + + internal override char[] GetValues() => + [(char)(_e0 & ~0x20), _e0, (char)(_e1 & ~0x20), _e1]; + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal override bool ContainsCore(char value) + { + uint lowerCase = (uint)(value | 0x20); + return lowerCase == _uint0 || lowerCase == _uint1; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + [CompExactlyDependsOn(typeof(Sse2))] + internal override int IndexOfAny(ReadOnlySpan span) => + PackedSpanHelpers.IndexOfAnyIgnoreCase(ref MemoryMarshal.GetReference(span), _e0, _e1, span.Length); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + [CompExactlyDependsOn(typeof(Sse2))] + internal override int IndexOfAnyExcept(ReadOnlySpan span) => + PackedSpanHelpers.IndexOfAnyExceptIgnoreCase(ref MemoryMarshal.GetReference(span), _e0, _e1, span.Length); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + [CompExactlyDependsOn(typeof(Ssse3))] + [CompExactlyDependsOn(typeof(AdvSimd))] + [CompExactlyDependsOn(typeof(PackedSimd))] + internal override int LastIndexOfAny(ReadOnlySpan span) => + IndexOfAnyAsciiSearcher.LastIndexOfAny( + ref Unsafe.As(ref MemoryMarshal.GetReference(span)), span.Length, ref _state); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + [CompExactlyDependsOn(typeof(Ssse3))] + [CompExactlyDependsOn(typeof(AdvSimd))] + [CompExactlyDependsOn(typeof(PackedSimd))] + internal override int LastIndexOfAnyExcept(ReadOnlySpan span) => + IndexOfAnyAsciiSearcher.LastIndexOfAny( + ref Unsafe.As(ref MemoryMarshal.GetReference(span)), span.Length, ref _state); + } +} diff --git a/src/libraries/System.Private.CoreLib/src/System/SearchValues/Any2CharPackedSearchValues.cs b/src/libraries/System.Private.CoreLib/src/System/SearchValues/Any2CharPackedSearchValues.cs new file mode 100644 index 00000000000000..d951b8e2375e0f --- /dev/null +++ b/src/libraries/System.Private.CoreLib/src/System/SearchValues/Any2CharPackedSearchValues.cs @@ -0,0 +1,42 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; +using System.Runtime.Intrinsics.X86; + +namespace System.Buffers +{ + internal sealed class Any2CharPackedSearchValues : SearchValues + { + private readonly char _e0, _e1; + + public Any2CharPackedSearchValues(char value0, char value1) => + (_e0, _e1) = (value0, value1); + + internal override char[] GetValues() => + [_e0, _e1]; + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal override bool ContainsCore(char value) => + value == _e0 || value == _e1; + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + [CompExactlyDependsOn(typeof(Sse2))] + internal override int IndexOfAny(ReadOnlySpan span) => + PackedSpanHelpers.IndexOfAny(ref MemoryMarshal.GetReference(span), _e0, _e1, span.Length); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + [CompExactlyDependsOn(typeof(Sse2))] + internal override int IndexOfAnyExcept(ReadOnlySpan span) => + PackedSpanHelpers.IndexOfAnyExcept(ref MemoryMarshal.GetReference(span), _e0, _e1, span.Length); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal override int LastIndexOfAny(ReadOnlySpan span) => + span.LastIndexOfAny(_e0, _e1); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal override int LastIndexOfAnyExcept(ReadOnlySpan span) => + span.LastIndexOfAnyExcept(_e0, _e1); + } +} diff --git a/src/libraries/System.Private.CoreLib/src/System/SearchValues/Any2CharSearchValues.cs b/src/libraries/System.Private.CoreLib/src/System/SearchValues/Any2CharSearchValues.cs deleted file mode 100644 index a2bfde7f6fab94..00000000000000 --- a/src/libraries/System.Private.CoreLib/src/System/SearchValues/Any2CharSearchValues.cs +++ /dev/null @@ -1,51 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -using System.Runtime.CompilerServices; -using System.Runtime.InteropServices; - -namespace System.Buffers -{ - internal sealed class Any2CharSearchValues : SearchValues - where TShouldUsePacked : struct, SearchValues.IRuntimeConst - { - private char _e0, _e1; - - public Any2CharSearchValues(char value0, char value1) => - (_e0, _e1) = (value0, value1); - - internal override char[] GetValues() => new[] { _e0, _e1 }; - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal override bool ContainsCore(char value) => - value == _e0 || value == _e1; - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal override int IndexOfAny(ReadOnlySpan span) => - (PackedSpanHelpers.PackedIndexOfIsSupported && TShouldUsePacked.Value) - ? PackedSpanHelpers.IndexOfAny(ref MemoryMarshal.GetReference(span), _e0, _e1, span.Length) - : SpanHelpers.NonPackedIndexOfAnyValueType>( - ref Unsafe.As(ref MemoryMarshal.GetReference(span)), - Unsafe.As(ref _e0), - Unsafe.As(ref _e1), - span.Length); - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal override int IndexOfAnyExcept(ReadOnlySpan span) => - (PackedSpanHelpers.PackedIndexOfIsSupported && TShouldUsePacked.Value) - ? PackedSpanHelpers.IndexOfAnyExcept(ref MemoryMarshal.GetReference(span), _e0, _e1, span.Length) - : SpanHelpers.NonPackedIndexOfAnyValueType>( - ref Unsafe.As(ref MemoryMarshal.GetReference(span)), - Unsafe.As(ref _e0), - Unsafe.As(ref _e1), - span.Length); - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal override int LastIndexOfAny(ReadOnlySpan span) => - span.LastIndexOfAny(_e0, _e1); - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal override int LastIndexOfAnyExcept(ReadOnlySpan span) => - span.LastIndexOfAnyExcept(_e0, _e1); - } -} diff --git a/src/libraries/System.Private.CoreLib/src/System/SearchValues/Any2SearchValues.cs b/src/libraries/System.Private.CoreLib/src/System/SearchValues/Any2SearchValues.cs new file mode 100644 index 00000000000000..640a15f05c8f2b --- /dev/null +++ b/src/libraries/System.Private.CoreLib/src/System/SearchValues/Any2SearchValues.cs @@ -0,0 +1,52 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Diagnostics; +using System.Numerics; +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; + +#pragma warning disable 8500 // address of managed types + +namespace System.Buffers +{ + internal sealed class Any2SearchValues : SearchValues + where T : struct, IEquatable + where TImpl : struct, INumber + { + private readonly TImpl _e0, _e1; + + public Any2SearchValues(ReadOnlySpan values) + { + Debug.Assert(Unsafe.SizeOf() == Unsafe.SizeOf()); + Debug.Assert(values.Length == 2); + (_e0, _e1) = (values[0], values[1]); + } + + internal override unsafe T[] GetValues() + { + TImpl e0 = _e0, e1 = _e1; + return new[] { *(T*)&e0, *(T*)&e1 }; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal override unsafe bool ContainsCore(T value) => + *(TImpl*)&value == _e0 || *(TImpl*)&value == _e1; + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal override int IndexOfAny(ReadOnlySpan span) => + SpanHelpers.NonPackedIndexOfAnyValueType>(ref Unsafe.As(ref MemoryMarshal.GetReference(span)), _e0, _e1, span.Length); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal override int IndexOfAnyExcept(ReadOnlySpan span) => + SpanHelpers.NonPackedIndexOfAnyValueType>(ref Unsafe.As(ref MemoryMarshal.GetReference(span)), _e0, _e1, span.Length); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal override int LastIndexOfAny(ReadOnlySpan span) => + SpanHelpers.LastIndexOfAnyValueType(ref Unsafe.As(ref MemoryMarshal.GetReference(span)), _e0, _e1, span.Length); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal override int LastIndexOfAnyExcept(ReadOnlySpan span) => + SpanHelpers.LastIndexOfAnyExceptValueType(ref Unsafe.As(ref MemoryMarshal.GetReference(span)), _e0, _e1, span.Length); + } +} diff --git a/src/libraries/System.Private.CoreLib/src/System/SearchValues/Any3ByteSearchValues.cs b/src/libraries/System.Private.CoreLib/src/System/SearchValues/Any3ByteSearchValues.cs deleted file mode 100644 index d7208dea435067..00000000000000 --- a/src/libraries/System.Private.CoreLib/src/System/SearchValues/Any3ByteSearchValues.cs +++ /dev/null @@ -1,41 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -using System.Diagnostics; -using System.Runtime.CompilerServices; - -namespace System.Buffers -{ - internal sealed class Any3ByteSearchValues : SearchValues - { - private readonly byte _e0, _e1, _e2; - - public Any3ByteSearchValues(ReadOnlySpan values) - { - Debug.Assert(values.Length == 3); - (_e0, _e1, _e2) = (values[0], values[1], values[2]); - } - - internal override byte[] GetValues() => new[] { _e0, _e1, _e2 }; - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal override bool ContainsCore(byte value) => - value == _e0 || value == _e1 || value == _e2; - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal override int IndexOfAny(ReadOnlySpan span) => - span.IndexOfAny(_e0, _e1, _e2); - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal override int IndexOfAnyExcept(ReadOnlySpan span) => - span.IndexOfAnyExcept(_e0, _e1, _e2); - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal override int LastIndexOfAny(ReadOnlySpan span) => - span.LastIndexOfAny(_e0, _e1, _e2); - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal override int LastIndexOfAnyExcept(ReadOnlySpan span) => - span.LastIndexOfAnyExcept(_e0, _e1, _e2); - } -} diff --git a/src/libraries/System.Private.CoreLib/src/System/SearchValues/Any3CharPackedSearchValues.cs b/src/libraries/System.Private.CoreLib/src/System/SearchValues/Any3CharPackedSearchValues.cs new file mode 100644 index 00000000000000..cebb695aa3fb79 --- /dev/null +++ b/src/libraries/System.Private.CoreLib/src/System/SearchValues/Any3CharPackedSearchValues.cs @@ -0,0 +1,42 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; +using System.Runtime.Intrinsics.X86; + +namespace System.Buffers +{ + internal sealed class Any3CharPackedSearchValues : SearchValues + { + private readonly char _e0, _e1, _e2; + + public Any3CharPackedSearchValues(char value0, char value1, char value2) => + (_e0, _e1, _e2) = (value0, value1, value2); + + internal override char[] GetValues() => + [_e0, _e1, _e2]; + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal override bool ContainsCore(char value) => + value == _e0 || value == _e1 || value == _e2; + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + [CompExactlyDependsOn(typeof(Sse2))] + internal override int IndexOfAny(ReadOnlySpan span) => + PackedSpanHelpers.IndexOfAny(ref MemoryMarshal.GetReference(span), _e0, _e1, _e2, span.Length); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + [CompExactlyDependsOn(typeof(Sse2))] + internal override int IndexOfAnyExcept(ReadOnlySpan span) => + PackedSpanHelpers.IndexOfAnyExcept(ref MemoryMarshal.GetReference(span), _e0, _e1, _e2, span.Length); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal override int LastIndexOfAny(ReadOnlySpan span) => + span.LastIndexOfAny(_e0, _e1, _e2); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal override int LastIndexOfAnyExcept(ReadOnlySpan span) => + span.LastIndexOfAnyExcept(_e0, _e1, _e2); + } +} diff --git a/src/libraries/System.Private.CoreLib/src/System/SearchValues/Any3CharSearchValues.cs b/src/libraries/System.Private.CoreLib/src/System/SearchValues/Any3CharSearchValues.cs deleted file mode 100644 index c69dbff8d0a4c7..00000000000000 --- a/src/libraries/System.Private.CoreLib/src/System/SearchValues/Any3CharSearchValues.cs +++ /dev/null @@ -1,53 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -using System.Runtime.CompilerServices; -using System.Runtime.InteropServices; - -namespace System.Buffers -{ - internal sealed class Any3CharSearchValues : SearchValues - where TShouldUsePacked : struct, SearchValues.IRuntimeConst - { - private char _e0, _e1, _e2; - - public Any3CharSearchValues(char value0, char value1, char value2) => - (_e0, _e1, _e2) = (value0, value1, value2); - - internal override char[] GetValues() => new[] { _e0, _e1, _e2 }; - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal override bool ContainsCore(char value) => - value == _e0 || value == _e1 || value == _e2; - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal override int IndexOfAny(ReadOnlySpan span) => - (PackedSpanHelpers.PackedIndexOfIsSupported && TShouldUsePacked.Value) - ? PackedSpanHelpers.IndexOfAny(ref MemoryMarshal.GetReference(span), _e0, _e1, _e2, span.Length) - : SpanHelpers.NonPackedIndexOfAnyValueType>( - ref Unsafe.As(ref MemoryMarshal.GetReference(span)), - Unsafe.As(ref _e0), - Unsafe.As(ref _e1), - Unsafe.As(ref _e2), - span.Length); - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal override int IndexOfAnyExcept(ReadOnlySpan span) => - (PackedSpanHelpers.PackedIndexOfIsSupported && TShouldUsePacked.Value) - ? PackedSpanHelpers.IndexOfAnyExcept(ref MemoryMarshal.GetReference(span), _e0, _e1, _e2, span.Length) - : SpanHelpers.NonPackedIndexOfAnyValueType>( - ref Unsafe.As(ref MemoryMarshal.GetReference(span)), - Unsafe.As(ref _e0), - Unsafe.As(ref _e1), - Unsafe.As(ref _e2), - span.Length); - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal override int LastIndexOfAny(ReadOnlySpan span) => - span.LastIndexOfAny(_e0, _e1, _e2); - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal override int LastIndexOfAnyExcept(ReadOnlySpan span) => - span.LastIndexOfAnyExcept(_e0, _e1, _e2); - } -} diff --git a/src/libraries/System.Private.CoreLib/src/System/SearchValues/Any3SearchValues.cs b/src/libraries/System.Private.CoreLib/src/System/SearchValues/Any3SearchValues.cs new file mode 100644 index 00000000000000..f166aef42703cd --- /dev/null +++ b/src/libraries/System.Private.CoreLib/src/System/SearchValues/Any3SearchValues.cs @@ -0,0 +1,52 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Diagnostics; +using System.Numerics; +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; + +#pragma warning disable 8500 // address of managed types + +namespace System.Buffers +{ + internal sealed class Any3SearchValues : SearchValues + where T : struct, IEquatable + where TImpl : struct, INumber + { + private readonly TImpl _e0, _e1, _e2; + + public Any3SearchValues(ReadOnlySpan values) + { + Debug.Assert(Unsafe.SizeOf() == Unsafe.SizeOf()); + Debug.Assert(values.Length == 3); + (_e0, _e1, _e2) = (values[0], values[1], values[2]); + } + + internal override unsafe T[] GetValues() + { + TImpl e0 = _e0, e1 = _e1, e2 = _e2; + return new[] { *(T*)&e0, *(T*)&e1, *(T*)&e2 }; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal override unsafe bool ContainsCore(T value) => + *(TImpl*)&value == _e0 || *(TImpl*)&value == _e1 || *(TImpl*)&value == _e2; + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal override int IndexOfAny(ReadOnlySpan span) => + SpanHelpers.NonPackedIndexOfAnyValueType>(ref Unsafe.As(ref MemoryMarshal.GetReference(span)), _e0, _e1, _e2, span.Length); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal override int IndexOfAnyExcept(ReadOnlySpan span) => + SpanHelpers.NonPackedIndexOfAnyValueType>(ref Unsafe.As(ref MemoryMarshal.GetReference(span)), _e0, _e1, _e2, span.Length); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal override int LastIndexOfAny(ReadOnlySpan span) => + SpanHelpers.LastIndexOfAnyValueType(ref Unsafe.As(ref MemoryMarshal.GetReference(span)), _e0, _e1, _e2, span.Length); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal override int LastIndexOfAnyExcept(ReadOnlySpan span) => + SpanHelpers.LastIndexOfAnyExceptValueType(ref Unsafe.As(ref MemoryMarshal.GetReference(span)), _e0, _e1, _e2, span.Length); + } +} diff --git a/src/libraries/System.Private.CoreLib/src/System/SearchValues/SearchValues.cs b/src/libraries/System.Private.CoreLib/src/System/SearchValues/SearchValues.cs index 13c6779a7d9851..7428c912c6cfda 100644 --- a/src/libraries/System.Private.CoreLib/src/System/SearchValues/SearchValues.cs +++ b/src/libraries/System.Private.CoreLib/src/System/SearchValues/SearchValues.cs @@ -34,7 +34,7 @@ public static SearchValues Create(ReadOnlySpan values) if (values.Length == 1) { - return new SingleByteSearchValues(values); + return new Any1SearchValues(values); } // RangeByteSearchValues is slower than SingleByteSearchValues, but faster than Any2ByteSearchValues @@ -48,8 +48,8 @@ public static SearchValues Create(ReadOnlySpan values) Debug.Assert(values.Length is 2 or 3 or 4 or 5); return values.Length switch { - 2 => new Any2ByteSearchValues(values), - 3 => new Any3ByteSearchValues(values), + 2 => new Any2SearchValues(values), + 3 => new Any3SearchValues(values), 4 => new Any4SearchValues(values), _ => new Any5SearchValues(values), }; @@ -75,12 +75,18 @@ public static SearchValues Create(ReadOnlySpan values) return new EmptySearchValues(); } + // Vector128 isn't valid. Treat the values as shorts instead. + ReadOnlySpan shortValues = MemoryMarshal.CreateReadOnlySpan( + ref Unsafe.As(ref MemoryMarshal.GetReference(values)), + values.Length); + if (values.Length == 1) { char value = values[0]; + return PackedSpanHelpers.PackedIndexOfIsSupported && PackedSpanHelpers.CanUsePackedIndexOf(value) - ? new SingleCharSearchValues(value) - : new SingleCharSearchValues(value); + ? new Any1CharPackedSearchValues(value) + : new Any1SearchValues(shortValues); } // RangeCharSearchValues is slower than SingleCharSearchValues, but faster than Any2CharSearchValues @@ -95,9 +101,18 @@ public static SearchValues Create(ReadOnlySpan values) { char value0 = values[0]; char value1 = values[1]; - return PackedSpanHelpers.PackedIndexOfIsSupported && PackedSpanHelpers.CanUsePackedIndexOf(value0) && PackedSpanHelpers.CanUsePackedIndexOf(value1) - ? new Any2CharSearchValues(value0, value1) - : new Any2CharSearchValues(value0, value1); + + if (PackedSpanHelpers.PackedIndexOfIsSupported && PackedSpanHelpers.CanUsePackedIndexOf(value0) && PackedSpanHelpers.CanUsePackedIndexOf(value1)) + { + // If the two values are the same ASCII letter with both cases, we can use an approach that + // reduces the number of comparisons by masking off the bit that differs between lower and upper case (0x20). + // While this most commonly applies to ASCII letters, it also works for other values that differ by 0x20 (e.g. "[{" => "{"). + return (value0 ^ value1) == 0x20 + ? new Any1CharPackedIgnoreCaseSearchValues((char)Math.Max(value0, value1)) + : new Any2CharPackedSearchValues(value0, value1); + } + + return new Any2SearchValues(shortValues); } if (values.Length == 3) @@ -105,24 +120,37 @@ public static SearchValues Create(ReadOnlySpan values) char value0 = values[0]; char value1 = values[1]; char value2 = values[2]; + return PackedSpanHelpers.PackedIndexOfIsSupported && PackedSpanHelpers.CanUsePackedIndexOf(value0) && PackedSpanHelpers.CanUsePackedIndexOf(value1) && PackedSpanHelpers.CanUsePackedIndexOf(value2) - ? new Any3CharSearchValues(value0, value1, value2) - : new Any3CharSearchValues(value0, value1, value2); + ? new Any3CharPackedSearchValues(value0, value1, value2) + : new Any3SearchValues(shortValues); } // IndexOfAnyAsciiSearcher for chars is slower than Any3CharSearchValues, but faster than Any4SearchValues if (IndexOfAnyAsciiSearcher.IsVectorizationSupported && maxInclusive < 128) { + // If the values are sets of 2 ASCII letters with both cases, we can use an approach that + // reduces the number of comparisons by masking off the bit that differs between lower and upper case (0x20). + // While this most commonly applies to ASCII letters, it also works for other values that differ by 0x20 (e.g. "[]{}" => "{}"). + if (PackedSpanHelpers.PackedIndexOfIsSupported && values.Length == 4 && minInclusive > 0) + { + Span copy = stackalloc char[4]; + values.CopyTo(copy); + copy.Sort(); + + if ((copy[0] ^ copy[2]) == 0x20 && + (copy[1] ^ copy[3]) == 0x20) + { + // We pick the higher two values (with the 0x20 bit set). "AaBb" => 'a', 'b' + return new Any2CharPackedIgnoreCaseSearchValues(copy[2], copy[3]); + } + } + return (Ssse3.IsSupported || PackedSimd.IsSupported) && minInclusive == 0 ? new AsciiCharSearchValues(values) : new AsciiCharSearchValues(values); } - // Vector128 isn't valid. Treat the values as shorts instead. - ReadOnlySpan shortValues = MemoryMarshal.CreateReadOnlySpan( - ref Unsafe.As(ref MemoryMarshal.GetReference(values)), - values.Length); - if (values.Length == 4) { return new Any4SearchValues(shortValues); diff --git a/src/libraries/System.Private.CoreLib/src/System/SearchValues/SingleByteSearchValues.cs b/src/libraries/System.Private.CoreLib/src/System/SearchValues/SingleByteSearchValues.cs deleted file mode 100644 index b768c2541562f7..00000000000000 --- a/src/libraries/System.Private.CoreLib/src/System/SearchValues/SingleByteSearchValues.cs +++ /dev/null @@ -1,41 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -using System.Diagnostics; -using System.Runtime.CompilerServices; - -namespace System.Buffers -{ - internal sealed class SingleByteSearchValues : SearchValues - { - private readonly byte _e0; - - public SingleByteSearchValues(ReadOnlySpan values) - { - Debug.Assert(values.Length == 1); - _e0 = values[0]; - } - - internal override byte[] GetValues() => new[] { _e0 }; - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal override bool ContainsCore(byte value) => - value == _e0; - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal override int IndexOfAny(ReadOnlySpan span) => - span.IndexOf(_e0); - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal override int IndexOfAnyExcept(ReadOnlySpan span) => - span.IndexOfAnyExcept(_e0); - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal override int LastIndexOfAny(ReadOnlySpan span) => - span.LastIndexOf(_e0); - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal override int LastIndexOfAnyExcept(ReadOnlySpan span) => - span.LastIndexOfAnyExcept(_e0); - } -} diff --git a/src/libraries/System.Private.CoreLib/src/System/SearchValues/SingleCharSearchValues.cs b/src/libraries/System.Private.CoreLib/src/System/SearchValues/SingleCharSearchValues.cs deleted file mode 100644 index b1348e3859b073..00000000000000 --- a/src/libraries/System.Private.CoreLib/src/System/SearchValues/SingleCharSearchValues.cs +++ /dev/null @@ -1,49 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -using System.Runtime.CompilerServices; -using System.Runtime.InteropServices; - -namespace System.Buffers -{ - internal sealed class SingleCharSearchValues : SearchValues - where TShouldUsePacked : struct, SearchValues.IRuntimeConst - { - private char _e0; - - public SingleCharSearchValues(char value) => - _e0 = value; - - internal override char[] GetValues() => new[] { _e0 }; - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal override bool ContainsCore(char value) => - value == _e0; - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal override int IndexOfAny(ReadOnlySpan span) => - (PackedSpanHelpers.PackedIndexOfIsSupported && TShouldUsePacked.Value) - ? PackedSpanHelpers.IndexOf(ref MemoryMarshal.GetReference(span), _e0, span.Length) - : SpanHelpers.NonPackedIndexOfValueType>( - ref Unsafe.As(ref MemoryMarshal.GetReference(span)), - Unsafe.As(ref _e0), - span.Length); - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal override int IndexOfAnyExcept(ReadOnlySpan span) => - (PackedSpanHelpers.PackedIndexOfIsSupported && TShouldUsePacked.Value) - ? PackedSpanHelpers.IndexOfAnyExcept(ref MemoryMarshal.GetReference(span), _e0, span.Length) - : SpanHelpers.NonPackedIndexOfValueType>( - ref Unsafe.As(ref MemoryMarshal.GetReference(span)), - Unsafe.As(ref _e0), - span.Length); - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal override int LastIndexOfAny(ReadOnlySpan span) => - span.LastIndexOf(_e0); - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal override int LastIndexOfAnyExcept(ReadOnlySpan span) => - span.LastIndexOfAnyExcept(_e0); - } -} diff --git a/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.Packed.cs b/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.Packed.cs index fca176fe438126..69ce8c1f7fada9 100644 --- a/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.Packed.cs +++ b/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.Packed.cs @@ -1,10 +1,8 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. -using System.Buffers.Binary; using System.Diagnostics; using System.Numerics; -using System.Runtime; using System.Runtime.CompilerServices; using System.Runtime.Intrinsics; using System.Runtime.Intrinsics.X86; @@ -37,22 +35,22 @@ public static unsafe bool CanUsePackedIndexOf(T value) [MethodImpl(MethodImplOptions.AggressiveInlining)] [CompExactlyDependsOn(typeof(Sse2))] public static int IndexOf(ref char searchSpace, char value, int length) => - IndexOf>(ref Unsafe.As(ref searchSpace), (short)value, length); + IndexOf, NopTransform>(ref Unsafe.As(ref searchSpace), (short)value, length); [MethodImpl(MethodImplOptions.AggressiveInlining)] [CompExactlyDependsOn(typeof(Sse2))] public static int IndexOfAnyExcept(ref char searchSpace, char value, int length) => - IndexOf>(ref Unsafe.As(ref searchSpace), (short)value, length); + IndexOf, NopTransform>(ref Unsafe.As(ref searchSpace), (short)value, length); [MethodImpl(MethodImplOptions.AggressiveInlining)] [CompExactlyDependsOn(typeof(Sse2))] public static int IndexOfAny(ref char searchSpace, char value0, char value1, int length) => - IndexOfAny>(ref Unsafe.As(ref searchSpace), (short)value0, (short)value1, length); + IndexOfAny, NopTransform>(ref Unsafe.As(ref searchSpace), (short)value0, (short)value1, length); [MethodImpl(MethodImplOptions.AggressiveInlining)] [CompExactlyDependsOn(typeof(Sse2))] public static int IndexOfAnyExcept(ref char searchSpace, char value0, char value1, int length) => - IndexOfAny>(ref Unsafe.As(ref searchSpace), (short)value0, (short)value1, length); + IndexOfAny, NopTransform>(ref Unsafe.As(ref searchSpace), (short)value0, (short)value1, length); [MethodImpl(MethodImplOptions.AggressiveInlining)] [CompExactlyDependsOn(typeof(Sse2))] @@ -64,6 +62,44 @@ public static int IndexOfAny(ref char searchSpace, char value0, char value1, cha public static int IndexOfAnyExcept(ref char searchSpace, char value0, char value1, char value2, int length) => IndexOfAny>(ref Unsafe.As(ref searchSpace), (short)value0, (short)value1, (short)value2, length); + [MethodImpl(MethodImplOptions.AggressiveInlining)] + [CompExactlyDependsOn(typeof(Sse2))] + public static int IndexOfAnyIgnoreCase(ref char searchSpace, char value, int length) + { + Debug.Assert((value | 0x20) == value); + + return IndexOf, Or20Transform>(ref Unsafe.As(ref searchSpace), (short)value, length); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + [CompExactlyDependsOn(typeof(Sse2))] + public static int IndexOfAnyExceptIgnoreCase(ref char searchSpace, char value, int length) + { + Debug.Assert((value | 0x20) == value); + + return IndexOf, Or20Transform>(ref Unsafe.As(ref searchSpace), (short)value, length); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + [CompExactlyDependsOn(typeof(Sse2))] + public static int IndexOfAnyIgnoreCase(ref char searchSpace, char value0, char value1, int length) + { + Debug.Assert((value0 | 0x20) == value0); + Debug.Assert((value1 | 0x20) == value1); + + return IndexOfAny, Or20Transform>(ref Unsafe.As(ref searchSpace), (short)value0, (short)value1, length); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + [CompExactlyDependsOn(typeof(Sse2))] + public static int IndexOfAnyExceptIgnoreCase(ref char searchSpace, char value0, char value1, int length) + { + Debug.Assert((value0 | 0x20) == value0); + Debug.Assert((value1 | 0x20) == value1); + + return IndexOfAny, Or20Transform>(ref Unsafe.As(ref searchSpace), (short)value0, (short)value1, length); + } + [MethodImpl(MethodImplOptions.AggressiveInlining)] [CompExactlyDependsOn(typeof(Sse2))] public static int IndexOfAnyInRange(ref char searchSpace, char lowInclusive, char rangeInclusive, int length) => @@ -277,8 +313,9 @@ public static bool Contains(ref short searchSpace, short value, int length) } [CompExactlyDependsOn(typeof(Sse2))] - private static int IndexOf(ref short searchSpace, short value, int length) + private static int IndexOf(ref short searchSpace, short value, int length) where TNegator : struct, SpanHelpers.INegator + where TTransform : struct, ITransform { Debug.Assert(CanUsePackedIndexOf(value)); @@ -290,10 +327,10 @@ private static int IndexOf(ref short searchSpace, short value, int len { length -= 4; - if (TNegator.NegateIfNeeded(searchSpace == value)) return 0; - if (TNegator.NegateIfNeeded(Unsafe.Add(ref searchSpace, 1) == value)) return 1; - if (TNegator.NegateIfNeeded(Unsafe.Add(ref searchSpace, 2) == value)) return 2; - if (TNegator.NegateIfNeeded(Unsafe.Add(ref searchSpace, 3) == value)) return 3; + if (TNegator.NegateIfNeeded(TTransform.TransformInput(searchSpace) == value)) return 0; + if (TNegator.NegateIfNeeded(TTransform.TransformInput(Unsafe.Add(ref searchSpace, 1)) == value)) return 1; + if (TNegator.NegateIfNeeded(TTransform.TransformInput(Unsafe.Add(ref searchSpace, 2)) == value)) return 2; + if (TNegator.NegateIfNeeded(TTransform.TransformInput(Unsafe.Add(ref searchSpace, 3)) == value)) return 3; offset = 4; } @@ -302,7 +339,7 @@ private static int IndexOf(ref short searchSpace, short value, int len { length -= 1; - if (TNegator.NegateIfNeeded(Unsafe.Add(ref searchSpace, offset) == value)) return (int)offset; + if (TNegator.NegateIfNeeded(TTransform.TransformInput(Unsafe.Add(ref searchSpace, offset)) == value)) return (int)offset; offset += 1; } @@ -329,7 +366,7 @@ private static int IndexOf(ref short searchSpace, short value, int len { Vector512 source0 = Vector512.LoadUnsafe(ref currentSearchSpace); Vector512 source1 = Vector512.LoadUnsafe(ref currentSearchSpace, (nuint)Vector512.Count); - Vector512 packedSource = PackSources(source0, source1); + Vector512 packedSource = TTransform.TransformInput(PackSources(source0, source1)); if (HasMatch(packedValue, packedSource)) { @@ -352,7 +389,7 @@ private static int IndexOf(ref short searchSpace, short value, int len Vector512 source0 = Vector512.LoadUnsafe(ref firstVector); Vector512 source1 = Vector512.LoadUnsafe(ref oneVectorAwayFromEnd); - Vector512 packedSource = PackSources(source0, source1); + Vector512 packedSource = TTransform.TransformInput(PackSources(source0, source1)); if (HasMatch(packedValue, packedSource)) { @@ -378,7 +415,7 @@ private static int IndexOf(ref short searchSpace, short value, int len { Vector256 source0 = Vector256.LoadUnsafe(ref currentSearchSpace); Vector256 source1 = Vector256.LoadUnsafe(ref currentSearchSpace, (nuint)Vector256.Count); - Vector256 packedSource = PackSources(source0, source1); + Vector256 packedSource = TTransform.TransformInput(PackSources(source0, source1)); Vector256 result = Vector256.Equals(packedValue, packedSource); result = NegateIfNeeded(result); @@ -403,7 +440,7 @@ private static int IndexOf(ref short searchSpace, short value, int len Vector256 source0 = Vector256.LoadUnsafe(ref firstVector); Vector256 source1 = Vector256.LoadUnsafe(ref oneVectorAwayFromEnd); - Vector256 packedSource = PackSources(source0, source1); + Vector256 packedSource = TTransform.TransformInput(PackSources(source0, source1)); Vector256 result = Vector256.Equals(packedValue, packedSource); result = NegateIfNeeded(result); @@ -438,7 +475,7 @@ private static int IndexOf(ref short searchSpace, short value, int len { Vector128 source0 = Vector128.LoadUnsafe(ref currentSearchSpace); Vector128 source1 = Vector128.LoadUnsafe(ref currentSearchSpace, (nuint)Vector128.Count); - Vector128 packedSource = PackSources(source0, source1); + Vector128 packedSource = TTransform.TransformInput(PackSources(source0, source1)); Vector128 result = Vector128.Equals(packedValue, packedSource); result = NegateIfNeeded(result); @@ -463,7 +500,7 @@ private static int IndexOf(ref short searchSpace, short value, int len Vector128 source0 = Vector128.LoadUnsafe(ref firstVector); Vector128 source1 = Vector128.LoadUnsafe(ref oneVectorAwayFromEnd); - Vector128 packedSource = PackSources(source0, source1); + Vector128 packedSource = TTransform.TransformInput(PackSources(source0, source1)); Vector128 result = Vector128.Equals(packedValue, packedSource); result = NegateIfNeeded(result); @@ -479,8 +516,9 @@ private static int IndexOf(ref short searchSpace, short value, int len } [CompExactlyDependsOn(typeof(Sse2))] - private static int IndexOfAny(ref short searchSpace, short value0, short value1, int length) + private static int IndexOfAny(ref short searchSpace, short value0, short value1, int length) where TNegator : struct, SpanHelpers.INegator + where TTransform : struct, ITransform { Debug.Assert(CanUsePackedIndexOf(value0)); Debug.Assert(CanUsePackedIndexOf(value1)); @@ -494,13 +532,13 @@ private static int IndexOfAny(ref short searchSpace, short value0, sho { length -= 4; - lookUp = searchSpace; + lookUp = TTransform.TransformInput(searchSpace); if (TNegator.NegateIfNeeded(lookUp == value0 || lookUp == value1)) return 0; - lookUp = Unsafe.Add(ref searchSpace, 1); + lookUp = TTransform.TransformInput(Unsafe.Add(ref searchSpace, 1)); if (TNegator.NegateIfNeeded(lookUp == value0 || lookUp == value1)) return 1; - lookUp = Unsafe.Add(ref searchSpace, 2); + lookUp = TTransform.TransformInput(Unsafe.Add(ref searchSpace, 2)); if (TNegator.NegateIfNeeded(lookUp == value0 || lookUp == value1)) return 2; - lookUp = Unsafe.Add(ref searchSpace, 3); + lookUp = TTransform.TransformInput(Unsafe.Add(ref searchSpace, 3)); if (TNegator.NegateIfNeeded(lookUp == value0 || lookUp == value1)) return 3; offset = 4; @@ -510,7 +548,7 @@ private static int IndexOfAny(ref short searchSpace, short value0, sho { length -= 1; - lookUp = Unsafe.Add(ref searchSpace, offset); + lookUp = TTransform.TransformInput(Unsafe.Add(ref searchSpace, offset)); if (TNegator.NegateIfNeeded(lookUp == value0 || lookUp == value1)) return (int)offset; offset += 1; @@ -538,7 +576,7 @@ private static int IndexOfAny(ref short searchSpace, short value0, sho { Vector512 source0 = Vector512.LoadUnsafe(ref currentSearchSpace); Vector512 source1 = Vector512.LoadUnsafe(ref currentSearchSpace, (nuint)Vector512.Count); - Vector512 packedSource = PackSources(source0, source1); + Vector512 packedSource = TTransform.TransformInput(PackSources(source0, source1)); Vector512 result = NegateIfNeeded(Vector512.Equals(packedValue0, packedSource) | Vector512.Equals(packedValue1, packedSource)); if (result != Vector512.Zero) @@ -562,7 +600,7 @@ private static int IndexOfAny(ref short searchSpace, short value0, sho Vector512 source0 = Vector512.LoadUnsafe(ref firstVector); Vector512 source1 = Vector512.LoadUnsafe(ref oneVectorAwayFromEnd); - Vector512 packedSource = PackSources(source0, source1); + Vector512 packedSource = TTransform.TransformInput(PackSources(source0, source1)); Vector512 result = NegateIfNeeded(Vector512.Equals(packedValue0, packedSource) | Vector512.Equals(packedValue1, packedSource)); if (result != Vector512.Zero) @@ -590,7 +628,7 @@ private static int IndexOfAny(ref short searchSpace, short value0, sho { Vector256 source0 = Vector256.LoadUnsafe(ref currentSearchSpace); Vector256 source1 = Vector256.LoadUnsafe(ref currentSearchSpace, (nuint)Vector256.Count); - Vector256 packedSource = PackSources(source0, source1); + Vector256 packedSource = TTransform.TransformInput(PackSources(source0, source1)); Vector256 result = Vector256.Equals(packedValue0, packedSource) | Vector256.Equals(packedValue1, packedSource); result = NegateIfNeeded(result); @@ -615,7 +653,7 @@ private static int IndexOfAny(ref short searchSpace, short value0, sho Vector256 source0 = Vector256.LoadUnsafe(ref firstVector); Vector256 source1 = Vector256.LoadUnsafe(ref oneVectorAwayFromEnd); - Vector256 packedSource = PackSources(source0, source1); + Vector256 packedSource = TTransform.TransformInput(PackSources(source0, source1)); Vector256 result = Vector256.Equals(packedValue0, packedSource) | Vector256.Equals(packedValue1, packedSource); result = NegateIfNeeded(result); @@ -651,7 +689,7 @@ private static int IndexOfAny(ref short searchSpace, short value0, sho { Vector128 source0 = Vector128.LoadUnsafe(ref currentSearchSpace); Vector128 source1 = Vector128.LoadUnsafe(ref currentSearchSpace, (nuint)Vector128.Count); - Vector128 packedSource = PackSources(source0, source1); + Vector128 packedSource = TTransform.TransformInput(PackSources(source0, source1)); Vector128 result = Vector128.Equals(packedValue0, packedSource) | Vector128.Equals(packedValue1, packedSource); result = NegateIfNeeded(result); @@ -676,7 +714,7 @@ private static int IndexOfAny(ref short searchSpace, short value0, sho Vector128 source0 = Vector128.LoadUnsafe(ref firstVector); Vector128 source1 = Vector128.LoadUnsafe(ref oneVectorAwayFromEnd); - Vector128 packedSource = PackSources(source0, source1); + Vector128 packedSource = TTransform.TransformInput(PackSources(source0, source1)); Vector128 result = Vector128.Equals(packedValue0, packedSource) | Vector128.Equals(packedValue1, packedSource); result = NegateIfNeeded(result); @@ -1283,5 +1321,29 @@ internal static Vector512 FixUpPackedVector512Result(Vector512 resul // We want to preserve the order of the two input vectors, so we deinterleave the packed value. return Avx512F.PermuteVar8x64(result.AsInt64(), Vector512.Create(0, 2, 4, 6, 1, 3, 5, 7)).AsByte(); } + + private interface ITransform + { + static abstract short TransformInput(short input); + static abstract Vector128 TransformInput(Vector128 input); + static abstract Vector256 TransformInput(Vector256 input); + static abstract Vector512 TransformInput(Vector512 input); + } + + private readonly struct NopTransform : ITransform + { + public static short TransformInput(short input) => input; + public static Vector128 TransformInput(Vector128 input) => input; + public static Vector256 TransformInput(Vector256 input) => input; + public static Vector512 TransformInput(Vector512 input) => input; + } + + private readonly struct Or20Transform : ITransform + { + public static short TransformInput(short input) => (short)(input | 0x20); + public static Vector128 TransformInput(Vector128 input) => input | Vector128.Create((byte)0x20); + public static Vector256 TransformInput(Vector256 input) => input | Vector256.Create((byte)0x20); + public static Vector512 TransformInput(Vector512 input) => input | Vector512.Create((byte)0x20); + } } } diff --git a/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.T.cs b/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.T.cs index 2cc6dc4405645d..da77d42320a98a 100644 --- a/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.T.cs +++ b/src/libraries/System.Private.CoreLib/src/System/SpanHelpers.T.cs @@ -1672,6 +1672,15 @@ private static unsafe int IndexOfAnyValueType(ref TValue searc { if (PackedSpanHelpers.PackedIndexOfIsSupported && typeof(TValue) == typeof(short) && PackedSpanHelpers.CanUsePackedIndexOf(value0) && PackedSpanHelpers.CanUsePackedIndexOf(value1)) { + if ((*(char*)&value0 ^ *(char*)&value1) == 0x20) + { + char lowerCase = (char)Math.Max(*(char*)&value0, *(char*)&value1); + + return typeof(TNegator) == typeof(DontNegate) + ? PackedSpanHelpers.IndexOfAnyIgnoreCase(ref Unsafe.As(ref searchSpace), lowerCase, length) + : PackedSpanHelpers.IndexOfAnyExceptIgnoreCase(ref Unsafe.As(ref searchSpace), lowerCase, length); + } + return typeof(TNegator) == typeof(DontNegate) ? PackedSpanHelpers.IndexOfAny(ref Unsafe.As(ref searchSpace), *(char*)&value0, *(char*)&value1, length) : PackedSpanHelpers.IndexOfAnyExcept(ref Unsafe.As(ref searchSpace), *(char*)&value0, *(char*)&value1, length); diff --git a/src/libraries/System.Private.CoreLib/src/System/String.Searching.cs b/src/libraries/System.Private.CoreLib/src/System/String.Searching.cs index 2affdb008575a3..56d471168632d9 100644 --- a/src/libraries/System.Private.CoreLib/src/System/String.Searching.cs +++ b/src/libraries/System.Private.CoreLib/src/System/String.Searching.cs @@ -84,10 +84,10 @@ private int IndexOfCharOrdinalIgnoreCase(char value) if (char.IsAsciiLetter(value)) { - char valueUc = (char)(value | 0x20); - char valueLc = (char)(value & ~0x20); + char valueLc = (char)(value | 0x20); + char valueUc = (char)(value & ~0x20); return PackedSpanHelpers.PackedIndexOfIsSupported - ? PackedSpanHelpers.IndexOfAny(ref _firstChar, valueLc, valueUc, Length) + ? PackedSpanHelpers.IndexOfAnyIgnoreCase(ref _firstChar, valueLc, Length) : SpanHelpers.IndexOfAnyChar(ref _firstChar, valueLc, valueUc, Length); } diff --git a/src/libraries/System.Text.RegularExpressions/src/System/Text/RegularExpressions/RegexCompiler.cs b/src/libraries/System.Text.RegularExpressions/src/System/Text/RegularExpressions/RegexCompiler.cs index 58abbb8b181eac..bac950c6db2f97 100644 --- a/src/libraries/System.Text.RegularExpressions/src/System/Text/RegularExpressions/RegexCompiler.cs +++ b/src/libraries/System.Text.RegularExpressions/src/System/Text/RegularExpressions/RegexCompiler.cs @@ -920,20 +920,10 @@ void EmitFixedSet_LeftToRight() Call(primarySet.Negated ? s_spanIndexOfAnyExceptCharCharChar : s_spanIndexOfAnyCharCharChar); break; - case 4 or 5: - // tmp = ...IndexOfAny("abcd"); - // Note that this case differs slightly from the source generator, where it might choose to use - // SearchValues instead of a literal, but there's extra cost to doing so for RegexCompiler so - // it just always uses IndexOfAny(span). - Ldstr(new string(primarySet.Chars)); - Call(s_stringAsSpanMethod); - Call(primarySet.Negated ? s_spanIndexOfAnyExceptSpan : s_spanIndexOfAnySpan); - break; - default: + // tmp = ...IndexOfAny(setChars); // tmp = ...IndexOfAny(s_searchValues); - LoadSearchValues(primarySet.Chars); - Call(primarySet.Negated ? s_spanIndexOfAnyExceptSearchValues : s_spanIndexOfAnySearchValues); + EmitIndexOfAnyWithSearchValuesOrLiteral(new string(primarySet.Chars), except: primarySet.Negated); break; } } @@ -3619,9 +3609,7 @@ literal.SetChars is not null || case (true, _): // startingPos = slice.IndexOfAny(literal.SetChars); - Ldstr(literal.SetChars); - Call(s_stringAsSpanMethod); - Call(s_spanIndexOfAnySpan); + EmitIndexOfAnyWithSearchValuesOrLiteral(literal.SetChars); break; case (false, 2): @@ -3634,9 +3622,7 @@ literal.SetChars is not null || case (false, _): // startingPos = slice.IndexOfAny($"{node.Ch}{literal.SetChars}"); - Ldstr($"{node.Ch}{literal.SetChars}"); - Call(s_stringAsSpanMethod); - Call(s_spanIndexOfAnySpan); + EmitIndexOfAnyWithSearchValuesOrLiteral($"{node.Ch}{literal.SetChars}"); break; } } @@ -3650,8 +3636,7 @@ literal.SetChars is not null || Array.Resize(ref asciiChars, asciiChars.Length + 1); asciiChars[^1] = node.Ch; } - LoadSearchValues(asciiChars); - Call(s_spanIndexOfAnySearchValues); + EmitIndexOfAnyWithSearchValuesOrLiteral(new string(asciiChars)); } else if (literal.Range.LowInclusive == literal.Range.HighInclusive) // single char from a RegexNode.One { @@ -5274,15 +5259,7 @@ void EmitIndexOf(RegexNode node, bool useLast, bool negate) return; default: - Ldstr(setChars.ToString()); - Call(s_stringAsSpanMethod); - Call((useLast, negated) switch - { - (false, false) => s_spanIndexOfAnySpan, - (false, true) => s_spanIndexOfAnyExceptSpan, - (true, false) => s_spanLastIndexOfAnySpan, - (true, true) => s_spanLastIndexOfAnyExceptSpan, - }); + EmitIndexOfAnyWithSearchValuesOrLiteral(setChars.ToString(), last: useLast, except: negated); return; } } @@ -5290,14 +5267,7 @@ void EmitIndexOf(RegexNode node, bool useLast, bool negate) // IndexOfAny{Except}(SearchValues) if (RegexCharClass.TryGetAsciiSetChars(node.Str, out char[]? asciiChars)) { - LoadSearchValues(asciiChars); - Call((useLast, negated) switch - { - (false, false) => s_spanIndexOfAnySearchValues, - (false, true) => s_spanIndexOfAnyExceptSearchValues, - (true, false) => s_spanLastIndexOfAnySearchValues, - (true, true) => s_spanLastIndexOfAnyExceptSearchValues, - }); + EmitIndexOfAnyWithSearchValuesOrLiteral(new string(asciiChars), last: useLast, except: negated); return; } } @@ -6192,6 +6162,38 @@ private void EmitTimeoutCheckIfNeeded() } } + /// Emits a call to either IndexOfAny("abcd") or IndexOfAny(SearchValues) depending on the . + private void EmitIndexOfAnyWithSearchValuesOrLiteral(string chars, bool last = false, bool except = false) + { + Debug.Assert(chars.Length > 3); + + // SearchValues is faster than a regular IndexOfAny("abcd") for sets of 4/5 values iff they are ASCII. + // Only emit SearchValues instances when we know they'll be faster to avoid increasing the startup cost too much. + if (chars.Length is 4 or 5 && !RegexCharClass.IsAscii(chars)) + { + Ldstr(chars); + Call(s_stringAsSpanMethod); + Call((last, except) switch + { + (false, false) => s_spanIndexOfAnySpan, + (false, true) => s_spanIndexOfAnyExceptSpan, + (true, false) => s_spanLastIndexOfAnySpan, + (true, true) => s_spanLastIndexOfAnyExceptSpan, + }); + } + else + { + LoadSearchValues(chars.ToCharArray()); + Call((last, except) switch + { + (false, false) => s_spanIndexOfAnySearchValues, + (false, true) => s_spanIndexOfAnyExceptSearchValues, + (true, false) => s_spanLastIndexOfAnySearchValues, + (true, true) => s_spanLastIndexOfAnyExceptSearchValues, + }); + } + } + /// /// Adds an entry in for the given and emits a load of that initialized value. ///