From d9beb1a75738ce2f18932ff7f191216b8121e9b4 Mon Sep 17 00:00:00 2001 From: EgorBo Date: Wed, 2 Mar 2022 20:59:59 +0300 Subject: [PATCH 1/5] Add OrdinalIgnoreCase support for importer_vectorization.cpp --- src/coreclr/jit/compiler.h | 7 +- src/coreclr/jit/importer_vectorization.cpp | 322 +++++++++++++++------ 2 files changed, 236 insertions(+), 93 deletions(-) diff --git a/src/coreclr/jit/compiler.h b/src/coreclr/jit/compiler.h index f9647e92c562fd..54557108b73b4c 100644 --- a/src/coreclr/jit/compiler.h +++ b/src/coreclr/jit/compiler.h @@ -4419,9 +4419,10 @@ class Compiler bool startsWith, WCHAR* cnsData, int len, - int dataOffset); - GenTree* impExpandHalfConstEqualsSWAR(GenTreeLclVar* data, WCHAR* cns, int len, int dataOffset); - GenTree* impExpandHalfConstEqualsSIMD(GenTreeLclVar* data, WCHAR* cns, int len, int dataOffset); + int dataOffset, + bool ignoreCase); + GenTree* impExpandHalfConstEqualsSWAR(GenTreeLclVar* data, WCHAR* cns, int len, int dataOffset, bool ignoreCase); + GenTree* impExpandHalfConstEqualsSIMD(GenTreeLclVar* data, WCHAR* cns, int len, int dataOffset, bool ignoreCase); GenTreeStrCon* impGetStrConFromSpan(GenTree* span); GenTree* impIntrinsic(GenTree* newobjThis, diff --git a/src/coreclr/jit/importer_vectorization.cpp b/src/coreclr/jit/importer_vectorization.cpp index 00a5389a8939ec..4519e1fac840c1 100644 --- a/src/coreclr/jit/importer_vectorization.cpp +++ b/src/coreclr/jit/importer_vectorization.cpp @@ -6,6 +6,10 @@ #pragma hdrstop #endif +// Mirrors StringComparer.cs +constexpr int StringComparer_Ordinal = 4; +constexpr int StringComparer_OrdinalIgnoreCase = 5; + //------------------------------------------------------------------------ // importer_vectorization.cpp // @@ -13,23 +17,98 @@ // e.g. the following APIs are currently supported: // // 1) String.Equals(string, string) -// 2) String.Equals(string, string, StringComparison.Ordinal) +// 2) String.Equals(string, string, Ordinal or OrdinalIgnoreCase) // 3) str.Equals(string) -// 4) str.Equals(String, StringComparison.Ordinal) -// 5) str.StartsWith(string, StringComparison.Ordinal) +// 4) str.Equals(String, Ordinal or OrdinalIgnoreCase) +// 5) str.StartsWith(string, Ordinal or OrdinalIgnoreCase) // 6) MemoryExtensions.SequenceEqual(ROS, ROS) -// 7) MemoryExtensions.Equals(ROS, ROS, StringComparison.Ordinal) +// 7) MemoryExtensions.Equals(ROS, ROS, Ordinal or OrdinalIgnoreCase) // 8) MemoryExtensions.StartsWith(ROS, ROS) -// 9) MemoryExtensions.StartsWith(ROS, ROS, StringComparison.Ordinal) +// 9) MemoryExtensions.StartsWith(ROS, ROS, Ordinal or OrdinalIgnoreCase) // // When one of the arguments is a constant string of a [0..32] size so we can inline // a vectorized comparison against it using SWAR or SIMD techniques (e.g. via two V256 vectors) // -// We might add these in future: -// 1) OrdinalIgnoreCase for everything above -// 2) Span.CopyTo -// 3) Spans/Arrays of bytes (e.g. UTF8) against a constant RVA data + +//------------------------------------------------------------------------ +// ConvertToLowerCase: Converts input ASCII data to lower case +// +// Arguments: +// input - Constant data to change casing to lower +// mask - Mask to apply to non-constant data, e.g.: +// input: [ h ][ i ][ 4 ][ - ][ A ] +// mask: [0x20][0x20][ 0x0][ 0x0][0x20] +// length - Length of input +// +// Return Value: +// false if input contains non-ASCII chars // +static bool ConvertToLowerCase(WCHAR* input, WCHAR* mask, int length) +{ + for (int i = 0; i < length; i++) + { + auto ch = (USHORT)input[i]; + if (ch > 127) + { + JITDUMP("Constant data contains non-ASCII char(s), give up.\n"); + return false; + } + + // Inside [0..127] range only [a-Z] and [A-Z] sub-ranges are + // eligible for case changing + if ((ch >= 'A') && (ch <= 'Z')) + { + input[i] |= 0x20; + mask[i] = 0x20; + } + else if ((ch >= 'a') && (ch <= 'z')) + { + mask[i] = 0x20; + } + else + { + mask[i] = 0; + } + } + return true; +} + +#if defined(FEATURE_HW_INTRINSICS) && defined(TARGET_64BIT) +//------------------------------------------------------------------------ +// CreateConstVector: a helper to create Vector128/256.Create() node +// +// Arguments: +// comp - Compiler object +// simdType - Vector type, either TYP_SIMD32 (xarch only) or TYP_SIMD16 +// cns - Constant data +// +// Return Value: +// GenTreeHWIntrinsic node representing Vector128/256.Create() +// +static GenTreeHWIntrinsic* CreateConstVector(Compiler* comp, var_types simdType, WCHAR* cns) +{ + const CorInfoType baseType = CORINFO_TYPE_ULONG; + + // We can use e.g. UINT here to support SIMD for 32bit as well, + // but it significantly complicates code, so 32bit support is left up-for-grabs + assert(sizeof(ssize_t) == 8); + +#ifdef TARGET_XARCH + if (simdType == TYP_SIMD32) + { + GenTree* long1 = comp->gtNewIconNode(*(ssize_t*)(cns + 0), TYP_LONG); + GenTree* long2 = comp->gtNewIconNode(*(ssize_t*)(cns + 4), TYP_LONG); + GenTree* long3 = comp->gtNewIconNode(*(ssize_t*)(cns + 8), TYP_LONG); + GenTree* long4 = comp->gtNewIconNode(*(ssize_t*)(cns + 12), TYP_LONG); + return comp->gtNewSimdHWIntrinsicNode(simdType, long1, long2, long3, long4, NI_Vector256_Create, baseType, 32); + } +#endif // TARGET_XARCH + + assert(simdType == TYP_SIMD16); + GenTree* long1 = comp->gtNewIconNode(*(ssize_t*)(cns + 0), TYP_LONG); + GenTree* long2 = comp->gtNewIconNode(*(ssize_t*)(cns + 4), TYP_LONG); + return comp->gtNewSimdHWIntrinsicNode(simdType, long1, long2, NI_Vector128_Create, baseType, 16); +} //------------------------------------------------------------------------ // impExpandHalfConstEqualsSIMD: Attempts to unroll and vectorize @@ -51,24 +130,26 @@ // } // // Arguments: -// data - Pointer to a data to vectorize -// cns - Constant data (array of 2-byte chars) -// len - Number of chars in the cns +// data - Pointer to a data to vectorize +// cns - Constant data (array of 2-byte chars) +// len - Number of chars in the cns // dataOffset - Offset for data +// ignoreCase - Ignore case mode (works only for ASCII cns) // // Return Value: // A pointer to the newly created SIMD node or nullptr if unrolling is not -// possible or not profitable +// possible, not profitable or constant data contains non-ASCII char(s) in 'ignoreCase' mode // // Notes: // This function doesn't check obj for null or its Length, it's just an internal helper // for impExpandHalfConstEquals // -GenTree* Compiler::impExpandHalfConstEqualsSIMD(GenTreeLclVar* data, WCHAR* cns, int len, int dataOffset) +GenTree* Compiler::impExpandHalfConstEqualsSIMD( + GenTreeLclVar* data, WCHAR* cns, int len, int dataOffset, bool ignoreCase) { - assert(len >= 8 && len <= 32); + constexpr int maxPossibleLength = 32; + assert(len >= 8 && len <= maxPossibleLength); -#if defined(FEATURE_HW_INTRINSICS) && defined(TARGET_64BIT) if (!compOpportunisticallyDependsOn(InstructionSet_Vector128)) { // We need SSE2 or ADVSIMD at least @@ -82,14 +163,26 @@ GenTree* Compiler::impExpandHalfConstEqualsSIMD(GenTreeLclVar* data, WCHAR* cns, NamedIntrinsic niZero; NamedIntrinsic niEquals; - NamedIntrinsic niCreate; - GenTree* cnsVec1; - GenTree* cnsVec2; + GenTree* cnsVec1 = nullptr; + GenTree* cnsVec2 = nullptr; + GenTree* toLowerVec1 = nullptr; + GenTree* toLowerVec2 = nullptr; // Optimization: don't use two vectors for Length == 8 or 16 bool useSingleVector = false; + WCHAR cnsValue[maxPossibleLength] = {}; + WCHAR toLowerMask[maxPossibleLength] = {}; + + CopyMemory((UINT8*)cnsValue, (UINT8*)cns, len * sizeof(WCHAR)); + + if (ignoreCase && !ConvertToLowerCase(cnsValue, toLowerMask, len)) + { + // value contains non-ASCII chars, we can't proceed further + return nullptr; + } + #if defined(TARGET_XARCH) if (compOpportunisticallyDependsOn(InstructionSet_Vector256) && len >= 16) { @@ -101,27 +194,21 @@ GenTree* Compiler::impExpandHalfConstEqualsSIMD(GenTreeLclVar* data, WCHAR* cns, niZero = NI_Vector256_get_Zero; niEquals = NI_Vector256_op_Equality; - niCreate = NI_Vector256_Create; // Special case: use a single vector for Length == 16 useSingleVector = len == 16; - assert(sizeof(ssize_t) == 8); // this code is guarded with TARGET_64BIT - GenTree* long1 = gtNewIconNode(*(ssize_t*)(cns + 0), TYP_LONG); - GenTree* long2 = gtNewIconNode(*(ssize_t*)(cns + 4), TYP_LONG); - GenTree* long3 = gtNewIconNode(*(ssize_t*)(cns + 8), TYP_LONG); - GenTree* long4 = gtNewIconNode(*(ssize_t*)(cns + 12), TYP_LONG); - cnsVec1 = gtNewSimdHWIntrinsicNode(simdType, long1, long2, long3, long4, niCreate, baseType, simdSize); + cnsVec1 = CreateConstVector(this, simdType, cnsValue); + cnsVec2 = CreateConstVector(this, simdType, cnsValue + len - 16); - // cnsVec2 most likely overlaps with cnsVec1: - GenTree* long5 = gtNewIconNode(*(ssize_t*)(cns + len - 16), TYP_LONG); - GenTree* long6 = gtNewIconNode(*(ssize_t*)(cns + len - 12), TYP_LONG); - GenTree* long7 = gtNewIconNode(*(ssize_t*)(cns + len - 8), TYP_LONG); - GenTree* long8 = gtNewIconNode(*(ssize_t*)(cns + len - 4), TYP_LONG); - cnsVec2 = gtNewSimdHWIntrinsicNode(simdType, long5, long6, long7, long8, niCreate, baseType, simdSize); + if (ignoreCase) + { + toLowerVec1 = CreateConstVector(this, simdType, toLowerMask); + toLowerVec2 = CreateConstVector(this, simdType, toLowerMask + len - 16); + } } else -#endif +#endif // TARGET_XARCH if (len <= 16) { // Handle [8..16] inputs via two Vector128 @@ -132,20 +219,18 @@ GenTree* Compiler::impExpandHalfConstEqualsSIMD(GenTreeLclVar* data, WCHAR* cns, niZero = NI_Vector128_get_Zero; niEquals = NI_Vector128_op_Equality; - niCreate = NI_Vector128_Create; // Special case: use a single vector for Length == 8 useSingleVector = len == 8; - assert(sizeof(ssize_t) == 8); // this code is guarded with TARGET_64BIT - GenTree* long1 = gtNewIconNode(*(ssize_t*)(cns + 0), TYP_LONG); - GenTree* long2 = gtNewIconNode(*(ssize_t*)(cns + 4), TYP_LONG); - cnsVec1 = gtNewSimdHWIntrinsicNode(simdType, long1, long2, niCreate, baseType, simdSize); + cnsVec1 = CreateConstVector(this, simdType, cnsValue); + cnsVec2 = CreateConstVector(this, simdType, cnsValue + len - 8); - // cnsVec2 most likely overlaps with cnsVec1: - GenTree* long3 = gtNewIconNode(*(ssize_t*)(cns + len - 8), TYP_LONG); - GenTree* long4 = gtNewIconNode(*(ssize_t*)(cns + len - 4), TYP_LONG); - cnsVec2 = gtNewSimdHWIntrinsicNode(simdType, long3, long4, niCreate, baseType, simdSize); + if (ignoreCase) + { + toLowerVec1 = CreateConstVector(this, simdType, toLowerMask); + toLowerVec2 = CreateConstVector(this, simdType, toLowerMask + len - 8); + } } else { @@ -179,15 +264,21 @@ GenTree* Compiler::impExpandHalfConstEqualsSIMD(GenTreeLclVar* data, WCHAR* cns, // vpxor xmm1, xmm1, xmmword ptr[reloc @RWD16] // + if (ignoreCase) + { + // Apply ASCII-only ToLowerCase mask (bitwise OR 0x20 for all a-Z chars) + assert((toLowerVec1 != nullptr) && (toLowerVec2 != nullptr)); + vec1 = gtNewSimdBinOpNode(GT_OR, simdType, vec1, toLowerVec1, baseType, simdSize, false); + vec2 = gtNewSimdBinOpNode(GT_OR, simdType, vec2, toLowerVec2, baseType, simdSize, false); + } + // ((v1 ^ cns1) | (v2 ^ cns2)) == zero GenTree* xor1 = gtNewSimdBinOpNode(GT_XOR, simdType, vec1, cnsVec1, baseType, simdSize, false); GenTree* xor2 = gtNewSimdBinOpNode(GT_XOR, simdType, vec2, cnsVec2, baseType, simdSize, false); GenTree* orr = gtNewSimdBinOpNode(GT_OR, simdType, xor1, xor2, baseType, simdSize, false); return gtNewSimdHWIntrinsicNode(TYP_BOOL, useSingleVector ? xor1 : orr, zero, niEquals, baseType, simdSize); -#else - return nullptr; -#endif } +#endif // defined(FEATURE_HW_INTRINSICS) && defined(TARGET_64BIT) //------------------------------------------------------------------------ // impCreateCompareInd: creates the following tree: @@ -199,22 +290,49 @@ GenTree* Compiler::impExpandHalfConstEqualsSIMD(GenTreeLclVar* data, WCHAR* cns, // | \--* CNS_INT // \--* CNS_INT // +// or in case of 'ignoreCase': +// +// * EQ int +// +--* OR int +// | +--* IND +// | | \--* ADD byref +// | | +--* +// | | \--* CNS_INT +// | \--* CNS_INT +// \--* CNS_INT +// // Arguments: -// comp - Compiler object -// obj - GenTree representing data pointer -// type - type for the IND node -// offset - offset for the data pointer -// value - constant value to compare against +// comp - Compiler object +// obj - GenTree representing data pointer +// type - Type for the IND node +// offset - Offset for the data pointer +// value - Constant value to compare against +// ignoreCase - Ignore case mode (works only for ASCII cns) // // Return Value: // A tree with indirect load and comparison +// nullptr in case of 'ignoreCase' mode and non-ASCII value // -static GenTree* impCreateCompareInd(Compiler* comp, GenTreeLclVar* obj, var_types type, ssize_t offset, ssize_t value) +static GenTree* impCreateCompareInd( + Compiler* comp, GenTreeLclVar* obj, var_types type, ssize_t offset, ssize_t value, bool ignoreCase) { GenTree* offsetTree = comp->gtNewIconNode(offset, TYP_I_IMPL); GenTree* addOffsetTree = comp->gtNewOperNode(GT_ADD, TYP_BYREF, obj, offsetTree); GenTree* indirTree = comp->gtNewIndir(type, addOffsetTree); - GenTree* valueTree = comp->gtNewIconNode(value, genActualType(type)); + + if (ignoreCase) + { + ssize_t mask; + if (!ConvertToLowerCase((WCHAR*)&value, (WCHAR*)&mask, sizeof(ssize_t) / sizeof(WCHAR))) + { + // value contains non-ASCII chars, we can't proceed further + return nullptr; + } + GenTree* toLowerMask = comp->gtNewIconNode(mask, genActualType(type)); + indirTree = comp->gtNewOperNode(GT_OR, genActualType(type), indirTree, toLowerMask); + } + + GenTree* valueTree = comp->gtNewIconNode(value, genActualType(type)); return comp->gtNewOperNode(GT_EQ, TYP_INT, indirTree, valueTree); } @@ -224,20 +342,22 @@ static GenTree* impCreateCompareInd(Compiler* comp, GenTreeLclVar* obj, var_type // using SWAR (a sort of SIMD but for GPR registers and instructions) // // Arguments: -// data - Pointer to a data to vectorize -// cns - Constant data (array of 2-byte chars) -// len - Number of chars in the cns +// data - Pointer to a data to vectorize +// cns - Constant data (array of 2-byte chars) +// len - Number of chars in the cns // dataOffset - Offset for data +// ignoreCase - Ignore case mode (works only for ASCII cns) // // Return Value: // A pointer to the newly created SWAR node or nullptr if unrolling is not -// possible or not profitable +// possible, not profitable or constant data contains non-ASCII char(s) in 'ignoreCase' mode // // Notes: // This function doesn't check obj for null or its Length, it's just an internal helper // for impExpandHalfConstEquals // -GenTree* Compiler::impExpandHalfConstEqualsSWAR(GenTreeLclVar* data, WCHAR* cns, int len, int dataOffset) +GenTree* Compiler::impExpandHalfConstEqualsSWAR( + GenTreeLclVar* data, WCHAR* cns, int len, int dataOffset, bool ignoreCase) { assert(len >= 1 && len <= 8); @@ -250,7 +370,7 @@ GenTree* Compiler::impExpandHalfConstEqualsSWAR(GenTreeLclVar* data, WCHAR* cns, // [ ch1 ] // [value] // - return impCreateCompareInd(this, data, TYP_SHORT, dataOffset, cns[0]); + return impCreateCompareInd(this, data, TYP_SHORT, dataOffset, cns[0], ignoreCase); } if (len == 2) { @@ -258,7 +378,7 @@ GenTree* Compiler::impExpandHalfConstEqualsSWAR(GenTreeLclVar* data, WCHAR* cns, // [ value ] // const UINT32 value = MAKEINT32(cns[0], cns[1]); - return impCreateCompareInd(this, data, TYP_INT, dataOffset, value); + return impCreateCompareInd(this, data, TYP_INT, dataOffset, value, ignoreCase); } #ifdef TARGET_64BIT if (len == 3) @@ -271,11 +391,16 @@ GenTree* Compiler::impExpandHalfConstEqualsSWAR(GenTreeLclVar* data, WCHAR* cns, // // where offset for value2 is 2 bytes (1 char) // - UINT32 value1 = MAKEINT32(cns[0], cns[1]); - UINT32 value2 = MAKEINT32(cns[1], cns[2]); - GenTree* firstIndir = impCreateCompareInd(this, data, TYP_INT, dataOffset, value1); - GenTree* secondIndir = - impCreateCompareInd(this, gtClone(data)->AsLclVar(), TYP_INT, dataOffset + sizeof(USHORT), value2); + UINT32 value1 = MAKEINT32(cns[0], cns[1]); + UINT32 value2 = MAKEINT32(cns[1], cns[2]); + GenTree* firstIndir = impCreateCompareInd(this, data, TYP_INT, dataOffset, value1, ignoreCase); + GenTree* secondIndir = impCreateCompareInd(this, gtClone(data)->AsLclVar(), TYP_INT, + dataOffset + sizeof(USHORT), value2, ignoreCase); + + if ((firstIndir == nullptr) || (secondIndir == nullptr)) + { + return nullptr; + } // TODO-Unroll-CQ: Consider merging two indirs via XOR instead of QMARK // e.g. gtNewOperNode(GT_XOR, TYP_INT, firstIndir, secondIndir); @@ -292,7 +417,7 @@ GenTree* Compiler::impExpandHalfConstEqualsSWAR(GenTreeLclVar* data, WCHAR* cns, // [ ch1 ][ ch2 ][ ch3 ][ ch4 ] // [ value ] // - return impCreateCompareInd(this, data, TYP_LONG, dataOffset, value1); + return impCreateCompareInd(this, data, TYP_LONG, dataOffset, value1, ignoreCase); } // For 5..7 value2 will overlap with value1, e.g. for Length == 6: @@ -302,10 +427,15 @@ GenTree* Compiler::impExpandHalfConstEqualsSWAR(GenTreeLclVar* data, WCHAR* cns, // [ value2 ] // UINT64 value2 = MAKEINT64(cns[len - 4], cns[len - 3], cns[len - 2], cns[len - 1]); - GenTree* firstIndir = impCreateCompareInd(this, data, TYP_LONG, dataOffset, value1); + GenTree* firstIndir = impCreateCompareInd(this, data, TYP_LONG, dataOffset, value1, ignoreCase); ssize_t offset = dataOffset + len * sizeof(WCHAR) - sizeof(UINT64); - GenTree* secondIndir = impCreateCompareInd(this, gtClone(data)->AsLclVar(), TYP_LONG, offset, value2); + GenTree* secondIndir = impCreateCompareInd(this, gtClone(data)->AsLclVar(), TYP_LONG, offset, value2, ignoreCase); + + if ((firstIndir == nullptr) || (secondIndir == nullptr)) + { + return nullptr; + } // TODO-Unroll-CQ: Consider merging two indirs via XOR instead of QMARK GenTreeColon* doubleIndirColon = gtNewColonNode(TYP_INT, secondIndir, gtNewFalse()); @@ -330,10 +460,11 @@ GenTree* Compiler::impExpandHalfConstEqualsSWAR(GenTreeLclVar* data, WCHAR* cns, // cns - Constant data (array of 2-byte chars) // len - Number of 2-byte chars in the cns // dataOffset - Offset for data +// ignoreCase - Ignore case mode (works only for ASCII cns) // // Return Value: -// A pointer to the newly created SIMD node or nullptr if unrolling is not -// possible or not profitable +// A pointer to the newly created SWAR/SIMD node or nullptr if unrolling is not +// possible, not profitable or constant data contains non-ASCII char(s) in 'ignoreCase' mode // GenTree* Compiler::impExpandHalfConstEquals(GenTreeLclVar* data, GenTree* lengthFld, @@ -341,7 +472,8 @@ GenTree* Compiler::impExpandHalfConstEquals(GenTreeLclVar* data, bool startsWith, WCHAR* cnsData, int len, - int dataOffset) + int dataOffset, + bool ignoreCase) { assert(len >= 0); @@ -378,12 +510,14 @@ GenTree* Compiler::impExpandHalfConstEquals(GenTreeLclVar* data, GenTree* indirCmp = nullptr; if (len < 8) // SWAR impl supports len == 8 but we'd better give it to SIMD { - indirCmp = impExpandHalfConstEqualsSWAR(gtClone(data)->AsLclVar(), cnsData, len, dataOffset); + indirCmp = impExpandHalfConstEqualsSWAR(gtClone(data)->AsLclVar(), cnsData, len, dataOffset, ignoreCase); } +#if defined(FEATURE_HW_INTRINSICS) && defined(TARGET_64BIT) else if (len <= 32) { - indirCmp = impExpandHalfConstEqualsSIMD(gtClone(data)->AsLclVar(), cnsData, len, dataOffset); + indirCmp = impExpandHalfConstEqualsSIMD(gtClone(data)->AsLclVar(), cnsData, len, dataOffset, ignoreCase); } +#endif if (indirCmp == nullptr) { @@ -461,16 +595,16 @@ GenTreeStrCon* Compiler::impGetStrConFromSpan(GenTree* span) // impStringEqualsOrStartsWith: The main entry-point for String methods // We're going to unroll & vectorize the following cases: // 1) String.Equals(obj, "cns") -// 2) String.Equals(obj, "cns", StringComparison.Ordinal) +// 2) String.Equals(obj, "cns", Ordinal or OrdinalIgnoreCase) // 3) String.Equals("cns", obj) -// 4) String.Equals("cns", obj, StringComparison.Ordinal) +// 4) String.Equals("cns", obj, Ordinal or OrdinalIgnoreCase) // 5) obj.Equals("cns") // 5) obj.Equals("cns") -// 6) obj.Equals("cns", StringComparison.Ordinal) +// 6) obj.Equals("cns", Ordinal or OrdinalIgnoreCase) // 7) "cns".Equals(obj) -// 8) "cns".Equals(obj, StringComparison.Ordinal) -// 9) obj.StartsWith("cns", StringComparison.Ordinal) -// 10) "cns".StartsWith(obj, StringComparison.Ordinal) +// 8) "cns".Equals(obj, Ordinal or OrdinalIgnoreCase) +// 9) obj.StartsWith("cns", Ordinal or OrdinalIgnoreCase) +// 10) "cns".StartsWith(obj, Ordinal or OrdinalIgnoreCase) // // For cases 5, 6 and 9 we don't emit "obj != null" // NOTE: String.Equals(object) is not supported currently @@ -488,13 +622,17 @@ GenTree* Compiler::impStringEqualsOrStartsWith(bool startsWith, CORINFO_SIG_INFO const bool isStatic = methodFlags & CORINFO_FLG_STATIC; const int argsCount = sig->numArgs + (isStatic ? 0 : 1); + bool ignoreCase = false; GenTree* op1; GenTree* op2; if (argsCount == 3) // overload with StringComparison { - if (!impStackTop(0).val->IsIntegralConst(4)) // StringComparison.Ordinal + if (impStackTop(0).val->IsIntegralConst(StringComparer_OrdinalIgnoreCase)) + { + ignoreCase = true; + } + else if (!impStackTop(0).val->IsIntegralConst(StringComparer_Ordinal)) { - // TODO-Unroll-CQ: Unroll & vectorize OrdinalIgnoreCase return nullptr; } op1 = impStackTop(2).val; @@ -533,8 +671,8 @@ GenTree* Compiler::impStringEqualsOrStartsWith(bool startsWith, CORINFO_SIG_INFO // for the following cases we should not check varStr for null: // // obj.Equals("cns") - // obj.Equals("cns", StringComparison.Ordinal) - // obj.StartsWith("cns", StringComparison.Ordinal) + // obj.Equals("cns", Ordinal or OrdinalIgnoreCase) + // obj.StartsWith("cns", Ordinal or OrdinalIgnoreCase) // // instead, it should throw NRE if it's null needsNullcheck = false; @@ -573,7 +711,7 @@ GenTree* Compiler::impStringEqualsOrStartsWith(bool startsWith, CORINFO_SIG_INFO varStrLcl = gtClone(varStrLcl)->AsLclVar(); GenTree* unrolled = impExpandHalfConstEquals(varStrLcl, lenNode, needsNullcheck, startsWith, (WCHAR*)str, cnsLength, - strLenOffset + sizeof(int)); + strLenOffset + sizeof(int), ignoreCase); if (unrolled != nullptr) { impAssignTempGen(varStrTmp, varStr); @@ -600,12 +738,12 @@ GenTree* Compiler::impStringEqualsOrStartsWith(bool startsWith, CORINFO_SIG_INFO // We're going to unroll & vectorize the following cases: // 1) MemoryExtensions.SequenceEqual(var, "cns") // 2) MemoryExtensions.SequenceEqual("cns", var) -// 3) MemoryExtensions.Equals(var, "cns", StringComparison.Ordinal) -// 4) MemoryExtensions.Equals("cns", var, StringComparison.Ordinal) +// 3) MemoryExtensions.Equals(var, "cns", Ordinal or OrdinalIgnoreCase) +// 4) MemoryExtensions.Equals("cns", var, Ordinal or OrdinalIgnoreCase) // 5) MemoryExtensions.StartsWith("cns", var) // 6) MemoryExtensions.StartsWith(var, "cns") -// 7) MemoryExtensions.StartsWith("cns", var, StringComparison.Ordinal) -// 8) MemoryExtensions.StartsWith(var, "cns", StringComparison.Ordinal) +// 7) MemoryExtensions.StartsWith("cns", var, Ordinal or OrdinalIgnoreCase) +// 8) MemoryExtensions.StartsWith(var, "cns", Ordinal or OrdinalIgnoreCase) // // Arguments: // startsWith - Is it StartsWith or Equals? @@ -620,13 +758,17 @@ GenTree* Compiler::impSpanEqualsOrStartsWith(bool startsWith, CORINFO_SIG_INFO* const bool isStatic = methodFlags & CORINFO_FLG_STATIC; const int argsCount = sig->numArgs + (isStatic ? 0 : 1); + bool ignoreCase = false; GenTree* op1; GenTree* op2; if (argsCount == 3) // overload with StringComparison { - if (!impStackTop(0).val->IsIntegralConst(4)) // StringComparison.Ordinal + if (impStackTop(0).val->IsIntegralConst(StringComparer_OrdinalIgnoreCase)) + { + ignoreCase = true; + } + else if (!impStackTop(0).val->IsIntegralConst(StringComparer_Ordinal)) { - // TODO-Unroll-CQ: Unroll & vectorize OrdinalIgnoreCase return nullptr; } op1 = impStackTop(2).val; @@ -712,7 +854,7 @@ GenTree* Compiler::impSpanEqualsOrStartsWith(bool startsWith, CORINFO_SIG_INFO* GenTreeField* spanData = gtNewFieldRef(TYP_BYREF, pointerHnd, spanObjRefLcl); GenTree* unrolled = - impExpandHalfConstEquals(spanDataTmpLcl, spanLength, false, startsWith, (WCHAR*)str, cnsLength, 0); + impExpandHalfConstEquals(spanDataTmpLcl, spanLength, false, startsWith, (WCHAR*)str, cnsLength, 0, ignoreCase); if (unrolled != nullptr) { // We succeeded, fill the placeholders: From fbec4073f6d8448b769bd844c90e89038e41b022 Mon Sep 17 00:00:00 2001 From: EgorBo Date: Thu, 3 Mar 2022 00:37:23 +0300 Subject: [PATCH 2/5] Optimize SWAR cases with XORs --- src/coreclr/jit/importer_vectorization.cpp | 43 ++++++++++------------ 1 file changed, 20 insertions(+), 23 deletions(-) diff --git a/src/coreclr/jit/importer_vectorization.cpp b/src/coreclr/jit/importer_vectorization.cpp index 4519e1fac840c1..06d2f844c0e3a2 100644 --- a/src/coreclr/jit/importer_vectorization.cpp +++ b/src/coreclr/jit/importer_vectorization.cpp @@ -54,17 +54,13 @@ static bool ConvertToLowerCase(WCHAR* input, WCHAR* mask, int length) return false; } - // Inside [0..127] range only [a-Z] and [A-Z] sub-ranges are - // eligible for case changing - if ((ch >= 'A') && (ch <= 'Z')) + // Inside [0..127] range only [a-z] and [A-Z] sub-ranges are + // eligible for case changing, we can't apply 0x20 bit for e.g. '-' + if (((ch >= 'A') && (ch <= 'Z')) || ((ch >= 'a') && (ch <= 'z'))) { input[i] |= 0x20; mask[i] = 0x20; } - else if ((ch >= 'a') && (ch <= 'z')) - { - mask[i] = 0x20; - } else { mask[i] = 0; @@ -308,13 +304,19 @@ GenTree* Compiler::impExpandHalfConstEqualsSIMD( // offset - Offset for the data pointer // value - Constant value to compare against // ignoreCase - Ignore case mode (works only for ASCII cns) +// useXor - Use GT_XOR instead of GT_EQ // // Return Value: // A tree with indirect load and comparison // nullptr in case of 'ignoreCase' mode and non-ASCII value // -static GenTree* impCreateCompareInd( - Compiler* comp, GenTreeLclVar* obj, var_types type, ssize_t offset, ssize_t value, bool ignoreCase) +static GenTree* impCreateCompareInd(Compiler* comp, + GenTreeLclVar* obj, + var_types type, + ssize_t offset, + ssize_t value, + bool ignoreCase, + bool useXor = false) { GenTree* offsetTree = comp->gtNewIconNode(offset, TYP_I_IMPL); GenTree* addOffsetTree = comp->gtNewOperNode(GT_ADD, TYP_BYREF, obj, offsetTree); @@ -333,7 +335,7 @@ static GenTree* impCreateCompareInd( } GenTree* valueTree = comp->gtNewIconNode(value, genActualType(type)); - return comp->gtNewOperNode(GT_EQ, TYP_INT, indirTree, valueTree); + return comp->gtNewOperNode(useXor ? GT_XOR : GT_EQ, TYP_INT, indirTree, valueTree); } //------------------------------------------------------------------------ @@ -393,20 +395,16 @@ GenTree* Compiler::impExpandHalfConstEqualsSWAR( // UINT32 value1 = MAKEINT32(cns[0], cns[1]); UINT32 value2 = MAKEINT32(cns[1], cns[2]); - GenTree* firstIndir = impCreateCompareInd(this, data, TYP_INT, dataOffset, value1, ignoreCase); + GenTree* firstIndir = impCreateCompareInd(this, data, TYP_INT, dataOffset, value1, ignoreCase, true); GenTree* secondIndir = impCreateCompareInd(this, gtClone(data)->AsLclVar(), TYP_INT, - dataOffset + sizeof(USHORT), value2, ignoreCase); + dataOffset + sizeof(USHORT), value2, ignoreCase, true); if ((firstIndir == nullptr) || (secondIndir == nullptr)) { return nullptr; } - // TODO-Unroll-CQ: Consider merging two indirs via XOR instead of QMARK - // e.g. gtNewOperNode(GT_XOR, TYP_INT, firstIndir, secondIndir); - // but it currently has CQ issues (redundant movs) - GenTreeColon* doubleIndirColon = gtNewColonNode(TYP_INT, secondIndir, gtNewFalse()); - return gtNewQmarkNode(TYP_INT, firstIndir, doubleIndirColon); + return gtNewOperNode(GT_EQ, TYP_INT, gtNewOperNode(GT_OR, TYP_INT, firstIndir, secondIndir), gtNewIconNode(0)); } assert(len >= 4 && len <= 8); @@ -427,19 +425,18 @@ GenTree* Compiler::impExpandHalfConstEqualsSWAR( // [ value2 ] // UINT64 value2 = MAKEINT64(cns[len - 4], cns[len - 3], cns[len - 2], cns[len - 1]); - GenTree* firstIndir = impCreateCompareInd(this, data, TYP_LONG, dataOffset, value1, ignoreCase); + GenTree* firstIndir = impCreateCompareInd(this, data, TYP_LONG, dataOffset, value1, ignoreCase, true); - ssize_t offset = dataOffset + len * sizeof(WCHAR) - sizeof(UINT64); - GenTree* secondIndir = impCreateCompareInd(this, gtClone(data)->AsLclVar(), TYP_LONG, offset, value2, ignoreCase); + ssize_t offset = dataOffset + len * sizeof(WCHAR) - sizeof(UINT64); + GenTree* secondIndir = + impCreateCompareInd(this, gtClone(data)->AsLclVar(), TYP_LONG, offset, value2, ignoreCase, true); if ((firstIndir == nullptr) || (secondIndir == nullptr)) { return nullptr; } - // TODO-Unroll-CQ: Consider merging two indirs via XOR instead of QMARK - GenTreeColon* doubleIndirColon = gtNewColonNode(TYP_INT, secondIndir, gtNewFalse()); - return gtNewQmarkNode(TYP_INT, firstIndir, doubleIndirColon); + return gtNewOperNode(GT_EQ, TYP_INT, gtNewOperNode(GT_OR, TYP_LONG, firstIndir, secondIndir), gtNewIconNode(0)); #else // TARGET_64BIT return nullptr; #endif From cc90fb89c165b77a3c5b1211192c71d43b157f37 Mon Sep 17 00:00:00 2001 From: EgorBo Date: Thu, 3 Mar 2022 01:05:19 +0300 Subject: [PATCH 3/5] Clean up --- src/coreclr/jit/importer_vectorization.cpp | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/src/coreclr/jit/importer_vectorization.cpp b/src/coreclr/jit/importer_vectorization.cpp index 06d2f844c0e3a2..7071ef027fa3b6 100644 --- a/src/coreclr/jit/importer_vectorization.cpp +++ b/src/coreclr/jit/importer_vectorization.cpp @@ -318,9 +318,10 @@ static GenTree* impCreateCompareInd(Compiler* comp, bool ignoreCase, bool useXor = false) { - GenTree* offsetTree = comp->gtNewIconNode(offset, TYP_I_IMPL); - GenTree* addOffsetTree = comp->gtNewOperNode(GT_ADD, TYP_BYREF, obj, offsetTree); - GenTree* indirTree = comp->gtNewIndir(type, addOffsetTree); + var_types actualType = genActualType(type); + GenTree* offsetTree = comp->gtNewIconNode(offset, TYP_I_IMPL); + GenTree* addOffsetTree = comp->gtNewOperNode(GT_ADD, TYP_BYREF, obj, offsetTree); + GenTree* indirTree = comp->gtNewIndir(type, addOffsetTree); if (ignoreCase) { @@ -330,12 +331,17 @@ static GenTree* impCreateCompareInd(Compiler* comp, // value contains non-ASCII chars, we can't proceed further return nullptr; } - GenTree* toLowerMask = comp->gtNewIconNode(mask, genActualType(type)); - indirTree = comp->gtNewOperNode(GT_OR, genActualType(type), indirTree, toLowerMask); + GenTree* toLowerMask = comp->gtNewIconNode(mask, actualType); + indirTree = comp->gtNewOperNode(GT_OR, actualType, indirTree, toLowerMask); } GenTree* valueTree = comp->gtNewIconNode(value, genActualType(type)); - return comp->gtNewOperNode(useXor ? GT_XOR : GT_EQ, TYP_INT, indirTree, valueTree); + if (useXor) + { + // XOR is better than CMP if we want to join multiple comparisons + return comp->gtNewOperNode(GT_XOR, actualType, indirTree, valueTree); + } + return comp->gtNewOperNode(GT_EQ, TYP_INT, indirTree, valueTree); } //------------------------------------------------------------------------ @@ -521,6 +527,7 @@ GenTree* Compiler::impExpandHalfConstEquals(GenTreeLclVar* data, JITDUMP("unable to compose indirCmp\n"); return nullptr; } + assert(indirCmp->TypeIs(TYP_INT, TYP_BOOL)); GenTreeColon* lenCheckColon = gtNewColonNode(TYP_INT, indirCmp, gtNewFalse()); From b001e8c478285cbcbcbb8026123006e5041f8ed5 Mon Sep 17 00:00:00 2001 From: EgorBo Date: Thu, 3 Mar 2022 02:00:40 +0300 Subject: [PATCH 4/5] fix assert --- src/coreclr/jit/importer_vectorization.cpp | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/coreclr/jit/importer_vectorization.cpp b/src/coreclr/jit/importer_vectorization.cpp index 7071ef027fa3b6..90bbadaf4bd732 100644 --- a/src/coreclr/jit/importer_vectorization.cpp +++ b/src/coreclr/jit/importer_vectorization.cpp @@ -335,7 +335,7 @@ static GenTree* impCreateCompareInd(Compiler* comp, indirTree = comp->gtNewOperNode(GT_OR, actualType, indirTree, toLowerMask); } - GenTree* valueTree = comp->gtNewIconNode(value, genActualType(type)); + GenTree* valueTree = comp->gtNewIconNode(value, actualType); if (useXor) { // XOR is better than CMP if we want to join multiple comparisons @@ -442,7 +442,8 @@ GenTree* Compiler::impExpandHalfConstEqualsSWAR( return nullptr; } - return gtNewOperNode(GT_EQ, TYP_INT, gtNewOperNode(GT_OR, TYP_LONG, firstIndir, secondIndir), gtNewIconNode(0)); + return gtNewOperNode(GT_EQ, TYP_INT, gtNewOperNode(GT_OR, TYP_LONG, firstIndir, secondIndir), + gtNewIconNode(0, TYP_LONG)); #else // TARGET_64BIT return nullptr; #endif From ab637c9ad06bc36b6c13846131bbd6ac87726be8 Mon Sep 17 00:00:00 2001 From: EgorBo Date: Wed, 9 Mar 2022 01:21:39 +0300 Subject: [PATCH 5/5] Clean up --- src/coreclr/jit/compiler.h | 39 ++++-- src/coreclr/jit/importer_vectorization.cpp | 132 ++++++++++----------- 2 files changed, 93 insertions(+), 78 deletions(-) diff --git a/src/coreclr/jit/compiler.h b/src/coreclr/jit/compiler.h index 422003eace83d3..64c8666bb36536 100644 --- a/src/coreclr/jit/compiler.h +++ b/src/coreclr/jit/compiler.h @@ -4413,18 +4413,37 @@ class Compiler void impResetLeaveBlock(BasicBlock* block, unsigned jmpAddr); GenTree* impTypeIsAssignable(GenTree* typeTo, GenTree* typeFrom); + // Mirrors StringComparison.cs + enum StringComparison + { + Ordinal = 4, + OrdinalIgnoreCase = 5 + }; + enum StringComparisonJoint + { + Eq, // (d1 == cns1) && (s2 == cns2) + Xor, // (d1 ^ cns1) | (s2 ^ cns2) + }; GenTree* impStringEqualsOrStartsWith(bool startsWith, CORINFO_SIG_INFO* sig, unsigned methodFlags); GenTree* impSpanEqualsOrStartsWith(bool startsWith, CORINFO_SIG_INFO* sig, unsigned methodFlags); - GenTree* impExpandHalfConstEquals(GenTreeLclVar* data, - GenTree* lengthFld, - bool checkForNull, - bool startsWith, - WCHAR* cnsData, - int len, - int dataOffset, - bool ignoreCase); - GenTree* impExpandHalfConstEqualsSWAR(GenTreeLclVar* data, WCHAR* cns, int len, int dataOffset, bool ignoreCase); - GenTree* impExpandHalfConstEqualsSIMD(GenTreeLclVar* data, WCHAR* cns, int len, int dataOffset, bool ignoreCase); + GenTree* impExpandHalfConstEquals(GenTreeLclVar* data, + GenTree* lengthFld, + bool checkForNull, + bool startsWith, + WCHAR* cnsData, + int len, + int dataOffset, + StringComparison cmpMode); + GenTree* impCreateCompareInd(GenTreeLclVar* obj, + var_types type, + ssize_t offset, + ssize_t value, + StringComparison ignoreCase, + StringComparisonJoint joint = Eq); + GenTree* impExpandHalfConstEqualsSWAR( + GenTreeLclVar* data, WCHAR* cns, int len, int dataOffset, StringComparison cmpMode); + GenTree* impExpandHalfConstEqualsSIMD( + GenTreeLclVar* data, WCHAR* cns, int len, int dataOffset, StringComparison cmpMode); GenTreeStrCon* impGetStrConFromSpan(GenTree* span); GenTree* impIntrinsic(GenTree* newobjThis, diff --git a/src/coreclr/jit/importer_vectorization.cpp b/src/coreclr/jit/importer_vectorization.cpp index 90bbadaf4bd732..631e3577ba5104 100644 --- a/src/coreclr/jit/importer_vectorization.cpp +++ b/src/coreclr/jit/importer_vectorization.cpp @@ -6,10 +6,6 @@ #pragma hdrstop #endif -// Mirrors StringComparer.cs -constexpr int StringComparer_Ordinal = 4; -constexpr int StringComparer_OrdinalIgnoreCase = 5; - //------------------------------------------------------------------------ // importer_vectorization.cpp // @@ -130,7 +126,7 @@ static GenTreeHWIntrinsic* CreateConstVector(Compiler* comp, var_types simdType, // cns - Constant data (array of 2-byte chars) // len - Number of chars in the cns // dataOffset - Offset for data -// ignoreCase - Ignore case mode (works only for ASCII cns) +// cmpMode - Ordinal or OrdinalIgnoreCase mode (works only for ASCII cns) // // Return Value: // A pointer to the newly created SIMD node or nullptr if unrolling is not @@ -141,7 +137,7 @@ static GenTreeHWIntrinsic* CreateConstVector(Compiler* comp, var_types simdType, // for impExpandHalfConstEquals // GenTree* Compiler::impExpandHalfConstEqualsSIMD( - GenTreeLclVar* data, WCHAR* cns, int len, int dataOffset, bool ignoreCase) + GenTreeLclVar* data, WCHAR* cns, int len, int dataOffset, StringComparison cmpMode) { constexpr int maxPossibleLength = 32; assert(len >= 8 && len <= maxPossibleLength); @@ -173,7 +169,7 @@ GenTree* Compiler::impExpandHalfConstEqualsSIMD( CopyMemory((UINT8*)cnsValue, (UINT8*)cns, len * sizeof(WCHAR)); - if (ignoreCase && !ConvertToLowerCase(cnsValue, toLowerMask, len)) + if ((cmpMode == OrdinalIgnoreCase) && !ConvertToLowerCase(cnsValue, toLowerMask, len)) { // value contains non-ASCII chars, we can't proceed further return nullptr; @@ -197,7 +193,7 @@ GenTree* Compiler::impExpandHalfConstEqualsSIMD( cnsVec1 = CreateConstVector(this, simdType, cnsValue); cnsVec2 = CreateConstVector(this, simdType, cnsValue + len - 16); - if (ignoreCase) + if (cmpMode == OrdinalIgnoreCase) { toLowerVec1 = CreateConstVector(this, simdType, toLowerMask); toLowerVec2 = CreateConstVector(this, simdType, toLowerMask + len - 16); @@ -222,7 +218,7 @@ GenTree* Compiler::impExpandHalfConstEqualsSIMD( cnsVec1 = CreateConstVector(this, simdType, cnsValue); cnsVec2 = CreateConstVector(this, simdType, cnsValue + len - 8); - if (ignoreCase) + if (cmpMode == OrdinalIgnoreCase) { toLowerVec1 = CreateConstVector(this, simdType, toLowerMask); toLowerVec2 = CreateConstVector(this, simdType, toLowerMask + len - 8); @@ -260,7 +256,7 @@ GenTree* Compiler::impExpandHalfConstEqualsSIMD( // vpxor xmm1, xmm1, xmmword ptr[reloc @RWD16] // - if (ignoreCase) + if (cmpMode == OrdinalIgnoreCase) { // Apply ASCII-only ToLowerCase mask (bitwise OR 0x20 for all a-Z chars) assert((toLowerVec1 != nullptr) && (toLowerVec2 != nullptr)); @@ -303,27 +299,27 @@ GenTree* Compiler::impExpandHalfConstEqualsSIMD( // type - Type for the IND node // offset - Offset for the data pointer // value - Constant value to compare against -// ignoreCase - Ignore case mode (works only for ASCII cns) -// useXor - Use GT_XOR instead of GT_EQ +// cmpMode - Ordinal or OrdinalIgnoreCase mode (works only for ASCII cns) +// joint - Type of joint, can be Eq ((d1 == cns1) && (s2 == cns2)) +// or Xor (d1 ^ cns1) | (s2 ^ cns2). // // Return Value: // A tree with indirect load and comparison // nullptr in case of 'ignoreCase' mode and non-ASCII value // -static GenTree* impCreateCompareInd(Compiler* comp, - GenTreeLclVar* obj, - var_types type, - ssize_t offset, - ssize_t value, - bool ignoreCase, - bool useXor = false) +GenTree* Compiler::impCreateCompareInd(GenTreeLclVar* obj, + var_types type, + ssize_t offset, + ssize_t value, + StringComparison cmpMode, + StringComparisonJoint joint) { var_types actualType = genActualType(type); - GenTree* offsetTree = comp->gtNewIconNode(offset, TYP_I_IMPL); - GenTree* addOffsetTree = comp->gtNewOperNode(GT_ADD, TYP_BYREF, obj, offsetTree); - GenTree* indirTree = comp->gtNewIndir(type, addOffsetTree); + GenTree* offsetTree = gtNewIconNode(offset, TYP_I_IMPL); + GenTree* addOffsetTree = gtNewOperNode(GT_ADD, TYP_BYREF, obj, offsetTree); + GenTree* indirTree = gtNewIndir(type, addOffsetTree); - if (ignoreCase) + if (cmpMode == OrdinalIgnoreCase) { ssize_t mask; if (!ConvertToLowerCase((WCHAR*)&value, (WCHAR*)&mask, sizeof(ssize_t) / sizeof(WCHAR))) @@ -331,17 +327,18 @@ static GenTree* impCreateCompareInd(Compiler* comp, // value contains non-ASCII chars, we can't proceed further return nullptr; } - GenTree* toLowerMask = comp->gtNewIconNode(mask, actualType); - indirTree = comp->gtNewOperNode(GT_OR, actualType, indirTree, toLowerMask); + GenTree* toLowerMask = gtNewIconNode(mask, actualType); + indirTree = gtNewOperNode(GT_OR, actualType, indirTree, toLowerMask); } - GenTree* valueTree = comp->gtNewIconNode(value, actualType); - if (useXor) + GenTree* valueTree = gtNewIconNode(value, actualType); + if (joint == Xor) { // XOR is better than CMP if we want to join multiple comparisons - return comp->gtNewOperNode(GT_XOR, actualType, indirTree, valueTree); + return gtNewOperNode(GT_XOR, actualType, indirTree, valueTree); } - return comp->gtNewOperNode(GT_EQ, TYP_INT, indirTree, valueTree); + assert(joint == Eq); + return gtNewOperNode(GT_EQ, TYP_INT, indirTree, valueTree); } //------------------------------------------------------------------------ @@ -354,7 +351,7 @@ static GenTree* impCreateCompareInd(Compiler* comp, // cns - Constant data (array of 2-byte chars) // len - Number of chars in the cns // dataOffset - Offset for data -// ignoreCase - Ignore case mode (works only for ASCII cns) +// cmpMode - Ordinal or OrdinalIgnoreCase mode (works only for ASCII cns) // // Return Value: // A pointer to the newly created SWAR node or nullptr if unrolling is not @@ -365,7 +362,7 @@ static GenTree* impCreateCompareInd(Compiler* comp, // for impExpandHalfConstEquals // GenTree* Compiler::impExpandHalfConstEqualsSWAR( - GenTreeLclVar* data, WCHAR* cns, int len, int dataOffset, bool ignoreCase) + GenTreeLclVar* data, WCHAR* cns, int len, int dataOffset, StringComparison cmpMode) { assert(len >= 1 && len <= 8); @@ -378,7 +375,7 @@ GenTree* Compiler::impExpandHalfConstEqualsSWAR( // [ ch1 ] // [value] // - return impCreateCompareInd(this, data, TYP_SHORT, dataOffset, cns[0], ignoreCase); + return impCreateCompareInd(data, TYP_SHORT, dataOffset, cns[0], cmpMode); } if (len == 2) { @@ -386,7 +383,7 @@ GenTree* Compiler::impExpandHalfConstEqualsSWAR( // [ value ] // const UINT32 value = MAKEINT32(cns[0], cns[1]); - return impCreateCompareInd(this, data, TYP_INT, dataOffset, value, ignoreCase); + return impCreateCompareInd(data, TYP_INT, dataOffset, value, cmpMode); } #ifdef TARGET_64BIT if (len == 3) @@ -399,11 +396,11 @@ GenTree* Compiler::impExpandHalfConstEqualsSWAR( // // where offset for value2 is 2 bytes (1 char) // - UINT32 value1 = MAKEINT32(cns[0], cns[1]); - UINT32 value2 = MAKEINT32(cns[1], cns[2]); - GenTree* firstIndir = impCreateCompareInd(this, data, TYP_INT, dataOffset, value1, ignoreCase, true); - GenTree* secondIndir = impCreateCompareInd(this, gtClone(data)->AsLclVar(), TYP_INT, - dataOffset + sizeof(USHORT), value2, ignoreCase, true); + UINT32 value1 = MAKEINT32(cns[0], cns[1]); + UINT32 value2 = MAKEINT32(cns[1], cns[2]); + GenTree* firstIndir = impCreateCompareInd(data, TYP_INT, dataOffset, value1, cmpMode, Xor); + GenTree* secondIndir = + impCreateCompareInd(gtClone(data)->AsLclVar(), TYP_INT, dataOffset + sizeof(USHORT), value2, cmpMode, Xor); if ((firstIndir == nullptr) || (secondIndir == nullptr)) { @@ -421,7 +418,7 @@ GenTree* Compiler::impExpandHalfConstEqualsSWAR( // [ ch1 ][ ch2 ][ ch3 ][ ch4 ] // [ value ] // - return impCreateCompareInd(this, data, TYP_LONG, dataOffset, value1, ignoreCase); + return impCreateCompareInd(data, TYP_LONG, dataOffset, value1, cmpMode); } // For 5..7 value2 will overlap with value1, e.g. for Length == 6: @@ -431,11 +428,10 @@ GenTree* Compiler::impExpandHalfConstEqualsSWAR( // [ value2 ] // UINT64 value2 = MAKEINT64(cns[len - 4], cns[len - 3], cns[len - 2], cns[len - 1]); - GenTree* firstIndir = impCreateCompareInd(this, data, TYP_LONG, dataOffset, value1, ignoreCase, true); + GenTree* firstIndir = impCreateCompareInd(data, TYP_LONG, dataOffset, value1, cmpMode, Xor); - ssize_t offset = dataOffset + len * sizeof(WCHAR) - sizeof(UINT64); - GenTree* secondIndir = - impCreateCompareInd(this, gtClone(data)->AsLclVar(), TYP_LONG, offset, value2, ignoreCase, true); + ssize_t offset = dataOffset + len * sizeof(WCHAR) - sizeof(UINT64); + GenTree* secondIndir = impCreateCompareInd(gtClone(data)->AsLclVar(), TYP_LONG, offset, value2, cmpMode, Xor); if ((firstIndir == nullptr) || (secondIndir == nullptr)) { @@ -464,20 +460,20 @@ GenTree* Compiler::impExpandHalfConstEqualsSWAR( // cns - Constant data (array of 2-byte chars) // len - Number of 2-byte chars in the cns // dataOffset - Offset for data -// ignoreCase - Ignore case mode (works only for ASCII cns) +// cmpMode - Ordinal or OrdinalIgnoreCase mode (works only for ASCII cns) // // Return Value: // A pointer to the newly created SWAR/SIMD node or nullptr if unrolling is not // possible, not profitable or constant data contains non-ASCII char(s) in 'ignoreCase' mode // -GenTree* Compiler::impExpandHalfConstEquals(GenTreeLclVar* data, - GenTree* lengthFld, - bool checkForNull, - bool startsWith, - WCHAR* cnsData, - int len, - int dataOffset, - bool ignoreCase) +GenTree* Compiler::impExpandHalfConstEquals(GenTreeLclVar* data, + GenTree* lengthFld, + bool checkForNull, + bool startsWith, + WCHAR* cnsData, + int len, + int dataOffset, + StringComparison cmpMode) { assert(len >= 0); @@ -514,12 +510,12 @@ GenTree* Compiler::impExpandHalfConstEquals(GenTreeLclVar* data, GenTree* indirCmp = nullptr; if (len < 8) // SWAR impl supports len == 8 but we'd better give it to SIMD { - indirCmp = impExpandHalfConstEqualsSWAR(gtClone(data)->AsLclVar(), cnsData, len, dataOffset, ignoreCase); + indirCmp = impExpandHalfConstEqualsSWAR(gtClone(data)->AsLclVar(), cnsData, len, dataOffset, cmpMode); } #if defined(FEATURE_HW_INTRINSICS) && defined(TARGET_64BIT) else if (len <= 32) { - indirCmp = impExpandHalfConstEqualsSIMD(gtClone(data)->AsLclVar(), cnsData, len, dataOffset, ignoreCase); + indirCmp = impExpandHalfConstEqualsSIMD(gtClone(data)->AsLclVar(), cnsData, len, dataOffset, cmpMode); } #endif @@ -627,16 +623,16 @@ GenTree* Compiler::impStringEqualsOrStartsWith(bool startsWith, CORINFO_SIG_INFO const bool isStatic = methodFlags & CORINFO_FLG_STATIC; const int argsCount = sig->numArgs + (isStatic ? 0 : 1); - bool ignoreCase = false; - GenTree* op1; - GenTree* op2; + StringComparison cmpMode = Ordinal; + GenTree* op1; + GenTree* op2; if (argsCount == 3) // overload with StringComparison { - if (impStackTop(0).val->IsIntegralConst(StringComparer_OrdinalIgnoreCase)) + if (impStackTop(0).val->IsIntegralConst(OrdinalIgnoreCase)) { - ignoreCase = true; + cmpMode = OrdinalIgnoreCase; } - else if (!impStackTop(0).val->IsIntegralConst(StringComparer_Ordinal)) + else if (!impStackTop(0).val->IsIntegralConst(Ordinal)) { return nullptr; } @@ -716,7 +712,7 @@ GenTree* Compiler::impStringEqualsOrStartsWith(bool startsWith, CORINFO_SIG_INFO varStrLcl = gtClone(varStrLcl)->AsLclVar(); GenTree* unrolled = impExpandHalfConstEquals(varStrLcl, lenNode, needsNullcheck, startsWith, (WCHAR*)str, cnsLength, - strLenOffset + sizeof(int), ignoreCase); + strLenOffset + sizeof(int), cmpMode); if (unrolled != nullptr) { impAssignTempGen(varStrTmp, varStr); @@ -763,16 +759,16 @@ GenTree* Compiler::impSpanEqualsOrStartsWith(bool startsWith, CORINFO_SIG_INFO* const bool isStatic = methodFlags & CORINFO_FLG_STATIC; const int argsCount = sig->numArgs + (isStatic ? 0 : 1); - bool ignoreCase = false; - GenTree* op1; - GenTree* op2; + StringComparison cmpMode = Ordinal; + GenTree* op1; + GenTree* op2; if (argsCount == 3) // overload with StringComparison { - if (impStackTop(0).val->IsIntegralConst(StringComparer_OrdinalIgnoreCase)) + if (impStackTop(0).val->IsIntegralConst(OrdinalIgnoreCase)) { - ignoreCase = true; + cmpMode = OrdinalIgnoreCase; } - else if (!impStackTop(0).val->IsIntegralConst(StringComparer_Ordinal)) + else if (!impStackTop(0).val->IsIntegralConst(Ordinal)) { return nullptr; } @@ -859,7 +855,7 @@ GenTree* Compiler::impSpanEqualsOrStartsWith(bool startsWith, CORINFO_SIG_INFO* GenTreeField* spanData = gtNewFieldRef(TYP_BYREF, pointerHnd, spanObjRefLcl); GenTree* unrolled = - impExpandHalfConstEquals(spanDataTmpLcl, spanLength, false, startsWith, (WCHAR*)str, cnsLength, 0, ignoreCase); + impExpandHalfConstEquals(spanDataTmpLcl, spanLength, false, startsWith, (WCHAR*)str, cnsLength, 0, cmpMode); if (unrolled != nullptr) { // We succeeded, fill the placeholders: