diff --git a/src/coreclr/jit/codegen.h b/src/coreclr/jit/codegen.h index f7d9192cb588ec..8be7bcf4ddaae5 100644 --- a/src/coreclr/jit/codegen.h +++ b/src/coreclr/jit/codegen.h @@ -807,6 +807,7 @@ class CodeGen final : public CodeGenInterface void genCodeForDivMod(GenTreeOp* treeNode); void genCodeForMul(GenTreeOp* treeNode); + void genCodeForBitOp(GenTreeOp* treeNode); void genCodeForIncSaturate(GenTree* treeNode); void genCodeForMulHi(GenTreeOp* treeNode); void genLeaInstruction(GenTreeAddrMode* lea); diff --git a/src/coreclr/jit/codegenxarch.cpp b/src/coreclr/jit/codegenxarch.cpp index 5308e680efa374..e58c95e201845e 100644 --- a/src/coreclr/jit/codegenxarch.cpp +++ b/src/coreclr/jit/codegenxarch.cpp @@ -1129,6 +1129,51 @@ void CodeGen::genCodeForBinary(GenTreeOp* treeNode) genProduceReg(treeNode); } +//------------------------------------------------------------------------ +// genCodeForBitOp: Generate code for a GT_BIT_SET/GT_BIT_CLEAR/GT_BIT_INVERT operation, i.e. the +// value-producing `bts`/`btr`/`btc` instructions which set, reset, or complement a single bit of op1 +// selected by op2. +// +// Arguments: +// treeNode - the node to generate the code for +// +void CodeGen::genCodeForBitOp(GenTreeOp* treeNode) +{ + assert(treeNode->OperIs(GT_BIT_SET, GT_BIT_CLEAR, GT_BIT_INVERT)); + + GenTree* op1 = treeNode->gtGetOp1(); // value (read-modify-write destination) + GenTree* op2 = treeNode->gtGetOp2(); // bit index + + genConsumeOperands(treeNode); + + regNumber targetReg = treeNode->GetRegNum(); + var_types targetType = genActualType(treeNode); + emitAttr size = emitTypeSize(targetType); + emitter* emit = GetEmitter(); + + assert((targetType == TYP_INT) || (targetType == TYP_LONG)); + assert(op1->isUsedFromReg() && op2->isUsedFromReg()); + + instruction ins = treeNode->OperIs(GT_BIT_SET) ? INS_bts : treeNode->OperIs(GT_BIT_CLEAR) ? INS_btr : INS_btc; + + // These are read-modify-write: the `mov` below loads op1 (the value) into the destination and + // then `bts`/`btr`/`btc` reads the bit index from op2. LSRA marks op2 as delayFree except when + // op2 shares op1's interval and it's their last use -- i.e. `x (1 << x)`, where op1 and op2 + // are the same value (see AddDelayFreeUses). So the destination can only alias op2 when op1 and + // op2 hold the same value, in which case the `mov` writes that same value back into op2's + // register and nothing is clobbered before the bit-test reads it. When the operands are distinct + // values, delayFree guarantees the destination and op2 use different registers. + + // These are read-modify-write: the destination register also supplies the value operand. + inst_Mov(targetType, targetReg, op1->GetRegNum(), /* canSkip */ true); + + // The BT-family reg,reg encoding places the destination in the r/m slot and the bit index in + // the reg slot, so the operands are passed reversed (see the note in instrsxarch.h). + emit->emitIns_R_R(ins, size, op2->GetRegNum(), targetReg); + + genProduceReg(treeNode); +} + //------------------------------------------------------------------------ // genCodeForMul: Generate code for a MUL operation. // @@ -1892,6 +1937,12 @@ void CodeGen::genCodeForTreeNode(GenTree* treeNode) genCodeForBinary(treeNode->AsOp()); break; + case GT_BIT_SET: + case GT_BIT_CLEAR: + case GT_BIT_INVERT: + genCodeForBitOp(treeNode->AsOp()); + break; + case GT_MUL: if (varTypeIsFloating(treeNode->TypeGet())) { diff --git a/src/coreclr/jit/emitxarch.cpp b/src/coreclr/jit/emitxarch.cpp index 416c99d85d3584..c2a6df7a85560e 100644 --- a/src/coreclr/jit/emitxarch.cpp +++ b/src/coreclr/jit/emitxarch.cpp @@ -12999,6 +12999,15 @@ void emitter::emitDispIns( break; } + if ((ins == INS_bts) || (ins == INS_btr) || (ins == INS_btc)) + { + // The BT-family reg,reg encoding stores its operands reversed. Display them in + // the normal `dest, index` order. + printf("%s", emitRegName(id->idReg2(), tgtAttr)); + printf(", %s", emitRegName(id->idReg1(), srcAttr)); + break; + } + printf("%s", emitRegName(id->idReg1(), tgtAttr)); printf(", %s", emitRegName(id->idReg2(), srcAttr)); break; diff --git a/src/coreclr/jit/gtlist.h b/src/coreclr/jit/gtlist.h index be81ca9f22052d..e43ac686e1a9d5 100644 --- a/src/coreclr/jit/gtlist.h +++ b/src/coreclr/jit/gtlist.h @@ -298,14 +298,16 @@ GTNODE(SH3ADD_UW , GenTreeOp ,0,0,GTK_BINOP|DBK_NOTHIR) GTNODE(ADD_UW , GenTreeOp ,0,0,GTK_BINOP|DBK_NOTHIR) // Maps to riscv64 slli.uw instruction. Computes result = zext(op1[31..0]) << imm. GTNODE(SLLI_UW , GenTreeOp ,0,0,GTK_BINOP|DBK_NOTHIR) +#endif // TARGET_RISCV64 -// Maps to bset/bseti instruction. Computes result = op1 | (1 << op2) +#if defined(TARGET_RISCV64) || defined(TARGET_XARCH) +// Maps to riscv64 bset/bseti and xarch bts. Computes result = op1 | (1 << op2) GTNODE(BIT_SET , GenTreeOp ,0,0,GTK_BINOP|DBK_NOTHIR) -// Maps to bclr/bclri instruction. Computes result = op1 & ~(1 << op2) +// Maps to riscv64 bclr/bclri and xarch btr. Computes result = op1 & ~(1 << op2) GTNODE(BIT_CLEAR , GenTreeOp ,0,0,GTK_BINOP|DBK_NOTHIR) -// Maps to binv/binvi instruction. Computes result = op1 ^ (1 << op2) +// Maps to riscv64 binv/binvi and xarch btc. Computes result = op1 ^ (1 << op2) GTNODE(BIT_INVERT , GenTreeOp ,0,0,GTK_BINOP|DBK_NOTHIR) -#endif +#endif // TARGET_RISCV64 || TARGET_XARCH //----------------------------------------------------------------------------- // Other nodes that look like unary/binary operators: diff --git a/src/coreclr/jit/instrsxarch.h b/src/coreclr/jit/instrsxarch.h index 959aab27c0a5f9..b7232c66573278 100644 --- a/src/coreclr/jit/instrsxarch.h +++ b/src/coreclr/jit/instrsxarch.h @@ -95,6 +95,12 @@ INST4(lea, "lea", IUM_WR, BAD_CODE, BAD_CODE, // and the registers need to be reversed to get the correct encoding. INST3(bt, "bt", IUM_RD, 0x0F00A3, BAD_CODE, 0x0F00A3, 1C, 2X, INS_TT_NONE, Undefined_OF | Undefined_SF | Undefined_ZF | Undefined_AF | Undefined_PF | Writes_CF | Encoding_REX2) +// BTS/BTR/BTC are only emitted in their reg,reg form (like BT the registers are reversed to get +// the correct encoding). They read+write the first operand and write the old bit value to CF. +INST3(bts, "bts", IUM_RW, 0x0F00AB, BAD_CODE, 0x0F00AB, 1C, 2X, INS_TT_NONE, Undefined_OF | Undefined_SF | Undefined_ZF | Undefined_AF | Undefined_PF | Writes_CF | Encoding_REX2) +INST3(btr, "btr", IUM_RW, 0x0F00B3, BAD_CODE, 0x0F00B3, 1C, 2X, INS_TT_NONE, Undefined_OF | Undefined_SF | Undefined_ZF | Undefined_AF | Undefined_PF | Writes_CF | Encoding_REX2) +INST3(btc, "btc", IUM_RW, 0x0F00BB, BAD_CODE, 0x0F00BB, 1C, 2X, INS_TT_NONE, Undefined_OF | Undefined_SF | Undefined_ZF | Undefined_AF | Undefined_PF | Writes_CF | Encoding_REX2) + INST3(bsr, "bsr", IUM_WR, BAD_CODE, BAD_CODE, 0x0F00BD, 3C, 1C, INS_TT_NONE, Undefined_OF | Undefined_SF | Writes_ZF | Undefined_AF | Undefined_PF | Undefined_CF | Encoding_REX2) INST3(bsf, "bsf", IUM_WR, BAD_CODE, BAD_CODE, 0x0F00BC, 3C, 1C, INS_TT_NONE, Undefined_OF | Undefined_SF | Writes_ZF | Undefined_AF | Undefined_PF | Undefined_CF | Encoding_REX2) diff --git a/src/coreclr/jit/lower.cpp b/src/coreclr/jit/lower.cpp index 291114808b2bdc..7f890b3906440e 100644 --- a/src/coreclr/jit/lower.cpp +++ b/src/coreclr/jit/lower.cpp @@ -4237,6 +4237,34 @@ GenTree* Lowering::OptimizeConstCompare(GenTree* cmp) test->gtOp2 = bitOp->gtGetOp2(); return true; } + +#ifdef TARGET_XARCH + // Also recognize the arithmetic form `(x >> y) & 1`, i.e. AND(RSH|RSZ(x, y), 1), which + // tests bit `y` of `x` just like `x & (1 << y)`. Only bit 0 of the shifted value is kept so + // the shift kind is irrelevant, and `bt` masks the bit index modulo the operand size, which + // matches the C# masked-shift semantics even for an out-of-range `y`. Restricted to a + // variable index because a constant index keeps the shift, and `bt` has no immediate form + // here (a constant mask `test` is already optimal). + GenTree* shiftOp = test->gtOp1; + GenTree* oneOp = test->gtOp2; + if (!oneOp->IsIntegralConst(1)) + std::swap(shiftOp, oneOp); + + if (oneOp->IsIntegralConst(1) && shiftOp->OperIs(GT_RSH, GT_RSZ) && varTypeIsIntOrI(shiftOp) && + !shiftOp->gtGetOp2()->IsIntegralConst()) + { + BlockRange().Remove(oneOp); + BlockRange().Remove(shiftOp); + test->gtOp1 = shiftOp->gtGetOp1(); + test->gtOp2 = shiftOp->gtGetOp2(); + + // ContainCheckCompare is skipped when this transform succeeds, so clear any containment + // the value operand picked up from the removed shift (e.g. a `shrx` memory source) -- + // the reg,reg `bt` form requires it in a register. + test->gtOp1->ClearContained(); + return true; + } +#endif // TARGET_XARCH return false; }; @@ -10034,6 +10062,117 @@ void Lowering::ContainCheckRet(GenTreeUnOp* ret) #endif // FEATURE_MULTIREG_RET } +#if defined(TARGET_XARCH) || defined(TARGET_RISCV64) +//------------------------------------------------------------------------ +// TryLowerBitwiseOpToBitOp: Recognizes the single-bit-manipulation idioms with a variable +// (non-constant) bit index and rewrites them in place to the shared GT_BIT_* nodes: +// +// OR (X, LSH(1, Y)) -> BIT_SET (set bit Y of X) +// XOR(X, LSH(1, Y)) -> BIT_INVERT (complement bit Y of X) +// AND(X, NOT(LSH(1, Y))) -> BIT_CLEAR (reset bit Y of X) +// +// where op1 becomes the value (read-modify-write destination) and op2 becomes the bit index Y. +// The `1 << Y` sub-tree may appear on either side of the commutative operation. +// +// Arguments: +// binOp - a GT_OR, GT_XOR, or GT_AND node of TYP_INT or TYP_LONG +// +// Return Value: +// The rewritten node (== binOp) on success, or nullptr if the pattern did not match. +// +// Notes: +// Only the variable-index form is handled. A constant index folds to a constant mask that the +// plain `or`/`xor`/`and`-with-immediate form already handles optimally. +// +// The bit index is left as-is; callers are responsible for any target-specific masking of the +// index. x86's `bts`/`btr`/`btc` reg,reg form masks the index modulo the operand width (matching +// the C# masked-shift semantics of `1 << Y`), whereas RISC-V's `Zbs` ops operate on the full +// register and need an explicit `& 31` for 32-bit operands. +GenTree* Lowering::TryLowerBitwiseOpToBitOp(GenTreeOp* binOp) +{ + assert(binOp->OperIs(GT_OR, GT_XOR, GT_AND)); + + if (!binOp->TypeIs(TYP_INT, TYP_LONG)) + { + return nullptr; + } + + GenTree*& op1 = binOp->gtOp1; + GenTree*& op2 = binOp->gtOp2; + + bool isOp1Negated = op1->OperIs(GT_NOT); + bool isOp2Negated = op2->OperIs(GT_NOT); + + // For AND/`btr` the `1 << Y` must be negated (`~(1 << Y)`); for OR/XOR it must not be. + const bool wantNegated = binOp->OperIs(GT_AND); + GenTree* opp1 = isOp1Negated ? op1->AsUnOp()->gtGetOp1() : op1; + GenTree* opp2 = isOp2Negated ? op2->AsUnOp()->gtGetOp1() : op2; + + bool isOp1SingleBit = (isOp1Negated == wantNegated) && opp1->OperIs(GT_LSH) && opp1->gtGetOp1()->IsIntegralConst(1); + bool isOp2SingleBit = (isOp2Negated == wantNegated) && opp2->OperIs(GT_LSH) && opp2->gtGetOp1()->IsIntegralConst(1); + + if (!isOp1SingleBit && !isOp2SingleBit) + { + return nullptr; + } + + // Canonicalize so the `1 << Y` sub-tree is op2 and the value is op1. + if (isOp1SingleBit) + { + std::swap(op1, op2); + std::swap(isOp1Negated, isOp2Negated); + } + + GenTree* notNode = isOp2Negated ? op2 : nullptr; + GenTree* lshNode = (notNode != nullptr) ? op2->AsUnOp()->gtGetOp1() : op2; + + // The shifted value must match the width of the operation. + if (!lshNode->TypeIs(binOp->TypeGet())) + { + return nullptr; + } + + // A constant index folds to a constant mask, which the plain form already handles optimally. + GenTree* indexNode = lshNode->gtGetOp2(); + if (indexNode->IsIntegralConst()) + { + return nullptr; + } + + // Subsequent nodes may rely on CPU flags set by these nodes, in which case we cannot remove them. + if (((binOp->gtFlags & GTF_SET_FLAGS) != 0) || ((lshNode->gtFlags & GTF_SET_FLAGS) != 0) || + ((notNode != nullptr) && ((notNode->gtFlags & GTF_SET_FLAGS) != 0))) + { + return nullptr; + } + + static_assert(AreContiguous(GT_OR, GT_XOR, GT_AND), ""); + constexpr genTreeOps singleBitOpers[] = {GT_BIT_SET, GT_BIT_INVERT, GT_BIT_CLEAR}; + const genTreeOps newOper = singleBitOpers[binOp->OperGet() - GT_OR]; + + JITDUMP("Lower: optimize %s(X, %s)\n", GenTree::OpName(binOp->OperGet()), + (notNode != nullptr) ? "NOT(LSH(1, Y))" : "LSH(1, Y)"); + DISPNODE(binOp); + + // Rewrite in place: op1 stays the value, op2 becomes the bit index. Drop the `1`, the shift, and + // the optional NOT. + if (notNode != nullptr) + { + BlockRange().Remove(notNode); + } + BlockRange().Remove(lshNode->gtGetOp1()); + BlockRange().Remove(lshNode); + + op2 = indexNode; + binOp->ChangeOper(newOper); + + JITDUMP("to:\n"); + DISPNODE(binOp); + + return binOp; +} +#endif // TARGET_XARCH || TARGET_RISCV64 + //------------------------------------------------------------------------ // TryRemoveCast: // Try to remove a cast node by changing its operand. diff --git a/src/coreclr/jit/lower.h b/src/coreclr/jit/lower.h index 0313bf4871087d..16e6700c3b5d3d 100644 --- a/src/coreclr/jit/lower.h +++ b/src/coreclr/jit/lower.h @@ -453,6 +453,10 @@ class Lowering final : public Phase bool TryRemoveCast(GenTreeCast* node); bool TryRemoveBitCast(GenTreeUnOp* node); +#if defined(TARGET_XARCH) || defined(TARGET_RISCV64) + GenTree* TryLowerBitwiseOpToBitOp(GenTreeOp* binOp); +#endif // TARGET_XARCH || TARGET_RISCV64 + #ifdef TARGET_XARCH GenTree* TryLowerMulWithConstant(GenTreeOp* node); #endif // TARGET_XARCH @@ -510,6 +514,7 @@ class Lowering final : public Phase GenTree* TryLowerAndOpToResetLowestSetBit(GenTreeOp* andNode); GenTree* TryLowerAndOpToExtractLowestSetBit(GenTreeOp* andNode); GenTree* TryLowerAndOpToAndNot(GenTreeOp* andNode); + GenTree* TryLowerAndOpToZeroHighBits(GenTreeOp* andNode); GenTree* TryLowerXorOpToGetMaskUpToLowestSetBit(GenTreeOp* xorNode); void LowerBswapOp(GenTreeOp* node); GenTree* LowerHWIntrinsicDotInnerMulSum(GenTreeHWIntrinsic* node); diff --git a/src/coreclr/jit/lowerriscv64.cpp b/src/coreclr/jit/lowerriscv64.cpp index c16f12b5168387..f028fd8bda53a7 100644 --- a/src/coreclr/jit/lowerriscv64.cpp +++ b/src/coreclr/jit/lowerriscv64.cpp @@ -329,44 +329,13 @@ GenTree* Lowering::LowerBinaryArithmetic(GenTreeOp* binOp) } else // op2 is not constant { - GenTree* opp1 = isOp1Negated ? op1->gtGetOp1() : op1; - GenTree* opp2 = isOp2Negated ? op2->gtGetOp1() : op2; - - bool isOp1SingleBit = - (isOp1Negated == binOp->OperIs(GT_AND)) && opp1->OperIs(GT_LSH) && opp1->gtGetOp1()->IsIntegralConst(1); - bool isOp2SingleBit = - (isOp2Negated == binOp->OperIs(GT_AND)) && opp2->OperIs(GT_LSH) && opp2->gtGetOp1()->IsIntegralConst(1); - - if (isOp1SingleBit || isOp2SingleBit) + // a | (1 << b), a ^ (1 << b), a & ~(1 << b) => BIT_{SET,INVERT,CLEAR}(a, b) + if (TryLowerBitwiseOpToBitOp(binOp) != nullptr) { - // a | (1 << b), a ^ (1 << b), a & ~(1 << b) => BIT_{SET,INVERT,CLEAR}(a, b) - - if (isOp1SingleBit) - std::swap(op1, op2); - - if (binOp->OperIs(GT_AND)) - { - assert(op2->OperIs(GT_NOT)); - BlockRange().Remove(op2); - op2 = op2->gtGetOp1(); - } - - assert(binOp->OperIs(GT_OR, GT_XOR, GT_AND)); - static_assert(AreContiguous(GT_OR, GT_XOR, GT_AND), ""); - constexpr genTreeOps singleBitOpers[] = {GT_BIT_SET, GT_BIT_INVERT, GT_BIT_CLEAR}; - binOp->ChangeOper(singleBitOpers[binOp->OperGet() - GT_OR]); - - assert(op2->OperIs(GT_LSH)); - assert(op2->gtGetOp1()->IsIntegralConst(1)); - assert(!op2->gtGetOp2()->IsIntegralConst()); - BlockRange().Remove(op2->gtGetOp1()); - BlockRange().Remove(op2); - op2 = op2->gtGetOp2(); // shift amount becomes bit index - - assert(op1->TypeIs(TYP_INT, TYP_LONG)); - if (!op2->IsIntegralConst() && op1->TypeIs(TYP_INT)) + // binOp is now BIT_{SET,INVERT,CLEAR} with op2 as the (variable) bit index. + if (op1->TypeIs(TYP_INT)) { - // Zbs instructions don't have *w variants so wrap the bit index / shift amount to 0-31 manually + // Zbs instructions don't have *w variants so wrap the bit index to 0-31 manually. GenTreeIntCon* mask = m_compiler->gtNewIconNode(0x1F); mask->SetContained(); BlockRange().InsertAfter(op2, mask); diff --git a/src/coreclr/jit/lowerxarch.cpp b/src/coreclr/jit/lowerxarch.cpp index fdc6032f8a9209..0415784587d746 100644 --- a/src/coreclr/jit/lowerxarch.cpp +++ b/src/coreclr/jit/lowerxarch.cpp @@ -260,6 +260,15 @@ GenTree* Lowering::LowerMul(GenTreeOp* mul) // GenTree* Lowering::LowerBinaryArithmetic(GenTreeOp* binOp) { + if (m_compiler->opts.OptimizationEnabled() && varTypeIsIntegral(binOp) && binOp->OperIs(GT_OR, GT_XOR, GT_AND)) + { + GenTree* replacementNode = TryLowerBitwiseOpToBitOp(binOp); + if (replacementNode != nullptr) + { + return replacementNode->gtNext; + } + } + #ifdef FEATURE_HW_INTRINSICS if (m_compiler->opts.OptimizationEnabled() && varTypeIsIntegral(binOp)) { @@ -282,6 +291,12 @@ GenTree* Lowering::LowerBinaryArithmetic(GenTreeOp* binOp) { return replacementNode->gtNext; } + + replacementNode = TryLowerAndOpToZeroHighBits(binOp); + if (replacementNode != nullptr) + { + return replacementNode->gtNext; + } } else if (binOp->OperIs(GT_XOR)) { @@ -6295,7 +6310,8 @@ bool Lowering::TryInvertMask(GenTree* node, unsigned simdSize, var_types simdBas } //---------------------------------------------------------------------------------------------- -// Lowering::TryLowerAndOpToResetLowestSetBit: Lowers a tree AND(X, ADD(X, -1)) to HWIntrinsic::ResetLowestSetBit +// Lowering::TryLowerAndOpToResetLowestSetBit: Lowers a tree AND(X, ADD(X, -1)) (or the equivalent +// AND(X, SUB(X, 1))) to HWIntrinsic::ResetLowestSetBit // // Arguments: // andNode - GT_AND node of integral type @@ -6316,13 +6332,31 @@ GenTree* Lowering::TryLowerAndOpToResetLowestSetBit(GenTreeOp* andNode) } GenTree* op2 = andNode->gtGetOp2(); - if (!op2->OperIs(GT_ADD)) + + // op2 must be ADD(X, -1), or the equivalent, un-canonicalized SUB(X, 1). Global morph normally + // rewrites the subtraction into the addition form, but that only happens during global morph so + // a SUB introduced afterwards can still reach here. + ssize_t expectedConst; + if (op2->OperIs(GT_ADD)) + { + expectedConst = -1; + } + else if (op2->OperIs(GT_SUB)) + { + expectedConst = 1; + } + else + { + return nullptr; + } + + if (op2->gtOverflow()) { return nullptr; } GenTree* addOp2 = op2->gtGetOp2(); - if (!addOp2->IsIntegralConst(-1)) + if (!addOp2->IsIntegralConst(expectedConst)) { return nullptr; } @@ -6340,18 +6374,23 @@ GenTree* Lowering::TryLowerAndOpToResetLowestSetBit(GenTreeOp* andNode) return nullptr; } - NamedIntrinsic intrinsic; - if (op1->TypeIs(TYP_LONG) && m_compiler->compOpportunisticallyDependsOn(InstructionSet_AVX2_X64)) + if (!m_compiler->compOpportunisticallyDependsOn(InstructionSet_AVX2)) { - intrinsic = NamedIntrinsic::NI_AVX2_X64_ResetLowestSetBit; + return nullptr; } - else if (m_compiler->compOpportunisticallyDependsOn(InstructionSet_AVX2)) + + NamedIntrinsic intrinsic; +#if defined(TARGET_AMD64) + if (andNode->TypeIs(TYP_LONG)) { - intrinsic = NamedIntrinsic::NI_AVX2_ResetLowestSetBit; + intrinsic = NamedIntrinsic::NI_AVX2_X64_ResetLowestSetBit; } else +#endif // TARGET_AMD64 { - return nullptr; + // On x86 longs are decomposed before lowering, so only the 32-bit form is reachable here. + assert(andNode->TypeIs(TYP_INT)); + intrinsic = NamedIntrinsic::NI_AVX2_ResetLowestSetBit; } LIR::Use use; @@ -6425,18 +6464,23 @@ GenTree* Lowering::TryLowerAndOpToExtractLowestSetBit(GenTreeOp* andNode) return nullptr; } - NamedIntrinsic intrinsic; - if (andNode->TypeIs(TYP_LONG) && m_compiler->compOpportunisticallyDependsOn(InstructionSet_AVX2_X64)) + if (!m_compiler->compOpportunisticallyDependsOn(InstructionSet_AVX2)) { - intrinsic = NamedIntrinsic::NI_AVX2_X64_ExtractLowestSetBit; + return nullptr; } - else if (m_compiler->compOpportunisticallyDependsOn(InstructionSet_AVX2)) + + NamedIntrinsic intrinsic; +#if defined(TARGET_AMD64) + if (andNode->TypeIs(TYP_LONG)) { - intrinsic = NamedIntrinsic::NI_AVX2_ExtractLowestSetBit; + intrinsic = NamedIntrinsic::NI_AVX2_X64_ExtractLowestSetBit; } else +#endif // TARGET_AMD64 { - return nullptr; + // On x86 longs are decomposed before lowering, so only the 32-bit form is reachable here. + assert(andNode->TypeIs(TYP_INT)); + intrinsic = NamedIntrinsic::NI_AVX2_ExtractLowestSetBit; } LIR::Use use; @@ -6464,6 +6508,169 @@ GenTree* Lowering::TryLowerAndOpToExtractLowestSetBit(GenTreeOp* andNode) return blsiNode; } +//---------------------------------------------------------------------------------------------- +// Lowering::TryLowerAndOpToZeroHighBits: Lowers a tree AND(X, ADD(LSH(1, Y), -1)) (or the +// equivalent AND(X, SUB(LSH(1, Y), 1))) to HWIntrinsic::ZeroHighBits (the BMI2 `bzhi` instruction), +// which zeroes the bits of X starting at bit position Y. +// +// Arguments: +// andNode - GT_AND node of integral type +// +// Return Value: +// Returns the replacement node if one is created else nullptr indicating no replacement +// +// Notes: +// Performs containment checks on the replacement node if one is created. +// +// The bit index is masked to the operand width (`y & 31` for `int`, `y & 63` for `long`) before +// being handed to `bzhi`. C#'s `<<` masks the shift count modulo the operand width, so the mask +// `(1 << y) - 1` is well-defined for any `y` and this JIT already models `shl`/`shr` as masked +// (see `LowerShift`, which drops redundant `& 31`/`& 63`). `bzhi`, in contrast, leaves the source +// unchanged when the index is `>= width`, so without masking the index it would diverge from the +// source pattern for those inputs. Masking the index makes `bzhi` reproduce the result exactly. +GenTree* Lowering::TryLowerAndOpToZeroHighBits(GenTreeOp* andNode) +{ + assert(andNode->OperIs(GT_AND) && varTypeIsIntegral(andNode)); + + if (!andNode->TypeIs(TYP_INT, TYP_LONG)) + { + return nullptr; + } + + // The mask `(1 << y) - 1` may be on either side of the AND. Global morph normally canonicalizes + // the subtraction to ADD(LSH(1, y), -1), but that only happens during global morph, so a SUB + // introduced afterwards (or by a later phase) can still reach here; recognize both forms. + GenTree* srcNode = nullptr; + GenTree* maskNode = nullptr; + if (andNode->gtGetOp2()->OperIs(GT_ADD, GT_SUB)) + { + maskNode = andNode->gtGetOp2(); + srcNode = andNode->gtGetOp1(); + } + else if (andNode->gtGetOp1()->OperIs(GT_ADD, GT_SUB)) + { + maskNode = andNode->gtGetOp1(); + srcNode = andNode->gtGetOp2(); + } + else + { + return nullptr; + } + + // maskNode must be ADD(LSH(1, y), -1) or the equivalent, un-canonicalized SUB(LSH(1, y), 1). + GenTree* lshNode = nullptr; + GenTree* constNode = nullptr; + if (maskNode->OperIs(GT_ADD) && maskNode->gtGetOp2()->IsIntegralConst(-1) && maskNode->gtGetOp1()->OperIs(GT_LSH)) + { + lshNode = maskNode->gtGetOp1(); + constNode = maskNode->gtGetOp2(); + } + else if (maskNode->OperIs(GT_ADD) && maskNode->gtGetOp1()->IsIntegralConst(-1) && + maskNode->gtGetOp2()->OperIs(GT_LSH)) + { + lshNode = maskNode->gtGetOp2(); + constNode = maskNode->gtGetOp1(); + } + else if (maskNode->OperIs(GT_SUB) && maskNode->gtGetOp2()->IsIntegralConst(1) && + maskNode->gtGetOp1()->OperIs(GT_LSH)) + { + // Only SUB(LSH(1, y), 1) matches; SUB(1, LSH(1, y)) computes a different value. + lshNode = maskNode->gtGetOp1(); + constNode = maskNode->gtGetOp2(); + } + else + { + return nullptr; + } + + if (maskNode->gtOverflow()) + { + return nullptr; + } + + if (!lshNode->gtGetOp1()->IsIntegralConst(1)) + { + return nullptr; + } + GenTree* indexNode = lshNode->gtGetOp2(); + + // A constant shift count folds `(1 << Y) - 1` to a constant mask during morph, so a constant + // index should never reach here. Enforce that variable-index contract explicitly: `bzhi` takes + // its index in a register (there is no immediate form), so lowering a constant index would + // regress the optimal `and reg, imm` into `mov reg, imm` + `bzhi`. Bail and let the plain `and` + // stand -- this mirrors how the SUB form above guards against post-morph shapes. + if (indexNode->IsIntegralConst()) + { + return nullptr; + } + + // Subsequent nodes may rely on CPU flags set by these nodes in which case we cannot remove them + if (((andNode->gtFlags & GTF_SET_FLAGS) != 0) || ((maskNode->gtFlags & GTF_SET_FLAGS) != 0) || + ((lshNode->gtFlags & GTF_SET_FLAGS) != 0)) + { + return nullptr; + } + + if (!m_compiler->compOpportunisticallyDependsOn(InstructionSet_AVX2)) + { + return nullptr; + } + + NamedIntrinsic intrinsic; +#if defined(TARGET_AMD64) + if (andNode->TypeIs(TYP_LONG)) + { + intrinsic = NamedIntrinsic::NI_AVX2_X64_ZeroHighBits; + } + else +#endif // TARGET_AMD64 + { + // On x86 longs are decomposed before lowering, so only the 32-bit form is reachable here. + assert(andNode->TypeIs(TYP_INT)); + intrinsic = NamedIntrinsic::NI_AVX2_ZeroHighBits; + } + + LIR::Use use; + if (!BlockRange().TryGetUse(andNode, &use)) + { + return nullptr; + } + + // The backend expects op1 to be the index (encoded in VEX.vvvv) and op2 to be the value + // (which may be a memory operand), matching the import order for `ZeroHighBits`. + // + // Mask the index modulo the operand width so `bzhi` reproduces the C# masked-shift semantics of + // `1 << Y` even when `Y >= width` (where `bzhi` would otherwise leave the source unchanged). The + // index is guaranteed non-constant here (enforced by the bail above), so the mask is always + // applied to a variable and cannot be folded away. + GenTree* maskCns = m_compiler->gtNewIconNode(andNode->TypeIs(TYP_LONG) ? 63 : 31, genActualType(indexNode)); + GenTree* indexMask = m_compiler->gtNewOperNode(GT_AND, genActualType(indexNode), indexNode, maskCns); + + GenTreeHWIntrinsic* bzhiNode = + m_compiler->gtNewScalarHWIntrinsicNode(andNode->TypeGet(), indexMask, srcNode, intrinsic); + + JITDUMP("Lower: optimize AND(X, ADD(LSH(1, Y), -1))\n"); + DISPNODE(andNode); + JITDUMP("to:\n"); + DISPNODE(bzhiNode); + + BlockRange().InsertBefore(andNode, maskCns); + BlockRange().InsertBefore(andNode, indexMask); + BlockRange().InsertBefore(andNode, bzhiNode); + use.ReplaceWith(bzhiNode); + + BlockRange().Remove(andNode); + BlockRange().Remove(maskNode); + BlockRange().Remove(lshNode); + BlockRange().Remove(lshNode->gtGetOp1()); + BlockRange().Remove(constNode); + + ContainCheckBinary(indexMask->AsOp()); + ContainCheckHWIntrinsic(bzhiNode); + + return bzhiNode; +} + //---------------------------------------------------------------------------------------------- // Lowering::TryLowerAndOpToAndNot: Lowers a tree AND(X, NOT(Y)) to HWIntrinsic::AndNot // @@ -6551,8 +6758,8 @@ GenTree* Lowering::TryLowerAndOpToAndNot(GenTreeOp* andNode) } //---------------------------------------------------------------------------------------------- -// Lowering::TryLowerXorOpToGetMaskUpToLowestSetBit: Lowers a tree XOR(X, ADD(X, -1)) to -// HWIntrinsic::GetMaskUpToLowestSetBit +// Lowering::TryLowerXorOpToGetMaskUpToLowestSetBit: Lowers a tree XOR(X, ADD(X, -1)) (or the +// equivalent XOR(X, SUB(X, 1))) to HWIntrinsic::GetMaskUpToLowestSetBit // // Arguments: // xorNode - GT_XOR node of integral type @@ -6573,13 +6780,31 @@ GenTree* Lowering::TryLowerXorOpToGetMaskUpToLowestSetBit(GenTreeOp* xorNode) } GenTree* op2 = xorNode->gtGetOp2(); - if (!op2->OperIs(GT_ADD)) + + // op2 must be ADD(X, -1), or the equivalent, un-canonicalized SUB(X, 1). Global morph normally + // rewrites the subtraction into the addition form, but that only happens during global morph so + // a SUB introduced afterwards can still reach here. + ssize_t expectedConst; + if (op2->OperIs(GT_ADD)) + { + expectedConst = -1; + } + else if (op2->OperIs(GT_SUB)) + { + expectedConst = 1; + } + else + { + return nullptr; + } + + if (op2->gtOverflow()) { return nullptr; } GenTree* addOp2 = op2->gtGetOp2(); - if (!addOp2->IsIntegralConst(-1)) + if (!addOp2->IsIntegralConst(expectedConst)) { return nullptr; } @@ -6597,18 +6822,23 @@ GenTree* Lowering::TryLowerXorOpToGetMaskUpToLowestSetBit(GenTreeOp* xorNode) return nullptr; } - NamedIntrinsic intrinsic; - if (xorNode->TypeIs(TYP_LONG) && m_compiler->compOpportunisticallyDependsOn(InstructionSet_AVX2_X64)) + if (!m_compiler->compOpportunisticallyDependsOn(InstructionSet_AVX2)) { - intrinsic = NamedIntrinsic::NI_AVX2_X64_GetMaskUpToLowestSetBit; + return nullptr; } - else if (m_compiler->compOpportunisticallyDependsOn(InstructionSet_AVX2)) + + NamedIntrinsic intrinsic; +#if defined(TARGET_AMD64) + if (xorNode->TypeIs(TYP_LONG)) { - intrinsic = NamedIntrinsic::NI_AVX2_GetMaskUpToLowestSetBit; + intrinsic = NamedIntrinsic::NI_AVX2_X64_GetMaskUpToLowestSetBit; } else +#endif // TARGET_AMD64 { - return nullptr; + // On x86 longs are decomposed before lowering, so only the 32-bit form is reachable here. + assert(xorNode->TypeIs(TYP_INT)); + intrinsic = NamedIntrinsic::NI_AVX2_GetMaskUpToLowestSetBit; } LIR::Use use; diff --git a/src/coreclr/jit/lsraxarch.cpp b/src/coreclr/jit/lsraxarch.cpp index a3b7f34d568027..c423b86e2ab480 100644 --- a/src/coreclr/jit/lsraxarch.cpp +++ b/src/coreclr/jit/lsraxarch.cpp @@ -315,6 +315,9 @@ int LinearScan::BuildNode(GenTree* tree) case GT_AND: case GT_OR: case GT_XOR: + case GT_BIT_SET: + case GT_BIT_CLEAR: + case GT_BIT_INVERT: srcCount = BuildBinaryUses(tree->AsOp()); assert(dstCount == 1); BuildDef(tree); diff --git a/src/tests/JIT/opt/InstructionCombining/Bzhi.cs b/src/tests/JIT/opt/InstructionCombining/Bzhi.cs new file mode 100644 index 00000000000000..66e366574815ab --- /dev/null +++ b/src/tests/JIT/opt/InstructionCombining/Bzhi.cs @@ -0,0 +1,60 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Runtime.CompilerServices; +using Xunit; + +// Correctness coverage for the xarch `bzhi` (ZeroHighBits) lowering. +// +// The transform is opportunistic on AVX2, so this test intentionally avoids disasm checks (which +// would be flaky on hardware without AVX2) and instead validates that the optimized expression +// produces the same results as an independent oracle. +// +// C#'s `<<` masks the shift count modulo the operand width, so `x & ((1 << y) - 1)` is well-defined +// for *any* y (e.g. `1 << 32 == 1 << 0`). The lowering must reproduce that for out-of-range y too, +// so the index range below deliberately spans well past the operand width. The oracle routes the +// shift through a NoInlining `1 << y` helper: that emits a bare (masked) `shl` which is never itself +// recognized as `bzhi`, so it is an independent reference rather than a copy of the candidate. +public static class Bzhi +{ + [MethodImpl(MethodImplOptions.NoInlining)] static int ShlOne(int y) => 1 << y; + [MethodImpl(MethodImplOptions.NoInlining)] static long ShlOneL(int y) => 1L << y; + + // Oracle for `x & ((1 << y) - 1)` using masked-shift (C#-defined) semantics. + static int ZeroHighRef(int x, int y) => x & (ShlOne(y) - 1); + static long ZeroHighRef(long x, int y) => x & (ShlOneL(y) - 1); + + // bzhi candidates (variable index). + [MethodImpl(MethodImplOptions.NoInlining)] static int Bzhi_I(int x, int y) => x & ((1 << y) - 1); + [MethodImpl(MethodImplOptions.NoInlining)] static long Bzhi_L(long x, int y) => x & ((1L << y) - 1); + + [Fact] + public static void Test() + { + var rng = new Random(12345); + + // Boundary indices around and beyond both operand widths (32 and 64). + int[] boundaries = { 0, 1, 15, 30, 31, 32, 33, 63, 64, 65, 95, 96, 127, 128, 200, 255 }; + + for (int i = 0; i < 5000; i++) + { + uint xu = (uint)rng.Next() ^ ((uint)rng.Next() << 1); + int xi = (int)xu; + ulong xul = ((ulong)xu << 32) | (uint)rng.Next(); + long xl = (long)xul; + + foreach (int y in boundaries) + { + Assert.Equal(ZeroHighRef(xi, y), Bzhi_I(xi, y)); + Assert.Equal(ZeroHighRef(xl, y), Bzhi_L(xl, y)); + } + + // Random indices spanning past the operand width to exercise masked-shift semantics. + int yi = rng.Next(0, 256); + int yl = rng.Next(0, 256); + Assert.Equal(ZeroHighRef(xi, yi), Bzhi_I(xi, yi)); + Assert.Equal(ZeroHighRef(xl, yl), Bzhi_L(xl, yl)); + } + } +} diff --git a/src/tests/JIT/opt/InstructionCombining/Bzhi.csproj b/src/tests/JIT/opt/InstructionCombining/Bzhi.csproj new file mode 100644 index 00000000000000..54dfc20c185b6d --- /dev/null +++ b/src/tests/JIT/opt/InstructionCombining/Bzhi.csproj @@ -0,0 +1,15 @@ + + + + true + + + None + True + + + + + + + diff --git a/src/tests/JIT/opt/InstructionCombining/CheckedBitOps.cs b/src/tests/JIT/opt/InstructionCombining/CheckedBitOps.cs new file mode 100644 index 00000000000000..3bc3b15c709c5f --- /dev/null +++ b/src/tests/JIT/opt/InstructionCombining/CheckedBitOps.cs @@ -0,0 +1,52 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Runtime.CompilerServices; +using Xunit; + +// Correctness coverage for the xarch BMI lowerings when the `- 1` subtraction is a checked +// (overflowing) operation. `blsr` (AND(X, SUB(X, 1))), `blsmsk` (XOR(X, SUB(X, 1))) and `bzhi` +// (AND(X, SUB(LSH(1, Y), 1))) all fold the subtraction away, so they must not fire when it is a +// checked subtraction that would throw -- otherwise the observable OverflowException is dropped. +public static class CheckedBitOps +{ + // blsr: x & (x - 1). Overflows for x == INT_MIN / LONG_MIN. + [MethodImpl(MethodImplOptions.NoInlining)] + static int ResetLowestChecked(int x) => x & checked(x - 1); + + [MethodImpl(MethodImplOptions.NoInlining)] + static long ResetLowestChecked(long x) => x & checked(x - 1); + + // blsmsk: x ^ (x - 1). Overflows for x == INT_MIN / LONG_MIN. + [MethodImpl(MethodImplOptions.NoInlining)] + static int MaskUpToChecked(int x) => x ^ checked(x - 1); + + [MethodImpl(MethodImplOptions.NoInlining)] + static long MaskUpToChecked(long x) => x ^ checked(x - 1); + + // bzhi: x & ((1 << y) - 1). Overflows when (1 << y) == INT_MIN (y == 31) / LONG_MIN (y == 63). + [MethodImpl(MethodImplOptions.NoInlining)] + static int ZeroHighChecked(int x, int y) => x & checked((1 << y) - 1); + + [MethodImpl(MethodImplOptions.NoInlining)] + static long ZeroHighChecked(long x, int y) => x & checked((1L << y) - 1); + + [Fact] + public static void Test() + { + Assert.Throws(() => ResetLowestChecked(int.MinValue)); + Assert.Throws(() => ResetLowestChecked(long.MinValue)); + + Assert.Throws(() => MaskUpToChecked(int.MinValue)); + Assert.Throws(() => MaskUpToChecked(long.MinValue)); + + Assert.Throws(() => ZeroHighChecked(0x1234, 31)); + Assert.Throws(() => ZeroHighChecked(0x1234L, 63)); + + // The non-overflowing inputs must still compute the expected result. + Assert.Equal(0b1100 & 0b1011, ResetLowestChecked(0b1100)); + Assert.Equal(0b1100 ^ 0b1011, MaskUpToChecked(0b1100)); + Assert.Equal(0xFF & ((1 << 4) - 1), ZeroHighChecked(0xFF, 4)); + } +} diff --git a/src/tests/JIT/opt/InstructionCombining/CheckedBitOps.csproj b/src/tests/JIT/opt/InstructionCombining/CheckedBitOps.csproj new file mode 100644 index 00000000000000..d860b95ade8e28 --- /dev/null +++ b/src/tests/JIT/opt/InstructionCombining/CheckedBitOps.csproj @@ -0,0 +1,15 @@ + + + + true + + + None + True + + + + + + + diff --git a/src/tests/JIT/opt/InstructionCombining/SingleBit.cs b/src/tests/JIT/opt/InstructionCombining/SingleBit.cs index 4f0c267720850f..48a0b8517109c2 100644 --- a/src/tests/JIT/opt/InstructionCombining/SingleBit.cs +++ b/src/tests/JIT/opt/InstructionCombining/SingleBit.cs @@ -8,13 +8,34 @@ public static class SingleBit { [MethodImpl(MethodImplOptions.NoInlining)] - static int Set(int a, int b) => a | (1 << b); + static int Set(int a, int b) + { + // X64: bts {{[a-z0-9]+}}, {{[a-z0-9]+}} + return a | (1 << b); + } + + [MethodImpl(MethodImplOptions.NoInlining)] + static int SetSwap(int a, int b) + { + // X64: bts {{[a-z0-9]+}}, {{[a-z0-9]+}} + return (1 << b) | a; + } [MethodImpl(MethodImplOptions.NoInlining)] - static int SetSwap(int a, int b) => (1 << b) | a ; + static long SetLong(long a, int b) + { + // X64: bts {{[a-z0-9]+}}, {{[a-z0-9]+}} + return a | (1L << b); + } [MethodImpl(MethodImplOptions.NoInlining)] - static int Set10(int a) => a | (1 << 10); + static int Set10(int a) + { + // A constant bit index folds to a plain 'or' with an immediate; it must not use 'bts'. + // X64-NOT: bts + // X64: or {{[a-z0-9]+}}, {{(1024|0x400)}} + return a | (1 << 10); + } [MethodImpl(MethodImplOptions.NoInlining)] static int Set11(int a) => a | (1 << 11); @@ -27,10 +48,25 @@ public static class SingleBit [MethodImpl(MethodImplOptions.NoInlining)] - static int Clear(int a, int b) => a & ~(1 << b); + static int Clear(int a, int b) + { + // X64: btr {{[a-z0-9]+}}, {{[a-z0-9]+}} + return a & ~(1 << b); + } + + [MethodImpl(MethodImplOptions.NoInlining)] + static int ClearSwap(int a, int b) + { + // X64: btr {{[a-z0-9]+}}, {{[a-z0-9]+}} + return ~(1 << b) & a; + } [MethodImpl(MethodImplOptions.NoInlining)] - static int ClearSwap(int a, int b) => ~(1 << b) & a; + static long ClearLong(long a, int b) + { + // X64: btr {{[a-z0-9]+}}, {{[a-z0-9]+}} + return a & ~(1L << b); + } [MethodImpl(MethodImplOptions.NoInlining)] static int Clear10(int a) => a & ~(1 << 10); @@ -49,10 +85,25 @@ public static class SingleBit [MethodImpl(MethodImplOptions.NoInlining)] - static int Invert(int a, int b) => a ^ (1 << b); + static int Invert(int a, int b) + { + // X64: btc {{[a-z0-9]+}}, {{[a-z0-9]+}} + return a ^ (1 << b); + } + + [MethodImpl(MethodImplOptions.NoInlining)] + static int InvertSwap(int a, int b) + { + // X64: btc {{[a-z0-9]+}}, {{[a-z0-9]+}} + return (1 << b) ^ a; + } [MethodImpl(MethodImplOptions.NoInlining)] - static int InvertSwap(int a, int b) => (1 << b) ^ a; + static long InvertLong(long a, int b) + { + // X64: btc {{[a-z0-9]+}}, {{[a-z0-9]+}} + return a ^ (1L << b); + } [MethodImpl(MethodImplOptions.NoInlining)] static int Invert10(int a) => a ^ (1 << 10); @@ -66,6 +117,68 @@ public static class SingleBit [MethodImpl(MethodImplOptions.NoInlining)] static int InvertNegatedBit(int a, int b) => ~(1 << a) ^ (1 << b); + // Bit-test recognition: testing a single bit to feed a branch should become 'bt'. Both the + // '(x >> y) & 1' and 'x & (1 << y)' shapes select bit 'y', and 'bt' masks the index modulo the + // operand size, matching the C# masked-shift semantics even for an out-of-range 'y'. + + [MethodImpl(MethodImplOptions.NoInlining)] + static int TestBitShr(int a, int b) + { + // X64: bt {{[a-z0-9]+}}, {{[a-z0-9]+}} + if (((a >> b) & 1) != 0) + { + return 100; + } + return 200; + } + + [MethodImpl(MethodImplOptions.NoInlining)] + static int TestBitShrEq(int a, int b) + { + // X64: bt {{[a-z0-9]+}}, {{[a-z0-9]+}} + if (((a >> b) & 1) == 0) + { + return 100; + } + return 200; + } + + [MethodImpl(MethodImplOptions.NoInlining)] + static int TestBitShrLong(long a, int b) + { + // X64: bt {{[a-z0-9]+}}, {{[a-z0-9]+}} + if (((a >> b) & 1) != 0) + { + return 100; + } + return 200; + } + + [MethodImpl(MethodImplOptions.NoInlining)] + static int TestBitMask(int a, int b) + { + // X64: bt {{[a-z0-9]+}}, {{[a-z0-9]+}} + if ((a & (1 << b)) != 0) + { + return 100; + } + return 200; + } + + [MethodImpl(MethodImplOptions.NoInlining)] + static int TestBitShrConst(int a) + { + // A constant bit index keeps the shift folded into a 'test' with an immediate; 'bt' has no + // immediate form here so it must not be used. + // X64-NOT: bt + // X64: test {{[a-z0-9]+}}, {{(32|0x20)}} + if (((a >> 5) & 1) != 0) + { + return 100; + } + return 200; + } + [Fact] public static void Test() { @@ -104,5 +217,31 @@ public static void Test() Assert.Equal(int.MinValue, Invert31(0)); Assert.Equal(-1, InvertNegatedBit(0, 0 + 32)); Assert.Equal(-4, InvertNegatedBit(0, 1 + 32)); + + // Long variants (exercise the 64-bit bts/btr/btc forms). The reg,reg encoding masks the + // bit index modulo 64, matching the C# masked-shift semantics even for out-of-range indices. + Assert.Equal(0x1_00000000L, SetLong(0, 32)); + Assert.Equal(0x1_00000000L, SetLong(0, 32 + 64)); + Assert.Equal(unchecked((long)0x8000000000000000UL), SetLong(0, 63)); + Assert.Equal(0x12345078L, ClearLong(0x1_12345078L, 32)); + Assert.Equal(0L, ClearLong(unchecked((long)0x8000000000000000UL), 63)); + Assert.Equal(0x1_00000000L, InvertLong(0, 32)); + Assert.Equal(0L, InvertLong(0x1_00000000L, 32)); + Assert.Equal(unchecked((long)0x8000000000000000UL), InvertLong(0, 63)); + + // Bit-test recognition. Exercise a set and a clear bit, plus an out-of-range (masked) index. + Assert.Equal(100, TestBitShr(0b1000, 3)); + Assert.Equal(200, TestBitShr(0b1000, 2)); + Assert.Equal(100, TestBitShr(0b1000, 3 + 32)); + Assert.Equal(200, TestBitShrEq(0b1000, 3)); + Assert.Equal(100, TestBitShrEq(0b1000, 2)); + Assert.Equal(100, TestBitShrLong(0x1_00000000L, 32)); + Assert.Equal(200, TestBitShrLong(0x1_00000000L, 33)); + Assert.Equal(100, TestBitShrLong(0x1_00000000L, 32 + 64)); + Assert.Equal(100, TestBitMask(0b1000, 3)); + Assert.Equal(200, TestBitMask(0b1000, 2)); + Assert.Equal(100, TestBitMask(0b1000, 3 + 32)); + Assert.Equal(100, TestBitShrConst(0b100000)); + Assert.Equal(200, TestBitShrConst(0b010000)); } } \ No newline at end of file diff --git a/src/tests/JIT/opt/InstructionCombining/SingleBit.csproj b/src/tests/JIT/opt/InstructionCombining/SingleBit.csproj index ebb108bc571dfe..1abbe59d35840d 100644 --- a/src/tests/JIT/opt/InstructionCombining/SingleBit.csproj +++ b/src/tests/JIT/opt/InstructionCombining/SingleBit.csproj @@ -8,7 +8,9 @@ True - + + true +