diff --git a/src/coreclr/jit/codegen.h b/src/coreclr/jit/codegen.h index 7820c437429dad..28451d1cbd609e 100644 --- a/src/coreclr/jit/codegen.h +++ b/src/coreclr/jit/codegen.h @@ -1233,6 +1233,7 @@ XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX void genCodeForLclFld(GenTreeLclFld* tree); void genCodeForStoreLclFld(GenTreeLclFld* tree); void genCodeForStoreLclVar(GenTreeLclVar* tree); + void genCodeForCStoreLclVar(GenTreeLclVar* tree); void genCodeForReturnTrap(GenTreeOp* tree); void genCodeForJcc(GenTreeCC* tree); void genCodeForSetcc(GenTreeCC* setcc); diff --git a/src/coreclr/jit/codegenarm64.cpp b/src/coreclr/jit/codegenarm64.cpp index 69fc1885da3e9e..ab03dfc4a64399 100644 --- a/src/coreclr/jit/codegenarm64.cpp +++ b/src/coreclr/jit/codegenarm64.cpp @@ -2647,6 +2647,39 @@ void CodeGen::genCodeForStoreLclVar(GenTreeLclVar* lclNode) } } +//------------------------------------------------------------------------ +// genCodeForStoreLclVar: Produce code for a GT_CSTORE_LCL_VAR node. +// +// Arguments: +// lclNode - the GT_CSTORE_LCL_VAR node +// +void CodeGen::genCodeForCStoreLclVar(GenTreeLclVar* lclNode) +{ + emitter* emit = GetEmitter(); + + insCond cond = ((lclNode->gtFlags & GTF_CSEL_T) != 0) ? INS_COND_EQ : INS_COND_NE; + + emitAttr attr = emitActualTypeSize(lclNode->TypeGet()); + + // AHTODO: do we need this? + // // TODO-CQ: Currently we cannot do this for all handles because of + // // https://github.com/dotnet/runtime/issues/60712 + // if (con->ImmedValNeedsReloc(compiler)) + // { + // attr = EA_SET_FLG(attr, EA_CNS_RELOC_FLG); + // } + // if (targetType == TYP_BYREF) + // { + // attr = EA_SET_FLG(attr, EA_BYREF_FLG); + // } + + regNumber targetReg = lclNode->GetRegNum(); + regNumber srcReg = genConsumeReg(lclNode->gtGetOp1()); + + emit->emitIns_R_R_R_COND(INS_csel, attr, targetReg, srcReg, targetReg, cond); + regSet.verifyRegUsed(targetReg); +} + //------------------------------------------------------------------------ // genSimpleReturn: Generates code for simple return statement for arm64. // diff --git a/src/coreclr/jit/codegenarmarch.cpp b/src/coreclr/jit/codegenarmarch.cpp index 01fbaff7a8c4d7..17e3b435e78359 100644 --- a/src/coreclr/jit/codegenarmarch.cpp +++ b/src/coreclr/jit/codegenarmarch.cpp @@ -279,6 +279,10 @@ void CodeGen::genCodeForTreeNode(GenTree* treeNode) genCodeForStoreLclVar(treeNode->AsLclVar()); break; + case GT_CSTORE_LCL_VAR: + genCodeForCStoreLclVar(treeNode->AsLclVar()); + break; + case GT_RETFILT: case GT_RETURN: genReturn(treeNode); diff --git a/src/coreclr/jit/compiler.cpp b/src/coreclr/jit/compiler.cpp index 0920016eac1f58..4f5ec3b0027f64 100644 --- a/src/coreclr/jit/compiler.cpp +++ b/src/coreclr/jit/compiler.cpp @@ -9134,6 +9134,7 @@ void cTreeFlags(Compiler* comp, GenTree* tree) case GT_LCL_FLD_ADDR: case GT_STORE_LCL_FLD: case GT_STORE_LCL_VAR: + case GT_CSTORE_LCL_VAR: if (tree->gtFlags & GTF_VAR_DEF) { chars += printf("[VAR_DEF]"); diff --git a/src/coreclr/jit/compiler.h b/src/coreclr/jit/compiler.h index 2eb6c68b0c53a6..c43867d3c6e263 100644 --- a/src/coreclr/jit/compiler.h +++ b/src/coreclr/jit/compiler.h @@ -11506,6 +11506,7 @@ class GenTreeVisitor // Lclvar unary operators case GT_STORE_LCL_VAR: + case GT_CSTORE_LCL_VAR: case GT_STORE_LCL_FLD: if (TVisitor::DoLclVarsOnly) { diff --git a/src/coreclr/jit/decomposelongs.cpp b/src/coreclr/jit/decomposelongs.cpp index 9b75e4c73727b9..264bf69a842d6c 100644 --- a/src/coreclr/jit/decomposelongs.cpp +++ b/src/coreclr/jit/decomposelongs.cpp @@ -174,6 +174,7 @@ GenTree* DecomposeLongs::DecomposeNode(GenTree* tree) case GT_STORE_LCL_VAR: nextNode = DecomposeStoreLclVar(use); break; + // case GT_CSTORE_LCL_VAR: Do nothing? case GT_CAST: nextNode = DecomposeCast(use); diff --git a/src/coreclr/jit/gcinfo.cpp b/src/coreclr/jit/gcinfo.cpp index 85e7f688c2f2ef..05e197e37f3f7b 100644 --- a/src/coreclr/jit/gcinfo.cpp +++ b/src/coreclr/jit/gcinfo.cpp @@ -278,6 +278,7 @@ GCInfo::WriteBarrierForm GCInfo::gcIsWriteBarrierCandidate(GenTree* tgt, GenTree case GT_LCL_VAR: /* Definitely not in the managed heap */ case GT_LCL_FLD: case GT_STORE_LCL_VAR: + case GT_CSTORE_LCL_VAR: case GT_STORE_LCL_FLD: return WBF_NoBarrier; diff --git a/src/coreclr/jit/gentree.cpp b/src/coreclr/jit/gentree.cpp index 86b2ee82d7a1d1..92b3f45a8a02fc 100644 --- a/src/coreclr/jit/gentree.cpp +++ b/src/coreclr/jit/gentree.cpp @@ -9600,6 +9600,7 @@ void Compiler::gtDispNode(GenTree* tree, IndentStack* indentStack, _In_ _In_opt_ case GT_LCL_FLD_ADDR: case GT_STORE_LCL_FLD: case GT_STORE_LCL_VAR: + case GT_CSTORE_LCL_VAR: if (tree->gtFlags & GTF_VAR_USEASG) { printf("U"); @@ -9799,7 +9800,7 @@ void Compiler::gtDispNode(GenTree* tree, IndentStack* indentStack, _In_ _In_opt_ { layout = tree->AsBlk()->GetLayout(); } - else if (tree->OperIs(GT_LCL_VAR, GT_STORE_LCL_VAR)) + else if (tree->OperIs(GT_LCL_VAR, GT_STORE_LCL_VAR, GT_CSTORE_LCL_VAR)) { LclVarDsc* varDsc = lvaGetDesc(tree->AsLclVar()); @@ -9844,7 +9845,7 @@ void Compiler::gtDispNode(GenTree* tree, IndentStack* indentStack, _In_ _In_opt_ } } - if (tree->gtOper == GT_LCL_VAR || tree->gtOper == GT_STORE_LCL_VAR) + if (tree->gtOper == GT_LCL_VAR || tree->gtOper == GT_STORE_LCL_VAR || tree->gtOper == GT_CSTORE_LCL_VAR) { LclVarDsc* varDsc = lvaGetDesc(tree->AsLclVarCommon()); if (varDsc->IsAddressExposed()) @@ -10539,6 +10540,7 @@ void Compiler::gtDispLeaf(GenTree* tree, IndentStack* indentStack) case GT_LCL_VAR: case GT_LCL_VAR_ADDR: case GT_STORE_LCL_VAR: + case GT_CSTORE_LCL_VAR: { printf(" "); const unsigned varNum = tree->AsLclVarCommon()->GetLclNum(); diff --git a/src/coreclr/jit/gentree.h b/src/coreclr/jit/gentree.h index 98cdcc0d15f58d..bda522c8691d22 100644 --- a/src/coreclr/jit/gentree.h +++ b/src/coreclr/jit/gentree.h @@ -400,6 +400,9 @@ enum GenTreeFlags : unsigned int GTF_LATE_ARG = 0x00010000, // The specified node is evaluated to a temp in the arg list, and this temp is added to gtCallLateArgs. GTF_SPILL = 0x00020000, // Needs to be spilled here + GTF_CSEL_T = 0x00040000, // Conditional op should check for True flags + GTF_CSEL_F = 0x00080000, // Conditional op should check for False flags + // The extra flag GTF_IS_IN_CSE is used to tell the consumer of the side effect flags // that we are calling in the context of performing a CSE, thus we // should allow the run-once side effects of running a class constructor. @@ -1165,7 +1168,7 @@ struct GenTree static bool OperIsLocal(genTreeOps gtOper) { static_assert_no_msg( - OpersAreContiguous(GT_PHI_ARG, GT_LCL_VAR, GT_LCL_FLD, GT_STORE_LCL_VAR, GT_STORE_LCL_FLD)); + OpersAreContiguous(GT_PHI_ARG, GT_LCL_VAR, GT_LCL_FLD, GT_STORE_LCL_VAR, GT_CSTORE_LCL_VAR, GT_STORE_LCL_FLD)); return (GT_PHI_ARG <= gtOper) && (gtOper <= GT_STORE_LCL_FLD); } @@ -1186,7 +1189,7 @@ struct GenTree static bool OperIsScalarLocal(genTreeOps gtOper) { - return (gtOper == GT_LCL_VAR || gtOper == GT_STORE_LCL_VAR); + return (gtOper == GT_LCL_VAR || gtOper == GT_STORE_LCL_VAR || gtOper == GT_CSTORE_LCL_VAR); } static bool OperIsNonPhiLocal(genTreeOps gtOper) @@ -1201,7 +1204,7 @@ struct GenTree static bool OperIsLocalStore(genTreeOps gtOper) { - return (gtOper == GT_STORE_LCL_VAR || gtOper == GT_STORE_LCL_FLD); + return (gtOper == GT_STORE_LCL_VAR || gtOper == GT_STORE_LCL_FLD || gtOper == GT_CSTORE_LCL_VAR); } static bool OperIsAddrMode(genTreeOps gtOper) @@ -1573,7 +1576,7 @@ struct GenTree static bool OperIsStore(genTreeOps gtOper) { - return (gtOper == GT_STOREIND || gtOper == GT_STORE_LCL_VAR || gtOper == GT_STORE_LCL_FLD || + return (gtOper == GT_STOREIND || gtOper == GT_STORE_LCL_VAR || gtOper == GT_CSTORE_LCL_VAR || gtOper == GT_STORE_LCL_FLD || OperIsStoreBlk(gtOper) || OperIsAtomicOp(gtOper)); } @@ -2934,7 +2937,7 @@ struct GenTreeOp : public GenTreeUnOp : GenTreeUnOp(oper, type DEBUGARG(largeNode)), gtOp2(nullptr) { // Unary operators with optional arguments: - assert(oper == GT_NOP || oper == GT_RETURN || oper == GT_RETFILT || OperIsBlk(oper)); + assert(oper == GT_NOP || oper == GT_RETURN || oper == GT_RETFILT || OperIsBlk(oper) || OperIsLocal(oper) || OperIsLocalAddr(oper)); } // returns true if we will use the division by constant optimization for this node. @@ -3298,8 +3301,8 @@ struct GenTreeStrCon : public GenTree }; // Common supertype of LCL_VAR, LCL_FLD, REG_VAR, PHI_ARG -// This inherits from UnOp because lclvar stores are Unops -struct GenTreeLclVarCommon : public GenTreeUnOp +// This inherits from GenTreeOp because CSTORE_LCL_VAR has 2 ops +struct GenTreeLclVarCommon : public GenTreeOp { private: unsigned _gtLclNum; // The local number. An index into the Compiler::lvaTable array. @@ -3307,7 +3310,7 @@ struct GenTreeLclVarCommon : public GenTreeUnOp public: GenTreeLclVarCommon(genTreeOps oper, var_types type, unsigned lclNum DEBUGARG(bool largeNode = false)) - : GenTreeUnOp(oper, type DEBUGARG(largeNode)) + : GenTreeOp(oper, type DEBUGARG(largeNode)) { SetLclNum(lclNum); } @@ -3341,7 +3344,7 @@ struct GenTreeLclVarCommon : public GenTreeUnOp } #if DEBUGGABLE_GENTREE - GenTreeLclVarCommon() : GenTreeUnOp() + GenTreeLclVarCommon() : GenTreeOp() { } #endif @@ -8298,7 +8301,7 @@ inline GenTreeFlags GenTree::GetRegSpillFlagByIdx(int regIndex) const inline GenTreeFlags GenTree::GetLastUseBit(int regIndex) const { assert(regIndex < 4); - assert(OperIs(GT_LCL_VAR, GT_STORE_LCL_VAR, GT_COPY, GT_RELOAD)); + assert(OperIs(GT_LCL_VAR, GT_STORE_LCL_VAR, GT_CSTORE_LCL_VAR, GT_COPY, GT_RELOAD)); static_assert_no_msg((1 << MULTIREG_LAST_USE_SHIFT) == GTF_VAR_MULTIREG_DEATH0); return (GenTreeFlags)(1 << (MULTIREG_LAST_USE_SHIFT + regIndex)); } @@ -8317,7 +8320,7 @@ inline GenTreeFlags GenTree::GetLastUseBit(int regIndex) const // inline bool GenTree::IsLastUse(int regIndex) const { - assert(OperIs(GT_LCL_VAR, GT_STORE_LCL_VAR, GT_COPY, GT_RELOAD)); + assert(OperIs(GT_LCL_VAR, GT_STORE_LCL_VAR, GT_CSTORE_LCL_VAR, GT_COPY, GT_RELOAD)); return (gtFlags & GetLastUseBit(regIndex)) != 0; } diff --git a/src/coreclr/jit/gtlist.h b/src/coreclr/jit/gtlist.h index 1d0be8bd167664..92a958b252f05f 100644 --- a/src/coreclr/jit/gtlist.h +++ b/src/coreclr/jit/gtlist.h @@ -24,6 +24,7 @@ GTNODE(PHI_ARG , GenTreePhiArg ,0,GTK_LEAF) // phi(phi GTNODE(LCL_VAR , GenTreeLclVar ,0,GTK_LEAF) // local variable GTNODE(LCL_FLD , GenTreeLclFld ,0,GTK_LEAF) // field in a non-primitive variable GTNODE(STORE_LCL_VAR , GenTreeLclVar ,0,GTK_UNOP|GTK_NOVALUE) // store to local variable +GTNODE(CSTORE_LCL_VAR , GenTreeLclVar ,0,GTK_BINOP|GTK_NOVALUE) // conditional store to local variable GTNODE(STORE_LCL_FLD , GenTreeLclFld ,0,GTK_UNOP|GTK_NOVALUE) // store to a part of the variable GTNODE(LCL_VAR_ADDR , GenTreeLclVar ,0,GTK_LEAF) // address of local variable GTNODE(LCL_FLD_ADDR , GenTreeLclFld ,0,GTK_LEAF) // address of field in a non-primitive variable diff --git a/src/coreclr/jit/gtstructs.h b/src/coreclr/jit/gtstructs.h index 63730117790201..4e44b326e43bda 100644 --- a/src/coreclr/jit/gtstructs.h +++ b/src/coreclr/jit/gtstructs.h @@ -60,8 +60,8 @@ GTSTRUCT_1(IntCon , GT_CNS_INT) GTSTRUCT_1(LngCon , GT_CNS_LNG) GTSTRUCT_1(DblCon , GT_CNS_DBL) GTSTRUCT_1(StrCon , GT_CNS_STR) -GTSTRUCT_N(LclVarCommon, GT_LCL_VAR, GT_LCL_FLD, GT_PHI_ARG, GT_STORE_LCL_VAR, GT_STORE_LCL_FLD, GT_LCL_VAR_ADDR, GT_LCL_FLD_ADDR) -GTSTRUCT_3(LclVar , GT_LCL_VAR, GT_LCL_VAR_ADDR, GT_STORE_LCL_VAR) +GTSTRUCT_N(LclVarCommon, GT_LCL_VAR, GT_LCL_FLD, GT_PHI_ARG, GT_STORE_LCL_VAR, GT_CSTORE_LCL_VAR, GT_STORE_LCL_FLD, GT_LCL_VAR_ADDR, GT_LCL_FLD_ADDR) +GTSTRUCT_4(LclVar , GT_LCL_VAR, GT_LCL_VAR_ADDR, GT_STORE_LCL_VAR, GT_CSTORE_LCL_VAR) GTSTRUCT_3(LclFld , GT_LCL_FLD, GT_STORE_LCL_FLD, GT_LCL_FLD_ADDR) GTSTRUCT_1(Cast , GT_CAST) GTSTRUCT_1(Box , GT_BOX) diff --git a/src/coreclr/jit/lclvars.cpp b/src/coreclr/jit/lclvars.cpp index cd13f96e83d83d..8da37f94921a4c 100644 --- a/src/coreclr/jit/lclvars.cpp +++ b/src/coreclr/jit/lclvars.cpp @@ -4537,6 +4537,7 @@ void Compiler::lvaComputeRefCounts(bool isRecompute, bool setSlotNumbers) case GT_LCL_VAR_ADDR: case GT_LCL_FLD_ADDR: case GT_STORE_LCL_VAR: + case GT_CSTORE_LCL_VAR: case GT_STORE_LCL_FLD: { LclVarDsc* varDsc = lvaGetDesc(node->AsLclVarCommon()); diff --git a/src/coreclr/jit/liveness.cpp b/src/coreclr/jit/liveness.cpp index ba1a653b93f2b4..1bc832f76cc455 100644 --- a/src/coreclr/jit/liveness.cpp +++ b/src/coreclr/jit/liveness.cpp @@ -51,7 +51,7 @@ void Compiler::fgMarkUseDef(GenTreeLclVarCommon* tree) { // If this is an enregisterable variable that is not marked doNotEnregister, // we should only see direct references (not ADDRs). - assert(varDsc->lvDoNotEnregister || tree->OperIs(GT_LCL_VAR, GT_STORE_LCL_VAR)); + assert(varDsc->lvDoNotEnregister || tree->OperIs(GT_LCL_VAR, GT_STORE_LCL_VAR, GT_CSTORE_LCL_VAR)); } if (isUse && !VarSetOps::IsMember(this, fgCurDefSet, varDsc->lvVarIndex)) @@ -220,6 +220,7 @@ void Compiler::fgPerNodeLocalVarLiveness(GenTree* tree) case GT_LCL_VAR_ADDR: case GT_LCL_FLD_ADDR: case GT_STORE_LCL_VAR: + case GT_CSTORE_LCL_VAR: case GT_STORE_LCL_FLD: fgMarkUseDef(tree->AsLclVarCommon()); break; @@ -2013,6 +2014,7 @@ void Compiler::fgComputeLifeLIR(VARSET_TP& life, BasicBlock* block, VARSET_VALAR break; case GT_STORE_LCL_VAR: + case GT_CSTORE_LCL_VAR: case GT_STORE_LCL_FLD: { GenTreeLclVarCommon* const lclVarNode = node->AsLclVarCommon(); diff --git a/src/coreclr/jit/lower.cpp b/src/coreclr/jit/lower.cpp index 4a68f8195cfb5b..196fa346270c49 100644 --- a/src/coreclr/jit/lower.cpp +++ b/src/coreclr/jit/lower.cpp @@ -3174,6 +3174,140 @@ GenTree* Lowering::LowerCompare(GenTree* cmp) return cmp->gtNext; } + +//------------------------------------------------------------------------ +// Lowering::LowerIfCompares: Merges a if+branch block with a following conditional block +// +// Arguments: +// block - Block to be lowered +// +// Return Value: +// Nonev +// + +void Lowering::LowerIfCompares(BasicBlock* block) +{ + GenTree* last = block->lastNode(); + BasicBlock *middle_block = nullptr; + + // Arm only for now. +#ifdef TARGET_ARM64 + + // Does the block ends by branching via a JTRUE after a compare? + if (block->bbJumpKind == BBJ_COND && block->NumSucc() == 2 && + last->OperIs(GT_JTRUE) && last->gtGetOp1()->OperIs(GT_EQ, GT_NE)) + { + // Is the false path (next) a diversion which then unconditionally + // goes back to the true path (JumpDest)? + if (block->bbNext->NumSucc() == 1 && block->bbNext->bbJumpKind == BBJ_NONE && + block->bbNext->bbNext == block->bbJumpDest) + { + middle_block = block->bbNext; + } + } + + // middle_block is the diversion block. + // Can all the nodes within it be made to conditionally execute? + if (middle_block) + { + GenTree* node = LIR::AsRange(middle_block).FirstNode(); + while (node != nullptr) + { + // Check the block only contains certain instructions + switch (node->gtOper) + { + case GT_IL_OFFSET: + case GT_CNS_INT: + case GT_STORE_LCL_VAR: + case GT_LCL_VAR: + // These instructions are allowed. + node = node->gtNext; + break; + // TODO: Detect compares to allow chaining of blocks + // case GT_EQ: + // case GT_NE: + default: + // Cannot optimise this block + node = nullptr; + middle_block = nullptr; + break; + } + } + } + + // Conditionally execute the middle block and merge it into the main one. + if (middle_block) + { + GenTree* cmp = last->gtGetOp1(); + + // Remove the JTRUE node + LIR::AsRange(block).Remove(last); + assert(cmp->gtNext == nullptr); + + // The compare generates flags, not a register. + cmp->gtType = TYP_VOID; + cmp->gtFlags |= GTF_SET_FLAGS; + + // Iterate over the middle block, rewriting nodes to be conditional + GenTreeFlags csel_flag = cmp->OperIs(GT_EQ) ? GTF_CSEL_F : GTF_CSEL_T; + GenTree* node = LIR::AsRange(middle_block).FirstNode(); + while (node != nullptr) + { + switch (node->gtOper) + { + case GT_STORE_LCL_VAR: + { + // Replace the store with a conditional store. + // Op1, the true path input, is the existing store input. + // Op2, the false path, is the current value of the store output. + + // Switch to a cstore. + node->SetOper(GT_CSTORE_LCL_VAR); + // ...Alternatively we could keep it as a GT_STORE_LCL_VAR. + // Code elsewhere has already been changed to allow local variable nodes to take 2 args. + // In the codegen for GT_STORE_LCL_VAR, we can check gtFlags, and if a CSEL flag is set + // then assert that Op2 is set, and generate the csel instead of a store. + // Advantage of this is we don't need to create explicit conditional node types + // (there will be a few more conditional node types to add later, eg CEQ) + + // Mark with the compare type. + node->gtFlags |= csel_flag; + + // Create a new node for Op2. + GenTree* base = new (comp, GT_LCL_VAR) GenTreeLclVar(GT_LCL_VAR, node->TypeGet(), node->AsLclVarCommon()->GetLclNum()); + BlockRange().InsertBefore(node, base); + node->AsOp()->gtOp2 = base; + + break; + } + + // TODO: Replace these with conditional versions to allow chaining. + // case GT_EQ: + // case GT_NE: + + case GT_IL_OFFSET: + case GT_CNS_INT: + case GT_LCL_VAR: + // Nothing needs updating here. + break; + + default: + assert(false && "Invalid node found in conditional block."); + break; + } + node = node->gtNext; + } + + // Merge the original and middle blocks. + comp->fgRemoveAllRefPreds(block->bbJumpDest, block); + block->bbJumpKind = BBJ_NONE; + block->bbJumpDest = block->bbNext; + comp->fgCompactBlocks(block, block->bbNext); + } +#endif +} + + //------------------------------------------------------------------------ // Lowering::LowerJTrue: Lowers a JTRUE node. // @@ -6404,6 +6538,19 @@ PhaseStatus Lowering::DoPhase() comp->lvSetMinOptsDoNotEnreg(); } + // Iterate through the blocks lowering any if+branch blocks. + // Do this before general lowering so that JCMP nodes have not been created. + // TODO: For now this reverse iterates - that may or may not be the best option. + // TODO: Is this the best place to do this? Do we need a full phase? + // TODO: Do we want to add a reverse block iterator? + // TODO: Can JCMP detection be removed once this does everything? + BasicBlock* block = comp->fgLastBB; + while (block != nullptr) + { + LowerIfCompares(block); + block=block->bbPrev; + } + for (BasicBlock* const block : comp->Blocks()) { /* Make the block publicly available */ @@ -6852,6 +6999,7 @@ void Lowering::ContainCheckNode(GenTree* node) switch (node->gtOper) { case GT_STORE_LCL_VAR: + case GT_CSTORE_LCL_VAR: case GT_STORE_LCL_FLD: ContainCheckStoreLoc(node->AsLclVarCommon()); break; diff --git a/src/coreclr/jit/lower.h b/src/coreclr/jit/lower.h index 1a0bb8b5ed9926..8da6e10595e2d2 100644 --- a/src/coreclr/jit/lower.h +++ b/src/coreclr/jit/lower.h @@ -165,6 +165,7 @@ class Lowering final : public Phase GenTree* LowerFloatArg(GenTree** pArg, fgArgTabEntry* info); GenTree* LowerFloatArgReg(GenTree* arg, regNumber regNum); #endif + void LowerIfCompares(BasicBlock* block); void InsertPInvokeCallProlog(GenTreeCall* call); void InsertPInvokeCallEpilog(GenTreeCall* call); diff --git a/src/coreclr/jit/lowerarmarch.cpp b/src/coreclr/jit/lowerarmarch.cpp index 978f3777869489..690bed777cb875 100644 --- a/src/coreclr/jit/lowerarmarch.cpp +++ b/src/coreclr/jit/lowerarmarch.cpp @@ -132,6 +132,7 @@ bool Lowering::IsContainableImmed(GenTree* parentNode, GenTree* childNode) const #ifdef TARGET_ARM64 case GT_STORE_LCL_FLD: case GT_STORE_LCL_VAR: + case GT_CSTORE_LCL_VAR: if (immVal == 0) return true; break; @@ -158,6 +159,7 @@ bool Lowering::IsContainableImmed(GenTree* parentNode, GenTree* childNode) const void Lowering::LowerStoreLoc(GenTreeLclVarCommon* storeLoc) { GenTree* op1 = storeLoc->gtGetOp1(); + assert(storeLoc->gtOper != GT_CSTORE_LCL_VAR); if ((storeLoc->gtOper == GT_STORE_LCL_VAR) && (op1->gtOper == GT_CNS_INT)) { // Try to widen the ops if they are going into a local var. diff --git a/src/coreclr/jit/lsra.cpp b/src/coreclr/jit/lsra.cpp index c9d1942097afd9..f006743a111e10 100644 --- a/src/coreclr/jit/lsra.cpp +++ b/src/coreclr/jit/lsra.cpp @@ -160,7 +160,7 @@ void lsraAssignRegToTree(GenTree* tree, regNumber reg, unsigned regIdx) tree->AsHWIntrinsic()->SetOtherReg(reg); } #endif // FEATURE_HW_INTRINSICS - else if (tree->OperIs(GT_LCL_VAR, GT_STORE_LCL_VAR)) + else if (tree->OperIs(GT_LCL_VAR, GT_STORE_LCL_VAR, GT_CSTORE_LCL_VAR)) { tree->AsLclVar()->SetRegNumByIdx(reg, regIdx); } @@ -6905,7 +6905,7 @@ void LinearScan::resolveRegisters() { writeRegisters(currentRefPosition, treeNode); - if (treeNode->OperIs(GT_LCL_VAR, GT_STORE_LCL_VAR) && currentRefPosition->getInterval()->isLocalVar) + if (treeNode->OperIs(GT_LCL_VAR, GT_STORE_LCL_VAR, GT_CSTORE_LCL_VAR) && currentRefPosition->getInterval()->isLocalVar) { resolveLocalRef(block, treeNode->AsLclVar(), currentRefPosition); } diff --git a/src/coreclr/jit/lsra.h b/src/coreclr/jit/lsra.h index 56f00145b68ba3..dbe409c4730135 100644 --- a/src/coreclr/jit/lsra.h +++ b/src/coreclr/jit/lsra.h @@ -1062,7 +1062,7 @@ class LinearScan : public LinearScanInterface var_types type = tree->TypeGet(); if (type == TYP_STRUCT) { - assert(tree->OperIs(GT_LCL_VAR, GT_STORE_LCL_VAR)); + assert(tree->OperIs(GT_LCL_VAR, GT_STORE_LCL_VAR, GT_CSTORE_LCL_VAR)); GenTreeLclVar* lclVar = tree->AsLclVar(); LclVarDsc* varDsc = compiler->lvaGetDesc(lclVar); type = varDsc->GetRegisterType(lclVar); diff --git a/src/coreclr/jit/lsraarm64.cpp b/src/coreclr/jit/lsraarm64.cpp index 3a3d0ee23241fd..72f003c158e0c9 100644 --- a/src/coreclr/jit/lsraarm64.cpp +++ b/src/coreclr/jit/lsraarm64.cpp @@ -45,7 +45,7 @@ XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX int LinearScan::BuildNode(GenTree* tree) { assert(!tree->isContained()); - int srcCount; + int srcCount = 0; int dstCount = 0; regMaskTP dstCandidates = RBM_NONE; regMaskTP killMask = RBM_NONE; @@ -103,6 +103,12 @@ int LinearScan::BuildNode(GenTree* tree) } break; + case GT_CSTORE_LCL_VAR: + // Take into account the conditional. + BuildUse(tree->gtGetOp2()); + srcCount++; + FALLTHROUGH; + case GT_STORE_LCL_VAR: if (tree->IsMultiRegLclVar() && isCandidateMultiRegLclVar(tree->AsLclVar())) { @@ -111,7 +117,7 @@ int LinearScan::BuildNode(GenTree* tree) FALLTHROUGH; case GT_STORE_LCL_FLD: - srcCount = BuildStoreLoc(tree->AsLclVarCommon()); + srcCount += BuildStoreLoc(tree->AsLclVarCommon()); break; case GT_FIELD_LIST: diff --git a/src/coreclr/jit/lsrabuild.cpp b/src/coreclr/jit/lsrabuild.cpp index 04df846e367e74..b12261ee70bacd 100644 --- a/src/coreclr/jit/lsrabuild.cpp +++ b/src/coreclr/jit/lsrabuild.cpp @@ -3460,7 +3460,7 @@ int LinearScan::BuildMultiRegStoreLoc(GenTreeLclVar* storeLoc) // BuildStoreLoc: Set register requirements for a store of a lclVar // // Arguments: -// storeLoc - the local store (GT_STORE_LCL_FLD or GT_STORE_LCL_VAR) +// storeLoc - the local store (GT_STORE_LCL_FLD, GT_STORE_LCL_VAR or GT_CSTORE_LCL_VAR) // // Notes: // This involves: diff --git a/src/coreclr/jit/morph.cpp b/src/coreclr/jit/morph.cpp index 1ff41e2fa15c36..f6f72e9679d6ce 100644 --- a/src/coreclr/jit/morph.cpp +++ b/src/coreclr/jit/morph.cpp @@ -15622,7 +15622,7 @@ void Compiler::fgMorphTreeDone(GenTree* tree, // // But we shouldn't be running local assertion prop on these, // as local prop gets disabled when we run global prop. - assert(!tree->OperIs(GT_STORE_LCL_VAR, GT_STORE_LCL_FLD)); + assert(!tree->OperIs(GT_STORE_LCL_VAR, GT_STORE_LCL_FLD, GT_CSTORE_LCL_VAR)); // DefinesLocal can return true for some BLK op uses, so // check what gets assigned only when we're at an assignment.