Skip to content

JIT: Unify LowerBlockStore for init across targets #128683

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
EgorBo merged 4 commits into from
Jun 1, 2026
Merged

JIT: Unify LowerBlockStore for init across targets #128683

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
5a4a5c4
JIT: Unify LowerBlockStore for init across targets
EgorBo May 28, 2026
b27f550
JIT: Simplify getKillSetForBlockStore after RepInstr removal
EgorBo May 28, 2026
7a92aee
Merge remote-tracking branch 'origin/main' into unify-lower-init-blks…
EgorBo May 28, 2026
0750b8b
clean up
EgorBo May 28, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 0 additions & 2 deletions src/coreclr/jit/codegen.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1175,7 +1175,6 @@ class CodeGen final : public CodeGenInterface
void genCodeForReturnTrap(GenTreeOp* tree);
void genCodeForStoreInd(GenTreeStoreInd* tree);
void genCodeForSwap(GenTreeOp* tree);
void genCodeForCpBlkRepMovs(GenTreeBlk* cpBlkNode);
void genCodeForCpBlkUnroll(GenTreeBlk* cpBlkNode);
void genCodeForPhysReg(GenTreePhysReg* tree);
#ifdef SWIFT_SUPPORT
Expand Down Expand Up @@ -1252,7 +1251,6 @@ class CodeGen final : public CodeGenInterface

void genCodeForStoreBlk(GenTreeBlk* storeBlkNode);
void genCodeForInitBlkLoop(GenTreeBlk* initBlkNode);
void genCodeForInitBlkRepStos(GenTreeBlk* initBlkNode);
void genCodeForInitBlkUnroll(GenTreeBlk* initBlkNode);
unsigned genEmitJumpTable(GenTree* treeNode, bool relativeAddr);
void genJumpTable(GenTree* tree);
Expand Down
42 changes: 0 additions & 42 deletions src/coreclr/jit/codegenxarch.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3047,17 +3047,6 @@ void CodeGen::genCodeForStoreBlk(GenTreeBlk* storeBlkNode)
genCodeForInitBlkLoop(storeBlkNode);
break;

case GenTreeBlk::BlkOpKindRepInstr:
assert(!storeBlkNode->gtBlkOpGcUnsafe);
if (isCopyBlk)
{
genCodeForCpBlkRepMovs(storeBlkNode);
}
else
{
genCodeForInitBlkRepStos(storeBlkNode);
}
break;
case GenTreeBlk::BlkOpKindUnrollMemmove:
case GenTreeBlk::BlkOpKindUnroll:
if (isCopyBlk)
Expand Down Expand Up @@ -3091,19 +3080,6 @@ void CodeGen::genCodeForStoreBlk(GenTreeBlk* storeBlkNode)
}
}

//
//------------------------------------------------------------------------
// genCodeForInitBlkRepStos: Generate code for InitBlk using rep stos.
//
// Arguments:
// initBlkNode - The Block store for which we are generating code.
//
void CodeGen::genCodeForInitBlkRepStos(GenTreeBlk* initBlkNode)
{
genConsumeBlockOp(initBlkNode, REG_RDI, REG_RAX, REG_RCX);
instGen(INS_r_stosb);
}

//----------------------------------------------------------------------------------
// genCodeForInitBlkUnroll: Generate unrolled block initialization code.
//
Expand Down Expand Up @@ -3681,24 +3657,6 @@ void CodeGen::genCodeForCpBlkUnroll(GenTreeBlk* node)
}
}

//----------------------------------------------------------------------------------
// genCodeForCpBlkRepMovs - Generate code for CpBlk by using rep movs
//
// Arguments:
// cpBlkNode - the GT_STORE_[BLK|OBJ|DYN_BLK]
//
// Preconditions:
// The register assignments have been set appropriately.
// This is validated by genConsumeBlockOp().
//
void CodeGen::genCodeForCpBlkRepMovs(GenTreeBlk* cpBlkNode)
{
// Destination address goes in RDI, source address goes in RSE, and size goes in RCX.
// genConsumeBlockOp takes care of this for us.
genConsumeBlockOp(cpBlkNode, REG_RDI, REG_RSI, REG_RCX);
instGen(INS_r_movsb);
}

//------------------------------------------------------------------------
// CodeGen::genMove8IfNeeded: Conditionally move 8 bytes of a struct to the argument area
//
Expand Down
1 change: 0 additions & 1 deletion src/coreclr/jit/compiler.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11553,7 +11553,6 @@ class Compiler
STRESS_MODE(IF_CONVERSION_COST) \
STRESS_MODE(IF_CONVERSION_INNER_LOOPS) \
STRESS_MODE(POISON_IMPLICIT_BYREFS) \
STRESS_MODE(STORE_BLOCK_UNROLLING) \
STRESS_MODE(THREE_OPT_LAYOUT) \
STRESS_MODE(COUNT)

Expand Down
5 changes: 0 additions & 5 deletions src/coreclr/jit/gentree.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14021,11 +14021,6 @@ void Compiler::gtDispTree(GenTree* tree,
{
switch (tree->AsBlk()->gtBlkOpKind)
{
#ifdef TARGET_XARCH
case GenTreeBlk::BlkOpKindRepInstr:
printf(" (RepInstr)");
break;
#endif
case GenTreeBlk::BlkOpKindUnroll:
printf(" (Unroll)");
break;
Expand Down
3 changes: 0 additions & 3 deletions src/coreclr/jit/gentree.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8112,9 +8112,6 @@ struct GenTreeBlk : public GenTreeIndir
enum
{
BlkOpKindInvalid,
#ifdef TARGET_XARCH
BlkOpKindRepInstr,
#endif
BlkOpKindLoop,
BlkOpKindUnroll,
BlkOpKindUnrollMemmove,
Expand Down
92 changes: 91 additions & 1 deletion src/coreclr/jit/lower.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10328,6 +10328,96 @@ void Lowering::LowerCopyBlockStore(GenTreeBlk* blkNode)
#endif // TARGET_WASM
}

#ifndef TARGET_WASM
//------------------------------------------------------------------------
// LowerInitBlockStore: Shared lowering of a block init store (memset).
//
// Arguments:
// blkNode - The block store node to lower. Must be an init (not copy).
//
void Lowering::LowerInitBlockStore(GenTreeBlk* blkNode)
{
assert(blkNode->OperIsInitBlkOp());

#ifdef TARGET_XARCH
TryCreateAddrMode(blkNode->Addr(), false, blkNode);
#endif

GenTree* dstAddr = blkNode->Addr();
GenTree* src = blkNode->Data();
const unsigned size = blkNode->Size();

if (src->OperIs(GT_INIT_VAL))
{
src->SetContained();
src = src->AsUnOp()->gtGetOp1();
}

#ifdef TARGET_ARM64
// On arm64 SIMD stores guarantee 8-byte atomicity when data is 8-byte aligned
const bool canUseSimd = true;
#else
const bool canUseSimd = !blkNode->IsOnHeapAndContainsReferences();
#endif

if (src->OperIs(GT_CNS_INT) && (size <= m_compiler->getUnrollThreshold(Compiler::UnrollKind::Memset, canUseSimd)))
{
// The fill value of an initblk is interpreted to hold a value of (unsigned int8)
ssize_t fill = src->AsIntCon()->IconValue() & 0xFF;

bool useSimdFill = false;

#ifdef TARGET_XARCH
useSimdFill = canUseSimd && (size >= XMM_REGSIZE_BYTES);
#endif

if (useSimdFill)
{
// We're going to use SIMD (and only SIMD - we don't want to occupy a GPR register
// with a fill value just to handle the remainder when we can do that with
// an overlapped SIMD load).
src->SetContained();
}
else if (fill == 0)
{
#if FEATURE_HAS_ZERO_REG
// We can use the hardware zero register as the contained source.
src->SetContained();
#endif
}
#ifdef TARGET_64BIT
else if (size >= REGSIZE_BYTES)
{
fill *= 0x0101010101010101LL;
src->gtType = TYP_LONG;
}
#endif
else
{
fill *= 0x01010101;
}

blkNode->gtBlkOpKind = GenTreeBlk::BlkOpKindUnroll;
src->AsIntCon()->SetIconValue(fill);
ContainBlockStoreAddress(blkNode, size, dstAddr, nullptr);
return;
}

if (blkNode->IsZeroingGcPointersOnHeap())
{
blkNode->gtBlkOpKind = GenTreeBlk::BlkOpKindLoop;
#if FEATURE_HAS_ZERO_REG
// We can use the hardware zero register as the contained source.
src->SetContained();
#endif
}
else
{
LowerBlockStoreAsHelperCall(blkNode);
Comment thread
EgorBo marked this conversation as resolved.
}
}
#endif // !TARGET_WASM

//------------------------------------------------------------------------
// LowerBlockStoreAsHelperCall: Lower a block store node as a memset/memcpy call
//
Expand Down Expand Up @@ -12164,7 +12254,7 @@ void Lowering::LowerBlockStoreCommon(GenTreeBlk* blkNode)

if (blkNode->OperIsInitBlkOp())
{
LowerBlockStore(blkNode);
LowerInitBlockStore(blkNode);
}
else
{
Expand Down
2 changes: 1 addition & 1 deletion src/coreclr/jit/lower.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -436,10 +436,10 @@ class Lowering final : public Phase
bool TryLowerConstIntDivOrMod(GenTree* node, GenTree** nextNode);
GenTree* LowerSignedDivOrMod(GenTree* node);
void LowerDivOrMod(GenTreeOp* divMod);
void LowerBlockStore(GenTreeBlk* blkNode);
void LowerBlockStoreCommon(GenTreeBlk* blkNode);
void LowerBlockStoreAsHelperCall(GenTreeBlk* blkNode);
void LowerBlockStoreAsGcBulkCopyCall(GenTreeBlk* blkNode);
void LowerInitBlockStore(GenTreeBlk* blkNode);
void LowerCopyBlockStore(GenTreeBlk* blkNode);
bool TryDecomposeBlockStoreAsIndirs(GenTreeBlk* blkNode);
void LowerLclHeap(GenTree* node);
Expand Down
87 changes: 0 additions & 87 deletions src/coreclr/jit/lowerarmarch.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -709,93 +709,6 @@ GenTree* Lowering::LowerBinaryArithmetic(GenTreeOp* binOp)
return binOp->gtNext;
}

//------------------------------------------------------------------------
// LowerBlockStore: Lower a block store node
//
// Arguments:
// blkNode - The block store node to lower
//
void Lowering::LowerBlockStore(GenTreeBlk* blkNode)
{
assert(blkNode->OperIsInitBlkOp());
GenTree* dstAddr = blkNode->Addr();
GenTree* src = blkNode->Data();
unsigned size = blkNode->Size();

#ifdef DEBUG
// Use BlkOpKindLoop for more cases under stress mode
if (m_compiler->compStressCompile(Compiler::STRESS_STORE_BLOCK_UNROLLING, 50) && blkNode->OperIs(GT_STORE_BLK) &&
((blkNode->GetLayout()->GetSize() % TARGET_POINTER_SIZE) == 0) && src->IsIntegralConst(0))
{
blkNode->gtBlkOpKind = GenTreeBlk::BlkOpKindLoop;
#ifdef TARGET_ARM64
// On ARM64 we can just use REG_ZR instead of having to load
// the constant into a real register like on ARM32.
src->SetContained();
#endif
return;
}
#endif

if (src->OperIs(GT_INIT_VAL))
{
src->SetContained();
src = src->AsUnOp()->gtGetOp1();
}

if ((size <= m_compiler->getUnrollThreshold(Compiler::UnrollKind::Memset)) && src->OperIs(GT_CNS_INT))
{
blkNode->gtBlkOpKind = GenTreeBlk::BlkOpKindUnroll;

// The fill value of an initblk is interpreted to hold a
// value of (unsigned int8) however a constant of any size
// may practically reside on the evaluation stack. So extract
// the lower byte out of the initVal constant and replicate
// it to a larger constant whose size is sufficient to support
// the largest width store of the desired inline expansion.

ssize_t fill = src->AsIntCon()->IconValue() & 0xFF;

if (fill == 0)
{
#ifdef TARGET_ARM64
// On ARM64 we can just use REG_ZR instead of having to load
// the constant into a real register like on ARM32.
src->SetContained();
#endif
}
#ifdef TARGET_ARM64
else if (size >= REGSIZE_BYTES)
{
fill *= 0x0101010101010101LL;
src->gtType = TYP_LONG;
}
#endif
else
{
fill *= 0x01010101;
}

src->AsIntCon()->SetIconValue(fill);

ContainBlockStoreAddress(blkNode, size, dstAddr, nullptr);
}
else if (blkNode->IsZeroingGcPointersOnHeap())
{
blkNode->gtBlkOpKind = GenTreeBlk::BlkOpKindLoop;
#ifdef TARGET_ARM64
// On ARM64 we can just use REG_ZR instead of having to load
// the constant into a real register like on ARM32.
src->SetContained();
#endif
}
else
{
LowerBlockStoreAsHelperCall(blkNode);
return;
}
}

//------------------------------------------------------------------------
// ContainBlockStoreAddress: Attempt to contain an address used by an unrolled block store.
//
Expand Down
64 changes: 0 additions & 64 deletions src/coreclr/jit/lowerloongarch64.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -279,70 +279,6 @@ GenTree* Lowering::LowerStoreIndir(GenTreeStoreInd* node)
return node->gtNext;
}

//------------------------------------------------------------------------
// LowerBlockStore: Set block store type
//
// Arguments:
// blkNode - The block store node of interest
//
// Return Value:
// None.
//
void Lowering::LowerBlockStore(GenTreeBlk* blkNode)
{
assert(blkNode->OperIsInitBlkOp());
GenTree* dstAddr = blkNode->Addr();
GenTree* src = blkNode->Data();
unsigned size = blkNode->Size();

if (src->OperIs(GT_INIT_VAL))
{
src->SetContained();
src = src->AsUnOp()->gtGetOp1();
}

if ((size <= m_compiler->getUnrollThreshold(Compiler::UnrollKind::Memset)) && src->OperIs(GT_CNS_INT))
{
blkNode->gtBlkOpKind = GenTreeBlk::BlkOpKindUnroll;

// The fill value of an initblk is interpreted to hold a
// value of (unsigned int8) however a constant of any size
// may practically reside on the evaluation stack. So extract
// the lower byte out of the initVal constant and replicate
// it to a larger constant whose size is sufficient to support
// the largest width store of the desired inline expansion.

ssize_t fill = src->AsIntCon()->IconValue() & 0xFF;
if (fill == 0)
{
src->SetContained();
}
else if (size >= REGSIZE_BYTES)
{
fill *= 0x0101010101010101LL;
src->gtType = TYP_LONG;
}
else
{
fill *= 0x01010101;
}
src->AsIntCon()->SetIconValue(fill);

ContainBlockStoreAddress(blkNode, size, dstAddr, nullptr);
}
else if (blkNode->IsZeroingGcPointersOnHeap())
{
blkNode->gtBlkOpKind = GenTreeBlk::BlkOpKindLoop;
// We're going to use REG_R0 for zero
src->SetContained();
}
else
{
LowerBlockStoreAsHelperCall(blkNode);
return;
}
}

//------------------------------------------------------------------------
// ContainBlockStoreAddress: Attempt to contain an address used by an unrolled block store.
//
Expand Down
Loading
Loading