From 4782214fe6b05d1029c5e95a8e9f4ce1ea0d9f28 Mon Sep 17 00:00:00 2001 From: Tanner Gooding Date: Tue, 7 Jul 2026 09:26:36 -0700 Subject: [PATCH 01/11] JIT: Rematerialize spilled FP/SIMD/mask constants instead of stack spill/reload Floating-point, SIMD, and mask constants (CNS_DBL, CNS_VEC, CNS_MSK) have no GC liveness and can be rematerialized cheaply on xarch without a scratch register (zero via xorps, all-bits-set via pcmpeqd/vpternlogd, or a RIP-relative load from the constant's memory home). Rather than spilling such a value to the stack and reloading it, skip creating the spill temp and store in genProduceReg, and rematerialize the constant at the reload point in genUnspillRegIfNeeded. A shared predicate isRematerializableConstant gates both sites and is currently scoped to TARGET_XARCH; other targets can be enabled later for the subset of values that likewise need no scratch register. Values consumed directly from the spill temp as a contained memory operand (GTF_NOREG_AT_USE) continue to be spilled normally. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- src/coreclr/jit/codegen.h | 1 + src/coreclr/jit/codegenlinear.cpp | 98 +++++++++++++++++++++++++++---- 2 files changed, 88 insertions(+), 11 deletions(-) diff --git a/src/coreclr/jit/codegen.h b/src/coreclr/jit/codegen.h index f8f9eddf0c6322..f1dd80d0a46011 100644 --- a/src/coreclr/jit/codegen.h +++ b/src/coreclr/jit/codegen.h @@ -1121,6 +1121,7 @@ class CodeGen final : public CodeGenInterface unsigned varNum, var_types type, GenTreeLclVar* lclNode, regNumber regNum, bool reSpill, bool isLastUse); void genUnspillRegIfNeeded(GenTree* tree); void genUnspillRegIfNeeded(GenTree* tree, unsigned multiRegIndex); + bool isRematerializableConstant(GenTree* tree); regNumber genConsumeReg(GenTree* tree); regNumber genConsumeReg(GenTree* tree, unsigned multiRegIndex); void genCopyRegIfNeeded(GenTree* tree, regNumber needReg); diff --git a/src/coreclr/jit/codegenlinear.cpp b/src/coreclr/jit/codegenlinear.cpp index fdd1d3263b9968..cfc5409322f679 100644 --- a/src/coreclr/jit/codegenlinear.cpp +++ b/src/coreclr/jit/codegenlinear.cpp @@ -1404,23 +1404,85 @@ void CodeGen::genUnspillRegIfNeeded(GenTree* tree) } else { - // Here we may have a GT_RELOAD. - // The spill temp allocated for it is associated with the original tree that defined the - // register that it was spilled from. - // So we use 'unspillTree' to recover that spill temp. - TempDsc* t = regSet.rsUnspillInPlace(unspillTree, unspillTree->GetRegNum()); - emitAttr emitType = emitActualTypeSize(unspillTree->TypeGet()); - // Reload into the register specified by 'tree' which may be a GT_RELOAD. regNumber dstReg = tree->GetRegNum(); - GetEmitter()->emitIns_R_S(ins_Load(unspillTree->gtType), emitType, dstReg, t->tdTempNum(), 0); - regSet.tmpRlsTemp(t); - unspillTree->gtFlags &= ~GTF_SPILLED; - gcInfo.gcMarkRegPtrVal(dstReg, unspillTree->TypeGet()); +#ifdef TARGET_XARCH + // A floating-point, SIMD, or mask constant is not spilled to the stack (see + // genProduceReg); rematerialize it directly into the reload target register rather + // than loading it from a spill temp that was never created. Note that the GT_CNS_VEC + // and GT_CNS_MSK GenTree* overloads of genSetRegToConst materialize into the node's own + // register, so the explicit-register overloads are used to honor 'dstReg' (which may be + // a GT_RELOAD target register that differs from the original definition's register). + if (isRematerializableConstant(unspillTree) && ((unspillTree->gtFlags & GTF_NOREG_AT_USE) == 0)) + { + switch (unspillTree->OperGet()) + { + case GT_CNS_DBL: + genSetRegToConst(dstReg, unspillTree->TypeGet(), unspillTree); + break; + case GT_CNS_VEC: + genSetRegToConst(dstReg, unspillTree->TypeGet(), &unspillTree->AsVecCon()->gtSimdVal); + break; + case GT_CNS_MSK: + genSetRegToConst(dstReg, unspillTree->TypeGet(), &unspillTree->AsMskCon()->gtSimdMaskVal); + break; + default: + unreached(); + } + + unspillTree->gtFlags &= ~GTF_SPILLED; + } + else +#endif // TARGET_XARCH + { + // Here we may have a GT_RELOAD. + // The spill temp allocated for it is associated with the original tree that defined the + // register that it was spilled from. + // So we use 'unspillTree' to recover that spill temp. + TempDsc* t = regSet.rsUnspillInPlace(unspillTree, unspillTree->GetRegNum()); + emitAttr emitType = emitActualTypeSize(unspillTree->TypeGet()); + // Reload into the register specified by 'tree' which may be a GT_RELOAD. + GetEmitter()->emitIns_R_S(ins_Load(unspillTree->gtType), emitType, dstReg, t->tdTempNum(), 0); + regSet.tmpRlsTemp(t); + + unspillTree->gtFlags &= ~GTF_SPILLED; + gcInfo.gcMarkRegPtrVal(dstReg, unspillTree->TypeGet()); + } } } } +//------------------------------------------------------------------------ +// isRematerializableConstant: Determine whether a spilled constant tree temp can be +// rematerialized at its reload point instead of being spilled to and reloaded +// from the stack. +// +// Arguments: +// tree - the node that defines the value +// +// Return Value: +// True if 'tree' is a floating-point, SIMD, or mask constant that this target can +// rematerialize without requiring a scratch register or GC bookkeeping. +// +// Notes: +// These constants have no GC liveness. On xarch every such constant can be +// rematerialized without a scratch register (zero via `xorps`, all-bits-set via +// `pcmpeqd`/`vpternlogd`, or a RIP-relative load from the constant's memory home), +// so all of them are eligible. Other targets can be enabled later for the subset of +// values that likewise need no scratch register (for example, zero or fmov-immediate +// floating-point constants on arm64); loading arbitrary constants from the constant +// pool there requires a scratch address register that is not available at the reload +// point, so those remain spilled for now. +// +bool CodeGen::isRematerializableConstant(GenTree* tree) +{ +#ifdef TARGET_XARCH + return tree->OperIs(GT_CNS_DBL, GT_CNS_VEC, GT_CNS_MSK); +#else + return false; +#endif // TARGET_XARCH +} + //------------------------------------------------------------------------ // genCopyRegIfNeeded: Copy the given node into the specified register // @@ -2242,6 +2304,20 @@ void CodeGen::genProduceReg(GenTree* tree) } else { + // Floating-point, SIMD, and mask constants have no GC liveness and (on the targets + // for which isRematerializableConstant is enabled) can be rematerialized cheaply + // without a scratch register. Rather than spilling such a value to the stack and + // reloading it, skip creating the spill temp and emitting the store; + // genUnspillRegIfNeeded will rematerialize it at the reload point. If the value will + // instead be consumed directly from the spill temp as a contained memory operand + // (GTF_NOREG_AT_USE), it must still be spilled normally. + if (isRematerializableConstant(tree) && ((tree->gtFlags & GTF_NOREG_AT_USE) == 0)) + { + tree->gtFlags |= GTF_SPILLED; + tree->gtFlags &= ~GTF_SPILL; + return; + } + regSet.rsSpillTree(tree->GetRegNum(), tree); gcInfo.gcMarkRegSetNpt(genRegMask(tree->GetRegNum())); } From 2ba3933b595dbe343124f29cf4bfb2d752cd7e75 Mon Sep 17 00:00:00 2001 From: Tanner Gooding Date: Tue, 7 Jul 2026 10:12:43 -0700 Subject: [PATCH 02/11] JIT: Rematerialize spilled FP/SIMD/mask constants on Arm64 Extend the constant rematerialization added for xarch to Arm64, for the subset of CNS_DBL/CNS_VEC/CNS_MSK values that genSetRegToConst encodes directly into an instruction without a scratch register: 0.0/fmov-immediate doubles, zero/all-bits- set/movi-broadcast vectors, and all masks (pfalse/ptrue). Values that Arm64 would otherwise load from the constant pool via adrp/ldr need a scratch address register that is not reserved at the reload point, so those remain spilled. isRematerializableConstant gains an Arm64 branch mirroring the no-temp fast paths in genSetRegToConst, and genUnspillRegIfNeeded rematerializes into the reload target via the GenTree* overload of genSetRegToConst (which honors the explicit target register on Arm64). The genProduceReg skip-store site is already gated by the predicate. SPMI asmdiffs (coreclr_tests, windows-arm64): 554 methods improved, 0 regressions, -5432 bytes, with spill store/reload pairs replaced by movi/fmov/mvni/pfalse/ptrue. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- src/coreclr/jit/codegenlinear.cpp | 81 ++++++++++++++++++++++++++----- 1 file changed, 69 insertions(+), 12 deletions(-) diff --git a/src/coreclr/jit/codegenlinear.cpp b/src/coreclr/jit/codegenlinear.cpp index cfc5409322f679..c3dafddfc03356 100644 --- a/src/coreclr/jit/codegenlinear.cpp +++ b/src/coreclr/jit/codegenlinear.cpp @@ -1406,15 +1406,17 @@ void CodeGen::genUnspillRegIfNeeded(GenTree* tree) { regNumber dstReg = tree->GetRegNum(); -#ifdef TARGET_XARCH +#if defined(TARGET_XARCH) || defined(TARGET_ARM64) // A floating-point, SIMD, or mask constant is not spilled to the stack (see // genProduceReg); rematerialize it directly into the reload target register rather - // than loading it from a spill temp that was never created. Note that the GT_CNS_VEC - // and GT_CNS_MSK GenTree* overloads of genSetRegToConst materialize into the node's own - // register, so the explicit-register overloads are used to honor 'dstReg' (which may be - // a GT_RELOAD target register that differs from the original definition's register). + // than loading it from a spill temp that was never created. if (isRematerializableConstant(unspillTree) && ((unspillTree->gtFlags & GTF_NOREG_AT_USE) == 0)) { +#if defined(TARGET_XARCH) + // The GT_CNS_VEC and GT_CNS_MSK GenTree* overloads of genSetRegToConst materialize + // into the node's own register, so the explicit-register overloads are used to honor + // 'dstReg' (which may be a GT_RELOAD target register that differs from the original + // definition's register). switch (unspillTree->OperGet()) { case GT_CNS_DBL: @@ -1429,11 +1431,15 @@ void CodeGen::genUnspillRegIfNeeded(GenTree* tree) default: unreached(); } +#else // TARGET_ARM64 + // The arm64 genSetRegToConst honors the explicit target register for every operator. + genSetRegToConst(dstReg, unspillTree->TypeGet(), unspillTree); +#endif // TARGET_XARCH unspillTree->gtFlags &= ~GTF_SPILLED; } else -#endif // TARGET_XARCH +#endif // TARGET_XARCH || TARGET_ARM64 { // Here we may have a GT_RELOAD. // The spill temp allocated for it is associated with the original tree that defined the @@ -1468,16 +1474,67 @@ void CodeGen::genUnspillRegIfNeeded(GenTree* tree) // These constants have no GC liveness. On xarch every such constant can be // rematerialized without a scratch register (zero via `xorps`, all-bits-set via // `pcmpeqd`/`vpternlogd`, or a RIP-relative load from the constant's memory home), -// so all of them are eligible. Other targets can be enabled later for the subset of -// values that likewise need no scratch register (for example, zero or fmov-immediate -// floating-point constants on arm64); loading arbitrary constants from the constant -// pool there requires a scratch address register that is not available at the reload -// point, so those remain spilled for now. +// so all of them are eligible. +// +// On arm64 only the subset that genSetRegToConst encodes directly into an instruction +// is eligible; the remaining values are loaded from the constant pool via `adrp`/`ldr` +// which needs a scratch address register that is not reserved at the reload point. The +// conditions below must therefore stay in sync with the direct-encoding fast paths in +// genSetRegToConst: 0.0/fmov-immediate for CNS_DBL, zero/all-bits-set/movi-broadcast for +// CNS_VEC, and every CNS_MSK (encoded via `pfalse`/`ptrue`, which never needs a scratch). +// Other targets can be enabled later for their own scratch-free subset. // bool CodeGen::isRematerializableConstant(GenTree* tree) { -#ifdef TARGET_XARCH +#if defined(TARGET_XARCH) return tree->OperIs(GT_CNS_DBL, GT_CNS_VEC, GT_CNS_MSK); +#elif defined(TARGET_ARM64) + switch (tree->OperGet()) + { + case GT_CNS_DBL: + { + double constValue = tree->AsDblCon()->DconValue(); + return (*(int64_t*)&constValue == 0) || emitter::emitIns_valid_imm_for_fmov(constValue); + } + +#if defined(FEATURE_SIMD) + case GT_CNS_VEC: + { + // genSetRegToConst only encodes SIMD8/12/16 constants without a scratch register; any + // wider type falls through to its 'unreached' path, so reject those up front (mirroring + // the type switch in genSetRegToConst). + if (!tree->TypeIs(TYP_SIMD8, TYP_SIMD12, TYP_SIMD16)) + { + return false; + } + + GenTreeVecCon* vecCon = tree->AsVecCon(); + + if (vecCon->IsAllBitsSet() || vecCon->IsZero()) + { + return true; + } + + const bool is8 = tree->TypeIs(TYP_SIMD8); + simd16_t val = vecCon->gtSimd16Val; + + return (ElementsAreSame(val.i32, is8 ? 2 : 4) && + emitter::emitIns_valid_imm_for_movi(val.i32[0], EA_4BYTE)) || + (ElementsAreSame(val.i16, is8 ? 4 : 8) && + emitter::emitIns_valid_imm_for_movi(val.i16[0], EA_2BYTE)) || + (ElementsAreSame(val.i8, is8 ? 8 : 16) && + emitter::emitIns_valid_imm_for_movi(val.i8[0], EA_1BYTE)); + } +#endif // FEATURE_SIMD + +#if defined(FEATURE_MASKED_HW_INTRINSICS) + case GT_CNS_MSK: + return true; +#endif // FEATURE_MASKED_HW_INTRINSICS + + default: + return false; + } #else return false; #endif // TARGET_XARCH From 225ba7bb432c32656d355fd37318df20e33e676a Mon Sep 17 00:00:00 2001 From: Tanner Gooding Date: Tue, 7 Jul 2026 10:41:04 -0700 Subject: [PATCH 03/11] Fix formatting --- src/coreclr/jit/codegenlinear.cpp | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/coreclr/jit/codegenlinear.cpp b/src/coreclr/jit/codegenlinear.cpp index c3dafddfc03356..2549a2568cb9b7 100644 --- a/src/coreclr/jit/codegenlinear.cpp +++ b/src/coreclr/jit/codegenlinear.cpp @@ -1432,7 +1432,7 @@ void CodeGen::genUnspillRegIfNeeded(GenTree* tree) unreached(); } #else // TARGET_ARM64 - // The arm64 genSetRegToConst honors the explicit target register for every operator. + // The arm64 genSetRegToConst honors the explicit target register for every operator. genSetRegToConst(dstReg, unspillTree->TypeGet(), unspillTree); #endif // TARGET_XARCH @@ -1522,8 +1522,7 @@ bool CodeGen::isRematerializableConstant(GenTree* tree) emitter::emitIns_valid_imm_for_movi(val.i32[0], EA_4BYTE)) || (ElementsAreSame(val.i16, is8 ? 4 : 8) && emitter::emitIns_valid_imm_for_movi(val.i16[0], EA_2BYTE)) || - (ElementsAreSame(val.i8, is8 ? 8 : 16) && - emitter::emitIns_valid_imm_for_movi(val.i8[0], EA_1BYTE)); + (ElementsAreSame(val.i8, is8 ? 8 : 16) && emitter::emitIns_valid_imm_for_movi(val.i8[0], EA_1BYTE)); } #endif // FEATURE_SIMD From 893e0b9310145585c10d4d969dc09cecda979f8c Mon Sep 17 00:00:00 2001 From: Tanner Gooding Date: Tue, 7 Jul 2026 13:15:14 -0700 Subject: [PATCH 04/11] JIT: Elide dead constant definition when spilled value is rematerialized When a rematerializable FP/SIMD/mask constant is spilled immediately after its definition (GTF_SPILL with spill-after semantics), the value is rematerialized at each reload point and its stack store is skipped. The original definition therefore materializes a value into a register that is never read. Add a shared isRematerializedConstantSpill predicate and use it to also skip emitting the definition (genSetRegToConst) at the genCodeForTreeNode sites on xarch and arm64, in addition to the existing store-skip in genProduceReg. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- src/coreclr/jit/codegen.h | 1 + src/coreclr/jit/codegenarmarch.cpp | 9 +++++++- src/coreclr/jit/codegenlinear.cpp | 36 ++++++++++++++++++++++++++---- src/coreclr/jit/codegenxarch.cpp | 9 +++++++- 4 files changed, 49 insertions(+), 6 deletions(-) diff --git a/src/coreclr/jit/codegen.h b/src/coreclr/jit/codegen.h index f1dd80d0a46011..a963909ba8f7fb 100644 --- a/src/coreclr/jit/codegen.h +++ b/src/coreclr/jit/codegen.h @@ -1122,6 +1122,7 @@ class CodeGen final : public CodeGenInterface void genUnspillRegIfNeeded(GenTree* tree); void genUnspillRegIfNeeded(GenTree* tree, unsigned multiRegIndex); bool isRematerializableConstant(GenTree* tree); + bool isRematerializedConstantSpill(GenTree* tree); regNumber genConsumeReg(GenTree* tree); regNumber genConsumeReg(GenTree* tree, unsigned multiRegIndex); void genCopyRegIfNeeded(GenTree* tree, regNumber needReg); diff --git a/src/coreclr/jit/codegenarmarch.cpp b/src/coreclr/jit/codegenarmarch.cpp index e9dcc3d455a0c3..4bf1eacd5f5fb9 100644 --- a/src/coreclr/jit/codegenarmarch.cpp +++ b/src/coreclr/jit/codegenarmarch.cpp @@ -192,7 +192,14 @@ void CodeGen::genCodeForTreeNode(GenTree* treeNode) #if defined(FEATURE_MASKED_HW_INTRINSICS) case GT_CNS_MSK: #endif // FEATURE_MASKED_HW_INTRINSICS - genSetRegToConst(targetReg, targetType, treeNode); + // A rematerializable constant that is spilled right after its definition has no + // in-register use; its value is rematerialized at each reload point (see + // genUnspillRegIfNeeded) and its store is skipped (see genProduceReg). Skip emitting + // the definition too, since it would only materialize a value that is never read. + if (!isRematerializedConstantSpill(treeNode)) + { + genSetRegToConst(targetReg, targetType, treeNode); + } genProduceReg(treeNode); break; diff --git a/src/coreclr/jit/codegenlinear.cpp b/src/coreclr/jit/codegenlinear.cpp index 2549a2568cb9b7..fa65d5f4ed74cb 100644 --- a/src/coreclr/jit/codegenlinear.cpp +++ b/src/coreclr/jit/codegenlinear.cpp @@ -1539,6 +1539,34 @@ bool CodeGen::isRematerializableConstant(GenTree* tree) #endif // TARGET_XARCH } +//------------------------------------------------------------------------ +// isRematerializedConstantSpill: Determine whether a constant that LSRA has marked to +// spill will instead be handled entirely by rematerialization, so that both its stack +// store and its register definition can be elided. +// +// Arguments: +// tree - the node that defines the value +// +// Return Value: +// True if 'tree' is a rematerializable constant whose definition is spilled immediately +// (GTF_SPILL) and every reload will rematerialize the value rather than consume it from +// a spill temp (GTF_NOREG_AT_USE). +// +// Notes: +// GTF_SPILL on a definition carries spill-after semantics: LSRA stores the value right +// after it is produced and frees the register, so there is no in-register use of the +// definition and every consumer reloads it. When such a value is rematerializable, the +// reload is replaced by rematerialization (see genUnspillRegIfNeeded) and the stack store +// is skipped (see genProduceReg). In that case the definition itself only materializes a +// value into a register that is never read, so callers use this predicate to also elide +// the definition. GTF_NOREG_AT_USE means the value is consumed directly from the spill +// temp as a contained memory operand, which still requires the definition and its store. +// +bool CodeGen::isRematerializedConstantSpill(GenTree* tree) +{ + return ((tree->gtFlags & (GTF_SPILL | GTF_NOREG_AT_USE)) == GTF_SPILL) && isRematerializableConstant(tree); +} + //------------------------------------------------------------------------ // genCopyRegIfNeeded: Copy the given node into the specified register // @@ -2364,10 +2392,10 @@ void CodeGen::genProduceReg(GenTree* tree) // for which isRematerializableConstant is enabled) can be rematerialized cheaply // without a scratch register. Rather than spilling such a value to the stack and // reloading it, skip creating the spill temp and emitting the store; - // genUnspillRegIfNeeded will rematerialize it at the reload point. If the value will - // instead be consumed directly from the spill temp as a contained memory operand - // (GTF_NOREG_AT_USE), it must still be spilled normally. - if (isRematerializableConstant(tree) && ((tree->gtFlags & GTF_NOREG_AT_USE) == 0)) + // genUnspillRegIfNeeded will rematerialize it at the reload point. The definition + // that produced this value is likewise elided at the definition site (see + // genCodeForTreeNode), so nothing reads the register here. + if (isRematerializedConstantSpill(tree)) { tree->gtFlags |= GTF_SPILLED; tree->gtFlags &= ~GTF_SPILL; diff --git a/src/coreclr/jit/codegenxarch.cpp b/src/coreclr/jit/codegenxarch.cpp index ecb82d3a367eb1..84534f43797a4c 100644 --- a/src/coreclr/jit/codegenxarch.cpp +++ b/src/coreclr/jit/codegenxarch.cpp @@ -1846,7 +1846,14 @@ void CodeGen::genCodeForTreeNode(GenTree* treeNode) #if defined(FEATURE_MASKED_HW_INTRINSICS) case GT_CNS_MSK: #endif // FEATURE_MASKED_HW_INTRINSICS - genSetRegToConst(targetReg, targetType, treeNode); + // A rematerializable constant that is spilled right after its definition has no + // in-register use; its value is rematerialized at each reload point (see + // genUnspillRegIfNeeded) and its store is skipped (see genProduceReg). Skip emitting + // the definition too, since it would only materialize a value that is never read. + if (!isRematerializedConstantSpill(treeNode)) + { + genSetRegToConst(targetReg, targetType, treeNode); + } genProduceReg(treeNode); break; From 8b6f0aefd5d942a1dff3e27bfd25ab8b74831afd Mon Sep 17 00:00:00 2001 From: Tanner Gooding Date: Tue, 7 Jul 2026 13:54:10 -0700 Subject: [PATCH 05/11] JIT: Align data-section constant offsets to fix duplicate emission The data section packed entries sequentially with no per-entry alignment padding, so emitDataGenFind's ((curOffs % alignment) == 0) guard could reject a valid match when the existing constant landed at a misaligned offset, emitting a duplicate. Give each dataSection a dsOffset computed via AlignUp in the begin-functions and use it as the single source of truth for lookup, output, and display. This honors the requested alignment (a bug if it wasn't) while SMALL_CODE still gets tight packing via its smaller requested alignment. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- src/coreclr/jit/emit.cpp | 56 ++++++++++++++++++++++--------------- src/coreclr/jit/emit.h | 1 + src/coreclr/jit/emitarm.cpp | 9 ++---- 3 files changed, 37 insertions(+), 29 deletions(-) diff --git a/src/coreclr/jit/emit.cpp b/src/coreclr/jit/emit.cpp index d80cba4cb00e6c..06b9cb8937ff1c 100644 --- a/src/coreclr/jit/emit.cpp +++ b/src/coreclr/jit/emit.cpp @@ -6978,8 +6978,10 @@ unsigned emitter::emitEndCodeGen(Compiler* comp, dataChunk->flags = CORJIT_ALLOCMEM_READONLY_DATA | CORJIT_ALLOCMEM_HAS_POINTERS_TO_CODE; } - *dataChunkOffset = cumulativeOffset; - cumulativeOffset += sec->dsSize; + // The logical offset assigned to each section (see emitDataGenBeg) is what instructions + // reference and what emitDataOffsetToPtr maps back to a chunk, so use it here rather than + // recomputing a packed offset that would ignore inter-section alignment padding. + *dataChunkOffset = sec->dsOffset; } comp->Metrics.AllocatedHotCodeBytes = emitTotalHotCodeSize; @@ -7866,9 +7868,15 @@ UNATIVE_OFFSET emitter::emitDataGenBeg(unsigned size, unsigned alignment, var_ty // assert((size != 0) && ((size % dataSection::MIN_DATA_ALIGN) == 0)); - unsigned secOffs = emitConsDsc.dsdOffs; + // Place the constant at an offset that satisfies its required alignment. This ensures + // aligned loads of the constant address a properly aligned location, and it lets the + // duplicate-constant matching in emitDataGenFind reuse an existing entry (that search + // only considers candidates sitting at a suitably aligned offset). Compilers targeting + // SMALL_CODE request a smaller alignment in the const-emission helpers and so still get + // tight packing. + unsigned secOffs = AlignUp(emitConsDsc.dsdOffs, alignment); /* Advance the current offset */ - emitConsDsc.dsdOffs += size; + emitConsDsc.dsdOffs = secOffs + size; /* Allocate a data section descriptor and add it to the list */ @@ -7880,6 +7888,8 @@ UNATIVE_OFFSET emitter::emitDataGenBeg(unsigned size, unsigned alignment, var_ty secDesc->dsAlignment = alignment; + secDesc->dsOffset = secOffs; + secDesc->dsDataType = dataType; secDesc->dsNext = nullptr; @@ -7915,13 +7925,13 @@ UNATIVE_OFFSET emitter::emitBBTableDataGenBeg(unsigned numEntries, bool relative UNATIVE_OFFSET emittedSize = numEntries * elemSize; - /* Get hold of the current offset */ + /* Get hold of the current offset, aligned for the element size */ - secOffs = emitConsDsc.dsdOffs; + secOffs = AlignUp(emitConsDsc.dsdOffs, elemSize); /* Advance the current offset */ - emitConsDsc.dsdOffs += emittedSize; + emitConsDsc.dsdOffs = secOffs + emittedSize; /* Allocate a data section descriptor and add it to the list */ @@ -7933,6 +7943,8 @@ UNATIVE_OFFSET emitter::emitBBTableDataGenBeg(unsigned numEntries, bool relative secDesc->dsAlignment = elemSize; + secDesc->dsOffset = secOffs; + secDesc->dsDataType = TYP_UNKNOWN; secDesc->dsNext = nullptr; @@ -7962,9 +7974,9 @@ UNATIVE_OFFSET emitter::emitBBTableDataGenBeg(unsigned numEntries, bool relative // void emitter::emitAsyncResumeTable(unsigned numEntries, UNATIVE_OFFSET* dataSecOffs, emitter::dataSection** dataSec) { - UNATIVE_OFFSET secOffs = emitConsDsc.dsdOffs; unsigned emittedSize = sizeof(CORINFO_AsyncResumeInfo) * numEntries; - emitConsDsc.dsdOffs += emittedSize; + UNATIVE_OFFSET secOffs = AlignUp(emitConsDsc.dsdOffs, TARGET_POINTER_SIZE); + emitConsDsc.dsdOffs = secOffs + emittedSize; dataSection* secDesc = (dataSection*)emitGetMem(sizeof(dataSection)); secDesc->dsType = dataSection::asyncResumeInfo; @@ -7975,6 +7987,7 @@ void emitter::emitAsyncResumeTable(unsigned numEntries, UNATIVE_OFFSET* dataSecO secDesc->dsSize = emittedSize; secDesc->dsAlignment = TARGET_POINTER_SIZE; + secDesc->dsOffset = secOffs; secDesc->dsDataType = TYP_UNKNOWN; secDesc->dsNext = nullptr; @@ -8061,7 +8074,6 @@ UNATIVE_OFFSET emitter::emitDataGenFind(const void* cnsAddr, unsigned cnsSize, u { UNATIVE_OFFSET cnum = INVALID_UNATIVE_OFFSET; unsigned cmpCount = 0; - unsigned curOffs = 0; dataSection* secDesc = emitConsDsc.dsdList; while (secDesc != nullptr) { @@ -8071,11 +8083,17 @@ UNATIVE_OFFSET emitter::emitDataGenFind(const void* cnsAddr, unsigned cnsSize, u // We match the bit pattern, so the dataType can be different // Only match constants when the dsType is 'data' // - if ((secDesc->dsType == dataSection::data) && (secDesc->dsSize >= cnsSize) && ((curOffs % alignment) == 0)) + // The existing entry must also sit at an offset that satisfies the requested alignment; + // emitDataGenBeg aligns every entry's offset to its own alignment, so this normally holds + // for equally-aligned constants but can still fail when reusing a more-strictly-aligned + // request against a less-aligned entry. + // + if ((secDesc->dsType == dataSection::data) && (secDesc->dsSize >= cnsSize) && + ((secDesc->dsOffset % alignment) == 0)) { if (memcmp(cnsAddr, secDesc->Data(), cnsSize) == 0) { - cnum = curOffs; + cnum = secDesc->dsOffset; // We also might want to update the dsDataType // @@ -8092,7 +8110,6 @@ UNATIVE_OFFSET emitter::emitDataGenFind(const void* cnsAddr, unsigned cnsSize, u } } - curOffs += secDesc->dsSize; secDesc = secDesc->dsNext; if (++cmpCount > 64) @@ -8436,8 +8453,7 @@ void emitter::emitOutputDataSec(dataSecDsc* sec, AllocMemChunk* chunks) /* Walk and emit the contents of all the data blocks */ - size_t curOffs = 0; - AllocMemChunk* chunk = chunks; + AllocMemChunk* chunk = chunks; for (dataSection* dsc = sec->dsdList; dsc; dsc = dsc->dsNext, chunk++) { @@ -8539,7 +8555,7 @@ void emitter::emitOutputDataSec(dataSecDsc* sec, AllocMemChunk* chunks) #ifdef DEBUG if (EMITVERBOSE) { - printf(" section %3u, size %2u, RWD%2u:\t", secNum++, dscSize, curOffs); + printf(" section %3u, size %2u, RWD%2u:\t", secNum++, dscSize, dsc->dsOffset); for (size_t i = 0; i < dscSize; i++) { @@ -8565,8 +8581,6 @@ void emitter::emitOutputDataSec(dataSecDsc* sec, AllocMemChunk* chunks) } #endif // DEBUG } - - curOffs += dscSize; } } @@ -8586,8 +8600,7 @@ void emitter::emitDispDataSec(dataSecDsc* section, AllocMemChunk* dataChunks) { printf("\n"); - unsigned offset = 0; - AllocMemChunk* chunk = dataChunks; + AllocMemChunk* chunk = dataChunks; for (dataSection* data = section->dsdList; data != nullptr; data = data->dsNext, chunk++) { @@ -8600,9 +8613,8 @@ void emitter::emitDispDataSec(dataSecDsc* section, AllocMemChunk* dataChunks) const char* labelFormat = "%-7s"; char label[64]; - sprintf_s(label, ArrLen(label), "RWD%02u", offset); + sprintf_s(label, ArrLen(label), "RWD%02u", data->dsOffset); printf(labelFormat, label); - offset += data->dsSize; if ((data->dsType == dataSection::blockRelative32) || (data->dsType == dataSection::blockAbsoluteAddr)) { diff --git a/src/coreclr/jit/emit.h b/src/coreclr/jit/emit.h index 64306e35cd8a24..a736d5ab70b456 100644 --- a/src/coreclr/jit/emit.h +++ b/src/coreclr/jit/emit.h @@ -3627,6 +3627,7 @@ class emitter dataSection* dsNext; UNATIVE_OFFSET dsAlignment; + UNATIVE_OFFSET dsOffset; UNATIVE_OFFSET dsSize; sectionType dsType; var_types dsDataType; diff --git a/src/coreclr/jit/emitarm.cpp b/src/coreclr/jit/emitarm.cpp index 0fa1e878c73745..9dee7e935c8ac2 100644 --- a/src/coreclr/jit/emitarm.cpp +++ b/src/coreclr/jit/emitarm.cpp @@ -7259,24 +7259,19 @@ void emitter::emitDispInsHelp( emitDispReg(id->idReg1(), attr, true); imm = emitGetInsSC(id); { - dataSection* jdsc = nullptr; - NATIVE_OFFSET offs = 0; + dataSection* jdsc = nullptr; /* Find the appropriate entry in the data section list */ for (jdsc = emitConsDsc.dsdList; jdsc; jdsc = jdsc->dsNext) { - UNATIVE_OFFSET size = jdsc->dsSize; - /* Is this a label table? */ if (jdsc->dsType == dataSection::blockAbsoluteAddr) { - if (offs == imm) + if (jdsc->dsOffset == (UNATIVE_OFFSET)imm) break; } - - offs += size; } if (id->idIsDspReloc()) From b90683d6661f97df726d4582e74cc2c6ea77d425 Mon Sep 17 00:00:00 2001 From: Tanner Gooding Date: Tue, 7 Jul 2026 16:01:07 -0700 Subject: [PATCH 06/11] Apply formatting patch --- src/coreclr/jit/codegenarmarch.cpp | 8 ++++---- src/coreclr/jit/codegenxarch.cpp | 8 ++++---- src/native/external/libunwind/README.md | 0 3 files changed, 8 insertions(+), 8 deletions(-) mode change 120000 => 100644 src/native/external/libunwind/README.md diff --git a/src/coreclr/jit/codegenarmarch.cpp b/src/coreclr/jit/codegenarmarch.cpp index 4bf1eacd5f5fb9..2424d9f449825b 100644 --- a/src/coreclr/jit/codegenarmarch.cpp +++ b/src/coreclr/jit/codegenarmarch.cpp @@ -192,10 +192,10 @@ void CodeGen::genCodeForTreeNode(GenTree* treeNode) #if defined(FEATURE_MASKED_HW_INTRINSICS) case GT_CNS_MSK: #endif // FEATURE_MASKED_HW_INTRINSICS - // A rematerializable constant that is spilled right after its definition has no - // in-register use; its value is rematerialized at each reload point (see - // genUnspillRegIfNeeded) and its store is skipped (see genProduceReg). Skip emitting - // the definition too, since it would only materialize a value that is never read. + // A rematerializable constant that is spilled right after its definition has no + // in-register use; its value is rematerialized at each reload point (see + // genUnspillRegIfNeeded) and its store is skipped (see genProduceReg). Skip emitting + // the definition too, since it would only materialize a value that is never read. if (!isRematerializedConstantSpill(treeNode)) { genSetRegToConst(targetReg, targetType, treeNode); diff --git a/src/coreclr/jit/codegenxarch.cpp b/src/coreclr/jit/codegenxarch.cpp index 84534f43797a4c..b60bd52e98c447 100644 --- a/src/coreclr/jit/codegenxarch.cpp +++ b/src/coreclr/jit/codegenxarch.cpp @@ -1846,10 +1846,10 @@ void CodeGen::genCodeForTreeNode(GenTree* treeNode) #if defined(FEATURE_MASKED_HW_INTRINSICS) case GT_CNS_MSK: #endif // FEATURE_MASKED_HW_INTRINSICS - // A rematerializable constant that is spilled right after its definition has no - // in-register use; its value is rematerialized at each reload point (see - // genUnspillRegIfNeeded) and its store is skipped (see genProduceReg). Skip emitting - // the definition too, since it would only materialize a value that is never read. + // A rematerializable constant that is spilled right after its definition has no + // in-register use; its value is rematerialized at each reload point (see + // genUnspillRegIfNeeded) and its store is skipped (see genProduceReg). Skip emitting + // the definition too, since it would only materialize a value that is never read. if (!isRematerializedConstantSpill(treeNode)) { genSetRegToConst(targetReg, targetType, treeNode); diff --git a/src/native/external/libunwind/README.md b/src/native/external/libunwind/README.md deleted file mode 120000 index 100b93820ade4c..00000000000000 --- a/src/native/external/libunwind/README.md +++ /dev/null @@ -1 +0,0 @@ -README \ No newline at end of file diff --git a/src/native/external/libunwind/README.md b/src/native/external/libunwind/README.md new file mode 100644 index 00000000000000..100b93820ade4c --- /dev/null +++ b/src/native/external/libunwind/README.md @@ -0,0 +1 @@ +README \ No newline at end of file From 1fd8611af7a4979d8c657e3fd2cae92ae5ea8d23 Mon Sep 17 00:00:00 2001 From: Tanner Gooding Date: Tue, 7 Jul 2026 16:04:08 -0700 Subject: [PATCH 07/11] Respond to PR feedback --- src/coreclr/jit/emit.cpp | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/coreclr/jit/emit.cpp b/src/coreclr/jit/emit.cpp index 06b9cb8937ff1c..b58ae85e80df30 100644 --- a/src/coreclr/jit/emit.cpp +++ b/src/coreclr/jit/emit.cpp @@ -6964,7 +6964,6 @@ unsigned emitter::emitEndCodeGen(Compiler* comp, AllocMemChunk* dataChunk = emitDataChunks; unsigned* dataChunkOffset = emitDataChunkOffsets; - unsigned cumulativeOffset = 0; for (dataSection* sec = emitConsDsc.dsdList; sec != nullptr; sec = sec->dsNext, dataChunk++, dataChunkOffset++) { comp->Metrics.ReadOnlyDataBytes += sec->dsSize; @@ -8555,7 +8554,7 @@ void emitter::emitOutputDataSec(dataSecDsc* sec, AllocMemChunk* chunks) #ifdef DEBUG if (EMITVERBOSE) { - printf(" section %3u, size %2u, RWD%2u:\t", secNum++, dscSize, dsc->dsOffset); + printf(" section %3u, size %zu, RWD%zu:\t", secNum++, dscSize, (size_t)dsc->dsOffset); for (size_t i = 0; i < dscSize; i++) { @@ -8613,7 +8612,7 @@ void emitter::emitDispDataSec(dataSecDsc* section, AllocMemChunk* dataChunks) const char* labelFormat = "%-7s"; char label[64]; - sprintf_s(label, ArrLen(label), "RWD%02u", data->dsOffset); + sprintf_s(label, ArrLen(label), "RWD%02zu", (size_t)data->dsOffset); printf(labelFormat, label); if ((data->dsType == dataSection::blockRelative32) || (data->dsType == dataSection::blockAbsoluteAddr)) From 968e0c0326db9041aa7dcc921e91e7da925632bf Mon Sep 17 00:00:00 2001 From: Tanner Gooding Date: Tue, 7 Jul 2026 16:53:25 -0700 Subject: [PATCH 08/11] Undo change to unrelated file --- src/native/external/libunwind/README.md | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 120000 src/native/external/libunwind/README.md diff --git a/src/native/external/libunwind/README.md b/src/native/external/libunwind/README.md deleted file mode 100644 index 100b93820ade4c..00000000000000 --- a/src/native/external/libunwind/README.md +++ /dev/null @@ -1 +0,0 @@ -README \ No newline at end of file diff --git a/src/native/external/libunwind/README.md b/src/native/external/libunwind/README.md new file mode 120000 index 00000000000000..100b93820ade4c --- /dev/null +++ b/src/native/external/libunwind/README.md @@ -0,0 +1 @@ +README \ No newline at end of file From 0404d36ab473fb8382f82375f129bea4d013cb13 Mon Sep 17 00:00:00 2001 From: Tanner Gooding Date: Tue, 7 Jul 2026 20:37:32 -0700 Subject: [PATCH 09/11] JIT: Consume spilled rematerializable constants from their data-section home on xarch When a rematerializable FP/SIMD/mask constant is spilled after its definition (GTF_SPILL) and its use consumes the value from memory (GTF_NOREG_AT_USE), the value has a legal data-section home and never needs a stack spill temp. On xarch, redirect such uses to the constant's data-section location instead of creating a stack spill temp, eliminating the definition, the store, and the stack slot for optimized code. Non-xarch targets that cannot encode the constant as a memory operand retain the prior behavior. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- src/coreclr/jit/codegenlinear.cpp | 40 +++++++++++++++++++++---------- src/coreclr/jit/codegenxarch.cpp | 7 +++--- src/coreclr/jit/emitxarch.cpp | 7 +++++- src/coreclr/jit/instr.cpp | 5 +++- 4 files changed, 41 insertions(+), 18 deletions(-) diff --git a/src/coreclr/jit/codegenlinear.cpp b/src/coreclr/jit/codegenlinear.cpp index fa65d5f4ed74cb..297e74b553e2d3 100644 --- a/src/coreclr/jit/codegenlinear.cpp +++ b/src/coreclr/jit/codegenlinear.cpp @@ -1549,22 +1549,35 @@ bool CodeGen::isRematerializableConstant(GenTree* tree) // // Return Value: // True if 'tree' is a rematerializable constant whose definition is spilled immediately -// (GTF_SPILL) and every reload will rematerialize the value rather than consume it from -// a spill temp (GTF_NOREG_AT_USE). +// (GTF_SPILL) and whose value never needs to live in a stack spill temp, so both the store +// and the register definition can be elided. // // Notes: // GTF_SPILL on a definition carries spill-after semantics: LSRA stores the value right // after it is produced and frees the register, so there is no in-register use of the -// definition and every consumer reloads it. When such a value is rematerializable, the -// reload is replaced by rematerialization (see genUnspillRegIfNeeded) and the stack store -// is skipped (see genProduceReg). In that case the definition itself only materializes a -// value into a register that is never read, so callers use this predicate to also elide -// the definition. GTF_NOREG_AT_USE means the value is consumed directly from the spill -// temp as a contained memory operand, which still requires the definition and its store. +// definition. When such a value is rematerializable, the stack store is skipped (see +// genProduceReg) and the definition itself only materializes a value into a register that +// is never read, so callers use this predicate to also elide the definition. +// +// A reload that needs the value in a register rematerializes it (see genUnspillRegIfNeeded). +// A use that instead consumes the value directly from memory is marked GTF_NOREG_AT_USE; on +// xarch that memory operand can be the constant's data-section home (see genOperandDesc and +// emitInsBinary), so the definition and its spill store are still unnecessary. Targets that +// cannot encode the constant as a memory operand must materialize it into a register at the +// use, so there the definition and its store are retained for the GTF_NOREG_AT_USE case. // bool CodeGen::isRematerializedConstantSpill(GenTree* tree) { - return ((tree->gtFlags & (GTF_SPILL | GTF_NOREG_AT_USE)) == GTF_SPILL) && isRematerializableConstant(tree); + if (((tree->gtFlags & GTF_SPILL) == 0) || !isRematerializableConstant(tree)) + { + return false; + } + +#if defined(TARGET_XARCH) + return true; +#else + return (tree->gtFlags & GTF_NOREG_AT_USE) == 0; +#endif } //------------------------------------------------------------------------ @@ -2390,10 +2403,11 @@ void CodeGen::genProduceReg(GenTree* tree) { // Floating-point, SIMD, and mask constants have no GC liveness and (on the targets // for which isRematerializableConstant is enabled) can be rematerialized cheaply - // without a scratch register. Rather than spilling such a value to the stack and - // reloading it, skip creating the spill temp and emitting the store; - // genUnspillRegIfNeeded will rematerialize it at the reload point. The definition - // that produced this value is likewise elided at the definition site (see + // without a scratch register. Rather than spilling such a value to the stack, skip + // creating the spill temp and emitting the store; a reload rematerializes it into a + // register (see genUnspillRegIfNeeded) and, on xarch, a use that consumes it from + // memory reads the constant's data-section home directly (see genOperandDesc). The + // definition that produced this value is likewise elided at the definition site (see // genCodeForTreeNode), so nothing reads the register here. if (isRematerializedConstantSpill(tree)) { diff --git a/src/coreclr/jit/codegenxarch.cpp b/src/coreclr/jit/codegenxarch.cpp index b60bd52e98c447..3b386bbb4812d0 100644 --- a/src/coreclr/jit/codegenxarch.cpp +++ b/src/coreclr/jit/codegenxarch.cpp @@ -1847,9 +1847,10 @@ void CodeGen::genCodeForTreeNode(GenTree* treeNode) case GT_CNS_MSK: #endif // FEATURE_MASKED_HW_INTRINSICS // A rematerializable constant that is spilled right after its definition has no - // in-register use; its value is rematerialized at each reload point (see - // genUnspillRegIfNeeded) and its store is skipped (see genProduceReg). Skip emitting - // the definition too, since it would only materialize a value that is never read. + // in-register use; a reload rematerializes it (see genUnspillRegIfNeeded) or a use + // consumes it from its data-section home (see genOperandDesc), and its store is skipped + // (see genProduceReg). Skip emitting the definition too, since it would only materialize + // a value that is never read. if (!isRematerializedConstantSpill(treeNode)) { genSetRegToConst(targetReg, targetType, treeNode); diff --git a/src/coreclr/jit/emitxarch.cpp b/src/coreclr/jit/emitxarch.cpp index 90eee967367021..3af07673706350 100644 --- a/src/coreclr/jit/emitxarch.cpp +++ b/src/coreclr/jit/emitxarch.cpp @@ -6515,7 +6515,12 @@ regNumber emitter::emitInsBinary(instruction ins, emitAttr attr, GenTree* dst, G assert(!dst->isUsedFromMemory()); otherOp = dst; - if ((src->IsCnsIntOrI() || src->IsCnsFltOrDbl()) && !src->isUsedFromSpillTemp()) + // A floating-point constant is always emitted from its data-section home. When it is + // contained it has no spill temp, and a rematerializable constant that LSRA spilled with + // its use consuming the value from memory (GTF_NOREG_AT_USE) likewise has no spill temp + // (see genProduceReg), so route both to the constant (data section) path below rather + // than trying to read a spill temp that was never created. + if ((src->IsCnsIntOrI() && !src->isUsedFromSpillTemp()) || src->IsCnsFltOrDbl()) { assert(!src->isUsedFromMemory() || src->IsCnsFltOrDbl()); cnsOp = src; diff --git a/src/coreclr/jit/instr.cpp b/src/coreclr/jit/instr.cpp index 500e94832339cc..eb5774debe636c 100644 --- a/src/coreclr/jit/instr.cpp +++ b/src/coreclr/jit/instr.cpp @@ -1072,7 +1072,10 @@ CodeGen::OperandDesc CodeGen::genOperandDesc(instruction ins, GenTree* op) unsigned varNum = BAD_VAR_NUM; uint16_t offset = UINT16_MAX; - if (op->isUsedFromSpillTemp()) + // A rematerializable constant that LSRA spilled and whose use consumes it directly from + // memory (GTF_NOREG_AT_USE) has no stack spill temp (see genProduceReg); it is emitted + // from its data-section home instead, handled by the constant cases below. + if (op->isUsedFromSpillTemp() && !isRematerializableConstant(op)) { assert(op->IsRegOptional()); From 7a361072fb31a440499262a6ecbb54f9f937cd05 Mon Sep 17 00:00:00 2001 From: Tanner Gooding Date: Wed, 8 Jul 2026 00:07:55 -0700 Subject: [PATCH 10/11] JIT: Don't mark register-only materializable constants reg-optional on xarch Zero and all-bits-set FP/SIMD/mask constants are materialized directly into a register (xorps, pcmpeqd, etc.) and have no data-section home. Marking them reg-optional let the register allocator resolve the use to memory, forcing a data-section load and a new data-section entry that is strictly worse than rematerializing the value in a register. Report such constants as unsafe to mark reg-optional via IsSafeToMarkRegOptional so that a different operand can be chosen instead. Combined with the existing containment refusal for these constants, they now always keep a required register. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- src/coreclr/jit/lower.cpp | 66 +++++++++++++++++++++++++++++++++++++++ src/coreclr/jit/lower.h | 3 ++ 2 files changed, 69 insertions(+) diff --git a/src/coreclr/jit/lower.cpp b/src/coreclr/jit/lower.cpp index b55ca2619dee3b..7d85dcb00627f6 100644 --- a/src/coreclr/jit/lower.cpp +++ b/src/coreclr/jit/lower.cpp @@ -275,10 +275,27 @@ bool Lowering::IsSafeToContainMem(GenTree* grandparentNode, GenTree* parentNode, // interfere is due to it being address exposed. So this is the only unsafe // case. // +// In addition to interference, a constant that codegen materializes directly +// into a register without any memory access has no spill/data-section home to +// consume from, so reg-optionality is reported as unsafe for it as well (see +// IsConstantMaterializableInRegWithoutMemory). +// bool Lowering::IsSafeToMarkRegOptional(GenTree* parentNode, GenTree* childNode) const { if (!childNode->OperIs(GT_LCL_VAR)) { + // A constant that codegen materializes directly into a register without a memory + // access (e.g. zero via xorps, all-bits-set via pcmpeqd) has no data-section home. + // Marking it reg-optional would let the register allocator resolve the use to memory, + // forcing a data-section load (and a new data-section entry) that is strictly worse + // than rematerializing the value in a register. Report it as unsafe so that a + // different operand can be chosen for reg-optionality instead; such constants are + // likewise never made contained. + if (IsConstantMaterializableInRegWithoutMemory(childNode)) + { + return false; + } + // LIR edges never interfere. This includes GT_LCL_FLD, see the remarks above. return true; } @@ -295,6 +312,55 @@ bool Lowering::IsSafeToMarkRegOptional(GenTree* parentNode, GenTree* childNode) return false; } +//------------------------------------------------------------------------ +// IsConstantMaterializableInRegWithoutMemory: Determine whether a constant is +// materialized directly into a register by an instruction that does not access +// memory. +// +// Arguments: +// node - the node to check +// +// Return Value: +// True if 'node' is a floating-point, SIMD, or mask constant that codegen +// materializes with a register-only instruction (for example xorps for zero or +// pcmpeqd for all-bits-set) rather than a data-section load. +// +// Notes: +// Such constants must never be made contained or reg-optional. Otherwise the +// register allocator could resolve the use to a memory operand, forcing a +// data-section load (and a new data-section entry) that is strictly worse than +// rematerializing the value in a register. Reg-optionality is prevented via +// IsSafeToMarkRegOptional, which consults this predicate. On xarch this is exactly +// the zero and all-bits-set constants (see genSetRegToConst). Other targets always +// assign these constants a register (they are never reg-optional), so they return +// false here. +// +bool Lowering::IsConstantMaterializableInRegWithoutMemory(GenTree* node) const +{ +#if defined(TARGET_XARCH) + if (node->IsCnsFltOrDbl()) + { + return node->IsFloatPositiveZero() || node->IsFloatAllBitsSet(); + } +#if defined(FEATURE_SIMD) + if (node->IsCnsVec()) + { + // A TYP_SIMD32/TYP_SIMD64 constant only exists when the corresponding ISA + // (AVX2/AVX512) is available, so no ISA check is needed here. + return node->IsVectorZero() || node->IsVectorAllBitsSet(); + } +#endif // FEATURE_SIMD +#if defined(FEATURE_MASKED_HW_INTRINSICS) + if (node->IsCnsMsk()) + { + return node->IsMaskZero() || node->IsMaskAllBitsSet(); + } +#endif // FEATURE_MASKED_HW_INTRINSICS +#endif // TARGET_XARCH + + return false; +} + //------------------------------------------------------------------------ // LowerRange: // Lower the specified range of nodes. diff --git a/src/coreclr/jit/lower.h b/src/coreclr/jit/lower.h index 98620180db6f19..adf25b7e829a7a 100644 --- a/src/coreclr/jit/lower.h +++ b/src/coreclr/jit/lower.h @@ -627,6 +627,9 @@ class Lowering final : public Phase // Check if marking an operand of a node as reg-optional is safe. bool IsSafeToMarkRegOptional(GenTree* parentNode, GenTree* node) const; + // Check if a constant is materialized directly into a register without a memory access. + bool IsConstantMaterializableInRegWithoutMemory(GenTree* node) const; + // Checks if it's profitable to optimize an shift and rotate operations to set the zero flag. bool IsProfitableToSetZeroFlag(GenTree* op) const; From 36a6db65e1e468714cfd9d5e3c5269b65379afe8 Mon Sep 17 00:00:00 2001 From: Tanner Gooding Date: Wed, 8 Jul 2026 07:18:09 -0700 Subject: [PATCH 11/11] JIT: Only emit the all-bits-set division constant when it is used The xarch SIMD integer division codegen unconditionally emitted the all-bits-set (negative one) data-section constant used by the signed overflow check, even for unsigned division where that check is skipped. This left an orphaned data-section entry (an all-bits-set constant with no referencing instruction) in the generated code for unsigned SIMD division. Move the constant's emission inside the varTypeIsSigned branch, alongside the int.MinValue constant it is paired with, so it is only materialized when the signed overflow check that consumes it is emitted. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- src/coreclr/jit/hwintrinsiccodegenxarch.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/coreclr/jit/hwintrinsiccodegenxarch.cpp b/src/coreclr/jit/hwintrinsiccodegenxarch.cpp index 1c7849fb7b1d0f..54e4ddba497da7 100644 --- a/src/coreclr/jit/hwintrinsiccodegenxarch.cpp +++ b/src/coreclr/jit/hwintrinsiccodegenxarch.cpp @@ -2428,9 +2428,6 @@ void CodeGen::genBaseIntrinsic(GenTreeHWIntrinsic* node, insOpts instOptions) { divTypeSize = EA_32BYTE; } - simd_t negOneIntVec = simd_t::AllBitsSet(); - CORINFO_FIELD_HANDLE negOneFld = emit->emitSimdConst(&negOneIntVec, typeSize); - // div-by-zero check emit->emitIns_SIMD_R_R_R(INS_xorpd, typeSize, tmpReg2, tmpReg2, tmpReg2, instOptions); emit->emitIns_SIMD_R_R_R(INS_pcmpeqd, typeSize, tmpReg2, tmpReg2, op2Reg, instOptions); @@ -2448,6 +2445,9 @@ void CodeGen::genBaseIntrinsic(GenTreeHWIntrinsic* node, insOpts instOptions) } CORINFO_FIELD_HANDLE minValueFld = emit->emitSimdConst(&minValueInt, typeSize); + simd_t negOneIntVec = simd_t::AllBitsSet(); + CORINFO_FIELD_HANDLE negOneFld = emit->emitSimdConst(&negOneIntVec, typeSize); + emit->emitIns_SIMD_R_R_C(INS_pcmpeqd, typeSize, tmpReg2, op1Reg, minValueFld, 0, instOptions); emit->emitIns_SIMD_R_R_C(INS_pcmpeqd, typeSize, tmpReg3, op2Reg, negOneFld, 0, instOptions); emit->emitIns_SIMD_R_R_R(INS_pandd, typeSize, tmpReg2, tmpReg2, tmpReg3, instOptions);