From aaaed3781fc5d96df3c1e7a1ae5f0c7eba6b0551 Mon Sep 17 00:00:00 2001 From: Larry Ewing Date: Tue, 21 Jul 2026 17:30:56 -0500 Subject: [PATCH 01/10] JIT: Fix wasm scalar-field extraction from a SIMD-register parameter The parameter-register-to-local mapping in lowering maps a scalar field of an unenregisterable parameter to the local created for the containing register segment, retyping the local access to the field's scalar type. On x64/arm64 a scalar overlaps the low bytes of the vector register, so reading the register local at scalar width is a free, correct reinterpret. On wasm this is invalid: a v128 and a scalar (f64/f32/...) are distinct value-stack types with no implicit reinterpret, so 'local.get ' typed as a scalar produces invalid wasm (wasm-tools: 'expected f64, found v128'). This surfaces when crossgen2 compiles, e.g., Vector128.GetHashCode (whose GetElementUnsafe reads a scalar lane of the vector) inlined into GenericEqualityComparer>.GetHashCode. Guard the mapping on wasm: skip mapping a scalar field to a SIMD register segment so the access falls back to the (memory-homed) unenregisterable parameter and a normal scalar load is emitted. This is the floating-point ToScalar / SIMD->scalar var_type case noted as deferred in #130444. --- src/coreclr/jit/lower.cpp | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/src/coreclr/jit/lower.cpp b/src/coreclr/jit/lower.cpp index 1f64a48fc5d485..0d1785da282633 100644 --- a/src/coreclr/jit/lower.cpp +++ b/src/coreclr/jit/lower.cpp @@ -9222,6 +9222,19 @@ void Lowering::FindInducedParameterRegisterLocals() continue; } +#ifdef TARGET_WASM + // On wasm, a scalar and its containing vector are distinct value-stack types: unlike + // x64/arm64 (where the scalar overlaps the low bytes of the vector register), a scalar + // field cannot be read from a SIMD register local by simply retyping the local access. + // Skip mapping a scalar field to a SIMD register segment so the access falls back to the + // (unenregisterable / memory-homed) parameter and a normal scalar load is emitted; an + // explicit extract would be needed otherwise. + if (varTypeIsSIMD(segment.GetRegisterType()) && !varTypeIsSIMD(fld)) + { + continue; + } +#endif // TARGET_WASM + // Found a register segment this field is contained in regSegment = &segment; break; From 01fba8137864a61049420a86b01d2b4905547d23 Mon Sep 17 00:00:00 2001 From: Larry Ewing Date: Tue, 21 Jul 2026 18:34:12 -0500 Subject: [PATCH 02/10] Extract the scalar lane directly instead of falling back to memory Per review feedback (thanks @tannergooding, @AndyAyersMS): rather than skipping the parameter-register-local mapping for a scalar field of a SIMD register (which forces a spill/reload through the memory-homed parameter), extract the lane directly from the SIMD register local via gtNewSimdGetElementNode. This lowers to an explicit lane extract (e.g. f64x2.extract_lane 0) with no spill. The GetElement node is lowered by the subsequent per-block lowering pass. > [!NOTE] > This change was authored with the assistance of GitHub Copilot. --- src/coreclr/jit/lower.cpp | 32 +++++++++++++++++--------------- 1 file changed, 17 insertions(+), 15 deletions(-) diff --git a/src/coreclr/jit/lower.cpp b/src/coreclr/jit/lower.cpp index 0d1785da282633..9cb53e3f70c442 100644 --- a/src/coreclr/jit/lower.cpp +++ b/src/coreclr/jit/lower.cpp @@ -1843,7 +1843,7 @@ void Lowering::SplitArgumentBetweenRegistersAndStack(GenTreeCall* call, CallArg* { assert(arg->OperIsLocalRead()); - JITDUMP("Argument is a local\n", numRegs, stackSeg.Size); + JITDUMP("Argument is a local\n"); GenTreeLclVarCommon* lcl = arg->AsLclVarCommon(); @@ -9222,19 +9222,6 @@ void Lowering::FindInducedParameterRegisterLocals() continue; } -#ifdef TARGET_WASM - // On wasm, a scalar and its containing vector are distinct value-stack types: unlike - // x64/arm64 (where the scalar overlaps the low bytes of the vector register), a scalar - // field cannot be read from a SIMD register local by simply retyping the local access. - // Skip mapping a scalar field to a SIMD register segment so the access falls back to the - // (unenregisterable / memory-homed) parameter and a normal scalar load is emitted; an - // explicit extract would be needed otherwise. - if (varTypeIsSIMD(segment.GetRegisterType()) && !varTypeIsSIMD(fld)) - { - continue; - } -#endif // TARGET_WASM - // Found a register segment this field is contained in regSegment = &segment; break; @@ -9302,7 +9289,22 @@ void Lowering::FindInducedParameterRegisterLocals() GenTree* value = m_compiler->gtNewLclVarNode(remappedLclNum); - if (varTypeUsesFloatReg(value)) +#ifdef TARGET_WASM + if (varTypeIsSIMD(value) && !varTypeIsSIMD(fld)) + { + // On wasm a scalar and its containing vector are distinct value-stack types (unlike + // x64/arm64, where the scalar overlaps the low bytes of the vector register and can be read + // by simply retyping the access). Extract the scalar lane directly from the SIMD register + // local (e.g. f64x2.extract_lane), avoiding an invalid reinterpret or a spill/reload through + // memory. The GetElement node is lowered by the subsequent per-block lowering pass. + unsigned laneIndex = (fld->GetLclOffs() - regSegment->Offset) / genTypeSize(fld); + value = m_compiler->gtNewSimdGetElementNode(fld->TypeGet(), value, + m_compiler->gtNewIconNode((ssize_t)laneIndex), + fld->TypeGet(), genTypeSize(value)); + } + else +#endif // TARGET_WASM + if (varTypeUsesFloatReg(value)) { assert(fld->GetLclOffs() == regSegment->Offset); From e052bdd9c0a01a9a3c7c5edb20303a7e9db4413c Mon Sep 17 00:00:00 2001 From: Larry Ewing Date: Tue, 21 Jul 2026 20:10:26 -0500 Subject: [PATCH 03/10] Improve parameter register extraction Use direct SIMD lane extraction and scalar bit manipulation for fields carried in floating-point parameter registers. Keep physical promotion eligibility synchronized and retain the ARM32 fallback. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 22214765-2cd2-4951-b778-8793730f2bc1 --- src/coreclr/jit/lower.cpp | 39 ++++++++++++++++++++++++++---- src/coreclr/jit/promotion.cpp | 13 +++++++--- src/tests/JIT/opt/Unsafe/Unsafe.cs | 35 +++++++++++++++++++++++++++ 3 files changed, 79 insertions(+), 8 deletions(-) diff --git a/src/coreclr/jit/lower.cpp b/src/coreclr/jit/lower.cpp index 82680153e98589..05ed5f8e054d37 100644 --- a/src/coreclr/jit/lower.cpp +++ b/src/coreclr/jit/lower.cpp @@ -9214,13 +9214,23 @@ void Lowering::FindInducedParameterRegisterLocals() continue; } - // TODO-CQ: Float -> !float extractions are not supported - // TODO-CQ: Float -> float extractions with non-zero offset is not supported +#ifdef FEATURE_SIMD + if (varTypeIsSIMD(segment.GetRegisterType()) && + (varTypeIsSIMD(fld) ? (fld->GetLclOffs() != segment.Offset) + : (((fld->GetLclOffs() - segment.Offset) % genTypeSize(fld)) != 0))) + { + continue; + } +#endif // FEATURE_SIMD + +#ifdef TARGET_ARM + // The scalar extraction below can require TYP_LONG nodes, which are not legal after decomposition. if (genIsValidFloatReg(segment.GetRegister()) && (!varTypeUsesFloatReg(fld) || (fld->GetLclOffs() != segment.Offset))) { continue; } +#endif // TARGET_ARM // Found a register segment this field is contained in regSegment = &segment; @@ -9289,10 +9299,24 @@ void Lowering::FindInducedParameterRegisterLocals() GenTree* value = m_compiler->gtNewLclVarNode(remappedLclNum); - if (varTypeUsesFloatReg(value)) - { - assert(fld->GetLclOffs() == regSegment->Offset); + bool useSimdGetElement = false; +#ifdef FEATURE_SIMD + useSimdGetElement = varTypeIsSIMD(value) && !varTypeIsSIMD(fld); +#endif // FEATURE_SIMD + if (useSimdGetElement) + { +#ifdef FEATURE_SIMD + unsigned laneIndex = (fld->GetLclOffs() - regSegment->Offset) / genTypeSize(fld); + value = m_compiler->gtNewSimdGetElementNode(fld->TypeGet(), value, + m_compiler->gtNewIconNode((ssize_t)laneIndex), fld->TypeGet(), + genTypeSize(value)); +#else + unreached(); +#endif // FEATURE_SIMD + } + else if (varTypeUsesFloatReg(value) && varTypeUsesFloatReg(fld) && (fld->GetLclOffs() == regSegment->Offset)) + { value->gtType = fld->TypeGet(); #ifdef FEATURE_SIMD @@ -9307,6 +9331,11 @@ void Lowering::FindInducedParameterRegisterLocals() } else { + if (varTypeUsesFloatReg(value)) + { + value = m_compiler->gtNewBitCastNode(genTypeSize(value) == 8 ? TYP_LONG : TYP_INT, value); + } + var_types registerType = value->TypeGet(); if (fld->GetLclOffs() > regSegment->Offset) diff --git a/src/coreclr/jit/promotion.cpp b/src/coreclr/jit/promotion.cpp index 80562630ec1ed8..15be2ef4976df8 100644 --- a/src/coreclr/jit/promotion.cpp +++ b/src/coreclr/jit/promotion.cpp @@ -3052,21 +3052,28 @@ bool Promotion::MapsToParameterRegister(Compiler* comp, unsigned lclNum, unsigne for (const ABIPassingSegment& seg : abiInfo.Segments()) { - // This code corresponds to code in Lower::FindInducedParameterRegisterLocals + // This code corresponds to code in Lowering::FindInducedParameterRegisterLocals if ((offset < seg.Offset) || (offset + genTypeSize(accessType) > seg.Offset + seg.Size)) { continue; } - if (!genIsValidIntReg(seg.GetRegister()) && varTypeUsesFloatReg(accessType)) +#ifdef FEATURE_SIMD + if (varTypeIsSIMD(seg.GetRegisterType()) && + (varTypeIsSIMD(accessType) ? (offset != seg.Offset) + : (((offset - seg.Offset) % genTypeSize(accessType)) != 0))) { continue; } +#endif // FEATURE_SIMD - if (genIsValidFloatReg(seg.GetRegister()) && (offset != seg.Offset)) +#ifdef TARGET_ARM + // The scalar extraction in lowering can require TYP_LONG nodes, which are not legal after decomposition. + if (genIsValidFloatReg(seg.GetRegister()) && (!varTypeUsesFloatReg(accessType) || (offset != seg.Offset))) { continue; } +#endif // TARGET_ARM return true; } diff --git a/src/tests/JIT/opt/Unsafe/Unsafe.cs b/src/tests/JIT/opt/Unsafe/Unsafe.cs index b0c8296b2fc298..9104c4baf01287 100644 --- a/src/tests/JIT/opt/Unsafe/Unsafe.cs +++ b/src/tests/JIT/opt/Unsafe/Unsafe.cs @@ -3,6 +3,7 @@ using System; using System.Runtime.CompilerServices; +using System.Runtime.Intrinsics; using Xunit; namespace CodeGenTests @@ -30,6 +31,27 @@ static byte UnsafeAsNarrowCast_Long(long value) return Unsafe.As(ref value); } + [MethodImpl(MethodImplOptions.NoInlining)] + static float UnsafeAsSecondFloat_Double(double value) + { + // ARM64-FULL-LINE: lsr {{x[0-9]+}}, {{x[0-9]+}}, #32 + return Unsafe.Add(ref Unsafe.As(ref value), 1); + } + + [MethodImpl(MethodImplOptions.NoInlining)] + static int UnsafeAsInt_Vector128(Vector128 value) + { + // ARM64-FULL-LINE: smov {{x[0-9]+}}, {{v[0-9]+}}.s[2] + return Unsafe.Add(ref Unsafe.As, int>(ref value), 2); + } + + [MethodImpl(MethodImplOptions.NoInlining)] + static double UnsafeAsSecondDouble_Vector128(Vector128 value) + { + // ARM64-FULL-LINE: dup {{d[0-9]+}}, {{v[0-9]+}}.d[1] + return Unsafe.Add(ref Unsafe.As, double>(ref value), 1); + } + [Fact] public static int TestEntryPoint() { @@ -42,6 +64,19 @@ public static int TestEntryPoint() if (UnsafeAsNarrowCast_Long(255) != 255) return 0; + double doubleValue = BitConverter.Int64BitsToDouble( + ((long)BitConverter.SingleToInt32Bits(2.5f) << 32) | (uint)BitConverter.SingleToInt32Bits(1.25f)); + if (UnsafeAsSecondFloat_Double(doubleValue) != 2.5f) + return 0; + + Vector128 floatVector = Vector128.Create(1.0f, 2.0f, 3.0f, 4.0f); + if (UnsafeAsInt_Vector128(floatVector) != BitConverter.SingleToInt32Bits(3.0f)) + return 0; + + Vector128 doubleVector = Vector128.Create(5.0, 6.0); + if (UnsafeAsSecondDouble_Vector128(doubleVector) != 6.0) + return 0; + return 100; } } From 4a48eef943e5e6232fb6f99fbf75ccb624564a25 Mon Sep 17 00:00:00 2001 From: Larry Ewing Date: Wed, 22 Jul 2026 08:29:20 -0500 Subject: [PATCH 04/10] Optimize scalar FP parameter extraction Use SIMD lane extraction for aligned scalar floating-point fields while preserving the integer fallback for misaligned accesses. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 22214765-2cd2-4951-b778-8793730f2bc1 --- src/coreclr/jit/lower.cpp | 12 ++++++++++++ src/tests/JIT/opt/Unsafe/Unsafe.cs | 13 ++++++++++++- 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/src/coreclr/jit/lower.cpp b/src/coreclr/jit/lower.cpp index 05ed5f8e054d37..e19f4207789a92 100644 --- a/src/coreclr/jit/lower.cpp +++ b/src/coreclr/jit/lower.cpp @@ -9302,6 +9302,18 @@ void Lowering::FindInducedParameterRegisterLocals() bool useSimdGetElement = false; #ifdef FEATURE_SIMD useSimdGetElement = varTypeIsSIMD(value) && !varTypeIsSIMD(fld); + +#ifndef TARGET_WASM + if (!useSimdGetElement && varTypeIsFloating(value) && varTypeIsFloating(fld) && + (fld->GetLclOffs() != regSegment->Offset) && + (((fld->GetLclOffs() - regSegment->Offset) % genTypeSize(fld)) == 0)) + { + assert(value->TypeIs(TYP_DOUBLE)); + assert(fld->TypeIs(TYP_FLOAT)); + value->gtType = TYP_SIMD8; + useSimdGetElement = true; + } +#endif // !TARGET_WASM #endif // FEATURE_SIMD if (useSimdGetElement) diff --git a/src/tests/JIT/opt/Unsafe/Unsafe.cs b/src/tests/JIT/opt/Unsafe/Unsafe.cs index 9104c4baf01287..b260e5e2ea9fbd 100644 --- a/src/tests/JIT/opt/Unsafe/Unsafe.cs +++ b/src/tests/JIT/opt/Unsafe/Unsafe.cs @@ -34,10 +34,17 @@ static byte UnsafeAsNarrowCast_Long(long value) [MethodImpl(MethodImplOptions.NoInlining)] static float UnsafeAsSecondFloat_Double(double value) { - // ARM64-FULL-LINE: lsr {{x[0-9]+}}, {{x[0-9]+}}, #32 + // X64-FULL-LINE: {{v?movshdup}} {{xmm[0-9]+}}, {{xmm[0-9]+}} + // ARM64-FULL-LINE: dup {{s[0-9]+}}, {{v[0-9]+}}.s[1] return Unsafe.Add(ref Unsafe.As(ref value), 1); } + [MethodImpl(MethodImplOptions.NoInlining)] + static float UnsafeAsMisalignedFloat_Double(double value) + { + return Unsafe.As(ref Unsafe.AddByteOffset(ref Unsafe.As(ref value), 2)); + } + [MethodImpl(MethodImplOptions.NoInlining)] static int UnsafeAsInt_Vector128(Vector128 value) { @@ -69,6 +76,10 @@ public static int TestEntryPoint() if (UnsafeAsSecondFloat_Double(doubleValue) != 2.5f) return 0; + float expectedMisaligned = BitConverter.Int32BitsToSingle((int)(BitConverter.DoubleToInt64Bits(doubleValue) >> 16)); + if (UnsafeAsMisalignedFloat_Double(doubleValue) != expectedMisaligned) + return 0; + Vector128 floatVector = Vector128.Create(1.0f, 2.0f, 3.0f, 4.0f); if (UnsafeAsInt_Vector128(floatVector) != BitConverter.SingleToInt32Bits(3.0f)) return 0; From 1b494e4b93cd170206efcc42394e99f8ae247e3b Mon Sep 17 00:00:00 2001 From: Larry Ewing Date: Wed, 22 Jul 2026 12:04:30 -0500 Subject: [PATCH 05/10] Avoid reconstructing promoted parameter registers Keep induced-only promotion from partially decomposing struct operations when a field covers only part of an incoming parameter register segment. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 22214765-2cd2-4951-b778-8793730f2bc1 --- src/coreclr/jit/promotion.cpp | 42 +++++++++++++++++++++++++++++++++-- src/coreclr/jit/promotion.h | 6 ++++- 2 files changed, 45 insertions(+), 3 deletions(-) diff --git a/src/coreclr/jit/promotion.cpp b/src/coreclr/jit/promotion.cpp index 15be2ef4976df8..0cf025c79a5f0f 100644 --- a/src/coreclr/jit/promotion.cpp +++ b/src/coreclr/jit/promotion.cpp @@ -565,6 +565,25 @@ class LocalUses return 0; } + LclVarDsc* lcl = comp->lvaGetDesc(lclNum); + if (lcl->lvIsParam) + { + // Avoid decomposing a whole-struct operation when any induced access covers only part of a parameter + // register segment. Reconstructing that segment at the destination can be more expensive than preserving + // the original struct operation. Reject all induced promotions so we do not partially decompose it. + for (const PrimitiveAccess& inducedAccess : m_inducedAccesses) + { + bool requiresRegisterReconstruction; + if (Promotion::MapsToParameterRegister(comp, lclNum, inducedAccess.Offset, inducedAccess.AccessType, + &requiresRegisterReconstruction) && + requiresRegisterReconstruction) + { + JITDUMP("Not promoting induced accesses that require parameter register reconstruction\n"); + return 0; + } + } + } + int numReps = 0; JITDUMP("Picking induced promotions for V%02u\n", lclNum); for (PrimitiveAccess& inducedAccess : m_inducedAccesses) @@ -707,6 +726,13 @@ class LocalUses weight_t countOverlappedCallArgWtd = 0; weight_t countOverlappedStoredFromCallWtd = 0; + bool mapsToParameterRegister = false; + if (lcl->lvIsParam) + { + mapsToParameterRegister = + Promotion::MapsToParameterRegister(comp, lclNum, access.Offset, access.AccessType); + } + bool overlap = false; for (const Access& otherAccess : m_accesses) { @@ -778,7 +804,7 @@ class LocalUses else if (lcl->lvIsParam) { // For parameters, the backend may be able to map it directly from a register. - if (Promotion::MapsToParameterRegister(comp, lclNum, access.Offset, access.AccessType)) + if (mapsToParameterRegister) { // No promotion will result in a store to stack in the prolog. costWithout += COST_STRUCT_ACCESS_CYCLES * comp->fgFirstBB->getBBWeight(comp); @@ -3031,14 +3057,21 @@ GenTree* Promotion::EffectiveUser(Compiler::GenTreeStack& ancestors) // lclNum - Local being accessed into // offset - Offset being accessed at // accessType - Type of access +// requiresRegisterReconstruction - [out] Whether the access covers only part of its register segment // // Returns: // True if the access can be efficiently done via a parameter register. // -bool Promotion::MapsToParameterRegister(Compiler* comp, unsigned lclNum, unsigned offset, var_types accessType) +bool Promotion::MapsToParameterRegister( + Compiler* comp, unsigned lclNum, unsigned offset, var_types accessType, bool* requiresRegisterReconstruction) { assert(lclNum < comp->info.compArgsCount); + if (requiresRegisterReconstruction != nullptr) + { + *requiresRegisterReconstruction = false; + } + if (comp->opts.IsOSR()) { return false; @@ -3075,6 +3108,11 @@ bool Promotion::MapsToParameterRegister(Compiler* comp, unsigned lclNum, unsigne } #endif // TARGET_ARM + if (requiresRegisterReconstruction != nullptr) + { + *requiresRegisterReconstruction = (offset != seg.Offset) || (genTypeSize(accessType) != seg.Size); + } + return true; } diff --git a/src/coreclr/jit/promotion.h b/src/coreclr/jit/promotion.h index db24874ad69992..b0604d67b031bc 100644 --- a/src/coreclr/jit/promotion.h +++ b/src/coreclr/jit/promotion.h @@ -154,7 +154,11 @@ class Promotion static bool IsCandidateForPhysicalPromotion(LclVarDsc* dsc); static GenTree* EffectiveUser(Compiler::GenTreeStack& ancestors); - static bool MapsToParameterRegister(Compiler* comp, unsigned lclNum, unsigned offs, var_types accessType); + static bool MapsToParameterRegister(Compiler* comp, + unsigned lclNum, + unsigned offs, + var_types accessType, + bool* requiresRegisterReconstruction = nullptr); public: explicit Promotion(Compiler* compiler) : m_compiler(compiler) From 796b34b0e397971aa45640cc653dee95e2f8a31f Mon Sep 17 00:00:00 2001 From: Larry Ewing Date: Wed, 22 Jul 2026 13:45:29 -0500 Subject: [PATCH 06/10] Limit reconstruction guard to FP registers Keep integer parameter-register promotion unchanged while avoiding costly reconstruction of partial floating-point register segments. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 22214765-2cd2-4951-b778-8793730f2bc1 --- src/coreclr/jit/promotion.cpp | 35 +++++++++++++++++++++-------------- src/coreclr/jit/promotion.h | 2 +- 2 files changed, 22 insertions(+), 15 deletions(-) diff --git a/src/coreclr/jit/promotion.cpp b/src/coreclr/jit/promotion.cpp index 0cf025c79a5f0f..56a7e833aa8fd6 100644 --- a/src/coreclr/jit/promotion.cpp +++ b/src/coreclr/jit/promotion.cpp @@ -568,17 +568,18 @@ class LocalUses LclVarDsc* lcl = comp->lvaGetDesc(lclNum); if (lcl->lvIsParam) { - // Avoid decomposing a whole-struct operation when any induced access covers only part of a parameter - // register segment. Reconstructing that segment at the destination can be more expensive than preserving - // the original struct operation. Reject all induced promotions so we do not partially decompose it. + // Avoid decomposing a whole-struct operation when any induced access covers only part of a floating-point + // parameter register segment. Reconstructing that segment at the destination can be more expensive than + // preserving the original struct operation. Reject all induced promotions so we do not partially decompose + // it. for (const PrimitiveAccess& inducedAccess : m_inducedAccesses) { - bool requiresRegisterReconstruction; + bool requiresFloatingPointRegisterReconstruction; if (Promotion::MapsToParameterRegister(comp, lclNum, inducedAccess.Offset, inducedAccess.AccessType, - &requiresRegisterReconstruction) && - requiresRegisterReconstruction) + &requiresFloatingPointRegisterReconstruction) && + requiresFloatingPointRegisterReconstruction) { - JITDUMP("Not promoting induced accesses that require parameter register reconstruction\n"); + JITDUMP("Not promoting induced accesses that require floating-point register reconstruction\n"); return 0; } } @@ -3057,19 +3058,23 @@ GenTree* Promotion::EffectiveUser(Compiler::GenTreeStack& ancestors) // lclNum - Local being accessed into // offset - Offset being accessed at // accessType - Type of access -// requiresRegisterReconstruction - [out] Whether the access covers only part of its register segment +// requiresFloatingPointRegisterReconstruction - [out] Whether the access covers only part of its floating-point +// register segment // // Returns: // True if the access can be efficiently done via a parameter register. // -bool Promotion::MapsToParameterRegister( - Compiler* comp, unsigned lclNum, unsigned offset, var_types accessType, bool* requiresRegisterReconstruction) +bool Promotion::MapsToParameterRegister(Compiler* comp, + unsigned lclNum, + unsigned offset, + var_types accessType, + bool* requiresFloatingPointRegisterReconstruction) { assert(lclNum < comp->info.compArgsCount); - if (requiresRegisterReconstruction != nullptr) + if (requiresFloatingPointRegisterReconstruction != nullptr) { - *requiresRegisterReconstruction = false; + *requiresFloatingPointRegisterReconstruction = false; } if (comp->opts.IsOSR()) @@ -3108,9 +3113,11 @@ bool Promotion::MapsToParameterRegister( } #endif // TARGET_ARM - if (requiresRegisterReconstruction != nullptr) + if (requiresFloatingPointRegisterReconstruction != nullptr) { - *requiresRegisterReconstruction = (offset != seg.Offset) || (genTypeSize(accessType) != seg.Size); + *requiresFloatingPointRegisterReconstruction = + varTypeUsesFloatReg(seg.GetRegisterType()) && + ((offset != seg.Offset) || (genTypeSize(accessType) != seg.Size)); } return true; diff --git a/src/coreclr/jit/promotion.h b/src/coreclr/jit/promotion.h index b0604d67b031bc..46ced61bcf025f 100644 --- a/src/coreclr/jit/promotion.h +++ b/src/coreclr/jit/promotion.h @@ -158,7 +158,7 @@ class Promotion unsigned lclNum, unsigned offs, var_types accessType, - bool* requiresRegisterReconstruction = nullptr); + bool* requiresFloatingPointRegisterReconstruction = nullptr); public: explicit Promotion(Compiler* compiler) : m_compiler(compiler) From 674ceb560af3e414f4ce997a9d09b13a42a025f6 Mon Sep 17 00:00:00 2001 From: Larry Ewing Date: Wed, 22 Jul 2026 17:27:16 -0500 Subject: [PATCH 07/10] Remove parameter promotion reconstruction heuristic Drop the induced-access suppression that only helped contrived HFA forwarding cases and restore direct promotion costing. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 22214765-2cd2-4951-b778-8793730f2bc1 --- src/coreclr/jit/promotion.cpp | 49 ++--------------------------------- src/coreclr/jit/promotion.h | 6 +---- 2 files changed, 3 insertions(+), 52 deletions(-) diff --git a/src/coreclr/jit/promotion.cpp b/src/coreclr/jit/promotion.cpp index 56a7e833aa8fd6..15be2ef4976df8 100644 --- a/src/coreclr/jit/promotion.cpp +++ b/src/coreclr/jit/promotion.cpp @@ -565,26 +565,6 @@ class LocalUses return 0; } - LclVarDsc* lcl = comp->lvaGetDesc(lclNum); - if (lcl->lvIsParam) - { - // Avoid decomposing a whole-struct operation when any induced access covers only part of a floating-point - // parameter register segment. Reconstructing that segment at the destination can be more expensive than - // preserving the original struct operation. Reject all induced promotions so we do not partially decompose - // it. - for (const PrimitiveAccess& inducedAccess : m_inducedAccesses) - { - bool requiresFloatingPointRegisterReconstruction; - if (Promotion::MapsToParameterRegister(comp, lclNum, inducedAccess.Offset, inducedAccess.AccessType, - &requiresFloatingPointRegisterReconstruction) && - requiresFloatingPointRegisterReconstruction) - { - JITDUMP("Not promoting induced accesses that require floating-point register reconstruction\n"); - return 0; - } - } - } - int numReps = 0; JITDUMP("Picking induced promotions for V%02u\n", lclNum); for (PrimitiveAccess& inducedAccess : m_inducedAccesses) @@ -727,13 +707,6 @@ class LocalUses weight_t countOverlappedCallArgWtd = 0; weight_t countOverlappedStoredFromCallWtd = 0; - bool mapsToParameterRegister = false; - if (lcl->lvIsParam) - { - mapsToParameterRegister = - Promotion::MapsToParameterRegister(comp, lclNum, access.Offset, access.AccessType); - } - bool overlap = false; for (const Access& otherAccess : m_accesses) { @@ -805,7 +778,7 @@ class LocalUses else if (lcl->lvIsParam) { // For parameters, the backend may be able to map it directly from a register. - if (mapsToParameterRegister) + if (Promotion::MapsToParameterRegister(comp, lclNum, access.Offset, access.AccessType)) { // No promotion will result in a store to stack in the prolog. costWithout += COST_STRUCT_ACCESS_CYCLES * comp->fgFirstBB->getBBWeight(comp); @@ -3058,25 +3031,14 @@ GenTree* Promotion::EffectiveUser(Compiler::GenTreeStack& ancestors) // lclNum - Local being accessed into // offset - Offset being accessed at // accessType - Type of access -// requiresFloatingPointRegisterReconstruction - [out] Whether the access covers only part of its floating-point -// register segment // // Returns: // True if the access can be efficiently done via a parameter register. // -bool Promotion::MapsToParameterRegister(Compiler* comp, - unsigned lclNum, - unsigned offset, - var_types accessType, - bool* requiresFloatingPointRegisterReconstruction) +bool Promotion::MapsToParameterRegister(Compiler* comp, unsigned lclNum, unsigned offset, var_types accessType) { assert(lclNum < comp->info.compArgsCount); - if (requiresFloatingPointRegisterReconstruction != nullptr) - { - *requiresFloatingPointRegisterReconstruction = false; - } - if (comp->opts.IsOSR()) { return false; @@ -3113,13 +3075,6 @@ bool Promotion::MapsToParameterRegister(Compiler* comp, } #endif // TARGET_ARM - if (requiresFloatingPointRegisterReconstruction != nullptr) - { - *requiresFloatingPointRegisterReconstruction = - varTypeUsesFloatReg(seg.GetRegisterType()) && - ((offset != seg.Offset) || (genTypeSize(accessType) != seg.Size)); - } - return true; } diff --git a/src/coreclr/jit/promotion.h b/src/coreclr/jit/promotion.h index 46ced61bcf025f..db24874ad69992 100644 --- a/src/coreclr/jit/promotion.h +++ b/src/coreclr/jit/promotion.h @@ -154,11 +154,7 @@ class Promotion static bool IsCandidateForPhysicalPromotion(LclVarDsc* dsc); static GenTree* EffectiveUser(Compiler::GenTreeStack& ancestors); - static bool MapsToParameterRegister(Compiler* comp, - unsigned lclNum, - unsigned offs, - var_types accessType, - bool* requiresFloatingPointRegisterReconstruction = nullptr); + static bool MapsToParameterRegister(Compiler* comp, unsigned lclNum, unsigned offs, var_types accessType); public: explicit Promotion(Compiler* compiler) : m_compiler(compiler) From 202dda1796d079f9f1e3d605fed4bacaf9269cba Mon Sep 17 00:00:00 2001 From: Larry Ewing Date: Wed, 22 Jul 2026 17:27:25 -0500 Subject: [PATCH 08/10] Reconstruct float-pair register arguments directly Use a SIMD lane insertion for two float fields carried in one floating-point ABI register, avoiding spill/reload sequences for arguments and returns. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 22214765-2cd2-4951-b778-8793730f2bc1 --- src/coreclr/jit/lower.cpp | 66 +++++++++++++++++-- .../StructABI/FieldListFloatInsertion.cs | 40 +++++++++++ .../StructABI/FieldListFloatInsertion.csproj | 11 ++++ 3 files changed, 113 insertions(+), 4 deletions(-) create mode 100644 src/tests/JIT/Directed/StructABI/FieldListFloatInsertion.cs create mode 100644 src/tests/JIT/Directed/StructABI/FieldListFloatInsertion.csproj diff --git a/src/coreclr/jit/lower.cpp b/src/coreclr/jit/lower.cpp index e19f4207789a92..6df60546dffd1b 100644 --- a/src/coreclr/jit/lower.cpp +++ b/src/coreclr/jit/lower.cpp @@ -5199,6 +5199,29 @@ struct LowerFieldListRegisterInfo } }; +#ifdef FEATURE_SIMD +static bool IsFloatPairFieldListRegister(GenTreeFieldList::Use* use, + unsigned regStart, + unsigned regEnd, + var_types regType) +{ + if ((regType != TYP_DOUBLE) || (use == nullptr) || (use->GetType() != TYP_FLOAT) || (use->GetOffset() != regStart)) + { + return false; + } + + GenTreeFieldList::Use* secondUse = use->GetNext(); + if ((secondUse == nullptr) || (secondUse->GetType() != TYP_FLOAT) || + (secondUse->GetOffset() != regStart + genTypeSize(TYP_FLOAT))) + { + return false; + } + + GenTreeFieldList::Use* nextUse = secondUse->GetNext(); + return (nextUse == nullptr) || (nextUse->GetOffset() >= regEnd); +} +#endif // FEATURE_SIMD + //---------------------------------------------------------------------------------------------- // LowerRetFieldList: // Lower a returned FIELD_LIST node. @@ -5372,6 +5395,12 @@ bool Lowering::IsFieldListCompatibleWithRegisters(GenTreeFieldList* fieldList, return false; } +#ifdef FEATURE_SIMD + bool supportsFloatPairInsertion = IsFloatPairFieldListRegister(use, regStart, regEnd, regType); +#else + bool supportsFloatPairInsertion = false; +#endif // FEATURE_SIMD + do { unsigned fieldStart = use->GetOffset(); @@ -5396,12 +5425,15 @@ bool Lowering::IsFieldListCompatibleWithRegisters(GenTreeFieldList* fieldList, return false; } - // float -> float insertions are not yet supported + // A pair of floats can be directly reconstructed in an 8-byte floating-point register. if (varTypeUsesFloatReg(use->GetNode()) && varTypeUsesFloatReg(regType) && (fieldStart != regStart)) { - JITDUMP("it is not; field [%06u] requires an insertion into register %u\n", - Compiler::dspTreeID(use->GetNode()), i); - return false; + if (!supportsFloatPairInsertion) + { + JITDUMP("it is not; field [%06u] requires an insertion into register %u\n", + Compiler::dspTreeID(use->GetNode()), i); + return false; + } } // int -> float is currently only supported if we can do it as a single bitcast (i.e. without insertions @@ -5464,6 +5496,32 @@ void Lowering::LowerFieldListToFieldListOfRegisters(GenTreeFieldList* fieldLis GenTree* fieldListPrev = fieldList->gtPrev; +#ifdef FEATURE_SIMD + if (IsFloatPairFieldListRegister(use, regStart, regEnd, regType)) + { + GenTreeFieldList::Use* secondUse = use->GetNext(); + GenTree* value = m_compiler->gtNewSimdCreateScalarUnsafeNode(TYP_SIMD8, use->GetNode(), TYP_FLOAT, 8); + BlockRange().InsertBefore(fieldList, value); + + GenTree* index = m_compiler->gtNewIconNode(1); + BlockRange().InsertBefore(fieldList, index); + + value = m_compiler->gtNewSimdWithElementNode(TYP_SIMD8, value, index, secondUse->GetNode(), TYP_FLOAT, 8); + BlockRange().InsertBefore(fieldList, value); + + regEntry->SetNode(value); + regEntry->SetType(TYP_SIMD8); + regEntry->SetNext(secondUse->GetNext()); + use = regEntry->GetNext(); + + if (fieldListPrev->gtNext != fieldList) + { + LowerRange(fieldListPrev->gtNext, fieldList->gtPrev); + } + + continue; + } +#endif // FEATURE_SIMD do { unsigned fieldStart = use->GetOffset(); diff --git a/src/tests/JIT/Directed/StructABI/FieldListFloatInsertion.cs b/src/tests/JIT/Directed/StructABI/FieldListFloatInsertion.cs new file mode 100644 index 00000000000000..f6ef255562bd1d --- /dev/null +++ b/src/tests/JIT/Directed/StructABI/FieldListFloatInsertion.cs @@ -0,0 +1,40 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Runtime.CompilerServices; +using Xunit; + +public class FieldListFloatInsertion +{ + [MethodImpl(MethodImplOptions.NoInlining)] + private static FloatPair ReturnPair(float x, float y) + { + // X64-LINUX-FULL-LINE: {{v?insertps}} {{xmm[0-9]+}}, {{xmm[0-9]+}}, 16 + // X64-OSX-FULL-LINE: {{v?insertps}} {{xmm[0-9]+}}, {{xmm[0-9]+}}, 16 + return new FloatPair { X = x, Y = y }; + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static float PassPair(float x, float y) + { + // X64-LINUX-FULL-LINE: {{v?insertps}} {{xmm[0-9]+}}, {{xmm[0-9]+}}, 16 + // X64-OSX-FULL-LINE: {{v?insertps}} {{xmm[0-9]+}}, {{xmm[0-9]+}}, 16 + return SumPair(new FloatPair { X = x, Y = y }); + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static float SumPair(FloatPair value) => value.X + value.Y; + + [Fact] + public static int TestEntryPoint() + { + FloatPair value = ReturnPair(1.0f, 2.0f); + return (value.X == 1.0f) && (value.Y == 2.0f) && (PassPair(3.0f, 4.0f) == 7.0f) ? 100 : 0; + } + + private struct FloatPair + { + public float X; + public float Y; + } +} diff --git a/src/tests/JIT/Directed/StructABI/FieldListFloatInsertion.csproj b/src/tests/JIT/Directed/StructABI/FieldListFloatInsertion.csproj new file mode 100644 index 00000000000000..ed91f815fee721 --- /dev/null +++ b/src/tests/JIT/Directed/StructABI/FieldListFloatInsertion.csproj @@ -0,0 +1,11 @@ + + + None + True + + + + true + + + From 403fa129af008b16d5b24c56322801db11e81eac Mon Sep 17 00:00:00 2001 From: Larry Ewing Date: Wed, 22 Jul 2026 22:21:22 -0500 Subject: [PATCH 09/10] Use unpack for float-pair reconstruction Represent two scalar floats as an xarch unpack-low operation, producing smaller and faster code than lane insertion while preserving the exact ABI layout. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 22214765-2cd2-4951-b778-8793730f2bc1 --- src/coreclr/jit/lower.cpp | 27 ++++++++++--------- .../StructABI/FieldListFloatInsertion.cs | 8 +++--- 2 files changed, 19 insertions(+), 16 deletions(-) diff --git a/src/coreclr/jit/lower.cpp b/src/coreclr/jit/lower.cpp index 6df60546dffd1b..96a81c9a6f9bba 100644 --- a/src/coreclr/jit/lower.cpp +++ b/src/coreclr/jit/lower.cpp @@ -5199,7 +5199,7 @@ struct LowerFieldListRegisterInfo } }; -#ifdef FEATURE_SIMD +#if defined(TARGET_XARCH) static bool IsFloatPairFieldListRegister(GenTreeFieldList::Use* use, unsigned regStart, unsigned regEnd, @@ -5220,7 +5220,7 @@ static bool IsFloatPairFieldListRegister(GenTreeFieldList::Use* use, GenTreeFieldList::Use* nextUse = secondUse->GetNext(); return (nextUse == nullptr) || (nextUse->GetOffset() >= regEnd); } -#endif // FEATURE_SIMD +#endif // TARGET_XARCH //---------------------------------------------------------------------------------------------- // LowerRetFieldList: @@ -5395,11 +5395,11 @@ bool Lowering::IsFieldListCompatibleWithRegisters(GenTreeFieldList* fieldList, return false; } -#ifdef FEATURE_SIMD +#if defined(TARGET_XARCH) bool supportsFloatPairInsertion = IsFloatPairFieldListRegister(use, regStart, regEnd, regType); #else bool supportsFloatPairInsertion = false; -#endif // FEATURE_SIMD +#endif // TARGET_XARCH do { @@ -5496,21 +5496,24 @@ void Lowering::LowerFieldListToFieldListOfRegisters(GenTreeFieldList* fieldLis GenTree* fieldListPrev = fieldList->gtPrev; -#ifdef FEATURE_SIMD +#if defined(TARGET_XARCH) if (IsFloatPairFieldListRegister(use, regStart, regEnd, regType)) { GenTreeFieldList::Use* secondUse = use->GetNext(); - GenTree* value = m_compiler->gtNewSimdCreateScalarUnsafeNode(TYP_SIMD8, use->GetNode(), TYP_FLOAT, 8); - BlockRange().InsertBefore(fieldList, value); + GenTree* firstVector = + m_compiler->gtNewSimdCreateScalarUnsafeNode(TYP_SIMD16, use->GetNode(), TYP_FLOAT, 16); + BlockRange().InsertBefore(fieldList, firstVector); - GenTree* index = m_compiler->gtNewIconNode(1); - BlockRange().InsertBefore(fieldList, index); + GenTree* secondVector = + m_compiler->gtNewSimdCreateScalarUnsafeNode(TYP_SIMD16, secondUse->GetNode(), TYP_FLOAT, 16); + BlockRange().InsertBefore(fieldList, secondVector); - value = m_compiler->gtNewSimdWithElementNode(TYP_SIMD8, value, index, secondUse->GetNode(), TYP_FLOAT, 8); + GenTree* value = m_compiler->gtNewSimdHWIntrinsicNode(TYP_SIMD16, firstVector, secondVector, + NI_X86Base_UnpackLow, TYP_FLOAT, 16); BlockRange().InsertBefore(fieldList, value); regEntry->SetNode(value); - regEntry->SetType(TYP_SIMD8); + regEntry->SetType(TYP_SIMD16); regEntry->SetNext(secondUse->GetNext()); use = regEntry->GetNext(); @@ -5521,7 +5524,7 @@ void Lowering::LowerFieldListToFieldListOfRegisters(GenTreeFieldList* fieldLis continue; } -#endif // FEATURE_SIMD +#endif // TARGET_XARCH do { unsigned fieldStart = use->GetOffset(); diff --git a/src/tests/JIT/Directed/StructABI/FieldListFloatInsertion.cs b/src/tests/JIT/Directed/StructABI/FieldListFloatInsertion.cs index f6ef255562bd1d..4db2fc1f37f2b6 100644 --- a/src/tests/JIT/Directed/StructABI/FieldListFloatInsertion.cs +++ b/src/tests/JIT/Directed/StructABI/FieldListFloatInsertion.cs @@ -9,16 +9,16 @@ public class FieldListFloatInsertion [MethodImpl(MethodImplOptions.NoInlining)] private static FloatPair ReturnPair(float x, float y) { - // X64-LINUX-FULL-LINE: {{v?insertps}} {{xmm[0-9]+}}, {{xmm[0-9]+}}, 16 - // X64-OSX-FULL-LINE: {{v?insertps}} {{xmm[0-9]+}}, {{xmm[0-9]+}}, 16 + // X64-LINUX-FULL-LINE: {{v?unpcklps}} {{xmm[0-9]+}}, {{xmm[0-9]+}} + // X64-OSX-FULL-LINE: {{v?unpcklps}} {{xmm[0-9]+}}, {{xmm[0-9]+}} return new FloatPair { X = x, Y = y }; } [MethodImpl(MethodImplOptions.NoInlining)] private static float PassPair(float x, float y) { - // X64-LINUX-FULL-LINE: {{v?insertps}} {{xmm[0-9]+}}, {{xmm[0-9]+}}, 16 - // X64-OSX-FULL-LINE: {{v?insertps}} {{xmm[0-9]+}}, {{xmm[0-9]+}}, 16 + // X64-LINUX-FULL-LINE: {{v?unpcklps}} {{xmm[0-9]+}}, {{xmm[0-9]+}} + // X64-OSX-FULL-LINE: {{v?unpcklps}} {{xmm[0-9]+}}, {{xmm[0-9]+}} return SumPair(new FloatPair { X = x, Y = y }); } From 6abce498189408462eaea8d3e8709c5957eae0dc Mon Sep 17 00:00:00 2001 From: Larry Ewing Date: Thu, 23 Jul 2026 02:32:37 -0500 Subject: [PATCH 10/10] Fix constant float-pair reconstruction Use the lowering helper that removes scalar operands when CreateScalarUnsafe folds to a vector constant, preventing unmarked unused LIR values. Cover constant float-pair arguments in the directed StructABI test. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 22214765-2cd2-4951-b778-8793730f2bc1 --- src/coreclr/jit/lower.cpp | 13 +++---------- .../Directed/StructABI/FieldListFloatInsertion.cs | 13 ++++++++++++- 2 files changed, 15 insertions(+), 11 deletions(-) diff --git a/src/coreclr/jit/lower.cpp b/src/coreclr/jit/lower.cpp index aa267238edef4c..3efcf40d1fbb4c 100644 --- a/src/coreclr/jit/lower.cpp +++ b/src/coreclr/jit/lower.cpp @@ -5500,13 +5500,9 @@ void Lowering::LowerFieldListToFieldListOfRegisters(GenTreeFieldList* fieldLis if (IsFloatPairFieldListRegister(use, regStart, regEnd, regType)) { GenTreeFieldList::Use* secondUse = use->GetNext(); - GenTree* firstVector = - m_compiler->gtNewSimdCreateScalarUnsafeNode(TYP_SIMD16, use->GetNode(), TYP_FLOAT, 16); - BlockRange().InsertBefore(fieldList, firstVector); - + GenTree* firstVector = InsertNewSimdCreateScalarUnsafeNode(TYP_SIMD16, use->GetNode(), TYP_FLOAT, 16); GenTree* secondVector = - m_compiler->gtNewSimdCreateScalarUnsafeNode(TYP_SIMD16, secondUse->GetNode(), TYP_FLOAT, 16); - BlockRange().InsertBefore(fieldList, secondVector); + InsertNewSimdCreateScalarUnsafeNode(TYP_SIMD16, secondUse->GetNode(), TYP_FLOAT, 16); GenTree* value = m_compiler->gtNewSimdHWIntrinsicNode(TYP_SIMD16, firstVector, secondVector, NI_X86Base_UnpackLow, TYP_FLOAT, 16); @@ -5517,10 +5513,7 @@ void Lowering::LowerFieldListToFieldListOfRegisters(GenTreeFieldList* fieldLis regEntry->SetNext(secondUse->GetNext()); use = regEntry->GetNext(); - if (fieldListPrev->gtNext != fieldList) - { - LowerRange(fieldListPrev->gtNext, fieldList->gtPrev); - } + LowerNode(value); continue; } diff --git a/src/tests/JIT/Directed/StructABI/FieldListFloatInsertion.cs b/src/tests/JIT/Directed/StructABI/FieldListFloatInsertion.cs index 4db2fc1f37f2b6..71ee2034c8cc7e 100644 --- a/src/tests/JIT/Directed/StructABI/FieldListFloatInsertion.cs +++ b/src/tests/JIT/Directed/StructABI/FieldListFloatInsertion.cs @@ -22,6 +22,14 @@ private static float PassPair(float x, float y) return SumPair(new FloatPair { X = x, Y = y }); } + [MethodImpl(MethodImplOptions.NoInlining)] + private static float PassPairWithConstant(float y) + { + // X64-LINUX-FULL-LINE: {{v?unpcklps}} {{xmm[0-9]+}}, {{xmm[0-9]+}} + // X64-OSX-FULL-LINE: {{v?unpcklps}} {{xmm[0-9]+}}, {{xmm[0-9]+}} + return SumPair(new FloatPair { X = 0.0f, Y = y }); + } + [MethodImpl(MethodImplOptions.NoInlining)] private static float SumPair(FloatPair value) => value.X + value.Y; @@ -29,7 +37,10 @@ private static float PassPair(float x, float y) public static int TestEntryPoint() { FloatPair value = ReturnPair(1.0f, 2.0f); - return (value.X == 1.0f) && (value.Y == 2.0f) && (PassPair(3.0f, 4.0f) == 7.0f) ? 100 : 0; + return (value.X == 1.0f) && (value.Y == 2.0f) && (PassPair(3.0f, 4.0f) == 7.0f) && + (PassPairWithConstant(5.0f) == 5.0f) + ? 100 + : 0; } private struct FloatPair