From 22c5838e102c09846509277da45b3453ca01e5a9 Mon Sep 17 00:00:00 2001 From: Jakob Botsch Nielsen Date: Fri, 22 Aug 2025 13:52:14 +0200 Subject: [PATCH 01/14] JIT: Switch GetElement and WithElement to native sized indices --- src/coreclr/jit/decomposelongs.cpp | 2 +- src/coreclr/jit/gentree.cpp | 11 ++++++++--- src/coreclr/jit/hwintrinsic.cpp | 4 ++-- src/coreclr/jit/hwintrinsicarm64.cpp | 13 +++++++++---- src/coreclr/jit/hwintrinsiccodegenxarch.cpp | 4 ++++ src/coreclr/jit/hwintrinsicxarch.cpp | 10 +++++++--- src/coreclr/jit/lclmorph.cpp | 8 ++++---- src/coreclr/jit/lowerxarch.cpp | 2 +- src/coreclr/jit/valuenum.cpp | 18 ++++++++++-------- 9 files changed, 46 insertions(+), 26 deletions(-) diff --git a/src/coreclr/jit/decomposelongs.cpp b/src/coreclr/jit/decomposelongs.cpp index 70767278e390e1..d4b9fed941f9b7 100644 --- a/src/coreclr/jit/decomposelongs.cpp +++ b/src/coreclr/jit/decomposelongs.cpp @@ -1960,7 +1960,7 @@ GenTree* DecomposeLongs::DecomposeHWIntrinsicToScalar(LIR::Use& use, GenTreeHWIn simdTmpVar = m_compiler->gtNewLclLNode(simdTmpVarNum, simdTmpVar->TypeGet()); Range().InsertAfter(loResult, simdTmpVar); - GenTree* one = m_compiler->gtNewIconNode(1); + GenTree* one = m_compiler->gtNewIconNode(1, TYP_I_IMPL); GenTree* hiResult = m_compiler->gtNewSimdGetElementNode(TYP_INT, simdTmpVar, one, CORINFO_TYPE_INT, simdSize); Range().InsertAfter(simdTmpVar, one, hiResult); diff --git a/src/coreclr/jit/gentree.cpp b/src/coreclr/jit/gentree.cpp index 10bde780a8534f..b7b167bcfcf2fb 100644 --- a/src/coreclr/jit/gentree.cpp +++ b/src/coreclr/jit/gentree.cpp @@ -21484,18 +21484,21 @@ GenTree* Compiler::gtNewSimdBinOpNode( GenTree* op2Dup = fgMakeMultiUse(op2ToDup); assert(!varTypeIsArithmetic(op1Dup)); - op1Dup = gtNewSimdGetElementNode(TYP_LONG, op1Dup, gtNewIconNode(1), simdBaseJitType, simdSize); + op1Dup = + gtNewSimdGetElementNode(TYP_LONG, op1Dup, gtNewIconNode(1, TYP_I_IMPL), simdBaseJitType, simdSize); if (!varTypeIsArithmetic(op2Dup)) { - op2Dup = gtNewSimdGetElementNode(TYP_LONG, op2Dup, gtNewIconNode(1), simdBaseJitType, simdSize); + op2Dup = gtNewSimdGetElementNode(TYP_LONG, op2Dup, gtNewIconNode(1, TYP_I_IMPL), simdBaseJitType, + simdSize); } // upper = op1.GetElement(1) * op2.GetElement(1) GenTree* upper = gtNewOperNode(GT_MUL, TYP_LONG, op1Dup, op2Dup); // return Vector128.Create(lower, upper) - return gtNewSimdWithElementNode(type, lower, gtNewIconNode(1), upper, simdBaseJitType, simdSize); + return gtNewSimdWithElementNode(type, lower, gtNewIconNode(1, TYP_I_IMPL), upper, simdBaseJitType, + simdSize); } #endif // !TARGET_XARCH && !TARGET_ARM64 unreached(); @@ -23270,6 +23273,7 @@ GenTree* Compiler::gtNewSimdGetElementNode( var_types simdBaseType = JitType2PreciseVarType(simdBaseJitType); assert(varTypeIsArithmetic(simdBaseType)); + assert(op2->TypeIs(TYP_I_IMPL)); #if defined(TARGET_XARCH) if (op2->IsIntegralConst(0)) @@ -27748,6 +27752,7 @@ GenTree* Compiler::gtNewSimdWithElementNode( assert(varTypeIsArithmetic(simdBaseType)); assert(varTypeIsArithmetic(op3)); + assert(op2->TypeIs(TYP_I_IMPL)); #if defined(TARGET_XARCH) assert(!varTypeIsLong(simdBaseType) || compIsaSupportedDebugOnly(InstructionSet_X86Base_X64)); diff --git a/src/coreclr/jit/hwintrinsic.cpp b/src/coreclr/jit/hwintrinsic.cpp index ca815d3c14a8b6..fc23c7e04b490d 100644 --- a/src/coreclr/jit/hwintrinsic.cpp +++ b/src/coreclr/jit/hwintrinsic.cpp @@ -1590,7 +1590,7 @@ GenTree* Compiler::addRangeCheckForHWIntrinsic(GenTree* immOp, int immLowerBound // The value of (immUpperBound - immLowerBound + 1) is denoted as adjustedUpperBound. const ssize_t adjustedUpperBound = (ssize_t)immUpperBound - immLowerBound + 1; - GenTree* adjustedUpperBoundNode = gtNewIconNode(adjustedUpperBound, TYP_INT); + GenTree* adjustedUpperBoundNode = gtNewIconNode(adjustedUpperBound, TYP_I_IMPL); GenTree* immOpDup = nullptr; @@ -1599,7 +1599,7 @@ GenTree* Compiler::addRangeCheckForHWIntrinsic(GenTree* immOp, int immLowerBound if (immLowerBound != 0) { - immOpDup = gtNewOperNode(GT_SUB, TYP_INT, immOpDup, gtNewIconNode(immLowerBound, TYP_INT)); + immOpDup = gtNewOperNode(GT_SUB, TYP_I_IMPL, immOpDup, gtNewIconNode(immLowerBound, TYP_I_IMPL)); } GenTreeBoundsChk* hwIntrinsicChk = diff --git a/src/coreclr/jit/hwintrinsicarm64.cpp b/src/coreclr/jit/hwintrinsicarm64.cpp index a41e6219c20f84..ad7a144cc47f51 100644 --- a/src/coreclr/jit/hwintrinsicarm64.cpp +++ b/src/coreclr/jit/hwintrinsicarm64.cpp @@ -859,11 +859,11 @@ GenTree* Compiler::impSpecialIntrinsic(NamedIntrinsic intrinsic, op1 = gtNewSimdHWIntrinsicNode(retType, op1, NI_Vector64_ToVector128Unsafe, simdBaseJitType, 8); - GenTree* idx = gtNewIconNode(2, TYP_INT); + GenTree* idx = gtNewIconNode(2, TYP_I_IMPL); GenTree* zero = gtNewZeroConNode(TYP_FLOAT); op1 = gtNewSimdWithElementNode(retType, op1, idx, zero, simdBaseJitType, 16); - idx = gtNewIconNode(3, TYP_INT); + idx = gtNewIconNode(3, TYP_I_IMPL); zero = gtNewZeroConNode(TYP_FLOAT); retNode = gtNewSimdWithElementNode(retType, op1, idx, zero, simdBaseJitType, 16); @@ -887,7 +887,7 @@ GenTree* Compiler::impSpecialIntrinsic(NamedIntrinsic intrinsic, op1 = gtNewSimdHWIntrinsicNode(retType, op1, NI_Vector128_AsVector128Unsafe, simdBaseJitType, 12); - GenTree* idx = gtNewIconNode(3, TYP_INT); + GenTree* idx = gtNewIconNode(3, TYP_I_IMPL); GenTree* zero = gtNewZeroConNode(TYP_FLOAT); retNode = gtNewSimdWithElementNode(retType, op1, idx, zero, simdBaseJitType, 16); break; @@ -1345,7 +1345,8 @@ GenTree* Compiler::impSpecialIntrinsic(NamedIntrinsic intrinsic, op1 = impSIMDPopStack(); retNode = gtNewSimdDotProdNode(simdType, op1, op2, simdBaseJitType, simdSize); - retNode = gtNewSimdGetElementNode(retType, retNode, gtNewIconNode(0), simdBaseJitType, simdSize); + retNode = + gtNewSimdGetElementNode(retType, retNode, gtNewIconNode(0, TYP_I_IMPL), simdBaseJitType, simdSize); } break; } @@ -1628,6 +1629,8 @@ GenTree* Compiler::impSpecialIntrinsic(NamedIntrinsic intrinsic, op2 = impPopStack().val; op1 = impSIMDPopStack(); + op2 = gtFoldExpr(impImplicitIorI4Cast(op2, TYP_I_IMPL, /* zeroExtend */ true)); + retNode = gtNewSimdGetElementNode(retType, op1, op2, simdBaseJitType, simdSize); break; } @@ -2750,6 +2753,8 @@ GenTree* Compiler::impSpecialIntrinsic(NamedIntrinsic intrinsic, assert(numArgs == 3); GenTree* indexOp = impStackTop(1).val; + indexOp = gtFoldExpr(impImplicitIorI4Cast(indexOp, TYP_I_IMPL, /* zeroExtend */ true)); + if (!indexOp->OperIsConst()) { if (!opts.OptimizationEnabled()) diff --git a/src/coreclr/jit/hwintrinsiccodegenxarch.cpp b/src/coreclr/jit/hwintrinsiccodegenxarch.cpp index d2fba55f424d9c..1057660e6ae287 100644 --- a/src/coreclr/jit/hwintrinsiccodegenxarch.cpp +++ b/src/coreclr/jit/hwintrinsiccodegenxarch.cpp @@ -2001,6 +2001,8 @@ void CodeGen::genBaseIntrinsic(GenTreeHWIntrinsic* node, insOpts instOptions) assert(genStackLevel == 0); #endif // !FEATURE_FIXED_OUT_ARGS + assert(op2->TypeIs(TYP_I_IMPL)); + regNumber indexReg = op2->GetRegNum(); regNumber valueReg = op3->GetRegNum(); // New element value to be stored @@ -2035,6 +2037,8 @@ void CodeGen::genBaseIntrinsic(GenTreeHWIntrinsic* node, insOpts instOptions) simdType = TYP_SIMD16; } + assert(op2->TypeIs(TYP_I_IMPL)); + // Optimize the case of op1 is in memory and trying to access i'th element. if (!op1->isUsedFromReg()) { diff --git a/src/coreclr/jit/hwintrinsicxarch.cpp b/src/coreclr/jit/hwintrinsicxarch.cpp index ded77bd68d147e..b8f47f8c79f176 100644 --- a/src/coreclr/jit/hwintrinsicxarch.cpp +++ b/src/coreclr/jit/hwintrinsicxarch.cpp @@ -1573,11 +1573,11 @@ GenTree* Compiler::impSpecialIntrinsic(NamedIntrinsic intrinsic, op1 = gtNewSimdHWIntrinsicNode(retType, op1, NI_Vector128_AsVector128Unsafe, simdBaseJitType, 8); - GenTree* idx = gtNewIconNode(2, TYP_INT); + GenTree* idx = gtNewIconNode(2, TYP_I_IMPL); GenTree* zero = gtNewZeroConNode(TYP_FLOAT); op1 = gtNewSimdWithElementNode(retType, op1, idx, zero, simdBaseJitType, 16); - idx = gtNewIconNode(3, TYP_INT); + idx = gtNewIconNode(3, TYP_I_IMPL); zero = gtNewZeroConNode(TYP_FLOAT); retNode = gtNewSimdWithElementNode(retType, op1, idx, zero, simdBaseJitType, 16); @@ -1601,7 +1601,7 @@ GenTree* Compiler::impSpecialIntrinsic(NamedIntrinsic intrinsic, op1 = gtNewSimdHWIntrinsicNode(retType, op1, NI_Vector128_AsVector128Unsafe, simdBaseJitType, 12); - GenTree* idx = gtNewIconNode(3, TYP_INT); + GenTree* idx = gtNewIconNode(3, TYP_I_IMPL); GenTree* zero = gtNewZeroConNode(TYP_FLOAT); retNode = gtNewSimdWithElementNode(retType, op1, idx, zero, simdBaseJitType, 16); break; @@ -2705,6 +2705,8 @@ GenTree* Compiler::impSpecialIntrinsic(NamedIntrinsic intrinsic, op2 = impPopStack().val; op1 = impSIMDPopStack(); + op2 = gtFoldExpr(impImplicitIorI4Cast(op2, TYP_I_IMPL, /* zeroExtend */ true)); + retNode = gtNewSimdGetElementNode(retType, op1, op2, simdBaseJitType, simdSize); break; } @@ -4146,6 +4148,8 @@ GenTree* Compiler::impSpecialIntrinsic(NamedIntrinsic intrinsic, GenTree* indexOp = impPopStack().val; GenTree* vectorOp = impSIMDPopStack(); + indexOp = gtFoldExpr(impImplicitIorI4Cast(indexOp, TYP_I_IMPL, /* zeroExtend */ true)); + retNode = gtNewSimdWithElementNode(retType, vectorOp, indexOp, valueOp, simdBaseJitType, simdSize); break; } diff --git a/src/coreclr/jit/lclmorph.cpp b/src/coreclr/jit/lclmorph.cpp index cd464797e79890..d23a049a3d6ff8 100644 --- a/src/coreclr/jit/lclmorph.cpp +++ b/src/coreclr/jit/lclmorph.cpp @@ -1732,7 +1732,7 @@ class LocalAddressVisitor final : public GenTreeVisitor case TYP_FLOAT: { // Handle case 1 or the float field of case 2 - GenTree* indexNode = m_compiler->gtNewIconNode(offset / genTypeSize(elementType)); + GenTree* indexNode = m_compiler->gtNewIconNode(offset / genTypeSize(elementType), TYP_I_IMPL); hwiNode = m_compiler->gtNewSimdGetElementNode(elementType, lclNode, indexNode, CORINFO_TYPE_FLOAT, genTypeSize(varDsc)); break; @@ -1790,7 +1790,7 @@ class LocalAddressVisitor final : public GenTreeVisitor case TYP_FLOAT: { // Handle case 1 or the float field of case 2 - GenTree* indexNode = m_compiler->gtNewIconNode(offset / genTypeSize(elementType)); + GenTree* indexNode = m_compiler->gtNewIconNode(offset / genTypeSize(elementType), TYP_I_IMPL); hwiNode = m_compiler->gtNewSimdWithElementNode(varDsc->TypeGet(), simdLclNode, indexNode, elementNode, CORINFO_TYPE_FLOAT, genTypeSize(varDsc)); @@ -1810,11 +1810,11 @@ class LocalAddressVisitor final : public GenTreeVisitor NI_Vector128_AsVector128Unsafe, CORINFO_TYPE_FLOAT, 12); - GenTree* indexNode1 = m_compiler->gtNewIconNode(3, TYP_INT); + GenTree* indexNode1 = m_compiler->gtNewIconNode(3, TYP_I_IMPL); simdLclNode = m_compiler->gtNewSimdGetElementNode(TYP_FLOAT, simdLclNode, indexNode1, CORINFO_TYPE_FLOAT, 16); - GenTree* indexNode2 = m_compiler->gtNewIconNode(3, TYP_INT); + GenTree* indexNode2 = m_compiler->gtNewIconNode(3, TYP_I_IMPL); hwiNode = m_compiler->gtNewSimdWithElementNode(TYP_SIMD16, elementNode, indexNode2, simdLclNode, CORINFO_TYPE_FLOAT, 16); break; diff --git a/src/coreclr/jit/lowerxarch.cpp b/src/coreclr/jit/lowerxarch.cpp index f44117ba0ce00e..a32ce6a454902b 100644 --- a/src/coreclr/jit/lowerxarch.cpp +++ b/src/coreclr/jit/lowerxarch.cpp @@ -5235,7 +5235,7 @@ GenTree* Lowering::LowerHWIntrinsicGetElement(GenTreeHWIntrinsic* node) } else { - op2 = comp->gtNewIconNode(imm8); + op2 = comp->gtNewIconNode(imm8, TYP_I_IMPL); BlockRange().InsertBefore(node, op2); switch (simdBaseType) diff --git a/src/coreclr/jit/valuenum.cpp b/src/coreclr/jit/valuenum.cpp index 370601501f1763..912108a8cd65eb 100644 --- a/src/coreclr/jit/valuenum.cpp +++ b/src/coreclr/jit/valuenum.cpp @@ -8383,15 +8383,15 @@ ValueNum ValueNumStore::EvalHWIntrinsicFunBinary( case NI_Vector512_GetElement: #endif { - var_types simdType = TypeOfVN(arg0VN); - int32_t index = GetConstantInt32(arg1VN); + var_types simdType = TypeOfVN(arg0VN); + target_size_t index = ConstantValue(arg1VN); - if (static_cast(index) >= GenTreeVecCon::ElementCount(genTypeSize(simdType), baseType)) + if (index >= GenTreeVecCon::ElementCount(genTypeSize(simdType), baseType)) { // Nothing to fold for out of range indexes break; } - return EvaluateSimdGetElement(this, simdType, baseType, arg0VN, index); + return EvaluateSimdGetElement(this, simdType, baseType, arg0VN, static_cast(index)); } #if defined(TARGET_ARM64) @@ -9325,9 +9325,9 @@ ValueNum ValueNumStore::EvalHWIntrinsicFunTernary( break; } - int32_t index = GetConstantInt32(arg1VN); + target_size_t index = ConstantValue(arg1VN); - if (static_cast(index) >= GenTreeVecCon::ElementCount(genTypeSize(type), baseType)) + if (index >= GenTreeVecCon::ElementCount(genTypeSize(type), baseType)) { // Nothing to fold for out of range indexes break; @@ -9345,7 +9345,8 @@ ValueNum ValueNumStore::EvalHWIntrinsicFunTernary( { value = GetConstantDouble(arg2VN); } - return EvaluateSimdWithElementFloating(this, type, baseType, arg0VN, index, value); + return EvaluateSimdWithElementFloating(this, type, baseType, arg0VN, static_cast(index), + value); } else { @@ -9360,7 +9361,8 @@ ValueNum ValueNumStore::EvalHWIntrinsicFunTernary( { value = GetConstantInt32(arg2VN); } - return EvaluateSimdWithElementIntegral(this, type, baseType, arg0VN, index, value); + return EvaluateSimdWithElementIntegral(this, type, baseType, arg0VN, static_cast(index), + value); } } From 9538d985cf30ffb40c57f73a7b2fdd0cc68f5594 Mon Sep 17 00:00:00 2001 From: Jakob Botsch Nielsen Date: Mon, 25 Aug 2025 11:31:15 +0200 Subject: [PATCH 02/14] Make `addRangeCheckForHWIntrinsic` agnostic to type of index --- src/coreclr/jit/hwintrinsic.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/coreclr/jit/hwintrinsic.cpp b/src/coreclr/jit/hwintrinsic.cpp index fc23c7e04b490d..08d5a8cabe3f3f 100644 --- a/src/coreclr/jit/hwintrinsic.cpp +++ b/src/coreclr/jit/hwintrinsic.cpp @@ -1590,7 +1590,7 @@ GenTree* Compiler::addRangeCheckForHWIntrinsic(GenTree* immOp, int immLowerBound // The value of (immUpperBound - immLowerBound + 1) is denoted as adjustedUpperBound. const ssize_t adjustedUpperBound = (ssize_t)immUpperBound - immLowerBound + 1; - GenTree* adjustedUpperBoundNode = gtNewIconNode(adjustedUpperBound, TYP_I_IMPL); + GenTree* adjustedUpperBoundNode = gtNewIconNode(adjustedUpperBound, genActualType(immOp)); GenTree* immOpDup = nullptr; @@ -1599,7 +1599,7 @@ GenTree* Compiler::addRangeCheckForHWIntrinsic(GenTree* immOp, int immLowerBound if (immLowerBound != 0) { - immOpDup = gtNewOperNode(GT_SUB, TYP_I_IMPL, immOpDup, gtNewIconNode(immLowerBound, TYP_I_IMPL)); + immOpDup = gtNewOperNode(GT_SUB, genActualType(immOp), immOpDup, gtNewIconNode(immLowerBound, TYP_I_IMPL)); } GenTreeBoundsChk* hwIntrinsicChk = From 48fe4584ac83c3931f49a1400dde0077a26077e5 Mon Sep 17 00:00:00 2001 From: Jakob Botsch Nielsen Date: Mon, 25 Aug 2025 11:32:41 +0200 Subject: [PATCH 03/14] Missed one --- src/coreclr/jit/hwintrinsic.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/coreclr/jit/hwintrinsic.cpp b/src/coreclr/jit/hwintrinsic.cpp index 08d5a8cabe3f3f..20d85771597c39 100644 --- a/src/coreclr/jit/hwintrinsic.cpp +++ b/src/coreclr/jit/hwintrinsic.cpp @@ -1599,7 +1599,7 @@ GenTree* Compiler::addRangeCheckForHWIntrinsic(GenTree* immOp, int immLowerBound if (immLowerBound != 0) { - immOpDup = gtNewOperNode(GT_SUB, genActualType(immOp), immOpDup, gtNewIconNode(immLowerBound, TYP_I_IMPL)); + immOpDup = gtNewOperNode(GT_SUB, genActualType(immOp), immOpDup, gtNewIconNode(immLowerBound, genActualType(immOp))); } GenTreeBoundsChk* hwIntrinsicChk = From 1e1049056fe4dc9aa45fd89c5aa63ad6e9ee0620 Mon Sep 17 00:00:00 2001 From: Jakob Botsch Nielsen Date: Mon, 25 Aug 2025 11:49:01 +0200 Subject: [PATCH 04/14] Run jit-format --- src/coreclr/jit/hwintrinsic.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/coreclr/jit/hwintrinsic.cpp b/src/coreclr/jit/hwintrinsic.cpp index 20d85771597c39..137468d18d09b3 100644 --- a/src/coreclr/jit/hwintrinsic.cpp +++ b/src/coreclr/jit/hwintrinsic.cpp @@ -1599,7 +1599,8 @@ GenTree* Compiler::addRangeCheckForHWIntrinsic(GenTree* immOp, int immLowerBound if (immLowerBound != 0) { - immOpDup = gtNewOperNode(GT_SUB, genActualType(immOp), immOpDup, gtNewIconNode(immLowerBound, genActualType(immOp))); + immOpDup = + gtNewOperNode(GT_SUB, genActualType(immOp), immOpDup, gtNewIconNode(immLowerBound, genActualType(immOp))); } GenTreeBoundsChk* hwIntrinsicChk = From b4d4e92048a501cdff3dab7c88e02dfca229cf69 Mon Sep 17 00:00:00 2001 From: Jakob Botsch Nielsen Date: Mon, 25 Aug 2025 13:39:22 +0200 Subject: [PATCH 05/14] Insert cast when changing Extract -> GetElement --- src/coreclr/jit/lowerxarch.cpp | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/src/coreclr/jit/lowerxarch.cpp b/src/coreclr/jit/lowerxarch.cpp index a32ce6a454902b..0c9a3c63669479 100644 --- a/src/coreclr/jit/lowerxarch.cpp +++ b/src/coreclr/jit/lowerxarch.cpp @@ -2184,17 +2184,30 @@ GenTree* Lowering::LowerHWIntrinsic(GenTreeHWIntrinsic* node) if (!op2->OperIsConst()) { // Extract allows the full range while GetElement only allows - // 0-3, so we need to mask the index here so codegen works. + // 0-3, so we need to mask the index here so codegen works. We also may need + // an extension since GetElement takes a native sized index. - GenTree* msk = comp->gtNewIconNode(3, TYP_INT); + if (!op2->TypeIs(TYP_I_IMPL)) + { + GenTree* cast = comp->gtNewCastNode(TYP_I_IMPL, op2, true, TYP_I_IMPL); + BlockRange().InsertAfter(op2, cast); + LowerNode(cast); + op2 = cast; + } + + GenTree* msk = comp->gtNewIconNode(3, TYP_I_IMPL); BlockRange().InsertAfter(op2, msk); - GenTree* tmp = comp->gtNewOperNode(GT_AND, TYP_INT, op2, msk); + GenTree* tmp = comp->gtNewOperNode(GT_AND, TYP_I_IMPL, op2, msk); BlockRange().InsertAfter(msk, tmp); LowerNode(tmp); node->Op(2) = tmp; } + else + { + op2->gtType = TYP_I_IMPL; + } node->ChangeHWIntrinsicId(NI_Vector128_GetElement); return LowerNode(node); From 7a5be1caf853c0f10cff1d9007e129ed93c88d8a Mon Sep 17 00:00:00 2001 From: Jakob Botsch Nielsen Date: Mon, 25 Aug 2025 16:47:28 +0200 Subject: [PATCH 06/14] Revert "Insert cast when changing Extract -> GetElement" This reverts commit b4d4e92048a501cdff3dab7c88e02dfca229cf69. --- src/coreclr/jit/lowerxarch.cpp | 19 +++---------------- 1 file changed, 3 insertions(+), 16 deletions(-) diff --git a/src/coreclr/jit/lowerxarch.cpp b/src/coreclr/jit/lowerxarch.cpp index 0c9a3c63669479..a32ce6a454902b 100644 --- a/src/coreclr/jit/lowerxarch.cpp +++ b/src/coreclr/jit/lowerxarch.cpp @@ -2184,30 +2184,17 @@ GenTree* Lowering::LowerHWIntrinsic(GenTreeHWIntrinsic* node) if (!op2->OperIsConst()) { // Extract allows the full range while GetElement only allows - // 0-3, so we need to mask the index here so codegen works. We also may need - // an extension since GetElement takes a native sized index. + // 0-3, so we need to mask the index here so codegen works. - if (!op2->TypeIs(TYP_I_IMPL)) - { - GenTree* cast = comp->gtNewCastNode(TYP_I_IMPL, op2, true, TYP_I_IMPL); - BlockRange().InsertAfter(op2, cast); - LowerNode(cast); - op2 = cast; - } - - GenTree* msk = comp->gtNewIconNode(3, TYP_I_IMPL); + GenTree* msk = comp->gtNewIconNode(3, TYP_INT); BlockRange().InsertAfter(op2, msk); - GenTree* tmp = comp->gtNewOperNode(GT_AND, TYP_I_IMPL, op2, msk); + GenTree* tmp = comp->gtNewOperNode(GT_AND, TYP_INT, op2, msk); BlockRange().InsertAfter(msk, tmp); LowerNode(tmp); node->Op(2) = tmp; } - else - { - op2->gtType = TYP_I_IMPL; - } node->ChangeHWIntrinsicId(NI_Vector128_GetElement); return LowerNode(node); From 1249fa5f21d012f6a96fbf23ee4ae36f4f4c6132 Mon Sep 17 00:00:00 2001 From: Jakob Botsch Nielsen Date: Mon, 25 Aug 2025 16:47:30 +0200 Subject: [PATCH 07/14] Revert "Run jit-format" This reverts commit 1e1049056fe4dc9aa45fd89c5aa63ad6e9ee0620. --- src/coreclr/jit/hwintrinsic.cpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/coreclr/jit/hwintrinsic.cpp b/src/coreclr/jit/hwintrinsic.cpp index 137468d18d09b3..20d85771597c39 100644 --- a/src/coreclr/jit/hwintrinsic.cpp +++ b/src/coreclr/jit/hwintrinsic.cpp @@ -1599,8 +1599,7 @@ GenTree* Compiler::addRangeCheckForHWIntrinsic(GenTree* immOp, int immLowerBound if (immLowerBound != 0) { - immOpDup = - gtNewOperNode(GT_SUB, genActualType(immOp), immOpDup, gtNewIconNode(immLowerBound, genActualType(immOp))); + immOpDup = gtNewOperNode(GT_SUB, genActualType(immOp), immOpDup, gtNewIconNode(immLowerBound, genActualType(immOp))); } GenTreeBoundsChk* hwIntrinsicChk = From 076b2cc3ef37340fe63fd372c27af818c06cb909 Mon Sep 17 00:00:00 2001 From: Jakob Botsch Nielsen Date: Mon, 25 Aug 2025 16:47:32 +0200 Subject: [PATCH 08/14] Revert "Missed one" This reverts commit 48fe4584ac83c3931f49a1400dde0077a26077e5. --- src/coreclr/jit/hwintrinsic.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/coreclr/jit/hwintrinsic.cpp b/src/coreclr/jit/hwintrinsic.cpp index 20d85771597c39..08d5a8cabe3f3f 100644 --- a/src/coreclr/jit/hwintrinsic.cpp +++ b/src/coreclr/jit/hwintrinsic.cpp @@ -1599,7 +1599,7 @@ GenTree* Compiler::addRangeCheckForHWIntrinsic(GenTree* immOp, int immLowerBound if (immLowerBound != 0) { - immOpDup = gtNewOperNode(GT_SUB, genActualType(immOp), immOpDup, gtNewIconNode(immLowerBound, genActualType(immOp))); + immOpDup = gtNewOperNode(GT_SUB, genActualType(immOp), immOpDup, gtNewIconNode(immLowerBound, TYP_I_IMPL)); } GenTreeBoundsChk* hwIntrinsicChk = From 4cdc09d912a5e228025f7ca6530711e94ceb3d39 Mon Sep 17 00:00:00 2001 From: Jakob Botsch Nielsen Date: Mon, 25 Aug 2025 16:47:34 +0200 Subject: [PATCH 09/14] Revert "Make `addRangeCheckForHWIntrinsic` agnostic to type of index" This reverts commit 9538d985cf30ffb40c57f73a7b2fdd0cc68f5594. --- src/coreclr/jit/hwintrinsic.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/coreclr/jit/hwintrinsic.cpp b/src/coreclr/jit/hwintrinsic.cpp index 08d5a8cabe3f3f..fc23c7e04b490d 100644 --- a/src/coreclr/jit/hwintrinsic.cpp +++ b/src/coreclr/jit/hwintrinsic.cpp @@ -1590,7 +1590,7 @@ GenTree* Compiler::addRangeCheckForHWIntrinsic(GenTree* immOp, int immLowerBound // The value of (immUpperBound - immLowerBound + 1) is denoted as adjustedUpperBound. const ssize_t adjustedUpperBound = (ssize_t)immUpperBound - immLowerBound + 1; - GenTree* adjustedUpperBoundNode = gtNewIconNode(adjustedUpperBound, genActualType(immOp)); + GenTree* adjustedUpperBoundNode = gtNewIconNode(adjustedUpperBound, TYP_I_IMPL); GenTree* immOpDup = nullptr; @@ -1599,7 +1599,7 @@ GenTree* Compiler::addRangeCheckForHWIntrinsic(GenTree* immOp, int immLowerBound if (immLowerBound != 0) { - immOpDup = gtNewOperNode(GT_SUB, genActualType(immOp), immOpDup, gtNewIconNode(immLowerBound, TYP_I_IMPL)); + immOpDup = gtNewOperNode(GT_SUB, TYP_I_IMPL, immOpDup, gtNewIconNode(immLowerBound, TYP_I_IMPL)); } GenTreeBoundsChk* hwIntrinsicChk = From 04729775da6d8b4438a9148ce20df12c562c817d Mon Sep 17 00:00:00 2001 From: Jakob Botsch Nielsen Date: Mon, 25 Aug 2025 16:47:36 +0200 Subject: [PATCH 10/14] Revert "JIT: Switch GetElement and WithElement to native sized indices" This reverts commit 22c5838e102c09846509277da45b3453ca01e5a9. --- src/coreclr/jit/decomposelongs.cpp | 2 +- src/coreclr/jit/gentree.cpp | 11 +++-------- src/coreclr/jit/hwintrinsic.cpp | 4 ++-- src/coreclr/jit/hwintrinsicarm64.cpp | 13 ++++--------- src/coreclr/jit/hwintrinsiccodegenxarch.cpp | 4 ---- src/coreclr/jit/hwintrinsicxarch.cpp | 10 +++------- src/coreclr/jit/lclmorph.cpp | 8 ++++---- src/coreclr/jit/lowerxarch.cpp | 2 +- src/coreclr/jit/valuenum.cpp | 18 ++++++++---------- 9 files changed, 26 insertions(+), 46 deletions(-) diff --git a/src/coreclr/jit/decomposelongs.cpp b/src/coreclr/jit/decomposelongs.cpp index d4b9fed941f9b7..70767278e390e1 100644 --- a/src/coreclr/jit/decomposelongs.cpp +++ b/src/coreclr/jit/decomposelongs.cpp @@ -1960,7 +1960,7 @@ GenTree* DecomposeLongs::DecomposeHWIntrinsicToScalar(LIR::Use& use, GenTreeHWIn simdTmpVar = m_compiler->gtNewLclLNode(simdTmpVarNum, simdTmpVar->TypeGet()); Range().InsertAfter(loResult, simdTmpVar); - GenTree* one = m_compiler->gtNewIconNode(1, TYP_I_IMPL); + GenTree* one = m_compiler->gtNewIconNode(1); GenTree* hiResult = m_compiler->gtNewSimdGetElementNode(TYP_INT, simdTmpVar, one, CORINFO_TYPE_INT, simdSize); Range().InsertAfter(simdTmpVar, one, hiResult); diff --git a/src/coreclr/jit/gentree.cpp b/src/coreclr/jit/gentree.cpp index b7b167bcfcf2fb..10bde780a8534f 100644 --- a/src/coreclr/jit/gentree.cpp +++ b/src/coreclr/jit/gentree.cpp @@ -21484,21 +21484,18 @@ GenTree* Compiler::gtNewSimdBinOpNode( GenTree* op2Dup = fgMakeMultiUse(op2ToDup); assert(!varTypeIsArithmetic(op1Dup)); - op1Dup = - gtNewSimdGetElementNode(TYP_LONG, op1Dup, gtNewIconNode(1, TYP_I_IMPL), simdBaseJitType, simdSize); + op1Dup = gtNewSimdGetElementNode(TYP_LONG, op1Dup, gtNewIconNode(1), simdBaseJitType, simdSize); if (!varTypeIsArithmetic(op2Dup)) { - op2Dup = gtNewSimdGetElementNode(TYP_LONG, op2Dup, gtNewIconNode(1, TYP_I_IMPL), simdBaseJitType, - simdSize); + op2Dup = gtNewSimdGetElementNode(TYP_LONG, op2Dup, gtNewIconNode(1), simdBaseJitType, simdSize); } // upper = op1.GetElement(1) * op2.GetElement(1) GenTree* upper = gtNewOperNode(GT_MUL, TYP_LONG, op1Dup, op2Dup); // return Vector128.Create(lower, upper) - return gtNewSimdWithElementNode(type, lower, gtNewIconNode(1, TYP_I_IMPL), upper, simdBaseJitType, - simdSize); + return gtNewSimdWithElementNode(type, lower, gtNewIconNode(1), upper, simdBaseJitType, simdSize); } #endif // !TARGET_XARCH && !TARGET_ARM64 unreached(); @@ -23273,7 +23270,6 @@ GenTree* Compiler::gtNewSimdGetElementNode( var_types simdBaseType = JitType2PreciseVarType(simdBaseJitType); assert(varTypeIsArithmetic(simdBaseType)); - assert(op2->TypeIs(TYP_I_IMPL)); #if defined(TARGET_XARCH) if (op2->IsIntegralConst(0)) @@ -27752,7 +27748,6 @@ GenTree* Compiler::gtNewSimdWithElementNode( assert(varTypeIsArithmetic(simdBaseType)); assert(varTypeIsArithmetic(op3)); - assert(op2->TypeIs(TYP_I_IMPL)); #if defined(TARGET_XARCH) assert(!varTypeIsLong(simdBaseType) || compIsaSupportedDebugOnly(InstructionSet_X86Base_X64)); diff --git a/src/coreclr/jit/hwintrinsic.cpp b/src/coreclr/jit/hwintrinsic.cpp index fc23c7e04b490d..ca815d3c14a8b6 100644 --- a/src/coreclr/jit/hwintrinsic.cpp +++ b/src/coreclr/jit/hwintrinsic.cpp @@ -1590,7 +1590,7 @@ GenTree* Compiler::addRangeCheckForHWIntrinsic(GenTree* immOp, int immLowerBound // The value of (immUpperBound - immLowerBound + 1) is denoted as adjustedUpperBound. const ssize_t adjustedUpperBound = (ssize_t)immUpperBound - immLowerBound + 1; - GenTree* adjustedUpperBoundNode = gtNewIconNode(adjustedUpperBound, TYP_I_IMPL); + GenTree* adjustedUpperBoundNode = gtNewIconNode(adjustedUpperBound, TYP_INT); GenTree* immOpDup = nullptr; @@ -1599,7 +1599,7 @@ GenTree* Compiler::addRangeCheckForHWIntrinsic(GenTree* immOp, int immLowerBound if (immLowerBound != 0) { - immOpDup = gtNewOperNode(GT_SUB, TYP_I_IMPL, immOpDup, gtNewIconNode(immLowerBound, TYP_I_IMPL)); + immOpDup = gtNewOperNode(GT_SUB, TYP_INT, immOpDup, gtNewIconNode(immLowerBound, TYP_INT)); } GenTreeBoundsChk* hwIntrinsicChk = diff --git a/src/coreclr/jit/hwintrinsicarm64.cpp b/src/coreclr/jit/hwintrinsicarm64.cpp index ad7a144cc47f51..a41e6219c20f84 100644 --- a/src/coreclr/jit/hwintrinsicarm64.cpp +++ b/src/coreclr/jit/hwintrinsicarm64.cpp @@ -859,11 +859,11 @@ GenTree* Compiler::impSpecialIntrinsic(NamedIntrinsic intrinsic, op1 = gtNewSimdHWIntrinsicNode(retType, op1, NI_Vector64_ToVector128Unsafe, simdBaseJitType, 8); - GenTree* idx = gtNewIconNode(2, TYP_I_IMPL); + GenTree* idx = gtNewIconNode(2, TYP_INT); GenTree* zero = gtNewZeroConNode(TYP_FLOAT); op1 = gtNewSimdWithElementNode(retType, op1, idx, zero, simdBaseJitType, 16); - idx = gtNewIconNode(3, TYP_I_IMPL); + idx = gtNewIconNode(3, TYP_INT); zero = gtNewZeroConNode(TYP_FLOAT); retNode = gtNewSimdWithElementNode(retType, op1, idx, zero, simdBaseJitType, 16); @@ -887,7 +887,7 @@ GenTree* Compiler::impSpecialIntrinsic(NamedIntrinsic intrinsic, op1 = gtNewSimdHWIntrinsicNode(retType, op1, NI_Vector128_AsVector128Unsafe, simdBaseJitType, 12); - GenTree* idx = gtNewIconNode(3, TYP_I_IMPL); + GenTree* idx = gtNewIconNode(3, TYP_INT); GenTree* zero = gtNewZeroConNode(TYP_FLOAT); retNode = gtNewSimdWithElementNode(retType, op1, idx, zero, simdBaseJitType, 16); break; @@ -1345,8 +1345,7 @@ GenTree* Compiler::impSpecialIntrinsic(NamedIntrinsic intrinsic, op1 = impSIMDPopStack(); retNode = gtNewSimdDotProdNode(simdType, op1, op2, simdBaseJitType, simdSize); - retNode = - gtNewSimdGetElementNode(retType, retNode, gtNewIconNode(0, TYP_I_IMPL), simdBaseJitType, simdSize); + retNode = gtNewSimdGetElementNode(retType, retNode, gtNewIconNode(0), simdBaseJitType, simdSize); } break; } @@ -1629,8 +1628,6 @@ GenTree* Compiler::impSpecialIntrinsic(NamedIntrinsic intrinsic, op2 = impPopStack().val; op1 = impSIMDPopStack(); - op2 = gtFoldExpr(impImplicitIorI4Cast(op2, TYP_I_IMPL, /* zeroExtend */ true)); - retNode = gtNewSimdGetElementNode(retType, op1, op2, simdBaseJitType, simdSize); break; } @@ -2753,8 +2750,6 @@ GenTree* Compiler::impSpecialIntrinsic(NamedIntrinsic intrinsic, assert(numArgs == 3); GenTree* indexOp = impStackTop(1).val; - indexOp = gtFoldExpr(impImplicitIorI4Cast(indexOp, TYP_I_IMPL, /* zeroExtend */ true)); - if (!indexOp->OperIsConst()) { if (!opts.OptimizationEnabled()) diff --git a/src/coreclr/jit/hwintrinsiccodegenxarch.cpp b/src/coreclr/jit/hwintrinsiccodegenxarch.cpp index 1057660e6ae287..d2fba55f424d9c 100644 --- a/src/coreclr/jit/hwintrinsiccodegenxarch.cpp +++ b/src/coreclr/jit/hwintrinsiccodegenxarch.cpp @@ -2001,8 +2001,6 @@ void CodeGen::genBaseIntrinsic(GenTreeHWIntrinsic* node, insOpts instOptions) assert(genStackLevel == 0); #endif // !FEATURE_FIXED_OUT_ARGS - assert(op2->TypeIs(TYP_I_IMPL)); - regNumber indexReg = op2->GetRegNum(); regNumber valueReg = op3->GetRegNum(); // New element value to be stored @@ -2037,8 +2035,6 @@ void CodeGen::genBaseIntrinsic(GenTreeHWIntrinsic* node, insOpts instOptions) simdType = TYP_SIMD16; } - assert(op2->TypeIs(TYP_I_IMPL)); - // Optimize the case of op1 is in memory and trying to access i'th element. if (!op1->isUsedFromReg()) { diff --git a/src/coreclr/jit/hwintrinsicxarch.cpp b/src/coreclr/jit/hwintrinsicxarch.cpp index b8f47f8c79f176..ded77bd68d147e 100644 --- a/src/coreclr/jit/hwintrinsicxarch.cpp +++ b/src/coreclr/jit/hwintrinsicxarch.cpp @@ -1573,11 +1573,11 @@ GenTree* Compiler::impSpecialIntrinsic(NamedIntrinsic intrinsic, op1 = gtNewSimdHWIntrinsicNode(retType, op1, NI_Vector128_AsVector128Unsafe, simdBaseJitType, 8); - GenTree* idx = gtNewIconNode(2, TYP_I_IMPL); + GenTree* idx = gtNewIconNode(2, TYP_INT); GenTree* zero = gtNewZeroConNode(TYP_FLOAT); op1 = gtNewSimdWithElementNode(retType, op1, idx, zero, simdBaseJitType, 16); - idx = gtNewIconNode(3, TYP_I_IMPL); + idx = gtNewIconNode(3, TYP_INT); zero = gtNewZeroConNode(TYP_FLOAT); retNode = gtNewSimdWithElementNode(retType, op1, idx, zero, simdBaseJitType, 16); @@ -1601,7 +1601,7 @@ GenTree* Compiler::impSpecialIntrinsic(NamedIntrinsic intrinsic, op1 = gtNewSimdHWIntrinsicNode(retType, op1, NI_Vector128_AsVector128Unsafe, simdBaseJitType, 12); - GenTree* idx = gtNewIconNode(3, TYP_I_IMPL); + GenTree* idx = gtNewIconNode(3, TYP_INT); GenTree* zero = gtNewZeroConNode(TYP_FLOAT); retNode = gtNewSimdWithElementNode(retType, op1, idx, zero, simdBaseJitType, 16); break; @@ -2705,8 +2705,6 @@ GenTree* Compiler::impSpecialIntrinsic(NamedIntrinsic intrinsic, op2 = impPopStack().val; op1 = impSIMDPopStack(); - op2 = gtFoldExpr(impImplicitIorI4Cast(op2, TYP_I_IMPL, /* zeroExtend */ true)); - retNode = gtNewSimdGetElementNode(retType, op1, op2, simdBaseJitType, simdSize); break; } @@ -4148,8 +4146,6 @@ GenTree* Compiler::impSpecialIntrinsic(NamedIntrinsic intrinsic, GenTree* indexOp = impPopStack().val; GenTree* vectorOp = impSIMDPopStack(); - indexOp = gtFoldExpr(impImplicitIorI4Cast(indexOp, TYP_I_IMPL, /* zeroExtend */ true)); - retNode = gtNewSimdWithElementNode(retType, vectorOp, indexOp, valueOp, simdBaseJitType, simdSize); break; } diff --git a/src/coreclr/jit/lclmorph.cpp b/src/coreclr/jit/lclmorph.cpp index d23a049a3d6ff8..cd464797e79890 100644 --- a/src/coreclr/jit/lclmorph.cpp +++ b/src/coreclr/jit/lclmorph.cpp @@ -1732,7 +1732,7 @@ class LocalAddressVisitor final : public GenTreeVisitor case TYP_FLOAT: { // Handle case 1 or the float field of case 2 - GenTree* indexNode = m_compiler->gtNewIconNode(offset / genTypeSize(elementType), TYP_I_IMPL); + GenTree* indexNode = m_compiler->gtNewIconNode(offset / genTypeSize(elementType)); hwiNode = m_compiler->gtNewSimdGetElementNode(elementType, lclNode, indexNode, CORINFO_TYPE_FLOAT, genTypeSize(varDsc)); break; @@ -1790,7 +1790,7 @@ class LocalAddressVisitor final : public GenTreeVisitor case TYP_FLOAT: { // Handle case 1 or the float field of case 2 - GenTree* indexNode = m_compiler->gtNewIconNode(offset / genTypeSize(elementType), TYP_I_IMPL); + GenTree* indexNode = m_compiler->gtNewIconNode(offset / genTypeSize(elementType)); hwiNode = m_compiler->gtNewSimdWithElementNode(varDsc->TypeGet(), simdLclNode, indexNode, elementNode, CORINFO_TYPE_FLOAT, genTypeSize(varDsc)); @@ -1810,11 +1810,11 @@ class LocalAddressVisitor final : public GenTreeVisitor NI_Vector128_AsVector128Unsafe, CORINFO_TYPE_FLOAT, 12); - GenTree* indexNode1 = m_compiler->gtNewIconNode(3, TYP_I_IMPL); + GenTree* indexNode1 = m_compiler->gtNewIconNode(3, TYP_INT); simdLclNode = m_compiler->gtNewSimdGetElementNode(TYP_FLOAT, simdLclNode, indexNode1, CORINFO_TYPE_FLOAT, 16); - GenTree* indexNode2 = m_compiler->gtNewIconNode(3, TYP_I_IMPL); + GenTree* indexNode2 = m_compiler->gtNewIconNode(3, TYP_INT); hwiNode = m_compiler->gtNewSimdWithElementNode(TYP_SIMD16, elementNode, indexNode2, simdLclNode, CORINFO_TYPE_FLOAT, 16); break; diff --git a/src/coreclr/jit/lowerxarch.cpp b/src/coreclr/jit/lowerxarch.cpp index a32ce6a454902b..f44117ba0ce00e 100644 --- a/src/coreclr/jit/lowerxarch.cpp +++ b/src/coreclr/jit/lowerxarch.cpp @@ -5235,7 +5235,7 @@ GenTree* Lowering::LowerHWIntrinsicGetElement(GenTreeHWIntrinsic* node) } else { - op2 = comp->gtNewIconNode(imm8, TYP_I_IMPL); + op2 = comp->gtNewIconNode(imm8); BlockRange().InsertBefore(node, op2); switch (simdBaseType) diff --git a/src/coreclr/jit/valuenum.cpp b/src/coreclr/jit/valuenum.cpp index 912108a8cd65eb..370601501f1763 100644 --- a/src/coreclr/jit/valuenum.cpp +++ b/src/coreclr/jit/valuenum.cpp @@ -8383,15 +8383,15 @@ ValueNum ValueNumStore::EvalHWIntrinsicFunBinary( case NI_Vector512_GetElement: #endif { - var_types simdType = TypeOfVN(arg0VN); - target_size_t index = ConstantValue(arg1VN); + var_types simdType = TypeOfVN(arg0VN); + int32_t index = GetConstantInt32(arg1VN); - if (index >= GenTreeVecCon::ElementCount(genTypeSize(simdType), baseType)) + if (static_cast(index) >= GenTreeVecCon::ElementCount(genTypeSize(simdType), baseType)) { // Nothing to fold for out of range indexes break; } - return EvaluateSimdGetElement(this, simdType, baseType, arg0VN, static_cast(index)); + return EvaluateSimdGetElement(this, simdType, baseType, arg0VN, index); } #if defined(TARGET_ARM64) @@ -9325,9 +9325,9 @@ ValueNum ValueNumStore::EvalHWIntrinsicFunTernary( break; } - target_size_t index = ConstantValue(arg1VN); + int32_t index = GetConstantInt32(arg1VN); - if (index >= GenTreeVecCon::ElementCount(genTypeSize(type), baseType)) + if (static_cast(index) >= GenTreeVecCon::ElementCount(genTypeSize(type), baseType)) { // Nothing to fold for out of range indexes break; @@ -9345,8 +9345,7 @@ ValueNum ValueNumStore::EvalHWIntrinsicFunTernary( { value = GetConstantDouble(arg2VN); } - return EvaluateSimdWithElementFloating(this, type, baseType, arg0VN, static_cast(index), - value); + return EvaluateSimdWithElementFloating(this, type, baseType, arg0VN, index, value); } else { @@ -9361,8 +9360,7 @@ ValueNum ValueNumStore::EvalHWIntrinsicFunTernary( { value = GetConstantInt32(arg2VN); } - return EvaluateSimdWithElementIntegral(this, type, baseType, arg0VN, static_cast(index), - value); + return EvaluateSimdWithElementIntegral(this, type, baseType, arg0VN, index, value); } } From b4bfbe058afdd852f3f5d66bc8f2ea8eed9651c2 Mon Sep 17 00:00:00 2001 From: Jakob Botsch Nielsen Date: Mon, 25 Aug 2025 17:12:26 +0200 Subject: [PATCH 11/14] Do it in lowering instead --- src/coreclr/jit/hwintrinsiccodegenxarch.cpp | 4 +++ src/coreclr/jit/lower.cpp | 33 +++++++++++++++++++++ src/coreclr/jit/lower.h | 1 + src/coreclr/jit/lowerarmarch.cpp | 3 ++ src/coreclr/jit/lowerxarch.cpp | 5 ++++ 5 files changed, 46 insertions(+) diff --git a/src/coreclr/jit/hwintrinsiccodegenxarch.cpp b/src/coreclr/jit/hwintrinsiccodegenxarch.cpp index d2fba55f424d9c..1057660e6ae287 100644 --- a/src/coreclr/jit/hwintrinsiccodegenxarch.cpp +++ b/src/coreclr/jit/hwintrinsiccodegenxarch.cpp @@ -2001,6 +2001,8 @@ void CodeGen::genBaseIntrinsic(GenTreeHWIntrinsic* node, insOpts instOptions) assert(genStackLevel == 0); #endif // !FEATURE_FIXED_OUT_ARGS + assert(op2->TypeIs(TYP_I_IMPL)); + regNumber indexReg = op2->GetRegNum(); regNumber valueReg = op3->GetRegNum(); // New element value to be stored @@ -2035,6 +2037,8 @@ void CodeGen::genBaseIntrinsic(GenTreeHWIntrinsic* node, insOpts instOptions) simdType = TYP_SIMD16; } + assert(op2->TypeIs(TYP_I_IMPL)); + // Optimize the case of op1 is in memory and trying to access i'th element. if (!op1->isUsedFromReg()) { diff --git a/src/coreclr/jit/lower.cpp b/src/coreclr/jit/lower.cpp index 9e52cd07f388ff..45257667bdd01c 100644 --- a/src/coreclr/jit/lower.cpp +++ b/src/coreclr/jit/lower.cpp @@ -11682,6 +11682,39 @@ GenTree* Lowering::InsertNewSimdCreateScalarUnsafeNode(var_types simdType, } return result; } + +//---------------------------------------------------------------------------------------------- +// Lowering::NormalizeIndexToNativeSized: +// Prepare to use an index for address calculations by ensuring it is native sized. +// +// Arguments: +// index - The index that may be an int32 +// +// Returns: +// The node itself, or a cast added on top of the node to perform normalization. +// +// Remarks: +// May insert a cast or may bash the node type in place for constants. +// +GenTree* Lowering::NormalizeIndexToNativeSized(GenTree* index) +{ + if (index->TypeIs(TYP_I_IMPL)) + { + return index; + } + + if (index->OperIsConst()) + { + index->gtType = TYP_I_IMPL; + return index; + } + else + { + GenTree* cast = comp->gtNewCastNode(TYP_I_IMPL, index, true, TYP_I_IMPL); + BlockRange().InsertAfter(index, cast); + return cast; + } +} #endif // FEATURE_HW_INTRINSICS //---------------------------------------------------------------------------------------------- diff --git a/src/coreclr/jit/lower.h b/src/coreclr/jit/lower.h index b3afd8cceaeb28..d923a40f04af7f 100644 --- a/src/coreclr/jit/lower.h +++ b/src/coreclr/jit/lower.h @@ -463,6 +463,7 @@ class Lowering final : public Phase GenTree* op1, CorInfoType simdBaseJitType, unsigned simdSize); + GenTree* NormalizeIndexToNativeSized(GenTree* index); #endif // FEATURE_HW_INTRINSICS // Utility functions diff --git a/src/coreclr/jit/lowerarmarch.cpp b/src/coreclr/jit/lowerarmarch.cpp index 5fd53c25fbb137..f1d450661bdbcd 100644 --- a/src/coreclr/jit/lowerarmarch.cpp +++ b/src/coreclr/jit/lowerarmarch.cpp @@ -1684,6 +1684,9 @@ GenTree* Lowering::LowerHWIntrinsic(GenTreeHWIntrinsic* node) GenTree* op1 = node->Op(1); GenTree* op2 = node->Op(2); + op2 = NormalizeIndexToNativeSized(op2); + node->Op(2) = op2; + bool isContainableMemory = IsContainableMemoryOp(op1) && IsSafeToContainMem(node, op1); if (isContainableMemory || !op2->OperIsConst()) diff --git a/src/coreclr/jit/lowerxarch.cpp b/src/coreclr/jit/lowerxarch.cpp index f44117ba0ce00e..140041c927187a 100644 --- a/src/coreclr/jit/lowerxarch.cpp +++ b/src/coreclr/jit/lowerxarch.cpp @@ -4890,6 +4890,9 @@ GenTree* Lowering::LowerHWIntrinsicGetElement(GenTreeHWIntrinsic* node) return LowerNode(node); } + op2 = NormalizeIndexToNativeSized(op2); + node->Op(2) = op2; + uint32_t elemSize = genTypeSize(simdBaseType); uint32_t count = simdSize / elemSize; @@ -5337,6 +5340,8 @@ GenTree* Lowering::LowerHWIntrinsicWithElement(GenTreeHWIntrinsic* node) if (!op2->OperIsConst()) { + op2 = NormalizeIndexToNativeSized(op2); + node->Op(2) = op2; // We will specially handle WithElement in codegen when op2 isn't a constant ContainCheckHWIntrinsic(node); return node->gtNext; From e5edced4e5b5c72ef68eea5c41a53299b4a97469 Mon Sep 17 00:00:00 2001 From: Jakob Botsch Nielsen Date: Mon, 25 Aug 2025 17:14:40 +0200 Subject: [PATCH 12/14] Correct handling of small types --- src/coreclr/jit/lower.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/coreclr/jit/lower.cpp b/src/coreclr/jit/lower.cpp index 45257667bdd01c..190680d7144826 100644 --- a/src/coreclr/jit/lower.cpp +++ b/src/coreclr/jit/lower.cpp @@ -11698,7 +11698,7 @@ GenTree* Lowering::InsertNewSimdCreateScalarUnsafeNode(var_types simdType, // GenTree* Lowering::NormalizeIndexToNativeSized(GenTree* index) { - if (index->TypeIs(TYP_I_IMPL)) + if (genActualType(index) == TYP_I_IMPL) { return index; } From 20f053105a0b7e341fccdb82d443e6d9067f5cba Mon Sep 17 00:00:00 2001 From: Jakob Botsch Nielsen Date: Mon, 25 Aug 2025 17:25:42 +0200 Subject: [PATCH 13/14] Feedback --- src/coreclr/jit/lower.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/coreclr/jit/lower.cpp b/src/coreclr/jit/lower.cpp index 190680d7144826..f8c3a7bc0975e3 100644 --- a/src/coreclr/jit/lower.cpp +++ b/src/coreclr/jit/lower.cpp @@ -11694,7 +11694,8 @@ GenTree* Lowering::InsertNewSimdCreateScalarUnsafeNode(var_types simdType, // The node itself, or a cast added on top of the node to perform normalization. // // Remarks: -// May insert a cast or may bash the node type in place for constants. +// May insert a cast or may bash the node type in place for constants. Does +// not replace the use. // GenTree* Lowering::NormalizeIndexToNativeSized(GenTree* index) { From f15960a0d529389dbecc6e5120a28dcbdd4c0cb1 Mon Sep 17 00:00:00 2001 From: Jakob Botsch Nielsen Date: Mon, 25 Aug 2025 17:47:35 +0200 Subject: [PATCH 14/14] Fixup type for float/double extractions that stay as GetElement --- src/coreclr/jit/lowerxarch.cpp | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/coreclr/jit/lowerxarch.cpp b/src/coreclr/jit/lowerxarch.cpp index 140041c927187a..4dac70f9e6998a 100644 --- a/src/coreclr/jit/lowerxarch.cpp +++ b/src/coreclr/jit/lowerxarch.cpp @@ -5255,6 +5255,12 @@ GenTree* Lowering::LowerHWIntrinsicGetElement(GenTreeHWIntrinsic* node) { // We specially handle float and double for more efficient codegen resIntrinsic = NI_Vector128_GetElement; + // GetElement takes a native sized index after lowering, so change + // the type of the constant we inserted above. + // (This is generally only for the non constant index case, + // which is not the case here, but keep the index operand's + // type consistent) + op2->gtType = TYP_I_IMPL; break; }