diff --git a/src/coreclr/jit/simdashwintrinsic.cpp b/src/coreclr/jit/simdashwintrinsic.cpp index d50def6863f45d..693343c06dd7e9 100644 --- a/src/coreclr/jit/simdashwintrinsic.cpp +++ b/src/coreclr/jit/simdashwintrinsic.cpp @@ -880,56 +880,154 @@ GenTree* Compiler::impSimdAsHWIntrinsicSpecial(NamedIntrinsic intrinsic, case NI_VectorT128_op_Multiply: { - assert(baseType == TYP_INT); - NamedIntrinsic hwIntrinsic = NI_Illegal; + GenTree** broadcastOp = nullptr; - if (compOpportunisticallyDependsOn(InstructionSet_SSE41)) + if (varTypeIsArithmetic(op1->TypeGet())) { - hwIntrinsic = NI_SSE41_MultiplyLow; + broadcastOp = &op1; } - else + else if (varTypeIsArithmetic(op2->TypeGet())) { - // op1Dup = op1 - GenTree* op1Dup; - op1 = impCloneExpr(op1, &op1Dup, clsHnd, (unsigned)CHECK_SPILL_ALL, - nullptr DEBUGARG("Clone op1 for Vector.Multiply")); + broadcastOp = &op2; + } - // op2Dup = op2 - GenTree* op2Dup; - op2 = impCloneExpr(op2, &op2Dup, clsHnd, (unsigned)CHECK_SPILL_ALL, - nullptr DEBUGARG("Clone op2 for Vector.Multiply")); + if (broadcastOp != nullptr) + { + *broadcastOp = gtNewSimdCreateBroadcastNode(simdType, *broadcastOp, baseType, simdSize, + /* isSimdAsHWIntrinsic */ true); + } - // op1 = Sse2.ShiftRightLogical128BitLane(op1, 4) - op1 = gtNewSimdAsHWIntrinsicNode(retType, op1, gtNewIconNode(4, TYP_INT), - NI_SSE2_ShiftRightLogical128BitLane, baseType, simdSize); + switch (baseType) + { + case TYP_SHORT: + case TYP_USHORT: + { + hwIntrinsic = NI_SSE2_MultiplyLow; + break; + } - // op2 = Sse2.ShiftRightLogical128BitLane(op1, 4) - op2 = gtNewSimdAsHWIntrinsicNode(retType, op2, gtNewIconNode(4, TYP_INT), - NI_SSE2_ShiftRightLogical128BitLane, baseType, simdSize); + case TYP_INT: + case TYP_UINT: + { + if (compOpportunisticallyDependsOn(InstructionSet_SSE41)) + { + hwIntrinsic = NI_SSE41_MultiplyLow; + } + else + { + // op1Dup = op1 + GenTree* op1Dup; + op1 = impCloneExpr(op1, &op1Dup, clsHnd, (unsigned)CHECK_SPILL_ALL, + nullptr DEBUGARG("Clone op1 for Vector.Multiply")); + + // op2Dup = op2 + GenTree* op2Dup; + op2 = impCloneExpr(op2, &op2Dup, clsHnd, (unsigned)CHECK_SPILL_ALL, + nullptr DEBUGARG("Clone op2 for Vector.Multiply")); + + // op1 = Sse2.ShiftRightLogical128BitLane(op1, 4) + op1 = + gtNewSimdAsHWIntrinsicNode(retType, op1, gtNewIconNode(4, TYP_INT), + NI_SSE2_ShiftRightLogical128BitLane, baseType, simdSize); + + // op2 = Sse2.ShiftRightLogical128BitLane(op1, 4) + op2 = + gtNewSimdAsHWIntrinsicNode(retType, op2, gtNewIconNode(4, TYP_INT), + NI_SSE2_ShiftRightLogical128BitLane, baseType, simdSize); + + // op2 = Sse2.Multiply(op2.AsUInt64(), op1.AsUInt64()).AsInt32() + op2 = gtNewSimdAsHWIntrinsicNode(retType, op2, op1, NI_SSE2_Multiply, TYP_ULONG, + simdSize); + + // op2 = Sse2.Shuffle(op2, (0, 0, 2, 0)) + op2 = gtNewSimdAsHWIntrinsicNode(retType, op2, gtNewIconNode(SHUFFLE_XXZX, TYP_INT), + NI_SSE2_Shuffle, baseType, simdSize); + + // op1 = Sse2.Multiply(op1Dup.AsUInt64(), op2Dup.AsUInt64()).AsInt32() + op1 = gtNewSimdAsHWIntrinsicNode(retType, op1Dup, op2Dup, NI_SSE2_Multiply, TYP_ULONG, + simdSize); + + // op1 = Sse2.Shuffle(op1, (0, 0, 2, 0)) + op1 = gtNewSimdAsHWIntrinsicNode(retType, op1, gtNewIconNode(SHUFFLE_XXZX, TYP_INT), + NI_SSE2_Shuffle, baseType, simdSize); + + // result = Sse2.UnpackLow(op1, op2) + hwIntrinsic = NI_SSE2_UnpackLow; + } + break; + } - // op2 = Sse2.Multiply(op2.AsUInt64(), op1.AsUInt64()).AsInt32() - op2 = gtNewSimdAsHWIntrinsicNode(retType, op2, op1, NI_SSE2_Multiply, TYP_ULONG, simdSize); + case TYP_FLOAT: + { + hwIntrinsic = NI_SSE_Multiply; + break; + } - // op2 = Sse2.Shuffle(op2, (0, 0, 2, 0)) - op2 = gtNewSimdAsHWIntrinsicNode(retType, op2, gtNewIconNode(SHUFFLE_XXZX, TYP_INT), - NI_SSE2_Shuffle, baseType, simdSize); + case TYP_DOUBLE: + { + hwIntrinsic = NI_SSE2_Multiply; + break; + } - // op1 = Sse2.Multiply(op1Dup.AsUInt64(), op2Dup.AsUInt64()).AsInt32() - op1 = - gtNewSimdAsHWIntrinsicNode(retType, op1Dup, op2Dup, NI_SSE2_Multiply, TYP_ULONG, simdSize); + default: + { + unreached(); + } + } + + assert(hwIntrinsic != NI_Illegal); + return gtNewSimdAsHWIntrinsicNode(retType, op1, op2, hwIntrinsic, baseType, simdSize); + } - // op1 = Sse2.Shuffle(op1, (0, 0, 2, 0)) - op1 = gtNewSimdAsHWIntrinsicNode(retType, op1, gtNewIconNode(SHUFFLE_XXZX, TYP_INT), - NI_SSE2_Shuffle, baseType, simdSize); + case NI_VectorT256_op_Multiply: + { + NamedIntrinsic hwIntrinsic = NI_Illegal; + GenTree** broadcastOp = nullptr; - // result = Sse2.UnpackLow(op1, op2) - hwIntrinsic = NI_SSE2_UnpackLow; + if (varTypeIsArithmetic(op1->TypeGet())) + { + broadcastOp = &op1; + } + else if (varTypeIsArithmetic(op2->TypeGet())) + { + broadcastOp = &op2; } - assert(hwIntrinsic != NI_Illegal); + if (broadcastOp != nullptr) + { + *broadcastOp = gtNewSimdCreateBroadcastNode(simdType, *broadcastOp, baseType, simdSize, + /* isSimdAsHWIntrinsic */ true); + } + + switch (baseType) + { + case TYP_SHORT: + case TYP_USHORT: + case TYP_INT: + case TYP_UINT: + { + hwIntrinsic = NI_AVX2_MultiplyLow; + break; + } + + case TYP_FLOAT: + case TYP_DOUBLE: + { + hwIntrinsic = NI_AVX_Multiply; + break; + } + + default: + { + unreached(); + } + } + + assert(hwIntrinsic != NI_Illegal); return gtNewSimdAsHWIntrinsicNode(retType, op1, op2, hwIntrinsic, baseType, simdSize); } + #elif defined(TARGET_ARM64) case NI_Vector2_CreateBroadcast: case NI_Vector3_CreateBroadcast: @@ -970,6 +1068,83 @@ GenTree* Compiler::impSimdAsHWIntrinsicSpecial(NamedIntrinsic intrinsic, // result = ConditionalSelect(op1, op1Dup, op2Dup) return impSimdAsHWIntrinsicCndSel(clsHnd, retType, baseType, simdSize, op1, op1Dup, op2Dup); } + + case NI_VectorT128_op_Multiply: + { + NamedIntrinsic hwIntrinsic = NI_Illegal; + NamedIntrinsic scalarIntrinsic = NI_Illegal; + GenTree** scalarOp = nullptr; + + if (varTypeIsArithmetic(op1->TypeGet())) + { + // MultiplyByScalar requires the scalar op to be op2 + std::swap(op1, op2); + + scalarOp = &op2; + } + else if (varTypeIsArithmetic(op2->TypeGet())) + { + scalarOp = &op2; + } + + switch (baseType) + { + case TYP_BYTE: + case TYP_UBYTE: + { + if (scalarOp != nullptr) + { + *scalarOp = gtNewSimdCreateBroadcastNode(simdType, *scalarOp, baseType, simdSize, + /* isSimdAsHWIntrinsic */ true); + } + + hwIntrinsic = NI_AdvSimd_Multiply; + break; + } + + case TYP_SHORT: + case TYP_USHORT: + case TYP_INT: + case TYP_UINT: + case TYP_FLOAT: + { + if (scalarOp != nullptr) + { + hwIntrinsic = NI_AdvSimd_MultiplyByScalar; + *scalarOp = gtNewSimdAsHWIntrinsicNode(TYP_SIMD8, *scalarOp, + NI_Vector64_CreateScalarUnsafe, baseType, 8); + } + else + { + hwIntrinsic = NI_AdvSimd_Multiply; + } + break; + } + + case TYP_DOUBLE: + { + if (scalarOp != nullptr) + { + hwIntrinsic = NI_AdvSimd_Arm64_MultiplyByScalar; + *scalarOp = + gtNewSimdAsHWIntrinsicNode(TYP_SIMD8, *scalarOp, NI_Vector64_Create, baseType, 8); + } + else + { + hwIntrinsic = NI_AdvSimd_Arm64_Multiply; + } + break; + } + + default: + { + unreached(); + } + } + + assert(hwIntrinsic != NI_Illegal); + return gtNewSimdAsHWIntrinsicNode(retType, op1, op2, hwIntrinsic, baseType, simdSize); + } #else #error Unsupported platform #endif // !TARGET_XARCH && !TARGET_ARM64 diff --git a/src/coreclr/jit/simdashwintrinsiclistarm64.h b/src/coreclr/jit/simdashwintrinsiclistarm64.h index 494377bc7172fe..fc75eca9f3fc30 100644 --- a/src/coreclr/jit/simdashwintrinsiclistarm64.h +++ b/src/coreclr/jit/simdashwintrinsiclistarm64.h @@ -128,7 +128,7 @@ SIMD_AS_HWINTRINSIC_ID(VectorT128, op_Equality, SIMD_AS_HWINTRINSIC_ID(VectorT128, op_ExclusiveOr, 2, {NI_AdvSimd_Xor, NI_AdvSimd_Xor, NI_AdvSimd_Xor, NI_AdvSimd_Xor, NI_AdvSimd_Xor, NI_AdvSimd_Xor, NI_AdvSimd_Xor, NI_AdvSimd_Xor, NI_AdvSimd_Xor, NI_AdvSimd_Xor}, SimdAsHWIntrinsicFlag::None) SIMD_AS_HWINTRINSIC_ID(VectorT128, op_Explicit, 1, {NI_VectorT128_op_Explicit, NI_VectorT128_op_Explicit, NI_VectorT128_op_Explicit, NI_VectorT128_op_Explicit, NI_VectorT128_op_Explicit, NI_VectorT128_op_Explicit, NI_VectorT128_op_Explicit, NI_VectorT128_op_Explicit, NI_VectorT128_op_Explicit, NI_VectorT128_op_Explicit}, SimdAsHWIntrinsicFlag::None) SIMD_AS_HWINTRINSIC_ID(VectorT128, op_Inequality, 2, {NI_Vector128_op_Inequality, NI_Vector128_op_Inequality, NI_Vector128_op_Inequality, NI_Vector128_op_Inequality, NI_Vector128_op_Inequality, NI_Vector128_op_Inequality, NI_Vector128_op_Inequality, NI_Vector128_op_Inequality, NI_Vector128_op_Inequality, NI_Vector128_op_Inequality}, SimdAsHWIntrinsicFlag::None) -SIMD_AS_HWINTRINSIC_ID(VectorT128, op_Multiply, 2, {NI_AdvSimd_Multiply, NI_AdvSimd_Multiply, NI_AdvSimd_Multiply, NI_AdvSimd_Multiply, NI_AdvSimd_Multiply, NI_AdvSimd_Multiply, NI_Illegal, NI_Illegal, NI_AdvSimd_Multiply, NI_AdvSimd_Arm64_Multiply}, SimdAsHWIntrinsicFlag::None) +SIMD_AS_HWINTRINSIC_ID(VectorT128, op_Multiply, 2, {NI_VectorT128_op_Multiply, NI_VectorT128_op_Multiply, NI_VectorT128_op_Multiply, NI_VectorT128_op_Multiply, NI_VectorT128_op_Multiply, NI_VectorT128_op_Multiply, NI_Illegal, NI_Illegal, NI_VectorT128_op_Multiply, NI_VectorT128_op_Multiply}, SimdAsHWIntrinsicFlag::None) SIMD_AS_HWINTRINSIC_ID(VectorT128, op_Subtraction, 2, {NI_AdvSimd_Subtract, NI_AdvSimd_Subtract, NI_AdvSimd_Subtract, NI_AdvSimd_Subtract, NI_AdvSimd_Subtract, NI_AdvSimd_Subtract, NI_AdvSimd_Subtract, NI_AdvSimd_Subtract, NI_AdvSimd_Subtract, NI_AdvSimd_Arm64_Subtract}, SimdAsHWIntrinsicFlag::None) SIMD_AS_HWINTRINSIC_ID(VectorT128, SquareRoot, 1, {NI_Illegal, NI_Illegal, NI_Illegal, NI_Illegal, NI_Illegal, NI_Illegal, NI_Illegal, NI_Illegal, NI_AdvSimd_Arm64_Sqrt, NI_AdvSimd_Arm64_Sqrt}, SimdAsHWIntrinsicFlag::None) diff --git a/src/coreclr/jit/simdashwintrinsiclistxarch.h b/src/coreclr/jit/simdashwintrinsiclistxarch.h index 65584c3abb09e0..99e5c29ff8a9c4 100644 --- a/src/coreclr/jit/simdashwintrinsiclistxarch.h +++ b/src/coreclr/jit/simdashwintrinsiclistxarch.h @@ -128,7 +128,7 @@ SIMD_AS_HWINTRINSIC_ID(VectorT128, op_Equality, SIMD_AS_HWINTRINSIC_ID(VectorT128, op_ExclusiveOr, 2, {NI_SSE2_Xor, NI_SSE2_Xor, NI_SSE2_Xor, NI_SSE2_Xor, NI_SSE2_Xor, NI_SSE2_Xor, NI_SSE2_Xor, NI_SSE2_Xor, NI_SSE_Xor, NI_SSE2_Xor}, SimdAsHWIntrinsicFlag::None) SIMD_AS_HWINTRINSIC_ID(VectorT128, op_Explicit, 1, {NI_VectorT128_op_Explicit, NI_VectorT128_op_Explicit, NI_VectorT128_op_Explicit, NI_VectorT128_op_Explicit, NI_VectorT128_op_Explicit, NI_VectorT128_op_Explicit, NI_VectorT128_op_Explicit, NI_VectorT128_op_Explicit, NI_VectorT128_op_Explicit, NI_VectorT128_op_Explicit}, SimdAsHWIntrinsicFlag::None) SIMD_AS_HWINTRINSIC_ID(VectorT128, op_Inequality, 2, {NI_Vector128_op_Inequality, NI_Vector128_op_Inequality, NI_Vector128_op_Inequality, NI_Vector128_op_Inequality, NI_Vector128_op_Inequality, NI_Vector128_op_Inequality, NI_Vector128_op_Inequality, NI_Vector128_op_Inequality, NI_Vector128_op_Inequality, NI_Vector128_op_Inequality}, SimdAsHWIntrinsicFlag::None) -SIMD_AS_HWINTRINSIC_ID(VectorT128, op_Multiply, 2, {NI_Illegal, NI_Illegal, NI_SSE2_MultiplyLow, NI_Illegal, NI_VectorT128_op_Multiply, NI_Illegal, NI_Illegal, NI_Illegal, NI_SSE_Multiply, NI_SSE2_Multiply}, SimdAsHWIntrinsicFlag::None) +SIMD_AS_HWINTRINSIC_ID(VectorT128, op_Multiply, 2, {NI_Illegal, NI_Illegal, NI_VectorT128_op_Multiply, NI_VectorT128_op_Multiply, NI_VectorT128_op_Multiply, NI_VectorT128_op_Multiply, NI_Illegal, NI_Illegal, NI_VectorT128_op_Multiply, NI_VectorT128_op_Multiply}, SimdAsHWIntrinsicFlag::None) SIMD_AS_HWINTRINSIC_ID(VectorT128, op_Subtraction, 2, {NI_SSE2_Subtract, NI_SSE2_Subtract, NI_SSE2_Subtract, NI_SSE2_Subtract, NI_SSE2_Subtract, NI_SSE2_Subtract, NI_SSE2_Subtract, NI_SSE2_Subtract, NI_SSE_Subtract, NI_SSE2_Subtract}, SimdAsHWIntrinsicFlag::None) SIMD_AS_HWINTRINSIC_ID(VectorT128, SquareRoot, 1, {NI_Illegal, NI_Illegal, NI_Illegal, NI_Illegal, NI_Illegal, NI_Illegal, NI_Illegal, NI_Illegal, NI_SSE_Sqrt, NI_SSE2_Sqrt}, SimdAsHWIntrinsicFlag::None) @@ -165,7 +165,7 @@ SIMD_AS_HWINTRINSIC_ID(VectorT256, op_Equality, SIMD_AS_HWINTRINSIC_ID(VectorT256, op_ExclusiveOr, 2, {NI_AVX2_Xor, NI_AVX2_Xor, NI_AVX2_Xor, NI_AVX2_Xor, NI_AVX2_Xor, NI_AVX2_Xor, NI_AVX2_Xor, NI_AVX2_Xor, NI_AVX_Xor, NI_AVX_Xor}, SimdAsHWIntrinsicFlag::None) SIMD_AS_HWINTRINSIC_ID(VectorT256, op_Explicit, 1, {NI_VectorT256_op_Explicit, NI_VectorT256_op_Explicit, NI_VectorT256_op_Explicit, NI_VectorT256_op_Explicit, NI_VectorT256_op_Explicit, NI_VectorT256_op_Explicit, NI_VectorT256_op_Explicit, NI_VectorT256_op_Explicit, NI_VectorT256_op_Explicit, NI_VectorT256_op_Explicit}, SimdAsHWIntrinsicFlag::None) SIMD_AS_HWINTRINSIC_ID(VectorT256, op_Inequality, 2, {NI_Vector256_op_Inequality, NI_Vector256_op_Inequality, NI_Vector256_op_Inequality, NI_Vector256_op_Inequality, NI_Vector256_op_Inequality, NI_Vector256_op_Inequality, NI_Vector256_op_Inequality, NI_Vector256_op_Inequality, NI_Vector256_op_Inequality, NI_Vector256_op_Inequality}, SimdAsHWIntrinsicFlag::None) -SIMD_AS_HWINTRINSIC_ID(VectorT256, op_Multiply, 2, {NI_Illegal, NI_Illegal, NI_AVX2_MultiplyLow, NI_Illegal, NI_AVX2_MultiplyLow, NI_Illegal, NI_Illegal, NI_Illegal, NI_AVX_Multiply, NI_AVX_Multiply}, SimdAsHWIntrinsicFlag::None) +SIMD_AS_HWINTRINSIC_ID(VectorT256, op_Multiply, 2, {NI_Illegal, NI_Illegal, NI_VectorT256_op_Multiply, NI_VectorT256_op_Multiply, NI_VectorT256_op_Multiply, NI_VectorT256_op_Multiply, NI_Illegal, NI_Illegal, NI_VectorT256_op_Multiply, NI_VectorT256_op_Multiply}, SimdAsHWIntrinsicFlag::None) SIMD_AS_HWINTRINSIC_ID(VectorT256, op_Subtraction, 2, {NI_AVX2_Subtract, NI_AVX2_Subtract, NI_AVX2_Subtract, NI_AVX2_Subtract, NI_AVX2_Subtract, NI_AVX2_Subtract, NI_AVX2_Subtract, NI_AVX2_Subtract, NI_AVX_Subtract, NI_AVX_Subtract}, SimdAsHWIntrinsicFlag::None) SIMD_AS_HWINTRINSIC_ID(VectorT256, SquareRoot, 1, {NI_Illegal, NI_Illegal, NI_Illegal, NI_Illegal, NI_Illegal, NI_Illegal, NI_Illegal, NI_Illegal, NI_AVX_Sqrt, NI_AVX_Sqrt}, SimdAsHWIntrinsicFlag::None) diff --git a/src/libraries/System.Private.CoreLib/src/System/Numerics/Vector_1.cs b/src/libraries/System.Private.CoreLib/src/System/Numerics/Vector_1.cs index 6d8c7f32765418..d9056586a49afe 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Numerics/Vector_1.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Numerics/Vector_1.cs @@ -409,7 +409,7 @@ public readonly bool TryCopyTo(Span destination) /// The source vector. /// The scalar value. /// The scaled vector. - [MethodImpl(MethodImplOptions.AggressiveInlining)] + [Intrinsic] public static Vector operator *(Vector value, T factor) { Vector result = default; @@ -422,11 +422,11 @@ public readonly bool TryCopyTo(Span destination) return result; } - /// Multiplies a vector by the given scalar. /// The scalar value. /// The source vector. /// The scaled vector. + [Intrinsic] [MethodImpl(MethodImplOptions.AggressiveInlining)] public static Vector operator *(T factor, Vector value) => value * factor; @@ -450,6 +450,7 @@ public readonly bool TryCopyTo(Span destination) /// Negates a given vector. /// The source vector. /// The negated vector. + [MethodImpl(MethodImplOptions.AggressiveInlining)] public static Vector operator -(Vector value) => Zero - value; /// Returns a new vector by performing a bitwise-and operation on each of the elements in the given vectors.