diff --git a/src/mono/mono/mini/CMakeLists.txt b/src/mono/mono/mini/CMakeLists.txt index 98c8d1b166a369..cf3ca5b5de6c07 100644 --- a/src/mono/mono/mini/CMakeLists.txt +++ b/src/mono/mono/mini/CMakeLists.txt @@ -143,6 +143,7 @@ set(mini_common_sources arch-stubs.c llvm-runtime.h llvm-intrinsics.h + llvm-intrinsics-types.h type-checking.c lldb.h lldb.c diff --git a/src/mono/mono/mini/llvm-intrinsics-types.h b/src/mono/mono/mini/llvm-intrinsics-types.h new file mode 100644 index 00000000000000..f070ed982c912e --- /dev/null +++ b/src/mono/mono/mini/llvm-intrinsics-types.h @@ -0,0 +1,27 @@ +#ifndef __MONO_MINI_LLVM_INTRINSICS_TYPES_H__ +#define __MONO_MINI_LLVM_INTRINSICS_TYPES_H__ + +/* An intrinsic id. The lower 23 bits are used to store a mono-specific ID. The + * next 9 bits store overload tag bits. In the configuration of LLVM 9 we use, + * there are 7017 total intrinsics defined in IntrinsicEnums.inc, so only 13 + * bits are needed to label each intrinsic overload group. + */ +typedef enum { +#define INTRINS(id, llvm_id) INTRINS_ ## id, +#define INTRINS_OVR(id, llvm_id, ty) INTRINS_ ## id, +#define INTRINS_OVR_2_ARG(id, llvm_id, ty1, ty2) INTRINS_ ## id, +#define INTRINS_OVR_3_ARG(id, llvm_id, ty1, ty2, ty3) INTRINS_ ## id, +#define INTRINS_OVR_TAG(id, ...) INTRINS_ ## id, +#define INTRINS_OVR_TAG_KIND(id, ...) INTRINS_ ## id, +#include "llvm-intrinsics.h" + INTRINS_NUM +} IntrinsicId; + +enum { + XBINOP_FORCEINT_and, + XBINOP_FORCEINT_or, + XBINOP_FORCEINT_ornot, + XBINOP_FORCEINT_xor, +}; + +#endif /* __MONO_MINI_LLVM_INTRINSICS_TYPES_H__ */ diff --git a/src/mono/mono/mini/llvm-intrinsics.h b/src/mono/mono/mini/llvm-intrinsics.h index ffbe23dcfa5fe1..8c8d0a89112877 100644 --- a/src/mono/mono/mini/llvm-intrinsics.h +++ b/src/mono/mono/mini/llvm-intrinsics.h @@ -12,6 +12,20 @@ * To define an overloaded intrinsic with three arguments */ +#define Scalar INTRIN_scalar +#define V64 INTRIN_vector64 +#define V128 INTRIN_vector128 +#define I1 INTRIN_int8 +#define I2 INTRIN_int16 +#define I4 INTRIN_int32 +#define I8 INTRIN_int64 +#define R4 INTRIN_float32 +#define R8 INTRIN_float64 +#define Ftoi INTRIN_kind_ftoi +#define Widen INTRIN_kind_widen +#define WidenAcross INTRIN_kind_widen_across +#define Across INTRIN_kind_across + INTRINS_OVR_2_ARG(MEMSET, memset, LLVMPointerType (LLVMInt8Type (), 0), LLVMInt32Type ()) INTRINS_OVR_3_ARG(MEMCPY, memcpy, LLVMPointerType (LLVMInt8Type (), 0), LLVMPointerType (LLVMInt8Type (), 0), LLVMInt32Type () ) INTRINS_OVR_3_ARG(MEMMOVE, memmove, LLVMPointerType (LLVMInt8Type (), 0), LLVMPointerType (LLVMInt8Type (), 0), LLVMInt64Type ()) @@ -278,28 +292,181 @@ INTRINS(AARCH64_SHA256SU1, aarch64_crypto_sha256su1) INTRINS(AARCH64_SHA256H, aarch64_crypto_sha256h) INTRINS(AARCH64_SHA256H2, aarch64_crypto_sha256h2) INTRINS(AARCH64_PMULL64, aarch64_neon_pmull64) -INTRINS_OVR(AARCH64_ADV_SIMD_ABS_FLOAT, fabs, sse_r4_t) -INTRINS_OVR(AARCH64_ADV_SIMD_ABS_DOUBLE, fabs, sse_r8_t) -INTRINS_OVR(AARCH64_ADV_SIMD_ABS_INT8, aarch64_neon_abs, sse_i1_t) -INTRINS_OVR(AARCH64_ADV_SIMD_ABS_INT16, aarch64_neon_abs, sse_i2_t) -INTRINS_OVR(AARCH64_ADV_SIMD_ABS_INT32, aarch64_neon_abs, sse_i4_t) -INTRINS_OVR(AARCH64_ADV_SIMD_ABS_INT64, aarch64_neon_abs, sse_i8_t) -INTRINS_OVR(AARCH64_ADV_SIMD_ABS_SATURATE_INT8, aarch64_neon_sqabs, sse_i1_t) -INTRINS_OVR(AARCH64_ADV_SIMD_ABS_SATURATE_INT16, aarch64_neon_sqabs, sse_i2_t) -INTRINS_OVR(AARCH64_ADV_SIMD_ABS_SATURATE_INT32, aarch64_neon_sqabs, sse_i4_t) -INTRINS_OVR(AARCH64_ADV_SIMD_ABS_SATURATE_INT64, aarch64_neon_sqabs, sse_i8_t) -INTRINS_OVR_2_ARG(AARCH64_ADV_SIMD_ABS_COMPARE_GT_FLOAT, aarch64_neon_facgt, sse_i4_t, sse_r4_t) -INTRINS_OVR_2_ARG(AARCH64_ADV_SIMD_ABS_COMPARE_GT_DOUBLE, aarch64_neon_facgt, sse_i4_t, sse_r8_t) -INTRINS_OVR_2_ARG(AARCH64_ADV_SIMD_ABS_COMPARE_GTE_FLOAT, aarch64_neon_facge, sse_i4_t, sse_r4_t) -INTRINS_OVR_2_ARG(AARCH64_ADV_SIMD_ABS_COMPARE_GTE_DOUBLE, aarch64_neon_facge, sse_i4_t, sse_r8_t) -INTRINS_OVR_2_ARG(AARCH64_ADV_SIMD_ABS_COMPARE_LT_FLOAT, aarch64_neon_facgt, sse_i4_t, sse_r4_t) -INTRINS_OVR_2_ARG(AARCH64_ADV_SIMD_ABS_COMPARE_LT_DOUBLE, aarch64_neon_facgt, sse_i4_t, sse_r8_t) -INTRINS_OVR_2_ARG(AARCH64_ADV_SIMD_ABS_COMPARE_LTE_FLOAT, aarch64_neon_facge, sse_i4_t, sse_r4_t) -INTRINS_OVR_2_ARG(AARCH64_ADV_SIMD_ABS_COMPARE_LTE_DOUBLE, aarch64_neon_facge, sse_i4_t, sse_r8_t) + +INTRINS_OVR_TAG_KIND(AARCH64_ADV_SIMD_FACGE, aarch64_neon_facge, Ftoi, Scalar | V64 | V128 | I4 | I8) +INTRINS_OVR_TAG_KIND(AARCH64_ADV_SIMD_FACGT, aarch64_neon_facgt, Ftoi, Scalar | V64 | V128 | I4 | I8) + +INTRINS_OVR_TAG(AARCH64_ADV_SIMD_FABD_SCALAR, aarch64_sisd_fabd, Scalar | R4 | R8) + +INTRINS_OVR_TAG(AARCH64_ADV_SIMD_FABD, aarch64_neon_fabd, V64 | V128 | R4 | R8) +INTRINS_OVR_TAG(AARCH64_ADV_SIMD_UABD, aarch64_neon_uabd, V64 | V128 | I1 | I2 | I4) +INTRINS_OVR_TAG(AARCH64_ADV_SIMD_SABD, aarch64_neon_sabd, V64 | V128 | I1 | I2 | I4) + +INTRINS_OVR_TAG(AARCH64_ADV_SIMD_SQABS, aarch64_neon_sqabs, Scalar | V64 | V128 | I1 | I2 | I4 | I8) +INTRINS_OVR_TAG(AARCH64_ADV_SIMD_FABS, fabs, Scalar | V64 | V128 | R4 | R8) +INTRINS_OVR_TAG(AARCH64_ADV_SIMD_ABS, aarch64_neon_abs, Scalar | V64 | V128 | I1 | I2 | I4 | I8) + +INTRINS_OVR_TAG_KIND(AARCH64_ADV_SIMD_UADDLV, aarch64_neon_uaddlv, WidenAcross, V64 | V128 | I1 | I2 | I4) +INTRINS_OVR_TAG_KIND(AARCH64_ADV_SIMD_SADDLV, aarch64_neon_saddlv, WidenAcross, V64 | V128 | I1 | I2 | I4) + +INTRINS_OVR_TAG(AARCH64_ADV_SIMD_ADDP, aarch64_neon_addp, V64 | V128 | I1 | I2 | I4 | I8) +INTRINS_OVR_TAG(AARCH64_ADV_SIMD_FADDP, aarch64_neon_faddp, V64 | V128 | R4 | R8) + +INTRINS_OVR_TAG_KIND(AARCH64_ADV_SIMD_FMAXNMV, aarch64_neon_fmaxnmv, Across, V64 | V128 | R4 | R8) +INTRINS_OVR_TAG_KIND(AARCH64_ADV_SIMD_FMINNMV, aarch64_neon_fminnmv, Across, V64 | V128 | R4 | R8) + +INTRINS_OVR_TAG_KIND(AARCH64_ADV_SIMD_SADDV, aarch64_neon_saddv, Across, V64 | V128 | I1 | I2 | I4) +INTRINS_OVR_TAG_KIND(AARCH64_ADV_SIMD_UADDV, aarch64_neon_uaddv, Across, V64 | V128 | I1 | I2 | I4 | I8) +INTRINS_OVR_TAG_KIND(AARCH64_ADV_SIMD_SMAXV, aarch64_neon_smaxv, Across, V64 | V128 | I1 | I2 | I4) +INTRINS_OVR_TAG_KIND(AARCH64_ADV_SIMD_UMAXV, aarch64_neon_umaxv, Across, V64 | V128 | I1 | I2 | I4) +INTRINS_OVR_TAG_KIND(AARCH64_ADV_SIMD_SMINV, aarch64_neon_sminv, Across, V64 | V128 | I1 | I2 | I4) +INTRINS_OVR_TAG_KIND(AARCH64_ADV_SIMD_UMINV, aarch64_neon_uminv, Across, V64 | V128 | I1 | I2 | I4) +INTRINS_OVR_TAG_KIND(AARCH64_ADV_SIMD_FMAXV, aarch64_neon_fmaxv, Across, V64 | V128 | R4 | R8) +INTRINS_OVR_TAG_KIND(AARCH64_ADV_SIMD_FMINV, aarch64_neon_fminv, Across, V64 | V128 | R4 | R8) + +INTRINS_OVR_TAG_KIND(AARCH64_ADV_SIMD_SADDLP, aarch64_neon_saddlp, Widen, V64 | V128 | I1 | I2 | I4 | I8) +INTRINS_OVR_TAG_KIND(AARCH64_ADV_SIMD_UADDLP, aarch64_neon_uaddlp, Widen, V64 | V128 | I1 | I2 | I4 | I8) + +INTRINS_OVR_2_ARG(AARCH64_ADV_SIMD_FCVTXN, aarch64_neon_fcvtxn, v64_r4_t, v128_r8_t) + +INTRINS_OVR_TAG_KIND(AARCH64_ADV_SIMD_FCVTAS, aarch64_neon_fcvtas, Ftoi, Scalar | V64 | V128 | I4 | I8) +INTRINS_OVR_TAG_KIND(AARCH64_ADV_SIMD_FCVTNS, aarch64_neon_fcvtns, Ftoi, Scalar | V64 | V128 | I4 | I8) +INTRINS_OVR_TAG_KIND(AARCH64_ADV_SIMD_FCVTMS, aarch64_neon_fcvtms, Ftoi, Scalar | V64 | V128 | I4 | I8) +INTRINS_OVR_TAG_KIND(AARCH64_ADV_SIMD_FCVTPS, aarch64_neon_fcvtps, Ftoi, Scalar | V64 | V128 | I4 | I8) + +INTRINS_OVR_TAG_KIND(AARCH64_ADV_SIMD_FCVTAU, aarch64_neon_fcvtau, Ftoi, Scalar | V64 | V128 | I4 | I8) +INTRINS_OVR_TAG_KIND(AARCH64_ADV_SIMD_FCVTNU, aarch64_neon_fcvtnu, Ftoi, Scalar | V64 | V128 | I4 | I8) +INTRINS_OVR_TAG_KIND(AARCH64_ADV_SIMD_FCVTMU, aarch64_neon_fcvtmu, Ftoi, Scalar | V64 | V128 | I4 | I8) +INTRINS_OVR_TAG_KIND(AARCH64_ADV_SIMD_FCVTPU, aarch64_neon_fcvtpu, Ftoi, Scalar | V64 | V128 | I4 | I8) + +INTRINS_OVR_2_ARG(AARCH64_ADV_SIMD_SCALAR_SQXTUN, aarch64_neon_scalar_sqxtun, i4_t, i8_t) +INTRINS_OVR_2_ARG(AARCH64_ADV_SIMD_SCALAR_SQXTN, aarch64_neon_scalar_sqxtn, i4_t, i8_t) +INTRINS_OVR_2_ARG(AARCH64_ADV_SIMD_SCALAR_UQXTN, aarch64_neon_scalar_uqxtn, i4_t, i8_t) +INTRINS_OVR_TAG(AARCH64_ADV_SIMD_SQXTUN, aarch64_neon_sqxtun, V64 | I1 | I2 | I4) +INTRINS_OVR_TAG(AARCH64_ADV_SIMD_SQXTN, aarch64_neon_sqxtn, V64 | I1 | I2 | I4) +INTRINS_OVR_TAG(AARCH64_ADV_SIMD_UQXTN, aarch64_neon_uqxtn, V64 | I1 | I2 | I4) + +INTRINS_OVR_TAG(AARCH64_ADV_SIMD_SRHADD, aarch64_neon_srhadd, V64 | V128 | I1 | I2 | I4) +INTRINS_OVR_TAG(AARCH64_ADV_SIMD_URHADD, aarch64_neon_urhadd, V64 | V128 | I1 | I2 | I4) + +INTRINS_OVR_TAG(AARCH64_ADV_SIMD_FMA, fma, Scalar | V64 | V128 | R4 | R8) + +INTRINS_OVR_TAG(AARCH64_ADV_SIMD_SHADD, aarch64_neon_shadd, V64 | V128 | I1 | I2 | I4) +INTRINS_OVR_TAG(AARCH64_ADV_SIMD_UHADD, aarch64_neon_uhadd, V64 | V128 | I1 | I2 | I4) +INTRINS_OVR_TAG(AARCH64_ADV_SIMD_SHSUB, aarch64_neon_shsub, V64 | V128 | I1 | I2 | I4) +INTRINS_OVR_TAG(AARCH64_ADV_SIMD_UHSUB, aarch64_neon_uhsub, V64 | V128 | I1 | I2 | I4) + +INTRINS_OVR_TAG(AARCH64_ADV_SIMD_CLS, aarch64_neon_cls, V64 | V128 | I1 | I2 | I4 | I8) +INTRINS_OVR_TAG(AARCH64_ADV_SIMD_CLZ, ctlz, V64 | V128 | I1 | I2 | I4 | I8) + +INTRINS_OVR_TAG(AARCH64_ADV_SIMD_SMAX, aarch64_neon_smax, V64 | V128 | I1 | I2 | I4 | I8) +INTRINS_OVR_TAG(AARCH64_ADV_SIMD_UMAX, aarch64_neon_umax, V64 | V128 | I1 | I2 | I4 | I8) +INTRINS_OVR_TAG(AARCH64_ADV_SIMD_FMAX, aarch64_neon_fmax, Scalar | V64 | V128 | R4 | R8) +INTRINS_OVR_TAG(AARCH64_ADV_SIMD_SMIN, aarch64_neon_smin, V64 | V128 | I1 | I2 | I4 | I8) +INTRINS_OVR_TAG(AARCH64_ADV_SIMD_UMIN, aarch64_neon_umin, V64 | V128 | I1 | I2 | I4 | I8) +INTRINS_OVR_TAG(AARCH64_ADV_SIMD_FMIN, aarch64_neon_fmin, Scalar | V64 | V128 | R4 | R8) + +INTRINS_OVR_TAG(AARCH64_ADV_SIMD_FMAXP, aarch64_neon_fmaxp, V64 | V128 | R4 | R8) +INTRINS_OVR_TAG(AARCH64_ADV_SIMD_SMAXP, aarch64_neon_smaxp, V64 | V128 | I1 | I2 | I4 | I8) +INTRINS_OVR_TAG(AARCH64_ADV_SIMD_UMAXP, aarch64_neon_umaxp, V64 | V128 | I1 | I2 | I4 | I8) +INTRINS_OVR_TAG(AARCH64_ADV_SIMD_FMINP, aarch64_neon_fminp, V64 | V128 | R4 | R8) +INTRINS_OVR_TAG(AARCH64_ADV_SIMD_SMINP, aarch64_neon_sminp, V64 | V128 | I1 | I2 | I4 | I8) +INTRINS_OVR_TAG(AARCH64_ADV_SIMD_UMINP, aarch64_neon_uminp, V64 | V128 | I1 | I2 | I4 | I8) + +INTRINS_OVR_TAG(AARCH64_ADV_SIMD_FMAXNM, aarch64_neon_fmaxnm, Scalar | V64 | V128 | R4 | R8) +INTRINS_OVR_TAG(AARCH64_ADV_SIMD_FMINNM, aarch64_neon_fminnm, Scalar | V64 | V128 | R4 | R8) + +INTRINS_OVR_TAG(AARCH64_ADV_SIMD_FMAXNMP, aarch64_neon_fmaxnmp, V64 | V128 | R4 | R8) +INTRINS_OVR_TAG(AARCH64_ADV_SIMD_FMINNMP, aarch64_neon_fminnmp, V64 | V128 | R4 | R8) + +INTRINS_OVR_TAG(AARCH64_ADV_SIMD_SQDMULH, aarch64_neon_sqdmulh, Scalar | V64 | V128 | I2 | I4) + +INTRINS(AARCH64_ADV_SIMD_SQDMULL_SCALAR, aarch64_neon_sqdmulls_scalar) +INTRINS_OVR_TAG(AARCH64_ADV_SIMD_SQDMULL, aarch64_neon_sqdmull, V64 | V128 | I2 | I4 | I8) +INTRINS_OVR_TAG(AARCH64_ADV_SIMD_SQRDMULH, aarch64_neon_sqrdmulh, Scalar | V64 | V128 | I2 | I4) + +INTRINS_OVR_TAG(AARCH64_ADV_SIMD_SMULL, aarch64_neon_smull, V128 | I2 | I4 | I8) +INTRINS_OVR_TAG(AARCH64_ADV_SIMD_UMULL, aarch64_neon_umull, V128 | I2 | I4 | I8) + +INTRINS_OVR_TAG(AARCH64_ADV_SIMD_SQNEG, aarch64_neon_sqneg, Scalar | V64 | V128 | I1 | I2 | I4 | I8) + +INTRINS_OVR_TAG(AARCH64_ADV_SIMD_PMUL, aarch64_neon_pmul, V64 | V128 | I1) +INTRINS_OVR(AARCH64_ADV_SIMD_PMULL, aarch64_neon_pmull, v128_i2_t) + +INTRINS_OVR_TAG(AARCH64_ADV_SIMD_FMULX, aarch64_neon_fmulx, Scalar | V64 | V128 | R4 | R8) + +INTRINS_OVR_TAG(AARCH64_ADV_SIMD_CNT, ctpop, V64 | V128 | I1) + +INTRINS_OVR_TAG(AARCH64_ADV_SIMD_URECPE, aarch64_neon_urecpe, V64 | V128 | I4) +INTRINS_OVR_TAG(AARCH64_ADV_SIMD_FRECPE, aarch64_neon_frecpe, Scalar | V64 | V128 | R4 | R8) +INTRINS_OVR_TAG(AARCH64_ADV_SIMD_FRECPX, aarch64_neon_frecpx, Scalar | R4 | R8) +INTRINS_OVR_TAG(AARCH64_ADV_SIMD_URSQRTE, aarch64_neon_ursqrte, V64 | V128 | I4) +INTRINS_OVR_TAG(AARCH64_ADV_SIMD_FRSQRTE, aarch64_neon_frsqrte, Scalar | V64 | V128| R4 | R8) +INTRINS_OVR_TAG(AARCH64_ADV_SIMD_FRSQRTS, aarch64_neon_frsqrts, Scalar | V64 | V128 | R4 | R8) +INTRINS_OVR_TAG(AARCH64_ADV_SIMD_FRECPS, aarch64_neon_frecps, Scalar | V64 | V128 | R4 | R8) + +INTRINS_OVR_TAG(AARCH64_ADV_SIMD_RBIT, aarch64_neon_rbit, V64 | V128 | I1) + +INTRINS_OVR_TAG(AARCH64_ADV_SIMD_FRINTA, round, Scalar | V64 | V128 | R4 | R8) +INTRINS_OVR_TAG(AARCH64_ADV_SIMD_FRINTN, aarch64_neon_frintn, Scalar | V64 | V128 | R4 | R8) +INTRINS_OVR_TAG(AARCH64_ADV_SIMD_FRINTM, floor, Scalar | V64 | V128 | R4 | R8) +INTRINS_OVR_TAG(AARCH64_ADV_SIMD_FRINTP, ceil, Scalar | V64 | V128 | R4 | R8) +INTRINS_OVR_TAG(AARCH64_ADV_SIMD_FRINTZ, trunc, Scalar | V64 | V128 | R4 | R8) + +INTRINS_OVR_TAG(AARCH64_ADV_SIMD_SUQADD, aarch64_neon_suqadd, Scalar | V64 | V128 | I1 | I2 | I4 | I8) +INTRINS_OVR_TAG(AARCH64_ADV_SIMD_USQADD, aarch64_neon_usqadd, Scalar | V64 | V128 | I1 | I2 | I4 | I8) +INTRINS_OVR_TAG(AARCH64_ADV_SIMD_UQADD, aarch64_neon_uqadd, Scalar | V64 | V128 | I1 | I2 | I4 | I8) +INTRINS_OVR_TAG(AARCH64_ADV_SIMD_SQADD, aarch64_neon_sqadd, Scalar | V64 | V128 | I1 | I2 | I4 | I8) +INTRINS_OVR_TAG(AARCH64_ADV_SIMD_UQSUB, aarch64_neon_uqsub, Scalar | V64 | V128 | I1 | I2 | I4 | I8) +INTRINS_OVR_TAG(AARCH64_ADV_SIMD_SQSUB, aarch64_neon_sqsub, Scalar | V64 | V128 | I1 | I2 | I4 | I8) + +INTRINS_OVR_TAG(AARCH64_ADV_SIMD_RADDHN, aarch64_neon_raddhn, V64 | I1 | I2 | I4) +INTRINS_OVR_TAG(AARCH64_ADV_SIMD_RSUBHN, aarch64_neon_rsubhn, V64 | I1 | I2 | I4) + +INTRINS_OVR_TAG(AARCH64_ADV_SIMD_FSQRT, sqrt, Scalar | V64 | V128 | R4 | R8) + +INTRINS_OVR_TAG(AARCH64_ADV_SIMD_UQSHRN, aarch64_neon_uqshrn, V64 | I1 | I2 | I4) // Constant shift + +INTRINS_OVR_TAG(AARCH64_ADV_SIMD_RSHRN, aarch64_neon_rshrn, V64 | I1 | I2 | I4) // Constant shift +INTRINS_OVR_TAG(AARCH64_ADV_SIMD_SQRSHRN, aarch64_neon_sqrshrn, V64 | I1 | I2 | I4) // Constant shift +INTRINS_OVR_TAG(AARCH64_ADV_SIMD_SQRSHRUN, aarch64_neon_sqrshrun, V64 | I1 | I2 | I4) // Constant shift +INTRINS_OVR_TAG(AARCH64_ADV_SIMD_SQSHRN, aarch64_neon_sqshrn, V64 | I1 | I2 | I4) // Constant shift +INTRINS_OVR_TAG(AARCH64_ADV_SIMD_SQSHRUN, aarch64_neon_sqshrun, V64 | I1 | I2 | I4) // Constant shift +INTRINS_OVR_TAG(AARCH64_ADV_SIMD_UQRSHRN, aarch64_neon_uqrshrn, Scalar | V64 | I1 | I2 | I4) // Constant shift + +INTRINS_OVR_TAG(AARCH64_ADV_SIMD_SQRSHL, aarch64_neon_sqrshl, Scalar | V64 | V128 | I1 | I2 | I4 | I8) // Variable shift +INTRINS_OVR_TAG(AARCH64_ADV_SIMD_SQSHL, aarch64_neon_sqshl, Scalar | V64 | V128 | I1 | I2 | I4 | I8) // Variable shift +INTRINS_OVR_TAG(AARCH64_ADV_SIMD_SRSHL, aarch64_neon_srshl, V64 | V128 | I1 | I2 | I4 | I8) // Variable shift +INTRINS_OVR_TAG(AARCH64_ADV_SIMD_SSHL, aarch64_neon_sshl, V64 | V128 | I1 | I2 | I4 | I8) // Variable shift +INTRINS_OVR_TAG(AARCH64_ADV_SIMD_UQRSHL, aarch64_neon_uqrshl, Scalar | V64 | V128 | I1 | I2 | I4 | I8) // Variable shift +INTRINS_OVR_TAG(AARCH64_ADV_SIMD_UQSHL, aarch64_neon_uqshl, Scalar | V64 | V128 | I1 | I2 | I4 | I8) // Variable shift +INTRINS_OVR_TAG(AARCH64_ADV_SIMD_URSHL, aarch64_neon_urshl, V64 | V128 | I1 | I2 | I4 | I8) // Variable shift +INTRINS_OVR_TAG(AARCH64_ADV_SIMD_USHL, aarch64_neon_ushl, V64 | V128 | I1 | I2 | I4 | I8) // Variable shift + +INTRINS_OVR_TAG(AARCH64_ADV_SIMD_SQSHLU, aarch64_neon_sqshlu, Scalar | V64 | V128 | I1 | I2 | I4 | I8) // Constant shift + +INTRINS_OVR_TAG(AARCH64_ADV_SIMD_SLI, aarch64_neon_vsli, V64 | V128 | I1 | I2 | I4 | I8) // Constant shift +INTRINS_OVR_TAG(AARCH64_ADV_SIMD_SRI, aarch64_neon_vsri, V64 | V128 | I1 | I2 | I4 | I8) // Constant shift + +INTRINS_OVR_TAG(AARCH64_ADV_SIMD_TBX1, aarch64_neon_tbx1, V64 | V128 | I1) +INTRINS_OVR_TAG(AARCH64_ADV_SIMD_TBL1, aarch64_neon_tbl1, V64 | V128 | I1) #endif #undef INTRINS #undef INTRINS_OVR #undef INTRINS_OVR_2_ARG #undef INTRINS_OVR_3_ARG - +#undef INTRINS_OVR_TAG +#undef INTRINS_OVR_TAG_KIND +#undef Scalar +#undef V64 +#undef V128 +#undef I1 +#undef I2 +#undef I4 +#undef I8 +#undef R4 +#undef R8 +#undef Ftoi +#undef WidenAcross +#undef Across diff --git a/src/mono/mono/mini/mini-llvm-cpp.cpp b/src/mono/mono/mini/mini-llvm-cpp.cpp index fa1dea68c6a2e4..0392e2f0dcf249 100644 --- a/src/mono/mono/mini/mini-llvm-cpp.cpp +++ b/src/mono/mono/mini/mini-llvm-cpp.cpp @@ -633,9 +633,11 @@ get_intrins_id (IntrinsicId id) Intrinsic::ID intrins_id = Intrinsic::ID::not_intrinsic; switch (id) { #define INTRINS(id, llvm_id) case INTRINS_ ## id: intrins_id = Intrinsic::ID::llvm_id; break; -#define INTRINS_OVR(id, llvm_id, ty) case INTRINS_ ## id: intrins_id = Intrinsic::ID::llvm_id; break; -#define INTRINS_OVR_2_ARG(id, llvm_id, ty1, ty2) case INTRINS_ ## id: intrins_id = Intrinsic::ID::llvm_id; break; -#define INTRINS_OVR_3_ARG(id, llvm_id, ty1, ty2, ty3) case INTRINS_ ## id: intrins_id = Intrinsic::ID::llvm_id; break; +#define INTRINS_OVR(id, llvm_id, ty) INTRINS(id, llvm_id) +#define INTRINS_OVR_2_ARG(id, llvm_id, ty1, ty2) INTRINS(id, llvm_id) +#define INTRINS_OVR_3_ARG(id, llvm_id, ty1, ty2, ty3) INTRINS(id, llvm_id) +#define INTRINS_OVR_TAG(id, llvm_id, ...) INTRINS(id, llvm_id) +#define INTRINS_OVR_TAG_KIND(id, llvm_id, ...) INTRINS(id, llvm_id) #include "llvm-intrinsics.h" default: break; @@ -651,6 +653,8 @@ is_overloaded_intrins (IntrinsicId id) #define INTRINS_OVR(id, llvm_id, ty) case INTRINS_ ## id: return true; #define INTRINS_OVR_2_ARG(id, llvm_id, ty1, ty2) case INTRINS_ ## id: return true; #define INTRINS_OVR_3_ARG(id, llvm_id, ty1, ty2, ty3) case INTRINS_ ## id: return true; +#define INTRINS_OVR_TAG(id, llvm_id, ...) case INTRINS_ ## id: return true; +#define INTRINS_OVR_TAG_KIND(id, llvm_id, ...) case INTRINS_ ## id: return true; #include "llvm-intrinsics.h" default: break; @@ -694,11 +698,11 @@ mono_llvm_register_overloaded_intrinsic (LLVMModuleRef module, IntrinsicId id, L const int max_types = 5; g_assert (ntypes <= max_types); - Type *arr [max_types]; - for (int i = 0; i < ntypes; ++i) + Type *arr [max_types]; + for (int i = 0; i < ntypes; ++i) arr [i] = unwrap (types [i]); - auto f = Intrinsic::getDeclaration (unwrap (module), intrins_id, { arr, (size_t)ntypes }); - return wrap (f); + auto f = Intrinsic::getDeclaration (unwrap (module), intrins_id, { arr, (size_t)ntypes }); + return wrap (f); } unsigned int diff --git a/src/mono/mono/mini/mini-llvm-cpp.h b/src/mono/mono/mini/mini-llvm-cpp.h index 72cf7bfd63e7cd..baa90441be9094 100644 --- a/src/mono/mono/mini/mini-llvm-cpp.h +++ b/src/mono/mono/mini/mini-llvm-cpp.h @@ -16,22 +16,14 @@ #include "llvm-c/Core.h" #include "llvm-c/ExecutionEngine.h" +#include "llvm-intrinsics-types.h" + #ifdef HAVE_UNWIND_H #include #endif G_BEGIN_DECLS -typedef enum { -#define INTRINS(id, llvm_id) INTRINS_ ## id, -#define INTRINS_OVR(id, llvm_id, ty) INTRINS_ ## id, -#define INTRINS_OVR_2_ARG(id, llvm_id, ty1, ty2) INTRINS_ ## id, -#define INTRINS_OVR_3_ARG(id, llvm_id, ty1, ty2, ty3) INTRINS_ ## id, -#include "llvm-intrinsics.h" - INTRINS_NUM -} IntrinsicId; - - /* * Keep in sync with the enum in utils/mono-memory-model.h. */ diff --git a/src/mono/mono/mini/mini-llvm.c b/src/mono/mono/mini/mini-llvm.c index 7dcc4b71a810db..2ea44c5dcccdce 100644 --- a/src/mono/mono/mini/mini-llvm.c +++ b/src/mono/mono/mini/mini-llvm.c @@ -81,7 +81,6 @@ typedef struct { gboolean emit_dwarf; int max_got_offset; LLVMValueRef personality; - LLVMValueRef *intrins_by_id; gpointer gc_poll_cold_wrapper_compiled; /* For AOT */ @@ -254,6 +253,11 @@ mini_llvm_ins_info[] = { #define ctx_ok(ctx) (!(ctx)->cfg->disable_llvm) +enum { + MAX_VECTOR_ELEMS = 32, // 2 vectors * 128 bits per vector / 8 bits per element + ARM64_MAX_VECTOR_ELEMS = 16, +}; + static LLVMIntPredicate cond_to_llvm_cond [] = { LLVMIntEQ, LLVMIntNE, @@ -285,7 +289,10 @@ static LLVMRealPredicate fpcond_to_llvm_cond [] = { static MonoLLVMModule aot_module; static GHashTable *intrins_id_to_intrins; +static LLVMTypeRef i1_t, i2_t, i4_t, i8_t, r4_t, r8_t; static LLVMTypeRef sse_i1_t, sse_i2_t, sse_i4_t, sse_i8_t, sse_r4_t, sse_r8_t; +static LLVMTypeRef v64_i1_t, v64_i2_t, v64_i4_t, v64_i8_t, v64_r4_t, v64_r8_t; +static LLVMTypeRef v128_i1_t, v128_i2_t, v128_i4_t, v128_i8_t, v128_r4_t, v128_r8_t; static MonoLLVMModule *init_jit_module (void); @@ -303,6 +310,148 @@ static void create_aot_info_var (MonoLLVMModule *module); static void set_invariant_load_flag (LLVMValueRef v); static void set_nonnull_load_flag (LLVMValueRef v); +enum { + INTRIN_scalar = 1 << 0, + INTRIN_vector64 = 1 << 1, + INTRIN_vector128 = 1 << 2, + INTRIN_vectorwidths = 3, + INTRIN_vectormask = 0x7, + + INTRIN_int8 = 1 << 3, + INTRIN_int16 = 1 << 4, + INTRIN_int32 = 1 << 5, + INTRIN_int64 = 1 << 6, + INTRIN_float32 = 1 << 7, + INTRIN_float64 = 1 << 8, + INTRIN_elementwidths = 6, +}; + +typedef uint16_t llvm_ovr_tag_t; + +static LLVMTypeRef intrin_types [INTRIN_vectorwidths][INTRIN_elementwidths]; + +static const llvm_ovr_tag_t intrin_arm64_ovr [] = { + #define INTRINS(sym, ...) 0, + #define INTRINS_OVR(sym, ...) 0, + #define INTRINS_OVR_2_ARG(sym, ...) 0, + #define INTRINS_OVR_3_ARG(sym, ...) 0, + #define INTRINS_OVR_TAG(sym, _, spec) spec, + #define INTRINS_OVR_TAG_KIND(sym, _, kind, spec) spec, + #include "llvm-intrinsics.h" +}; + +enum { + INTRIN_kind_ftoi = 1, + INTRIN_kind_widen, + INTRIN_kind_widen_across, + INTRIN_kind_across, +}; + +static const uint8_t intrin_kind [] = { + #define INTRINS(sym, ...) 0, + #define INTRINS_OVR(sym, ...) 0, + #define INTRINS_OVR_2_ARG(sym, ...) 0, + #define INTRINS_OVR_3_ARG(sym, ...) 0, + #define INTRINS_OVR_TAG(sym, _, spec) 0, + #define INTRINS_OVR_TAG_KIND(sym, _, kind, spec) kind, + #include "llvm-intrinsics.h" +}; + +static inline llvm_ovr_tag_t +ovr_tag_force_scalar (llvm_ovr_tag_t tag) +{ + return (tag & ~INTRIN_vectormask) | INTRIN_scalar; +} + +static inline llvm_ovr_tag_t +ovr_tag_smaller_vector (llvm_ovr_tag_t tag) +{ + return (tag & ~INTRIN_vectormask) | ((tag & INTRIN_vectormask) >> 1); +} + +static inline llvm_ovr_tag_t +ovr_tag_smaller_elements (llvm_ovr_tag_t tag) +{ + return ((tag & ~INTRIN_vectormask) >> 1) | (tag & INTRIN_vectormask); +} + +static inline llvm_ovr_tag_t +ovr_tag_corresponding_integer (llvm_ovr_tag_t tag) +{ + return ((tag & ~INTRIN_vectormask) >> 2) | (tag & INTRIN_vectormask); +} + +static LLVMTypeRef +ovr_tag_to_llvm_type (llvm_ovr_tag_t tag) +{ + int vw = 0; + int ew = 0; + if (tag & INTRIN_vector64) vw = 1; + else if (tag & INTRIN_vector128) vw = 2; + + if (tag & INTRIN_int16) ew = 1; + else if (tag & INTRIN_int32) ew = 2; + else if (tag & INTRIN_int64) ew = 3; + else if (tag & INTRIN_float32) ew = 4; + else if (tag & INTRIN_float64) ew = 5; + return intrin_types [vw][ew]; +} + +static int +key_from_id_and_tag (int id, llvm_ovr_tag_t ovr_tag) +{ + return (((int) ovr_tag) << 23) | id; +} + +static llvm_ovr_tag_t +ovr_tag_from_mono_vector_class (MonoClass *klass) { + int size = mono_class_value_size (klass, NULL); + llvm_ovr_tag_t ret = 0; + switch (size) { + case 8: ret |= INTRIN_vector64; break; + case 16: ret |= INTRIN_vector128; break; + } + MonoType *etype = mono_class_get_context (klass)->class_inst->type_argv [0]; + switch (etype->type) { + case MONO_TYPE_I1: case MONO_TYPE_U1: ret |= INTRIN_int8; break; + case MONO_TYPE_I2: case MONO_TYPE_U2: ret |= INTRIN_int16; break; + case MONO_TYPE_I4: case MONO_TYPE_U4: ret |= INTRIN_int32; break; + case MONO_TYPE_I8: case MONO_TYPE_U8: ret |= INTRIN_int64; break; + case MONO_TYPE_R4: ret |= INTRIN_float32; break; + case MONO_TYPE_R8: ret |= INTRIN_float64; break; + } + return ret; +} + +static llvm_ovr_tag_t +ovr_tag_from_llvm_type (LLVMTypeRef type) +{ + llvm_ovr_tag_t ret = 0; + LLVMTypeKind kind = LLVMGetTypeKind (type); + LLVMTypeRef elem_t = NULL; + switch (kind) { + case LLVMVectorTypeKind: { + elem_t = LLVMGetElementType (type); + unsigned int bits = mono_llvm_get_prim_size_bits (type); + switch (bits) { + case 64: ret |= INTRIN_vector64; break; + case 128: ret |= INTRIN_vector128; break; + default: g_assert_not_reached (); + } + break; + } + default: + g_assert_not_reached (); + } + if (elem_t == i1_t) ret |= INTRIN_int8; + if (elem_t == i2_t) ret |= INTRIN_int16; + if (elem_t == i4_t) ret |= INTRIN_int32; + if (elem_t == i8_t) ret |= INTRIN_int64; + if (elem_t == r4_t) ret |= INTRIN_float32; + if (elem_t == r8_t) ret |= INTRIN_float64; + return ret; +} + static inline void set_failure (EmitContext *ctx, const char *message) { @@ -311,6 +460,12 @@ set_failure (EmitContext *ctx, const char *message) ctx->cfg->disable_llvm = TRUE; } +static LLVMValueRef +const_int1 (int v) +{ + return LLVMConstInt (LLVMInt1Type (), v ? 1 : 0, FALSE); +} + static LLVMValueRef const_int32 (int v) { @@ -4668,8 +4823,8 @@ emit_landing_pad (EmitContext *ctx, int group_index, int group_size) static LLVMValueRef create_const_vector (LLVMTypeRef t, const int *vals, int count) { - g_assert (count <= 16); - LLVMValueRef llvm_vals [16]; + g_assert (count <= MAX_VECTOR_ELEMS); + LLVMValueRef llvm_vals [MAX_VECTOR_ELEMS]; for (int i = 0; i < count; i++) llvm_vals [i] = LLVMConstInt (t, vals [i], FALSE); return LLVMConstVector (llvm_vals, count); @@ -4701,6 +4856,241 @@ create_const_vector_2_i32 (int v0, int v1) return LLVMConstVector (mask, 2); } +static LLVMValueRef +broadcast_element (EmitContext *ctx, LLVMValueRef elem, int count) +{ + LLVMTypeRef t = LLVMTypeOf (elem); + LLVMTypeRef init_vec_t = LLVMVectorType (t, 1); + LLVMValueRef undef = LLVMGetUndef (init_vec_t); + LLVMValueRef vec = LLVMBuildInsertElement (ctx->builder, undef, elem, const_int32 (0), ""); + LLVMValueRef select_zero = LLVMConstNull (LLVMVectorType (LLVMInt32Type (), count)); + return LLVMBuildShuffleVector (ctx->builder, vec, undef, select_zero, "broadcast"); +} + +static LLVMValueRef +broadcast_constant (int const_val, LLVMTypeRef elem_t, int count) +{ + int vals [MAX_VECTOR_ELEMS]; + for (int i = 0; i < count; ++i) + vals [i] = const_val; + return create_const_vector (elem_t, vals, count); +} + +static LLVMValueRef +create_shift_vector (EmitContext *ctx, LLVMValueRef type_donor, LLVMValueRef shiftamt) +{ + LLVMTypeRef t = LLVMTypeOf (type_donor); + unsigned int elems = LLVMGetVectorSize (t); + LLVMTypeRef elem_t = LLVMGetElementType (t); + shiftamt = convert_full (ctx, shiftamt, elem_t, TRUE); + shiftamt = broadcast_element (ctx, shiftamt, elems); + return shiftamt; +} + +static LLVMTypeRef +to_integral_vector_type (LLVMTypeRef t) +{ + unsigned int elems = LLVMGetVectorSize (t); + LLVMTypeRef elem_t = LLVMGetElementType (t); + unsigned int bits = mono_llvm_get_prim_size_bits (elem_t); + return LLVMVectorType (LLVMIntType (bits), elems); +} + +static LLVMValueRef +bitcast_to_integral (EmitContext *ctx, LLVMValueRef vec) +{ + LLVMTypeRef src_t = LLVMTypeOf (vec); + LLVMTypeRef dst_t = to_integral_vector_type (src_t); + if (dst_t != src_t) + return LLVMBuildBitCast (ctx->builder, vec, dst_t, "bc2i"); + return vec; +} + +static LLVMValueRef +extract_high_elements (EmitContext *ctx, LLVMValueRef src_vec) +{ + LLVMTypeRef src_t = LLVMTypeOf (src_vec); + unsigned int src_elems = LLVMGetVectorSize (src_t); + unsigned int dst_elems = src_elems / 2; + int mask [MAX_VECTOR_ELEMS] = { 0 }; + for (int i = 0; i < dst_elems; ++i) + mask [i] = dst_elems + i; + return LLVMBuildShuffleVector (ctx->builder, src_vec, LLVMGetUndef (src_t), create_const_vector_i32 (mask, dst_elems), "extract_high"); +} + +static LLVMValueRef +keep_lowest_element (EmitContext *ctx, LLVMTypeRef dst_t, LLVMValueRef vec) +{ + LLVMTypeRef t = LLVMTypeOf (vec); + g_assert (LLVMGetElementType (dst_t) == LLVMGetElementType (t)); + unsigned int elems = LLVMGetVectorSize (dst_t); + unsigned int src_elems = LLVMGetVectorSize (t); + int mask [MAX_VECTOR_ELEMS] = { 0 }; + mask [0] = 0; + for (unsigned int i = 1; i < elems; ++i) + mask [i] = src_elems; + return LLVMBuildShuffleVector (ctx->builder, vec, LLVMConstNull (t), create_const_vector_i32 (mask, elems), "keep_lowest"); +} + +static LLVMValueRef +concatenate_vectors (EmitContext *ctx, LLVMValueRef xs, LLVMValueRef ys) +{ + LLVMTypeRef t = LLVMTypeOf (xs); + unsigned int elems = LLVMGetVectorSize (t) * 2; + int mask [MAX_VECTOR_ELEMS] = { 0 }; + for (int i = 0; i < elems; ++i) + mask [i] = i; + return LLVMBuildShuffleVector (ctx->builder, xs, ys, create_const_vector_i32 (mask, elems), "concat_vecs"); +} + +static LLVMValueRef +scalar_from_vector (EmitContext *ctx, LLVMValueRef xs) +{ + return LLVMBuildExtractElement (ctx->builder, xs, const_int32 (0), "v2s"); +} + +static LLVMValueRef +vector_from_scalar (EmitContext *ctx, LLVMTypeRef type, LLVMValueRef x) +{ + return LLVMBuildInsertElement (ctx->builder, LLVMConstNull (type), x, const_int32 (0), "s2v"); +} + +typedef struct { + EmitContext *ctx; + MonoBasicBlock *bb; + LLVMBasicBlockRef continuation; + LLVMValueRef phi; + LLVMValueRef switch_ins; + LLVMBasicBlockRef tmp_block; + LLVMBasicBlockRef default_case; + LLVMTypeRef switch_index_type; + const char *name; + int max_cases; + int i; +} ImmediateUnrollCtx; + +static ImmediateUnrollCtx +immediate_unroll_begin ( + EmitContext *ctx, MonoBasicBlock *bb, int max_cases, + LLVMValueRef switch_index, LLVMTypeRef return_type, const char *name) +{ + LLVMBasicBlockRef default_case = gen_bb (ctx, name); + LLVMBasicBlockRef continuation = gen_bb (ctx, name); + LLVMValueRef switch_ins = LLVMBuildSwitch (ctx->builder, switch_index, default_case, max_cases); + LLVMPositionBuilderAtEnd (ctx->builder, continuation); + LLVMValueRef phi = LLVMBuildPhi (ctx->builder, return_type, name); + ImmediateUnrollCtx ictx = { 0 }; + ictx.ctx = ctx; + ictx.bb = bb; + ictx.continuation = continuation; + ictx.phi = phi; + ictx.switch_ins = switch_ins; + ictx.default_case = default_case; + ictx.switch_index_type = LLVMTypeOf (switch_index); + ictx.name = name; + ictx.max_cases = max_cases; + return ictx; +} + +static gboolean +immediate_unroll_next (ImmediateUnrollCtx *ictx, int *i) +{ + if (ictx->i >= ictx->max_cases) + return FALSE; + ictx->tmp_block = gen_bb (ictx->ctx, ictx->name); + LLVMPositionBuilderAtEnd (ictx->ctx->builder, ictx->tmp_block); + *i = ictx->i; + ++ictx->i; + return TRUE; +} + +static void +immediate_unroll_commit (ImmediateUnrollCtx *ictx, int switch_const, LLVMValueRef value) +{ + LLVMBuildBr (ictx->ctx->builder, ictx->continuation); + LLVMAddCase (ictx->switch_ins, LLVMConstInt (ictx->switch_index_type, switch_const, FALSE), ictx->tmp_block); + LLVMAddIncoming (ictx->phi, &value, &ictx->tmp_block, 1); +} + +static void +immediate_unroll_default (ImmediateUnrollCtx *ictx) +{ + LLVMPositionBuilderAtEnd (ictx->ctx->builder, ictx->default_case); +} + +static void +immediate_unroll_commit_default (ImmediateUnrollCtx *ictx, LLVMValueRef value) +{ + LLVMBuildBr (ictx->ctx->builder, ictx->continuation); + LLVMAddIncoming (ictx->phi, &value, &ictx->default_case, 1); +} + +static LLVMValueRef +immediate_unroll_end (ImmediateUnrollCtx *ictx, LLVMBasicBlockRef *continuation) +{ + EmitContext *ctx = ictx->ctx; + LLVMBuilderRef builder = ctx->builder; + LLVMPositionBuilderAtEnd (builder, ictx->continuation); + *continuation = ictx->continuation; + ctx->bblocks [ictx->bb->block_num].end_bblock = ictx->continuation; + return ictx->phi; +} + +typedef struct { + EmitContext *ctx; + LLVMTypeRef intermediate_type; + LLVMTypeRef return_type; + gboolean needs_fake_scalar_op; + llvm_ovr_tag_t ovr_tag; +} ScalarOpFromVectorOpCtx; + +static inline gboolean +check_needs_fake_scalar_op (MonoTypeEnum type) +{ +#if defined(TARGET_ARM64) + switch (type) { + case MONO_TYPE_U1: + case MONO_TYPE_I1: + case MONO_TYPE_U2: + case MONO_TYPE_I2: + return TRUE; + } +#endif + return FALSE; +} + +static ScalarOpFromVectorOpCtx +scalar_op_from_vector_op (EmitContext *ctx, LLVMTypeRef return_type, MonoInst *ins) +{ + ScalarOpFromVectorOpCtx ret = { 0 }; + ret.ctx = ctx; + ret.intermediate_type = return_type; + ret.return_type = return_type; + ret.needs_fake_scalar_op = check_needs_fake_scalar_op (inst_c1_type (ins)); + ret.ovr_tag = ovr_tag_from_llvm_type (return_type); + if (!ret.needs_fake_scalar_op) { + ret.ovr_tag = ovr_tag_force_scalar (ret.ovr_tag); + ret.intermediate_type = ovr_tag_to_llvm_type (ret.ovr_tag); + } + return ret; +} + +static void +scalar_op_from_vector_op_process_args (ScalarOpFromVectorOpCtx *sctx, LLVMValueRef *args, int num_args) +{ + if (!sctx->needs_fake_scalar_op) + for (int i = 0; i < num_args; ++i) + args [i] = scalar_from_vector (sctx->ctx, args [i]); +} + +static LLVMValueRef +scalar_op_from_vector_op_process_result (ScalarOpFromVectorOpCtx *sctx, LLVMValueRef result) +{ + if (sctx->needs_fake_scalar_op) + return keep_lowest_element (sctx->ctx, LLVMTypeOf (result), result); + return vector_from_scalar (sctx->ctx, sctx->return_type, result); +} + static void emit_llvmonly_handler_start (EmitContext *ctx, MonoBasicBlock *bb, LLVMBasicBlockRef cbb) { @@ -4902,21 +5292,26 @@ get_float_const (MonoCompile *cfg, float val) } static LLVMValueRef -call_intrins (EmitContext *ctx, int id, LLVMValueRef *args, const char *name) +call_overloaded_intrins (EmitContext *ctx, int id, llvm_ovr_tag_t ovr_tag, LLVMValueRef *args, const char *name) { - LLVMValueRef intrins = get_intrins (ctx, id); + int key = key_from_id_and_tag (id, ovr_tag); + LLVMValueRef intrins = get_intrins (ctx, key); int nargs = LLVMCountParamTypes (LLVMGetElementType (LLVMTypeOf (intrins))); - for (int i = 0; i < nargs; ++i) { LLVMTypeRef t1 = LLVMTypeOf (args [i]); LLVMTypeRef t2 = LLVMTypeOf (LLVMGetParam (intrins, i)); if (t1 != t2) args [i] = convert (ctx, args [i], t2); } - return LLVMBuildCall (ctx->builder, intrins, args, nargs, name); } +static LLVMValueRef +call_intrins (EmitContext *ctx, int id, LLVMValueRef *args, const char *name) +{ + return call_overloaded_intrins (ctx, id, 0, args, name); +} + static void process_bb (EmitContext *ctx, MonoBasicBlock *bb) { @@ -6967,9 +7362,6 @@ process_bb (EmitContext *ctx, MonoBasicBlock *bb) break; } - /* - * SIMD - */ #if defined(TARGET_X86) || defined(TARGET_AMD64) || defined(TARGET_ARM64) || defined(TARGET_WASM) case OP_EXPAND_I1: case OP_EXPAND_I2: @@ -6978,11 +7370,11 @@ process_bb (EmitContext *ctx, MonoBasicBlock *bb) case OP_EXPAND_R4: case OP_EXPAND_R8: { LLVMTypeRef t; - LLVMValueRef mask [32], v; + LLVMValueRef mask [MAX_VECTOR_ELEMS], v; int i; t = simd_class_to_llvm_type (ctx, ins->klass); - for (i = 0; i < 32; ++i) + for (i = 0; i < MAX_VECTOR_ELEMS; ++i) mask [i] = LLVMConstInt (LLVMInt32Type (), 0, FALSE); v = convert (ctx, values [ins->sreg1], LLVMGetElementType (t)); @@ -7011,9 +7403,161 @@ process_bb (EmitContext *ctx, MonoBasicBlock *bb) mono_llvm_build_aligned_store (builder, values [ins->sreg1], dest, FALSE, 1); break; } + case OP_XBINOP: + case OP_XBINOP_SCALAR: + case OP_XBINOP_BYSCALAR: { + gboolean scalar = ins->opcode == OP_XBINOP_SCALAR; + gboolean byscalar = ins->opcode == OP_XBINOP_BYSCALAR; + LLVMValueRef result = NULL; + LLVMValueRef args [] = { lhs, rhs }; + if (scalar) + for (int i = 0; i < 2; ++i) + args [i] = scalar_from_vector (ctx, args [i]); + if (byscalar) { + LLVMTypeRef t = LLVMTypeOf (args [0]); + unsigned int elems = LLVMGetVectorSize (t); + args [1] = broadcast_element (ctx, scalar_from_vector (ctx, args [1]), elems); + } + LLVMValueRef l = args [0]; + LLVMValueRef r = args [1]; + switch (ins->inst_c0) { + case OP_IADD: + result = LLVMBuildAdd (builder, l, r, ""); + break; + case OP_ISUB: + result = LLVMBuildSub (builder, l, r, ""); + break; + case OP_IMUL: + result = LLVMBuildMul (builder, l, r, ""); + break; + case OP_IAND: + result = LLVMBuildAnd (builder, l, r, ""); + break; + case OP_IOR: + result = LLVMBuildOr (builder, l, r, ""); + break; + case OP_IXOR: + result = LLVMBuildXor (builder, l, r, ""); + break; + case OP_FADD: + result = LLVMBuildFAdd (builder, l, r, ""); + break; + case OP_FSUB: + result = LLVMBuildFSub (builder, l, r, ""); + break; + case OP_FMUL: + result = LLVMBuildFMul (builder, l, r, ""); + break; + case OP_FDIV: + result = LLVMBuildFDiv (builder, l, r, ""); + break; + case OP_FMAX: + case OP_FMIN: { +#if defined(TARGET_X86) || defined(TARGET_AMD64) + LLVMValueRef args [] = { l, r }; + + gboolean is_r4 = ins->inst_c1 == MONO_TYPE_R4; + if (ins->inst_c0 == OP_FMAX) + result = call_intrins (ctx, is_r4 ? INTRINS_SSE_MAXPS : INTRINS_SSE_MAXPD, args, dname); + else + result = call_intrins (ctx, is_r4 ? INTRINS_SSE_MINPS : INTRINS_SSE_MINPD, args, dname); +#else + NOT_IMPLEMENTED; +#endif + break; + } + case OP_IMAX: { + gboolean is_unsigned = ins->inst_c1 == MONO_TYPE_U1 || ins->inst_c1 == MONO_TYPE_U2 || ins->inst_c1 == MONO_TYPE_U4 || ins->inst_c1 == MONO_TYPE_U8; + LLVMValueRef cmp = LLVMBuildICmp (builder, is_unsigned ? LLVMIntUGT : LLVMIntSGT, l, r, ""); + result = LLVMBuildSelect (builder, cmp, l, r, ""); + break; + } + case OP_IMIN: { + gboolean is_unsigned = ins->inst_c1 == MONO_TYPE_U1 || ins->inst_c1 == MONO_TYPE_U2 || ins->inst_c1 == MONO_TYPE_U4 || ins->inst_c1 == MONO_TYPE_U8; + LLVMValueRef cmp = LLVMBuildICmp (builder, is_unsigned ? LLVMIntULT : LLVMIntSLT, l, r, ""); + result = LLVMBuildSelect (builder, cmp, l, r, ""); + break; + } + + default: + g_assert_not_reached (); + } + if (scalar) + result = vector_from_scalar (ctx, LLVMTypeOf (lhs), result); + values [ins->dreg] = result; + break; + } + case OP_XBINOP_FORCEINT: { + LLVMTypeRef t = LLVMTypeOf (lhs); + LLVMTypeRef elem_t = LLVMGetElementType (t); + unsigned int elems = LLVMGetVectorSize (t); + unsigned int elem_bits = mono_llvm_get_prim_size_bits (elem_t); + LLVMTypeRef intermediate_elem_t = LLVMIntType (elem_bits); + LLVMTypeRef intermediate_t = LLVMVectorType (intermediate_elem_t, elems); + LLVMValueRef lhs_int = convert (ctx, lhs, intermediate_t); + LLVMValueRef rhs_int = convert (ctx, rhs, intermediate_t); + LLVMValueRef result = NULL; + switch (ins->inst_c0) { + case XBINOP_FORCEINT_and: + result = LLVMBuildAnd (builder, lhs_int, rhs_int, ""); + break; + case XBINOP_FORCEINT_or: + result = LLVMBuildOr (builder, lhs_int, rhs_int, ""); + break; + case XBINOP_FORCEINT_ornot: + result = LLVMBuildNot (builder, rhs_int, ""); + result = LLVMBuildOr (builder, result, lhs_int, ""); + break; + case XBINOP_FORCEINT_xor: + result = LLVMBuildXor (builder, lhs_int, rhs_int, ""); + break; + } + values [ins->dreg] = LLVMBuildBitCast (builder, result, t, ""); + break; + } + case OP_CREATE_SCALAR: + case OP_CREATE_SCALAR_UNSAFE: { + MonoTypeEnum primty = inst_c1_type (ins); + LLVMTypeRef type = simd_class_to_llvm_type (ctx, ins->klass); + // use undef vector (most likely empty but may contain garbage values) for OP_CREATE_SCALAR_UNSAFE + // and zero one for OP_CREATE_SCALAR + LLVMValueRef vector = (ins->opcode == OP_CREATE_SCALAR) ? LLVMConstNull (type) : LLVMGetUndef (type); + LLVMValueRef val = convert_full (ctx, lhs, primitive_type_to_llvm_type (primty), primitive_type_is_unsigned (primty)); + values [ins->dreg] = LLVMBuildInsertElement (builder, vector, val, const_int32 (0), ""); + break; + } + case OP_INSERT_I1: + values [ins->dreg] = LLVMBuildInsertElement (builder, values [ins->sreg1], convert (ctx, values [ins->sreg2], LLVMInt8Type ()), LLVMConstInt (LLVMInt32Type (), ins->inst_c0, FALSE), dname); + break; + case OP_INSERT_I2: + values [ins->dreg] = LLVMBuildInsertElement (builder, values [ins->sreg1], convert (ctx, values [ins->sreg2], LLVMInt16Type ()), LLVMConstInt (LLVMInt32Type (), ins->inst_c0, FALSE), dname); + break; + case OP_INSERT_I4: + values [ins->dreg] = LLVMBuildInsertElement (builder, values [ins->sreg1], convert (ctx, values [ins->sreg2], LLVMInt32Type ()), LLVMConstInt (LLVMInt32Type (), ins->inst_c0, FALSE), dname); + break; + case OP_INSERT_I8: + values [ins->dreg] = LLVMBuildInsertElement (builder, values [ins->sreg1], convert (ctx, values [ins->sreg2], LLVMInt64Type ()), LLVMConstInt (LLVMInt32Type (), ins->inst_c0, FALSE), dname); + break; + case OP_INSERT_R4: + values [ins->dreg] = LLVMBuildInsertElement (builder, values [ins->sreg1], convert (ctx, values [ins->sreg2], LLVMFloatType ()), LLVMConstInt (LLVMInt32Type (), ins->inst_c0, FALSE), dname); + break; + case OP_INSERT_R8: + values [ins->dreg] = LLVMBuildInsertElement (builder, values [ins->sreg1], convert (ctx, values [ins->sreg2], LLVMDoubleType ()), LLVMConstInt (LLVMInt32Type (), ins->inst_c0, FALSE), dname); + break; + case OP_XCAST: { + LLVMTypeRef t = simd_class_to_llvm_type (ctx, ins->klass); + + values [ins->dreg] = LLVMBuildBitCast (builder, lhs, t, ""); + break; + } + case OP_XCONCAT: { + values [ins->dreg] = concatenate_vectors (ctx, lhs, rhs); + break; + } #endif // defined(TARGET_X86) || defined(TARGET_AMD64) || defined(TARGET_ARM64) || defined(TARGET_WASM) #if defined(TARGET_X86) || defined(TARGET_AMD64) || defined(TARGET_WASM) + case OP_PADDB: case OP_PADDW: case OP_PADDD: @@ -7161,11 +7705,11 @@ process_bb (EmitContext *ctx, MonoBasicBlock *bb) case OP_PAVGB_UN: case OP_PAVGW_UN: { LLVMValueRef ones_vec; - LLVMValueRef ones [32]; + LLVMValueRef ones [MAX_VECTOR_ELEMS]; int vector_size = LLVMGetVectorSize (LLVMTypeOf (lhs)); LLVMTypeRef ext_elem_type = vector_size == 16 ? LLVMInt16Type () : LLVMInt32Type (); - for (int i = 0; i < 32; ++i) + for (int i = 0; i < MAX_VECTOR_ELEMS; ++i) ones [i] = LLVMConstInt (ext_elem_type, 1, FALSE); ones_vec = LLVMConstVector (ones, vector_size); @@ -7247,34 +7791,15 @@ process_bb (EmitContext *ctx, MonoBasicBlock *bb) values [ins->dreg] = LLVMBuildZExt (builder, values [ins->dreg], LLVMInt32Type (), ""); break; } - - case OP_INSERT_I1: - values [ins->dreg] = LLVMBuildInsertElement (builder, values [ins->sreg1], convert (ctx, values [ins->sreg2], LLVMInt8Type ()), LLVMConstInt (LLVMInt32Type (), ins->inst_c0, FALSE), dname); - break; - case OP_INSERT_I2: - values [ins->dreg] = LLVMBuildInsertElement (builder, values [ins->sreg1], convert (ctx, values [ins->sreg2], LLVMInt16Type ()), LLVMConstInt (LLVMInt32Type (), ins->inst_c0, FALSE), dname); - break; - case OP_INSERT_I4: - values [ins->dreg] = LLVMBuildInsertElement (builder, values [ins->sreg1], convert (ctx, values [ins->sreg2], LLVMInt32Type ()), LLVMConstInt (LLVMInt32Type (), ins->inst_c0, FALSE), dname); - break; - case OP_INSERT_I8: - values [ins->dreg] = LLVMBuildInsertElement (builder, values [ins->sreg1], convert (ctx, values [ins->sreg2], LLVMInt64Type ()), LLVMConstInt (LLVMInt32Type (), ins->inst_c0, FALSE), dname); - break; - case OP_INSERT_R4: - values [ins->dreg] = LLVMBuildInsertElement (builder, values [ins->sreg1], convert (ctx, values [ins->sreg2], LLVMFloatType ()), LLVMConstInt (LLVMInt32Type (), ins->inst_c0, FALSE), dname); - break; - case OP_INSERT_R8: - values [ins->dreg] = LLVMBuildInsertElement (builder, values [ins->sreg1], convert (ctx, values [ins->sreg2], LLVMDoubleType ()), LLVMConstInt (LLVMInt32Type (), ins->inst_c0, FALSE), dname); - break; - case OP_XINSERT_I2: { - LLVMBasicBlockRef bbs [64]; - LLVMValueRef switch_ins; - LLVMValueRef vector = lhs; - LLVMValueRef value = rhs; - LLVMValueRef index = values [ins->sreg3]; - LLVMValueRef phi_values [64]; - int nelems = LLVMGetVectorSize (LLVMTypeOf (lhs)); - int i; + case OP_XINSERT_I2: { + LLVMBasicBlockRef bbs [64]; + LLVMValueRef switch_ins; + LLVMValueRef vector = lhs; + LLVMValueRef value = rhs; + LLVMValueRef index = values [ins->sreg3]; + LLVMValueRef phi_values [64]; + int nelems = LLVMGetVectorSize (LLVMTypeOf (lhs)); + int i; /* * Many SIMD opcodes require an immediate operand, but can be called with a non-immediate. @@ -8578,18 +9103,6 @@ process_bb (EmitContext *ctx, MonoBasicBlock *bb) break; } - case OP_CREATE_SCALAR: - case OP_CREATE_SCALAR_UNSAFE: { - MonoTypeEnum primty = inst_c1_type (ins); - LLVMTypeRef type = simd_class_to_llvm_type (ctx, ins->klass); - // use undef vector (most likely empty but may contain garbage values) for OP_CREATE_SCALAR_UNSAFE - // and zero one for OP_CREATE_SCALAR - LLVMValueRef vector = (ins->opcode == OP_CREATE_SCALAR) ? LLVMConstNull (type) : LLVMGetUndef (type); - LLVMValueRef insert_pos = LLVMConstInt (LLVMInt32Type (), 0, FALSE); - LLVMValueRef val = convert_full (ctx, lhs, primitive_type_to_llvm_type (primty), primitive_type_is_unsigned (primty)); - values [ins->dreg] = LLVMBuildInsertElement (builder, vector, val, insert_pos, ""); - break; - } case OP_SSE41_ROUNDP: { LLVMValueRef args [] = { lhs, LLVMConstInt (LLVMInt32Type (), ins->inst_c0, FALSE) }; values [ins->dreg] = call_intrins (ctx, ins->inst_c1 == MONO_TYPE_R4 ? INTRINS_SSE_ROUNDPS : INTRINS_SSE_ROUNDPD, args, dname); @@ -8785,12 +9298,6 @@ process_bb (EmitContext *ctx, MonoBasicBlock *bb) } #endif - case OP_XCAST: { - LLVMTypeRef t = simd_class_to_llvm_type (ctx, ins->klass); - - values [ins->dreg] = LLVMBuildBitCast (builder, lhs, t, ""); - break; - } case OP_XCOMPARE_FP: { LLVMRealPredicate pred = fpcond_to_llvm_cond [ins->inst_c0]; LLVMValueRef cmp = LLVMBuildFCmp (builder, pred, lhs, rhs, ""); @@ -8811,7 +9318,7 @@ process_bb (EmitContext *ctx, MonoBasicBlock *bb) } case OP_XEQUAL: { LLVMTypeRef t; - LLVMValueRef cmp, mask [32], shuffle; + LLVMValueRef cmp, mask [MAX_VECTOR_ELEMS], shuffle; int nelems; #if defined(TARGET_WASM) && LLVM_API_VERSION >= 800 @@ -8882,68 +9389,6 @@ process_bb (EmitContext *ctx, MonoBasicBlock *bb) values [ins->dreg] = LLVMBuildZExt (builder, cmp_zero, LLVMInt8Type (), ""); break; } - case OP_XBINOP: { - switch (ins->inst_c0) { - case OP_IADD: - values [ins->dreg] = LLVMBuildAdd (builder, lhs, rhs, ""); - break; - case OP_ISUB: - values [ins->dreg] = LLVMBuildSub (builder, lhs, rhs, ""); - break; - case OP_IAND: - values [ins->dreg] = LLVMBuildAnd (builder, lhs, rhs, ""); - break; - case OP_IOR: - values [ins->dreg] = LLVMBuildOr (builder, lhs, rhs, ""); - break; - case OP_IXOR: - values [ins->dreg] = LLVMBuildXor (builder, lhs, rhs, ""); - break; - case OP_FADD: - values [ins->dreg] = LLVMBuildFAdd (builder, lhs, rhs, ""); - break; - case OP_FSUB: - values [ins->dreg] = LLVMBuildFSub (builder, lhs, rhs, ""); - break; - case OP_FMUL: - values [ins->dreg] = LLVMBuildFMul (builder, lhs, rhs, ""); - break; - case OP_FDIV: - values [ins->dreg] = LLVMBuildFDiv (builder, lhs, rhs, ""); - break; - case OP_FMAX: - case OP_FMIN: { -#if defined(TARGET_X86) || defined(TARGET_AMD64) - LLVMValueRef args [] = { lhs, rhs }; - - gboolean is_r4 = ins->inst_c1 == MONO_TYPE_R4; - if (ins->inst_c0 == OP_FMAX) - values [ins->dreg] = call_intrins (ctx, is_r4 ? INTRINS_SSE_MAXPS : INTRINS_SSE_MAXPD, args, dname); - else - values [ins->dreg] = call_intrins (ctx, is_r4 ? INTRINS_SSE_MINPS : INTRINS_SSE_MINPD, args, dname); -#else - NOT_IMPLEMENTED; -#endif - break; - } - case OP_IMAX: { - gboolean is_unsigned = ins->inst_c1 == MONO_TYPE_U1 || ins->inst_c1 == MONO_TYPE_U2 || ins->inst_c1 == MONO_TYPE_U4 || ins->inst_c1 == MONO_TYPE_U8; - LLVMValueRef cmp = LLVMBuildICmp (builder, is_unsigned ? LLVMIntUGT : LLVMIntSGT, lhs, rhs, ""); - values [ins->dreg] = LLVMBuildSelect (builder, cmp, lhs, rhs, ""); - break; - } - case OP_IMIN: { - gboolean is_unsigned = ins->inst_c1 == MONO_TYPE_U1 || ins->inst_c1 == MONO_TYPE_U2 || ins->inst_c1 == MONO_TYPE_U4 || ins->inst_c1 == MONO_TYPE_U8; - LLVMValueRef cmp = LLVMBuildICmp (builder, is_unsigned ? LLVMIntULT : LLVMIntSLT, lhs, rhs, ""); - values [ins->dreg] = LLVMBuildSelect (builder, cmp, lhs, rhs, ""); - } - break; - - default: - g_assert_not_reached (); - } - break; - } case OP_XEXTRACT_I32: case OP_XEXTRACT_I64: case OP_XEXTRACT_R8: @@ -9041,6 +9486,57 @@ process_bb (EmitContext *ctx, MonoBasicBlock *bb) #endif #if defined(TARGET_ARM64) + case OP_EXTRACT_VAR_I1: + case OP_EXTRACT_VAR_U1: + case OP_EXTRACT_VAR_I2: + case OP_EXTRACT_VAR_U2: + case OP_EXTRACT_VAR_I4: + case OP_EXTRACT_VAR_R4: + case OP_EXTRACT_VAR_R8: + case OP_EXTRACT_VAR_I8: + case OP_EXTRACT_U1: + case OP_EXTRACT_I1: + case OP_EXTRACT_U2: + case OP_EXTRACT_I2: + case OP_EXTRACT_I4: + case OP_EXTRACT_I8: + case OP_EXTRACT_R4: + case OP_EXTRACT_R8: { + gboolean sext = FALSE; + gboolean zext = FALSE; + switch (ins->opcode) { + case OP_EXTRACT_U1: case OP_EXTRACT_U2: zext = TRUE; break; + case OP_EXTRACT_I1: case OP_EXTRACT_I2: sext = TRUE; break; + case OP_EXTRACT_VAR_U1: case OP_EXTRACT_VAR_U2: zext = TRUE; break; + case OP_EXTRACT_VAR_I1: case OP_EXTRACT_VAR_I2: sext = TRUE; break; + } + LLVMValueRef element_ix = NULL; + switch (ins->opcode) { + case OP_EXTRACT_VAR_I1: + case OP_EXTRACT_VAR_U1: + case OP_EXTRACT_VAR_I2: + case OP_EXTRACT_VAR_U2: + case OP_EXTRACT_VAR_I4: + case OP_EXTRACT_VAR_R4: + case OP_EXTRACT_VAR_R8: + case OP_EXTRACT_VAR_I8: + element_ix = rhs; + break; + default: + element_ix = const_int32 (ins->inst_c0); + } + LLVMValueRef result = LLVMBuildExtractElement (builder, lhs, element_ix, "extract"); + /* TODO: Scalar types smaller than i32 seem to be + * normalized to i32 via zero or sign extension. + * Is this still necessary? + */ + if (zext) + result = LLVMBuildZExt (builder, result, i4_t, "extract_zext"); + else if (sext) + result = LLVMBuildSExt (builder, result, i4_t, "extract_sext"); + values [ins->dreg] = result; + break; + } case OP_XOP_I4_I4: case OP_XOP_I8_I8: { IntrinsicId id = (IntrinsicId)0; @@ -9071,14 +9567,6 @@ process_bb (EmitContext *ctx, MonoBasicBlock *bb) case SIMD_OP_AES_ENC: id = INTRINS_AARCH64_AESE; break; case SIMD_OP_ARM64_SHA1SU1: id = INTRINS_AARCH64_SHA1SU1; break; case SIMD_OP_ARM64_SHA256SU0: id = INTRINS_AARCH64_SHA256SU0; break; - case SIMD_OP_ARM64_FABSOLUTE_COMPARE_GREATER_THAN: id = INTRINS_AARCH64_ADV_SIMD_ABS_COMPARE_GT_FLOAT; break; - case SIMD_OP_ARM64_DABSOLUTE_COMPARE_GREATER_THAN: id = INTRINS_AARCH64_ADV_SIMD_ABS_COMPARE_GT_DOUBLE; break; - case SIMD_OP_ARM64_FABSOLUTE_COMPARE_GREATER_THAN_OR_EQUAL: id = INTRINS_AARCH64_ADV_SIMD_ABS_COMPARE_GTE_FLOAT; break; - case SIMD_OP_ARM64_DABSOLUTE_COMPARE_GREATER_THAN_OR_EQUAL: id = INTRINS_AARCH64_ADV_SIMD_ABS_COMPARE_GTE_DOUBLE; break; - case SIMD_OP_ARM64_FABSOLUTE_COMPARE_LESS_THAN: id = INTRINS_AARCH64_ADV_SIMD_ABS_COMPARE_LT_FLOAT; break; - case SIMD_OP_ARM64_DABSOLUTE_COMPARE_LESS_THAN: id = INTRINS_AARCH64_ADV_SIMD_ABS_COMPARE_LT_DOUBLE; break; - case SIMD_OP_ARM64_FABSOLUTE_COMPARE_LESS_THAN_OR_EQUAL: id = INTRINS_AARCH64_ADV_SIMD_ABS_COMPARE_LTE_FLOAT; break; - case SIMD_OP_ARM64_DABSOLUTE_COMPARE_LESS_THAN_OR_EQUAL: id = INTRINS_AARCH64_ADV_SIMD_ABS_COMPARE_LTE_DOUBLE; break; case SIMD_OP_ARM64_PMULL64_LOWER: id = INTRINS_AARCH64_PMULL64; getElement = TRUE; @@ -9128,34 +9616,236 @@ process_bb (EmitContext *ctx, MonoBasicBlock *bb) } case OP_XOP_X_X: { IntrinsicId id = (IntrinsicId)0; + LLVMTypeRef ret_t = simd_class_to_llvm_type (ctx, ins->klass); gboolean getLowerElement = FALSE; - switch (ins->inst_c0) { - case SIMD_OP_AES_IMC: id = INTRINS_AARCH64_AESIMC; break; - case SIMD_OP_ARM64_AES_AESMC: id = INTRINS_AARCH64_AESMC; break; - case SIMD_OP_ARM64_FABS: id = INTRINS_AARCH64_ADV_SIMD_ABS_FLOAT; break; - case SIMD_OP_ARM64_DABS: id = INTRINS_AARCH64_ADV_SIMD_ABS_DOUBLE; break; - case SIMD_OP_ARM64_I8ABS: id = INTRINS_AARCH64_ADV_SIMD_ABS_INT8; break; - case SIMD_OP_ARM64_I16ABS: id = INTRINS_AARCH64_ADV_SIMD_ABS_INT16; break; - case SIMD_OP_ARM64_I32ABS: id = INTRINS_AARCH64_ADV_SIMD_ABS_INT32; break; - case SIMD_OP_ARM64_I64ABS: id = INTRINS_AARCH64_ADV_SIMD_ABS_INT64; break; - case SIMD_OP_ARM64_I8ABS_SATURATE: id = INTRINS_AARCH64_ADV_SIMD_ABS_SATURATE_INT8; break; - case SIMD_OP_ARM64_I16ABS_SATURATE: id = INTRINS_AARCH64_ADV_SIMD_ABS_SATURATE_INT16; break; - case SIMD_OP_ARM64_I32ABS_SATURATE: id = INTRINS_AARCH64_ADV_SIMD_ABS_SATURATE_INT32; break; - case SIMD_OP_ARM64_I64ABS_SATURATE: id = INTRINS_AARCH64_ADV_SIMD_ABS_SATURATE_INT64; break; - case SIMD_OP_ARM64_SHA1H: id = INTRINS_AARCH64_SHA1H; getLowerElement = TRUE; break; - default: g_assert_not_reached (); break; + switch (ins->opcode) { + default: + switch (ins->inst_c0) { + case SIMD_OP_AES_IMC: id = INTRINS_AARCH64_AESIMC; break; + case SIMD_OP_ARM64_AES_AESMC: id = INTRINS_AARCH64_AESMC; break; + case SIMD_OP_ARM64_SHA1H: id = INTRINS_AARCH64_SHA1H; getLowerElement = TRUE; break; + default: g_assert_not_reached (); break; + } } LLVMValueRef arg0 = lhs; - LLVMValueRef result; if (getLowerElement) arg0 = LLVMBuildExtractElement (ctx->builder, arg0, const_int32 (0), ""); - result = call_intrins (ctx, id, &arg0, ""); - if (getLowerElement) { - LLVMTypeRef t = simd_class_to_llvm_type (ctx, ins->klass); - result = LLVMBuildInsertElement (ctx->builder, LLVMConstNull (t), result, const_int32 (0), ""); + LLVMValueRef result = call_intrins (ctx, id, &arg0, ""); + if (getLowerElement) + result = vector_from_scalar (ctx, ret_t, result); + values [ins->dreg] = result; + break; + } + case OP_XINSERT_I1: + case OP_XINSERT_I2: + case OP_XINSERT_I4: + case OP_XINSERT_I8: + case OP_XINSERT_R4: + case OP_XINSERT_R8: { + /* TODO: Scalar types smaller than i32 seem to be + * normalized to i32 via zero or sign extension. + * Is this still necessary? + */ + LLVMTypeRef t = LLVMTypeOf (lhs); + LLVMTypeRef elem_t = LLVMGetElementType (t); + MonoTypeEnum primty = inst_c1_type (ins); + LLVMValueRef val = convert_full (ctx, rhs, elem_t, primitive_type_is_unsigned (primty)); + LLVMValueRef result = LLVMBuildInsertElement (builder, lhs, val, arg3, "xinsert"); + values [ins->dreg] = result; + break; + } + case OP_XCOMPARE_FP_SCALAR: + case OP_XCOMPARE_FP: { + g_assert (LLVMTypeOf (lhs) == LLVMTypeOf (rhs)); + gboolean scalar = ins->opcode == OP_XCOMPARE_FP_SCALAR; + LLVMRealPredicate pred = fpcond_to_llvm_cond [ins->inst_c0]; + LLVMTypeRef ret_t = LLVMTypeOf (lhs); + LLVMTypeRef reti_t = to_integral_vector_type (ret_t); + LLVMValueRef args [] = { lhs, rhs }; + if (scalar) + for (int i = 0; i < 2; ++i) + args [i] = scalar_from_vector (ctx, args [i]); + LLVMValueRef result = LLVMBuildFCmp (builder, pred, args [0], args [1], "xcompare_fp"); + if (scalar) + result = vector_from_scalar (ctx, LLVMVectorType (LLVMIntType (1), LLVMGetVectorSize (reti_t)), result); + result = LLVMBuildSExt (builder, result, reti_t, ""); + result = LLVMBuildBitCast (builder, result, ret_t, ""); + values [ins->dreg] = result; + break; + } + case OP_XCOMPARE_SCALAR: + case OP_XCOMPARE: { + g_assert (LLVMTypeOf (lhs) == LLVMTypeOf (rhs)); + gboolean scalar = ins->opcode == OP_XCOMPARE_SCALAR; + LLVMIntPredicate pred = cond_to_llvm_cond [ins->inst_c0]; + LLVMTypeRef ret_t = LLVMTypeOf (lhs); + LLVMValueRef args [] = { lhs, rhs }; + if (scalar) + for (int i = 0; i < 2; ++i) + args [i] = scalar_from_vector (ctx, args [i]); + LLVMValueRef result = LLVMBuildICmp (builder, pred, args [0], args [1], "xcompare"); + if (scalar) + result = vector_from_scalar (ctx, LLVMVectorType (LLVMIntType (1), LLVMGetVectorSize (ret_t)), result); + values [ins->dreg] = LLVMBuildSExt (builder, result, ret_t, ""); + break; + } + case OP_ARM64_EXT: { + LLVMTypeRef ret_t = LLVMTypeOf (lhs); + unsigned int elems = LLVMGetVectorSize (ret_t); + const int unrolled_mask [MAX_VECTOR_ELEMS] = { + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, + 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31 + }; + enum { ARM64_EXT_MAX_INDEX = MAX_VECTOR_ELEMS / 2 }; + int max_index = elems; + LLVMValueRef index = arg3; + LLVMValueRef default_value = lhs; + ImmediateUnrollCtx ictx = immediate_unroll_begin (ctx, bb, max_index, index, ret_t, "arm64_ext"); + int i = 0; + while (immediate_unroll_next (&ictx, &i)) { + LLVMValueRef mask = create_const_vector_i32 (&unrolled_mask [i], elems); + LLVMValueRef result = LLVMBuildShuffleVector (builder, lhs, rhs, mask, "arm64_ext"); + immediate_unroll_commit (&ictx, i, result); + } + immediate_unroll_default (&ictx); + immediate_unroll_commit_default (&ictx, default_value); + values [ins->dreg] = immediate_unroll_end (&ictx, &cbb); + break; + } + case OP_ARM64_MVN: { + LLVMTypeRef ret_t = LLVMTypeOf (lhs); + LLVMValueRef result = bitcast_to_integral (ctx, lhs); + result = LLVMBuildNot (builder, result, "arm64_mvn"); + result = convert (ctx, result, ret_t); + values [ins->dreg] = result; + break; + } + case OP_ARM64_BIC: { + LLVMTypeRef ret_t = LLVMTypeOf (lhs); + LLVMValueRef result = bitcast_to_integral (ctx, lhs); + LLVMValueRef mask = bitcast_to_integral (ctx, rhs); + mask = LLVMBuildNot (builder, mask, ""); + result = LLVMBuildAnd (builder, mask, result, "arm64_bic"); + result = convert (ctx, result, ret_t); + values [ins->dreg] = result; + break; + } + case OP_ARM64_BSL: { + LLVMTypeRef ret_t = LLVMTypeOf (rhs); + LLVMValueRef select = bitcast_to_integral (ctx, lhs); + LLVMValueRef left = bitcast_to_integral (ctx, rhs); + LLVMValueRef right = bitcast_to_integral (ctx, arg3); + LLVMValueRef result1 = LLVMBuildAnd (builder, select, left, "arm64_bsl"); + LLVMValueRef result2 = LLVMBuildAnd (builder, LLVMBuildNot (builder, select, ""), right, ""); + LLVMValueRef result = LLVMBuildOr (builder, result1, result2, ""); + result = convert (ctx, result, ret_t); + values [ins->dreg] = result; + break; + } + case OP_ARM64_CMTST: { + LLVMTypeRef ret_t = simd_class_to_llvm_type (ctx, ins->klass); + LLVMValueRef l = bitcast_to_integral (ctx, lhs); + LLVMValueRef r = bitcast_to_integral (ctx, rhs); + LLVMValueRef result = LLVMBuildAnd (builder, l, r, "arm64_cmtst"); + LLVMTypeRef t = LLVMTypeOf (l); + result = LLVMBuildICmp (builder, LLVMIntNE, result, LLVMConstNull (t), ""); + result = LLVMBuildSExt (builder, result, t, ""); + result = convert (ctx, result, ret_t); + values [ins->dreg] = result; + break; + } + case OP_ARM64_FCVTL: + case OP_ARM64_FCVTL2: { + LLVMTypeRef ret_t = simd_class_to_llvm_type (ctx, ins->klass); + gboolean high = ins->opcode == OP_ARM64_FCVTL2; + LLVMValueRef result = lhs; + if (high) + result = extract_high_elements (ctx, result); + result = LLVMBuildFPExt (builder, result, ret_t, "arm64_fcvtl"); + values [ins->dreg] = result; + break; + } + case OP_ARM64_FCVTXN: + case OP_ARM64_FCVTXN2: + case OP_ARM64_FCVTN: + case OP_ARM64_FCVTN2: { + gboolean high = FALSE; + int iid = 0; + switch (ins->opcode) { + case OP_ARM64_FCVTXN2: high = TRUE; case OP_ARM64_FCVTXN: iid = INTRINS_AARCH64_ADV_SIMD_FCVTXN; break; + case OP_ARM64_FCVTN2: high = TRUE; break; + } + LLVMValueRef result = lhs; + if (high) + result = rhs; + if (iid) + result = call_intrins (ctx, iid, &result, ""); + else + result = LLVMBuildFPTrunc (builder, result, v64_r4_t, ""); + if (high) + result = concatenate_vectors (ctx, lhs, result); + values [ins->dreg] = result; + break; + } + case OP_ARM64_UCVTF: + case OP_ARM64_SCVTF: + case OP_ARM64_UCVTF_SCALAR: + case OP_ARM64_SCVTF_SCALAR: { + LLVMTypeRef ret_t = simd_class_to_llvm_type (ctx, ins->klass); + gboolean scalar = FALSE; + gboolean is_unsigned = FALSE; + switch (ins->opcode) { + case OP_ARM64_UCVTF_SCALAR: scalar = TRUE; case OP_ARM64_UCVTF: is_unsigned = TRUE; break; + case OP_ARM64_SCVTF_SCALAR: scalar = TRUE; break; + } + LLVMValueRef result = lhs; + LLVMTypeRef cvt_t = ret_t; + if (scalar) { + result = scalar_from_vector (ctx, result); + cvt_t = LLVMGetElementType (ret_t); } + if (is_unsigned) + result = LLVMBuildUIToFP (builder, result, cvt_t, "arm64_ucvtf"); + else + result = LLVMBuildSIToFP (builder, result, cvt_t, "arm64_scvtf"); + if (scalar) + result = vector_from_scalar (ctx, ret_t, result); + values [ins->dreg] = result; + break; + } + case OP_ARM64_FCVTZS: + case OP_ARM64_FCVTZS_SCALAR: + case OP_ARM64_FCVTZU: + case OP_ARM64_FCVTZU_SCALAR: { + LLVMTypeRef ret_t = simd_class_to_llvm_type (ctx, ins->klass); + gboolean scalar = FALSE; + gboolean is_unsigned = FALSE; + switch (ins->opcode) { + case OP_ARM64_FCVTZU_SCALAR: scalar = TRUE; case OP_ARM64_FCVTZU: is_unsigned = TRUE; break; + case OP_ARM64_FCVTZS_SCALAR: scalar = TRUE; break; + } + LLVMValueRef result = lhs; + LLVMTypeRef cvt_t = ret_t; + if (scalar) { + result = scalar_from_vector (ctx, result); + cvt_t = LLVMGetElementType (ret_t); + } + if (is_unsigned) + result = LLVMBuildFPToUI (builder, result, cvt_t, "arm64_fcvtzu"); + else + result = LLVMBuildFPToSI (builder, result, cvt_t, "arm64_fcvtzs"); + if (scalar) + result = vector_from_scalar (ctx, ret_t, result); + values [ins->dreg] = result; + break; + } + case OP_ARM64_SELECT_SCALAR: { + LLVMValueRef result = LLVMBuildExtractElement (builder, lhs, rhs, ""); + LLVMTypeRef elem_t = LLVMTypeOf (result); + unsigned int elem_bits = mono_llvm_get_prim_size_bits (elem_t); + LLVMTypeRef t = LLVMVectorType (elem_t, 64 / elem_bits); + result = vector_from_scalar (ctx, t, result); values [ins->dreg] = result; - break; } case OP_LSCNT32: @@ -9174,7 +9864,7 @@ process_bb (EmitContext *ctx, MonoBasicBlock *bb) LLVMValueRef xor = LLVMBuildXor (builder, shr, lhs, ""); LLVMValueRef mul = LLVMBuildShl (builder, xor, one, ""); LLVMValueRef add = LLVMBuildOr (builder, mul, one, ""); - + LLVMValueRef args [2]; args [0] = add; args [1] = LLVMConstInt (LLVMInt1Type (), 0, FALSE); @@ -9197,6 +9887,1055 @@ process_bb (EmitContext *ctx, MonoBasicBlock *bb) values [ins->dreg] = LLVMBuildTrunc (builder, hi64, LLVMInt64Type (), ""); break; } + case OP_ARM64_XNARROW_SCALAR: { + // Unfortunately, @llvm.aarch64.neon.scalar.sqxtun isn't available for i8 or i16. + LLVMTypeRef ret_t = simd_class_to_llvm_type (ctx, ins->klass); + llvm_ovr_tag_t ovr_tag = ovr_tag_from_llvm_type (ret_t); + LLVMTypeRef elem_t = LLVMGetElementType (ret_t); + LLVMValueRef result = NULL; + int iid = ins->inst_c0; + int scalar_iid = 0; + switch (iid) { + case INTRINS_AARCH64_ADV_SIMD_SQXTUN: scalar_iid = INTRINS_AARCH64_ADV_SIMD_SCALAR_SQXTUN; break; + case INTRINS_AARCH64_ADV_SIMD_SQXTN: scalar_iid = INTRINS_AARCH64_ADV_SIMD_SCALAR_SQXTN; break; + case INTRINS_AARCH64_ADV_SIMD_UQXTN: scalar_iid = INTRINS_AARCH64_ADV_SIMD_SCALAR_UQXTN; break; + default: g_assert_not_reached (); + } + if (elem_t == i4_t) { + LLVMValueRef arg = scalar_from_vector (ctx, lhs); + result = call_intrins (ctx, scalar_iid, &arg, "arm64_xnarrow_scalar"); + result = vector_from_scalar (ctx, ret_t, result); + } else { + LLVMTypeRef arg_t = LLVMTypeOf (lhs); + LLVMTypeRef argelem_t = LLVMGetElementType (arg_t); + unsigned int argelems = LLVMGetVectorSize (arg_t); + LLVMValueRef arg = keep_lowest_element (ctx, LLVMVectorType (argelem_t, argelems * 2), lhs); + result = call_overloaded_intrins (ctx, iid, ovr_tag, &arg, "arm64_xnarrow_scalar"); + result = keep_lowest_element (ctx, LLVMTypeOf (result), result); + } + values [ins->dreg] = result; + break; + } + case OP_ARM64_SQXTUN2: + case OP_ARM64_UQXTN2: + case OP_ARM64_SQXTN2: + case OP_ARM64_XTN: + case OP_ARM64_XTN2: { + llvm_ovr_tag_t ovr_tag = ovr_tag_from_mono_vector_class (ins->klass); + gboolean high = FALSE; + int iid = 0; + switch (ins->opcode) { + case OP_ARM64_SQXTUN2: high = TRUE; iid = INTRINS_AARCH64_ADV_SIMD_SQXTUN; break; + case OP_ARM64_UQXTN2: high = TRUE; iid = INTRINS_AARCH64_ADV_SIMD_UQXTN; break; + case OP_ARM64_SQXTN2: high = TRUE; iid = INTRINS_AARCH64_ADV_SIMD_SQXTN; break; + case OP_ARM64_XTN2: high = TRUE; break; + } + LLVMValueRef result = lhs; + if (high) { + result = rhs; + ovr_tag = ovr_tag_smaller_vector (ovr_tag); + } + LLVMTypeRef t = LLVMTypeOf (result); + LLVMTypeRef elem_t = LLVMGetElementType (t); + unsigned int elems = LLVMGetVectorSize (t); + unsigned int elem_bits = mono_llvm_get_prim_size_bits (elem_t); + LLVMTypeRef result_t = LLVMVectorType (LLVMIntType (elem_bits / 2), elems); + if (iid != 0) + result = call_overloaded_intrins (ctx, iid, ovr_tag, &result, ""); + else + result = LLVMBuildTrunc (builder, result, result_t, "arm64_xtn"); + if (high) + result = concatenate_vectors (ctx, lhs, result); + values [ins->dreg] = result; + break; + } + case OP_ARM64_CLZ: { + llvm_ovr_tag_t ovr_tag = ovr_tag_from_mono_vector_class (ins->klass); + LLVMValueRef args [] = { lhs, const_int1 (0) }; + LLVMValueRef result = call_overloaded_intrins (ctx, INTRINS_AARCH64_ADV_SIMD_CLZ, ovr_tag, args, ""); + values [ins->dreg] = result; + break; + } + case OP_ARM64_FMSUB: + case OP_ARM64_FMSUB_BYSCALAR: + case OP_ARM64_FMSUB_SCALAR: + case OP_ARM64_FNMSUB_SCALAR: + case OP_ARM64_FMADD: + case OP_ARM64_FMADD_BYSCALAR: + case OP_ARM64_FMADD_SCALAR: + case OP_ARM64_FNMADD_SCALAR: { + llvm_ovr_tag_t ovr_tag = ovr_tag_from_mono_vector_class (ins->klass); + gboolean scalar = FALSE; + gboolean negate = FALSE; + gboolean subtract = FALSE; + gboolean byscalar = FALSE; + switch (ins->opcode) { + case OP_ARM64_FMSUB: subtract = TRUE; break; + case OP_ARM64_FMSUB_BYSCALAR: subtract = TRUE; byscalar = TRUE; break; + case OP_ARM64_FMSUB_SCALAR: subtract = TRUE; scalar = TRUE; break; + case OP_ARM64_FNMSUB_SCALAR: subtract = TRUE; scalar = TRUE; negate = TRUE; break; + case OP_ARM64_FMADD: break; + case OP_ARM64_FMADD_BYSCALAR: byscalar = TRUE; break; + case OP_ARM64_FMADD_SCALAR: scalar = TRUE; break; + case OP_ARM64_FNMADD_SCALAR: scalar = TRUE; negate = TRUE; break; + } + // llvm.fma argument order: mulop1, mulop2, addend + LLVMValueRef args [] = { rhs, arg3, lhs }; + if (byscalar) { + unsigned int elems = LLVMGetVectorSize (LLVMTypeOf (args [0])); + args [1] = broadcast_element (ctx, scalar_from_vector (ctx, args [1]), elems); + } + if (scalar) { + ovr_tag = ovr_tag_force_scalar (ovr_tag); + for (int i = 0; i < 3; ++i) + args [i] = scalar_from_vector (ctx, args [i]); + } + if (subtract) + args [0] = LLVMBuildFNeg (builder, args [0], "arm64_fma_sub"); + if (negate) { + args [0] = LLVMBuildFNeg (builder, args [0], "arm64_fma_negate"); + args [2] = LLVMBuildFNeg (builder, args [2], "arm64_fma_negate"); + } + LLVMValueRef result = call_overloaded_intrins (ctx, INTRINS_AARCH64_ADV_SIMD_FMA, ovr_tag, args, "arm64_fma"); + if (scalar) + result = vector_from_scalar (ctx, LLVMTypeOf (lhs), result); + values [ins->dreg] = result; + break; + } + case OP_ARM64_SQDMULL: + case OP_ARM64_SQDMULL_BYSCALAR: + case OP_ARM64_SQDMULL2: + case OP_ARM64_SQDMULL2_BYSCALAR: + case OP_ARM64_SQDMLAL: + case OP_ARM64_SQDMLAL_BYSCALAR: + case OP_ARM64_SQDMLAL2: + case OP_ARM64_SQDMLAL2_BYSCALAR: + case OP_ARM64_SQDMLSL: + case OP_ARM64_SQDMLSL_BYSCALAR: + case OP_ARM64_SQDMLSL2: + case OP_ARM64_SQDMLSL2_BYSCALAR: { + llvm_ovr_tag_t ovr_tag = ovr_tag_from_mono_vector_class (ins->klass); + gboolean scalar = FALSE; + gboolean add = FALSE; + gboolean subtract = FALSE; + gboolean high = FALSE; + switch (ins->opcode) { + case OP_ARM64_SQDMULL_BYSCALAR: scalar = TRUE; case OP_ARM64_SQDMULL: break; + case OP_ARM64_SQDMULL2_BYSCALAR: scalar = TRUE; case OP_ARM64_SQDMULL2: high = TRUE; break; + case OP_ARM64_SQDMLAL_BYSCALAR: scalar = TRUE; case OP_ARM64_SQDMLAL: add = TRUE; break; + case OP_ARM64_SQDMLAL2_BYSCALAR: scalar = TRUE; case OP_ARM64_SQDMLAL2: high = TRUE; add = TRUE; break; + case OP_ARM64_SQDMLSL_BYSCALAR: scalar = TRUE; case OP_ARM64_SQDMLSL: subtract = TRUE; break; + case OP_ARM64_SQDMLSL2_BYSCALAR: scalar = TRUE; case OP_ARM64_SQDMLSL2: high = TRUE; subtract = TRUE; break; + } + int iid = 0; + if (add) + iid = INTRINS_AARCH64_ADV_SIMD_SQADD; + else if (subtract) + iid = INTRINS_AARCH64_ADV_SIMD_SQSUB; + LLVMValueRef mul1 = lhs; + LLVMValueRef mul2 = rhs; + if (iid != 0) { + mul1 = rhs; + mul2 = arg3; + } + if (scalar) { + LLVMTypeRef t = LLVMTypeOf (mul1); + unsigned int elems = LLVMGetVectorSize (t); + mul2 = broadcast_element (ctx, scalar_from_vector (ctx, mul2), elems); + } + LLVMValueRef args [] = { mul1, mul2 }; + if (high) + for (int i = 0; i < 2; ++i) + args [i] = extract_high_elements (ctx, args [i]); + LLVMValueRef result = call_overloaded_intrins (ctx, INTRINS_AARCH64_ADV_SIMD_SQDMULL, ovr_tag, args, ""); + LLVMValueRef args2 [] = { lhs, result }; + if (iid != 0) + result = call_overloaded_intrins (ctx, iid, ovr_tag, args2, ""); + values [ins->dreg] = result; + break; + } + case OP_ARM64_SQDMULL_SCALAR: + case OP_ARM64_SQDMLAL_SCALAR: + case OP_ARM64_SQDMLSL_SCALAR: { + /* + * define dso_local i32 @__vqdmlslh_lane_s16(i32, i16, <4 x i16>, i32) local_unnamed_addr #0 { + * %5 = insertelement <4 x i16> undef, i16 %1, i64 0 + * %6 = shufflevector <4 x i16> %2, <4 x i16> undef, <4 x i32> + * %7 = tail call <4 x i32> @llvm.aarch64.neon.sqdmull.v4i32(<4 x i16> %5, <4 x i16> %6) + * %8 = extractelement <4 x i32> %7, i64 0 + * %9 = tail call i32 @llvm.aarch64.neon.sqsub.i32(i32 %0, i32 %8) + * ret i32 %9 + * } + * + * define dso_local i64 @__vqdmlals_s32(i64, i32, i32) local_unnamed_addr #0 { + * %4 = tail call i64 @llvm.aarch64.neon.sqdmulls.scalar(i32 %1, i32 %2) #2 + * %5 = tail call i64 @llvm.aarch64.neon.sqadd.i64(i64 %0, i64 %4) #2 + * ret i64 %5 + * } + */ + int mulid = INTRINS_AARCH64_ADV_SIMD_SQDMULL; + int iid = 0; + gboolean scalar_mul_result = FALSE; + gboolean scalar_acc_result = FALSE; + switch (ins->opcode) { + case OP_ARM64_SQDMLAL_SCALAR: iid = INTRINS_AARCH64_ADV_SIMD_SQADD; break; + case OP_ARM64_SQDMLSL_SCALAR: iid = INTRINS_AARCH64_ADV_SIMD_SQSUB; break; + } + LLVMTypeRef ret_t = simd_class_to_llvm_type (ctx, ins->klass); + LLVMValueRef mularg = lhs; + LLVMValueRef selected_scalar = rhs; + if (iid != 0) { + mularg = rhs; + selected_scalar = arg3; + } + llvm_ovr_tag_t multag = ovr_tag_smaller_elements (ovr_tag_from_llvm_type (ret_t)); + llvm_ovr_tag_t iidtag = ovr_tag_force_scalar (ovr_tag_from_llvm_type (ret_t)); + LLVMTypeRef mularg_t = ovr_tag_to_llvm_type (multag); + if (multag & INTRIN_int32) { + /* The (i32, i32) -> i64 variant of aarch64_neon_sqdmull has + * a unique, non-overloaded name. + */ + mulid = INTRINS_AARCH64_ADV_SIMD_SQDMULL_SCALAR; + multag = 0; + iidtag = INTRIN_int64 | INTRIN_scalar; + scalar_mul_result = TRUE; + scalar_acc_result = TRUE; + } else if (multag & INTRIN_int16) { + /* We were passed a (<4 x i16>, <4 x i16>) but the + * widening multiplication intrinsic will yield a <4 x i32>. + */ + multag = INTRIN_int32 | INTRIN_vector128; + } else + g_assert_not_reached (); + + if (scalar_mul_result) { + mularg = scalar_from_vector (ctx, mularg); + selected_scalar = scalar_from_vector (ctx, selected_scalar); + } else { + mularg = keep_lowest_element (ctx, mularg_t, mularg); + selected_scalar = keep_lowest_element (ctx, mularg_t, selected_scalar); + } + LLVMValueRef mulargs [] = { mularg, selected_scalar }; + LLVMValueRef result = call_overloaded_intrins (ctx, mulid, multag, mulargs, "arm64_sqdmull_scalar"); + + if (iid != 0) { + LLVMValueRef acc = scalar_from_vector (ctx, lhs); + if (!scalar_mul_result) + result = scalar_from_vector (ctx, result); + LLVMValueRef subargs [] = { acc, result }; + result = call_overloaded_intrins (ctx, iid, iidtag, subargs, "arm64_sqdmlxl_scalar"); + scalar_acc_result = TRUE; + } + if (scalar_acc_result) + result = vector_from_scalar (ctx, ret_t, result); + else + result = keep_lowest_element (ctx, ret_t, result); + values [ins->dreg] = result; + break; + } + case OP_ARM64_FMUL_SEL: { + LLVMValueRef mul2 = LLVMBuildExtractElement (builder, rhs, arg3, ""); + LLVMValueRef mul1 = scalar_from_vector (ctx, lhs); + LLVMValueRef result = LLVMBuildFMul (builder, mul1, mul2, "arm64_fmul_sel"); + result = vector_from_scalar (ctx, LLVMTypeOf (lhs), result); + values [ins->dreg] = result; + break; + } + case OP_ARM64_MLA: + case OP_ARM64_MLA_SCALAR: + case OP_ARM64_MLS: + case OP_ARM64_MLS_SCALAR: { + gboolean scalar = FALSE; + gboolean add = FALSE; + switch (ins->opcode) { + case OP_ARM64_MLA_SCALAR: scalar = TRUE; case OP_ARM64_MLA: add = TRUE; break; + case OP_ARM64_MLS_SCALAR: scalar = TRUE; case OP_ARM64_MLS: break; + } + LLVMTypeRef mul_t = LLVMTypeOf (rhs); + unsigned int elems = LLVMGetVectorSize (mul_t); + LLVMValueRef mul2 = arg3; + if (scalar) + mul2 = broadcast_element (ctx, scalar_from_vector (ctx, mul2), elems); + LLVMValueRef result = LLVMBuildMul (builder, rhs, mul2, ""); + if (add) + result = LLVMBuildAdd (builder, lhs, result, ""); + else + result = LLVMBuildSub (builder, lhs, result, ""); + values [ins->dreg] = result; + break; + } + case OP_ARM64_SMULL: + case OP_ARM64_SMULL_SCALAR: + case OP_ARM64_SMULL2: + case OP_ARM64_SMULL2_SCALAR: + case OP_ARM64_UMULL: + case OP_ARM64_UMULL_SCALAR: + case OP_ARM64_UMULL2: + case OP_ARM64_UMULL2_SCALAR: + case OP_ARM64_SMLAL: + case OP_ARM64_SMLAL_SCALAR: + case OP_ARM64_SMLAL2: + case OP_ARM64_SMLAL2_SCALAR: + case OP_ARM64_UMLAL: + case OP_ARM64_UMLAL_SCALAR: + case OP_ARM64_UMLAL2: + case OP_ARM64_UMLAL2_SCALAR: + case OP_ARM64_SMLSL: + case OP_ARM64_SMLSL_SCALAR: + case OP_ARM64_SMLSL2: + case OP_ARM64_SMLSL2_SCALAR: + case OP_ARM64_UMLSL: + case OP_ARM64_UMLSL_SCALAR: + case OP_ARM64_UMLSL2: + case OP_ARM64_UMLSL2_SCALAR: { + llvm_ovr_tag_t ovr_tag = ovr_tag_from_mono_vector_class (ins->klass); + gboolean is_unsigned = FALSE; + gboolean high = FALSE; + gboolean add = FALSE; + gboolean subtract = FALSE; + gboolean scalar = FALSE; + int opcode = ins->opcode; + switch (opcode) { + case OP_ARM64_SMULL_SCALAR: scalar = TRUE; opcode = OP_ARM64_SMULL; break; + case OP_ARM64_UMULL_SCALAR: scalar = TRUE; opcode = OP_ARM64_UMULL; break; + case OP_ARM64_SMLAL_SCALAR: scalar = TRUE; opcode = OP_ARM64_SMLAL; break; + case OP_ARM64_UMLAL_SCALAR: scalar = TRUE; opcode = OP_ARM64_UMLAL; break; + case OP_ARM64_SMLSL_SCALAR: scalar = TRUE; opcode = OP_ARM64_SMLSL; break; + case OP_ARM64_UMLSL_SCALAR: scalar = TRUE; opcode = OP_ARM64_UMLSL; break; + case OP_ARM64_SMULL2_SCALAR: scalar = TRUE; opcode = OP_ARM64_SMULL2; break; + case OP_ARM64_UMULL2_SCALAR: scalar = TRUE; opcode = OP_ARM64_UMULL2; break; + case OP_ARM64_SMLAL2_SCALAR: scalar = TRUE; opcode = OP_ARM64_SMLAL2; break; + case OP_ARM64_UMLAL2_SCALAR: scalar = TRUE; opcode = OP_ARM64_UMLAL2; break; + case OP_ARM64_SMLSL2_SCALAR: scalar = TRUE; opcode = OP_ARM64_SMLSL2; break; + case OP_ARM64_UMLSL2_SCALAR: scalar = TRUE; opcode = OP_ARM64_UMLSL2; break; + } + switch (opcode) { + case OP_ARM64_SMULL2: high = TRUE; case OP_ARM64_SMULL: break; + case OP_ARM64_UMULL2: high = TRUE; case OP_ARM64_UMULL: is_unsigned = TRUE; break; + case OP_ARM64_SMLAL2: high = TRUE; case OP_ARM64_SMLAL: add = TRUE; break; + case OP_ARM64_UMLAL2: high = TRUE; case OP_ARM64_UMLAL: add = TRUE; is_unsigned = TRUE; break; + case OP_ARM64_SMLSL2: high = TRUE; case OP_ARM64_SMLSL: subtract = TRUE; break; + case OP_ARM64_UMLSL2: high = TRUE; case OP_ARM64_UMLSL: subtract = TRUE; is_unsigned = TRUE; break; + } + int iid = is_unsigned ? INTRINS_AARCH64_ADV_SIMD_UMULL : INTRINS_AARCH64_ADV_SIMD_SMULL; + LLVMValueRef intrin_args [] = { lhs, rhs }; + if (add || subtract) { + intrin_args [0] = rhs; + intrin_args [1] = arg3; + } + if (scalar) { + LLVMValueRef sarg = intrin_args [1]; + LLVMTypeRef t = LLVMTypeOf (intrin_args [0]); + unsigned int elems = LLVMGetVectorSize (t); + sarg = broadcast_element (ctx, scalar_from_vector (ctx, sarg), elems); + intrin_args [1] = sarg; + } + if (high) + for (int i = 0; i < 2; ++i) + intrin_args [i] = extract_high_elements (ctx, intrin_args [i]); + LLVMValueRef result = call_overloaded_intrins (ctx, iid, ovr_tag, intrin_args, ""); + if (add) + result = LLVMBuildAdd (builder, lhs, result, ""); + if (subtract) + result = LLVMBuildSub (builder, lhs, result, ""); + values [ins->dreg] = result; + break; + } + case OP_ARM64_XNEG: + case OP_ARM64_XNEG_SCALAR: { + gboolean scalar = ins->opcode == OP_ARM64_XNEG_SCALAR; + gboolean is_float = FALSE; + switch (inst_c1_type (ins)) { + case MONO_TYPE_R4: case MONO_TYPE_R8: is_float = TRUE; + } + LLVMValueRef result = lhs; + if (scalar) + result = scalar_from_vector (ctx, result); + if (is_float) + result = LLVMBuildFNeg (builder, result, "arm64_xneg"); + else + result = LLVMBuildNeg (builder, result, "arm64_xneg"); + if (scalar) + result = vector_from_scalar (ctx, LLVMTypeOf (lhs), result); + values [ins->dreg] = result; + break; + } + case OP_ARM64_PMULL: + case OP_ARM64_PMULL2: { + gboolean high = ins->opcode == OP_ARM64_PMULL2; + LLVMValueRef args [] = { lhs, rhs }; + if (high) + for (int i = 0; i < 2; ++i) + args [i] = extract_high_elements (ctx, args [i]); + LLVMValueRef result = call_intrins (ctx, INTRINS_AARCH64_ADV_SIMD_PMULL, args, "arm64_pmull"); + values [ins->dreg] = result; + break; + } + case OP_ARM64_REVN: { + LLVMTypeRef t = LLVMTypeOf (lhs); + LLVMTypeRef elem_t = LLVMGetElementType (t); + unsigned int group_bits = mono_llvm_get_prim_size_bits (elem_t); + unsigned int vec_bits = mono_llvm_get_prim_size_bits (t); + unsigned int tmp_bits = ins->inst_c0; + unsigned int tmp_elements = vec_bits / tmp_bits; + const int cycle8 [] = { 7, 6, 5, 4, 3, 2, 1, 0, 15, 14, 13, 12, 11, 10, 9, 8 }; + const int cycle4 [] = { 3, 2, 1, 0, 7, 6, 5, 4, 11, 10, 9, 8, 15, 14, 13, 12 }; + const int cycle2 [] = { 1, 0, 3, 2, 5, 4, 7, 6, 9, 8, 11, 10, 13, 12, 15, 14 }; + const int *cycle = NULL; + switch (group_bits / tmp_bits) { + case 2: cycle = cycle2; break; + case 4: cycle = cycle4; break; + case 8: cycle = cycle8; break; + default: g_assert_not_reached (); + } + g_assert (tmp_elements <= ARM64_MAX_VECTOR_ELEMS); + LLVMTypeRef tmp_t = LLVMVectorType (LLVMIntType (tmp_bits), tmp_elements); + LLVMValueRef tmp = LLVMBuildBitCast (builder, lhs, tmp_t, "arm64_revn"); + LLVMValueRef result = LLVMBuildShuffleVector (builder, tmp, LLVMGetUndef (tmp_t), create_const_vector_i32 (cycle, tmp_elements), ""); + result = LLVMBuildBitCast (builder, result, t, ""); + values [ins->dreg] = result; + break; + } + case OP_ARM64_SHL: + case OP_ARM64_SSHR: + case OP_ARM64_SSRA: + case OP_ARM64_USHR: + case OP_ARM64_USRA: { + gboolean right = FALSE; + gboolean add = FALSE; + gboolean arith = FALSE; + switch (ins->opcode) { + case OP_ARM64_USHR: right = TRUE; break; + case OP_ARM64_USRA: right = TRUE; add = TRUE; break; + case OP_ARM64_SSHR: arith = TRUE; break; + case OP_ARM64_SSRA: arith = TRUE; add = TRUE; break; + } + LLVMValueRef shiftarg = lhs; + LLVMValueRef shift = rhs; + if (add) { + shiftarg = rhs; + shift = arg3; + } + shift = create_shift_vector (ctx, shiftarg, shift); + LLVMValueRef result = NULL; + if (right) + result = LLVMBuildLShr (builder, shiftarg, shift, ""); + else if (arith) + result = LLVMBuildAShr (builder, shiftarg, shift, ""); + else + result = LLVMBuildShl (builder, shiftarg, shift, ""); + if (add) + result = LLVMBuildAdd (builder, lhs, result, "arm64_usra"); + values [ins->dreg] = result; + break; + } + case OP_ARM64_SHRN: + case OP_ARM64_SHRN2: { + LLVMValueRef shiftarg = lhs; + LLVMValueRef shift = rhs; + gboolean high = ins->opcode == OP_ARM64_SHRN2; + if (high) { + shiftarg = rhs; + shift = arg3; + } + LLVMTypeRef arg_t = LLVMTypeOf (shiftarg); + LLVMTypeRef elem_t = LLVMGetElementType (arg_t); + unsigned int elems = LLVMGetVectorSize (arg_t); + unsigned int bits = mono_llvm_get_prim_size_bits (elem_t); + LLVMTypeRef trunc_t = LLVMVectorType (LLVMIntType (bits / 2), elems); + shift = create_shift_vector (ctx, shiftarg, shift); + LLVMValueRef result = LLVMBuildLShr (builder, shiftarg, shift, "shrn"); + result = LLVMBuildTrunc (builder, result, trunc_t, ""); + if (high) { + result = concatenate_vectors (ctx, lhs, result); + } + values [ins->dreg] = result; + break; + } + case OP_ARM64_SRSHR: + case OP_ARM64_SRSRA: + case OP_ARM64_URSHR: + case OP_ARM64_URSRA: { + llvm_ovr_tag_t ovr_tag = ovr_tag_from_mono_vector_class (ins->klass); + LLVMValueRef shiftarg = lhs; + LLVMValueRef shift = rhs; + gboolean right = FALSE; + gboolean add = FALSE; + switch (ins->opcode) { + case OP_ARM64_URSRA: add = TRUE; case OP_ARM64_URSHR: right = TRUE; break; + case OP_ARM64_SRSRA: add = TRUE; case OP_ARM64_SRSHR: right = TRUE; break; + } + int iid = 0; + switch (ins->opcode) { + case OP_ARM64_URSRA: case OP_ARM64_URSHR: iid = INTRINS_AARCH64_ADV_SIMD_URSHL; break; + case OP_ARM64_SRSRA: case OP_ARM64_SRSHR: iid = INTRINS_AARCH64_ADV_SIMD_SRSHL; break; + } + if (add) { + shiftarg = rhs; + shift = arg3; + } + if (right) + shift = LLVMBuildNeg (builder, shift, ""); + shift = create_shift_vector (ctx, shiftarg, shift); + LLVMValueRef args [] = { shiftarg, shift }; + LLVMValueRef result = call_overloaded_intrins (ctx, iid, ovr_tag, args, ""); + if (add) + result = LLVMBuildAdd (builder, result, lhs, ""); + values [ins->dreg] = result; + break; + } + case OP_ARM64_XNSHIFT_SCALAR: + case OP_ARM64_XNSHIFT: + case OP_ARM64_XNSHIFT2: { + LLVMTypeRef intrin_result_t = simd_class_to_llvm_type (ctx, ins->klass); + llvm_ovr_tag_t ovr_tag = ovr_tag_from_llvm_type (intrin_result_t); + LLVMValueRef shift_arg = lhs; + LLVMValueRef shift_amount = rhs; + gboolean high = FALSE; + gboolean scalar = FALSE; + int iid = ins->inst_c0; + switch (ins->opcode) { + case OP_ARM64_XNSHIFT_SCALAR: scalar = TRUE; break; + case OP_ARM64_XNSHIFT2: high = TRUE; break; + } + if (high) { + shift_arg = rhs; + shift_amount = arg3; + ovr_tag = ovr_tag_smaller_vector (ovr_tag); + intrin_result_t = ovr_tag_to_llvm_type (ovr_tag); + } + LLVMTypeRef shift_arg_t = LLVMTypeOf (shift_arg); + LLVMTypeRef shift_arg_elem_t = LLVMGetElementType (shift_arg_t); + unsigned int element_bits = mono_llvm_get_prim_size_bits (shift_arg_elem_t); + int range_min = 1; + int range_max = element_bits / 2; + if (scalar) { + unsigned int elems = LLVMGetVectorSize (shift_arg_t); + LLVMValueRef lo = scalar_from_vector (ctx, shift_arg); + shift_arg = vector_from_scalar (ctx, LLVMVectorType (shift_arg_elem_t, elems * 2), lo); + } + int max_index = range_max - range_min + 1; + ImmediateUnrollCtx ictx = immediate_unroll_begin (ctx, bb, max_index, shift_amount, intrin_result_t, "arm64_xnshift"); + int i = 0; + while (immediate_unroll_next (&ictx, &i)) { + int shift_const = i + range_min; + LLVMValueRef intrin_args [] = { shift_arg, const_int32 (shift_const) }; + LLVMValueRef result = call_overloaded_intrins (ctx, iid, ovr_tag, intrin_args, ""); + immediate_unroll_commit (&ictx, shift_const, result); + } + { + immediate_unroll_default (&ictx); + LLVMValueRef intrin_args [] = { shift_arg, const_int32 (range_max) }; + LLVMValueRef result = call_overloaded_intrins (ctx, iid, ovr_tag, intrin_args, ""); + immediate_unroll_commit_default (&ictx, result); + } + LLVMValueRef result = immediate_unroll_end (&ictx, &cbb); + if (high) + result = concatenate_vectors (ctx, lhs, result); + if (scalar) + result = keep_lowest_element (ctx, LLVMTypeOf (result), result); + values [ins->dreg] = result; + break; + } + case OP_ARM64_SQSHLU: + case OP_ARM64_SQSHLU_SCALAR: { + gboolean scalar = ins->opcode == OP_ARM64_SQSHLU_SCALAR; + LLVMTypeRef intrin_result_t = simd_class_to_llvm_type (ctx, ins->klass); + LLVMTypeRef elem_t = LLVMGetElementType (intrin_result_t); + unsigned int element_bits = mono_llvm_get_prim_size_bits (elem_t); + llvm_ovr_tag_t ovr_tag = ovr_tag_from_llvm_type (intrin_result_t); + int max_index = element_bits; + ScalarOpFromVectorOpCtx sctx = scalar_op_from_vector_op (ctx, intrin_result_t, ins); + intrin_result_t = scalar ? sctx.intermediate_type : intrin_result_t; + ovr_tag = scalar ? sctx.ovr_tag : ovr_tag; + ImmediateUnrollCtx ictx = immediate_unroll_begin (ctx, bb, max_index, rhs, intrin_result_t, "arm64_sqshlu"); + int i = 0; + while (immediate_unroll_next (&ictx, &i)) { + int shift_const = i; + LLVMValueRef args [2] = { lhs, create_shift_vector (ctx, lhs, const_int32 (shift_const)) }; + if (scalar) + scalar_op_from_vector_op_process_args (&sctx, args, 2); + LLVMValueRef result = call_overloaded_intrins (ctx, INTRINS_AARCH64_ADV_SIMD_SQSHLU, ovr_tag, args, ""); + immediate_unroll_commit (&ictx, shift_const, result); + } + { + immediate_unroll_default (&ictx); + LLVMValueRef srcarg = lhs; + if (scalar) + scalar_op_from_vector_op_process_args (&sctx, &srcarg, 1); + immediate_unroll_commit_default (&ictx, srcarg); + } + LLVMValueRef result = immediate_unroll_end (&ictx, &cbb); + if (scalar) + result = scalar_op_from_vector_op_process_result (&sctx, result); + values [ins->dreg] = result; + break; + } + case OP_ARM64_SSHLL: + case OP_ARM64_SSHLL2: + case OP_ARM64_USHLL: + case OP_ARM64_USHLL2: { + LLVMTypeRef ret_t = simd_class_to_llvm_type (ctx, ins->klass); + gboolean high = FALSE; + gboolean is_unsigned = FALSE; + switch (ins->opcode) { + case OP_ARM64_SSHLL2: high = TRUE; break; + case OP_ARM64_USHLL2: high = TRUE; case OP_ARM64_USHLL: is_unsigned = TRUE; break; + } + LLVMValueRef result = lhs; + if (high) + result = extract_high_elements (ctx, result); + if (is_unsigned) + result = LLVMBuildZExt (builder, result, ret_t, "arm64_ushll"); + else + result = LLVMBuildSExt (builder, result, ret_t, "arm64_ushll"); + result = LLVMBuildShl (builder, result, create_shift_vector (ctx, result, rhs), ""); + values [ins->dreg] = result; + break; + } + case OP_ARM64_SLI: + case OP_ARM64_SRI: { + LLVMTypeRef intrin_result_t = simd_class_to_llvm_type (ctx, ins->klass); + llvm_ovr_tag_t ovr_tag = ovr_tag_from_llvm_type (intrin_result_t); + unsigned int element_bits = mono_llvm_get_prim_size_bits (LLVMGetElementType (intrin_result_t)); + int range_min = 0; + int range_max = element_bits - 1; + if (ins->opcode == OP_ARM64_SRI) { + ++range_min; + ++range_max; + } + int iid = ins->opcode == OP_ARM64_SRI ? INTRINS_AARCH64_ADV_SIMD_SRI : INTRINS_AARCH64_ADV_SIMD_SLI; + int max_index = range_max - range_min + 1; + ImmediateUnrollCtx ictx = immediate_unroll_begin (ctx, bb, max_index, arg3, intrin_result_t, "arm64_ext"); + LLVMValueRef intrin_args [3] = { lhs, rhs, arg3 }; + int i = 0; + while (immediate_unroll_next (&ictx, &i)) { + int shift_const = i + range_min; + intrin_args [2] = const_int32 (shift_const); + LLVMValueRef result = call_overloaded_intrins (ctx, iid, ovr_tag, intrin_args, ""); + immediate_unroll_commit (&ictx, shift_const, result); + } + immediate_unroll_default (&ictx); + immediate_unroll_commit_default (&ictx, lhs); + LLVMValueRef result = immediate_unroll_end (&ictx, &cbb); + values [ins->dreg] = result; + break; + } + case OP_ARM64_SQRT_SCALAR: { + int iid = ins->inst_c0 == MONO_TYPE_R8 ? INTRINS_SQRT : INTRINS_SQRTF; + LLVMTypeRef t = LLVMTypeOf (lhs); + LLVMValueRef scalar = LLVMBuildExtractElement (builder, lhs, const_int32 (0), ""); + LLVMValueRef result = call_intrins (ctx, iid, &scalar, "arm64_sqrt_scalar"); + values [ins->dreg] = LLVMBuildInsertElement (builder, LLVMGetUndef (t), result, const_int32 (0), ""); + break; + } + case OP_ARM64_STP: + case OP_ARM64_STP_SCALAR: + case OP_ARM64_STNP: + case OP_ARM64_STNP_SCALAR: { + gboolean nontemporal = FALSE; + gboolean scalar = FALSE; + switch (ins->opcode) { + case OP_ARM64_STNP: nontemporal = TRUE; break; + case OP_ARM64_STNP_SCALAR: nontemporal = TRUE; scalar = TRUE; break; + case OP_ARM64_STP_SCALAR: scalar = TRUE; break; + } + LLVMTypeRef rhs_t = LLVMTypeOf (rhs); + LLVMValueRef val = NULL; + LLVMTypeRef dst_t = LLVMPointerType (rhs_t, 0); + if (scalar) + val = LLVMBuildShuffleVector (builder, rhs, arg3, create_const_vector_2_i32 (0, 2), ""); + else { + unsigned int rhs_elems = LLVMGetVectorSize (rhs_t); + LLVMTypeRef rhs_elt_t = LLVMGetElementType (rhs_t); + dst_t = LLVMPointerType (LLVMVectorType (rhs_elt_t, rhs_elems * 2), 0); + val = concatenate_vectors (ctx, rhs, arg3); + } + LLVMValueRef address = convert (ctx, lhs, dst_t); + LLVMValueRef store = mono_llvm_build_store (builder, val, address, FALSE, LLVM_BARRIER_NONE); + if (nontemporal) + set_nontemporal_flag (store); + break; + } + case OP_ARM64_LD1_INSERT: { + LLVMTypeRef ret_t = simd_class_to_llvm_type (ctx, ins->klass); + LLVMTypeRef elem_t = LLVMGetElementType (ret_t); + LLVMValueRef address = convert (ctx, arg3, LLVMPointerType (elem_t, 0)); + unsigned int alignment = mono_llvm_get_prim_size_bits (ret_t) / 8; + LLVMValueRef result = mono_llvm_build_aligned_load (builder, address, "arm64_ld1_insert", FALSE, alignment); + result = LLVMBuildInsertElement (builder, lhs, result, rhs, "arm64_ld1_insert"); + values [ins->dreg] = result; + break; + } + case OP_ARM64_LD1R: + case OP_ARM64_LD1: { + gboolean replicate = ins->opcode == OP_ARM64_LD1R; + LLVMTypeRef ret_t = simd_class_to_llvm_type (ctx, ins->klass); + unsigned int alignment = mono_llvm_get_prim_size_bits (ret_t) / 8; + LLVMValueRef address = lhs; + LLVMTypeRef address_t = LLVMPointerType (ret_t, 0); + if (replicate) { + LLVMTypeRef elem_t = LLVMGetElementType (ret_t); + address_t = LLVMPointerType (elem_t, 0); + } + address = convert (ctx, address, address_t); + LLVMValueRef result = mono_llvm_build_aligned_load (builder, address, "arm64_ld1", FALSE, alignment); + if (replicate) { + unsigned int elems = LLVMGetVectorSize (ret_t); + result = broadcast_element (ctx, result, elems); + } + values [ins->dreg] = result; + break; + } + case OP_ARM64_ST1: { + LLVMTypeRef t = LLVMTypeOf (rhs); + LLVMValueRef address = convert (ctx, lhs, LLVMPointerType (t, 0)); + unsigned int alignment = mono_llvm_get_prim_size_bits (t) / 8; + mono_llvm_build_aligned_store (builder, rhs, address, FALSE, alignment); + break; + } + case OP_ARM64_ST1_SCALAR: { + LLVMTypeRef t = LLVMGetElementType (LLVMTypeOf (rhs)); + LLVMValueRef val = LLVMBuildExtractElement (builder, rhs, arg3, "arm64_st1_scalar"); + LLVMValueRef address = convert (ctx, lhs, LLVMPointerType (t, 0)); + unsigned int alignment = mono_llvm_get_prim_size_bits (t) / 8; + mono_llvm_build_aligned_store (builder, val, address, FALSE, alignment); + break; + } + case OP_ARM64_ADDHN: + case OP_ARM64_ADDHN2: + case OP_ARM64_SUBHN: + case OP_ARM64_SUBHN2: + case OP_ARM64_RADDHN: + case OP_ARM64_RADDHN2: + case OP_ARM64_RSUBHN: + case OP_ARM64_RSUBHN2: { + LLVMValueRef args [2] = { lhs, rhs }; + gboolean high = FALSE; + gboolean subtract = FALSE; + int iid = 0; + switch (ins->opcode) { + case OP_ARM64_ADDHN2: high = TRUE; case OP_ARM64_ADDHN: break; + case OP_ARM64_SUBHN2: high = TRUE; case OP_ARM64_SUBHN: subtract = TRUE; break; + case OP_ARM64_RSUBHN2: high = TRUE; case OP_ARM64_RSUBHN: iid = INTRINS_AARCH64_ADV_SIMD_RSUBHN; break; + case OP_ARM64_RADDHN2: high = TRUE; case OP_ARM64_RADDHN: iid = INTRINS_AARCH64_ADV_SIMD_RADDHN; break; + } + llvm_ovr_tag_t ovr_tag = ovr_tag_from_mono_vector_class (ins->klass); + if (high) { + args [0] = rhs; + args [1] = arg3; + ovr_tag = ovr_tag_smaller_vector (ovr_tag); + } + LLVMValueRef result = NULL; + if (iid != 0) + result = call_overloaded_intrins (ctx, iid, ovr_tag, args, ""); + else { + LLVMTypeRef t = LLVMTypeOf (args [0]); + LLVMTypeRef elt_t = LLVMGetElementType (t); + unsigned int elems = LLVMGetVectorSize (t); + unsigned int elem_bits = mono_llvm_get_prim_size_bits (elt_t); + if (subtract) + result = LLVMBuildSub (builder, args [0], args [1], ""); + else + result = LLVMBuildAdd (builder, args [0], args [1], ""); + result = LLVMBuildLShr (builder, result, broadcast_constant (elem_bits / 2, elt_t, elems), ""); + result = LLVMBuildTrunc (builder, result, LLVMVectorType (LLVMIntType (elem_bits / 2), elems), ""); + } + if (high) + result = concatenate_vectors (ctx, lhs, result); + values [ins->dreg] = result; + break; + } + case OP_ARM64_SADD: + case OP_ARM64_UADD: + case OP_ARM64_SADD2: + case OP_ARM64_UADD2: + case OP_ARM64_SSUB: + case OP_ARM64_USUB: + case OP_ARM64_SSUB2: + case OP_ARM64_USUB2: { + LLVMTypeRef ret_t = simd_class_to_llvm_type (ctx, ins->klass); + gboolean is_unsigned = FALSE; + gboolean high = FALSE; + gboolean subtract = FALSE; + switch (ins->opcode) { + case OP_ARM64_SADD2: high = TRUE; case OP_ARM64_SADD: break; + case OP_ARM64_UADD2: high = TRUE; case OP_ARM64_UADD: is_unsigned = TRUE; break; + case OP_ARM64_SSUB2: high = TRUE; case OP_ARM64_SSUB: subtract = TRUE; break; + case OP_ARM64_USUB2: high = TRUE; case OP_ARM64_USUB: subtract = TRUE; is_unsigned = TRUE; break; + } + LLVMValueRef args [] = { lhs, rhs }; + for (int i = 0; i < 2; ++i) { + LLVMValueRef arg = args [i]; + LLVMTypeRef arg_t = LLVMTypeOf (arg); + if (high && arg_t != ret_t) + arg = extract_high_elements (ctx, arg); + if (is_unsigned) + arg = LLVMBuildZExt (builder, arg, ret_t, ""); + else + arg = LLVMBuildSExt (builder, arg, ret_t, ""); + args [i] = arg; + } + LLVMValueRef result = NULL; + if (subtract) + result = LLVMBuildSub (builder, args [0], args [1], "arm64_sub"); + else + result = LLVMBuildAdd (builder, args [0], args [1], "arm64_add"); + values [ins->dreg] = result; + break; + } + case OP_ARM64_SABAL: + case OP_ARM64_SABAL2: + case OP_ARM64_UABAL: + case OP_ARM64_UABAL2: + case OP_ARM64_SABDL: + case OP_ARM64_SABDL2: + case OP_ARM64_UABDL: + case OP_ARM64_UABDL2: + case OP_ARM64_SABA: + case OP_ARM64_UABA: + case OP_ARM64_SABD: + case OP_ARM64_UABD: { + LLVMTypeRef ret_t = simd_class_to_llvm_type (ctx, ins->klass); + gboolean is_unsigned = FALSE; + gboolean high = FALSE; + gboolean add = FALSE; + gboolean widen = FALSE; + switch (ins->opcode) { + case OP_ARM64_SABAL2: high = TRUE; case OP_ARM64_SABAL: widen = TRUE; add = TRUE; break; + case OP_ARM64_UABAL2: high = TRUE; case OP_ARM64_UABAL: widen = TRUE; add = TRUE; is_unsigned = TRUE; break; + case OP_ARM64_SABDL2: high = TRUE; case OP_ARM64_SABDL: widen = TRUE; break; + case OP_ARM64_UABDL2: high = TRUE; case OP_ARM64_UABDL: widen = TRUE; is_unsigned = TRUE; break; + case OP_ARM64_SABA: add = TRUE; break; + case OP_ARM64_UABA: add = TRUE; is_unsigned = TRUE; break; + case OP_ARM64_UABD: is_unsigned = TRUE; break; + } + LLVMValueRef args [] = { lhs, rhs }; + if (add) { + args [0] = rhs; + args [1] = arg3; + } + if (high) + for (int i = 0; i < 2; ++i) + args [i] = extract_high_elements (ctx, args [i]); + int iid = is_unsigned ? INTRINS_AARCH64_ADV_SIMD_UABD : INTRINS_AARCH64_ADV_SIMD_SABD; + llvm_ovr_tag_t ovr_tag = ovr_tag_from_llvm_type (LLVMTypeOf (args [0])); + LLVMValueRef result = call_overloaded_intrins (ctx, iid, ovr_tag, args, ""); + if (widen) + result = LLVMBuildZExt (builder, result, ret_t, ""); + if (add) + result = LLVMBuildAdd (builder, result, lhs, ""); + values [ins->dreg] = result; + break; + } + case OP_ARM64_XHORIZ: { + gboolean truncate = FALSE; + LLVMTypeRef arg_t = LLVMTypeOf (lhs); + LLVMTypeRef elem_t = LLVMGetElementType (arg_t); + LLVMTypeRef ret_t = simd_class_to_llvm_type (ctx, ins->klass); + llvm_ovr_tag_t ovr_tag = ovr_tag_from_llvm_type (arg_t); + if (elem_t == i1_t || elem_t == i2_t) + truncate = TRUE; + LLVMValueRef result = call_overloaded_intrins (ctx, ins->inst_c0, ovr_tag, &lhs, ""); + if (truncate) { + // @llvm.aarch64.neon.saddv.i32.v8i16 ought to return an i16, but doesn't in LLVM 9. + result = LLVMBuildTrunc (builder, result, elem_t, ""); + } + result = vector_from_scalar (ctx, ret_t, result); + values [ins->dreg] = result; + break; + } + case OP_ARM64_SADDLV: + case OP_ARM64_UADDLV: { + LLVMTypeRef arg_t = LLVMTypeOf (lhs); + LLVMTypeRef elem_t = LLVMGetElementType (arg_t); + LLVMTypeRef ret_t = simd_class_to_llvm_type (ctx, ins->klass); + llvm_ovr_tag_t ovr_tag = ovr_tag_from_llvm_type (arg_t); + gboolean truncate = elem_t == i1_t; + int iid = ins->opcode == OP_ARM64_UADDLV ? INTRINS_AARCH64_ADV_SIMD_UADDLV : INTRINS_AARCH64_ADV_SIMD_SADDLV; + LLVMValueRef result = call_overloaded_intrins (ctx, iid, ovr_tag, &lhs, ""); + if (truncate) { + // @llvm.aarch64.neon.saddlv.i32.v16i8 ought to return an i16, but doesn't in LLVM 9. + result = LLVMBuildTrunc (builder, result, i2_t, ""); + } + result = vector_from_scalar (ctx, ret_t, result); + values [ins->dreg] = result; + break; + } + case OP_ARM64_UADALP: + case OP_ARM64_SADALP: { + llvm_ovr_tag_t ovr_tag = ovr_tag_from_mono_vector_class (ins->klass); + int iid = ins->opcode == OP_ARM64_UADALP ? INTRINS_AARCH64_ADV_SIMD_UADDLP : INTRINS_AARCH64_ADV_SIMD_SADDLP; + LLVMValueRef result = call_overloaded_intrins (ctx, iid, ovr_tag, &rhs, ""); + result = LLVMBuildAdd (builder, result, lhs, ""); + values [ins->dreg] = result; + break; + } + case OP_ARM64_ADDP_SCALAR: { + llvm_ovr_tag_t ovr_tag = INTRIN_vector128 | INTRIN_int64; + LLVMValueRef result = call_overloaded_intrins (ctx, INTRINS_AARCH64_ADV_SIMD_UADDV, ovr_tag, &lhs, "arm64_addp_scalar"); + result = LLVMBuildInsertElement (builder, LLVMGetUndef (v64_i8_t), result, const_int32 (0), ""); + values [ins->dreg] = result; + break; + } + case OP_ARM64_FADDP_SCALAR: { + LLVMTypeRef ret_t = simd_class_to_llvm_type (ctx, ins->klass); + LLVMValueRef hi = LLVMBuildExtractElement (builder, lhs, const_int32 (0), ""); + LLVMValueRef lo = LLVMBuildExtractElement (builder, lhs, const_int32 (1), ""); + LLVMValueRef result = LLVMBuildFAdd (builder, hi, lo, "arm64_faddp_scalar"); + result = LLVMBuildInsertElement (builder, LLVMGetUndef (ret_t), result, const_int32 (0), ""); + values [ins->dreg] = result; + break; + } + case OP_ARM64_SXTL: + case OP_ARM64_SXTL2: + case OP_ARM64_UXTL: + case OP_ARM64_UXTL2: { + gboolean high = FALSE; + gboolean is_unsigned = FALSE; + switch (ins->opcode) { + case OP_ARM64_SXTL2: high = TRUE; break; + case OP_ARM64_UXTL2: high = TRUE; case OP_ARM64_UXTL: is_unsigned = TRUE; break; + } + LLVMTypeRef t = LLVMTypeOf (lhs); + unsigned int elem_bits = LLVMGetIntTypeWidth (LLVMGetElementType (t)); + unsigned int src_elems = LLVMGetVectorSize (t); + unsigned int dst_elems = src_elems; + LLVMValueRef arg = lhs; + if (high) { + arg = extract_high_elements (ctx, lhs); + dst_elems = LLVMGetVectorSize (LLVMTypeOf (arg)); + } + LLVMTypeRef result_t = LLVMVectorType (LLVMIntType (elem_bits * 2), dst_elems); + LLVMValueRef result = NULL; + if (is_unsigned) + result = LLVMBuildZExt (builder, arg, result_t, "arm64_uxtl"); + else + result = LLVMBuildSExt (builder, arg, result_t, "arm64_sxtl"); + values [ins->dreg] = result; + break; + } + case OP_ARM64_TRN1: + case OP_ARM64_TRN2: { + gboolean high = ins->opcode == OP_ARM64_TRN2; + LLVMTypeRef t = LLVMTypeOf (lhs); + unsigned int src_elems = LLVMGetVectorSize (t); + int mask [MAX_VECTOR_ELEMS] = { 0 }; + int laneix = high ? 1 : 0; + for (unsigned int i = 0; i < src_elems; i += 2) { + mask [i] = laneix; + mask [i + 1] = laneix + src_elems; + laneix += 2; + } + values [ins->dreg] = LLVMBuildShuffleVector (builder, lhs, rhs, create_const_vector_i32 (mask, src_elems), "arm64_uzp"); + break; + } + case OP_ARM64_UZP1: + case OP_ARM64_UZP2: { + gboolean high = ins->opcode == OP_ARM64_UZP2; + LLVMTypeRef t = LLVMTypeOf (lhs); + unsigned int src_elems = LLVMGetVectorSize (t); + int mask [MAX_VECTOR_ELEMS] = { 0 }; + int laneix = high ? 1 : 0; + for (unsigned int i = 0; i < src_elems; ++i) { + mask [i] = laneix; + laneix += 2; + } + values [ins->dreg] = LLVMBuildShuffleVector (builder, lhs, rhs, create_const_vector_i32 (mask, src_elems), "arm64_uzp"); + break; + } + case OP_ARM64_ZIP1: + case OP_ARM64_ZIP2: { + gboolean high = ins->opcode == OP_ARM64_ZIP2; + LLVMTypeRef t = LLVMTypeOf (lhs); + unsigned int src_elems = LLVMGetVectorSize (t); + int mask [MAX_VECTOR_ELEMS] = { 0 }; + int laneix = high ? src_elems / 2 : 0; + for (unsigned int i = 0; i < src_elems; i += 2) { + mask [i] = laneix; + mask [i + 1] = laneix + src_elems; + ++laneix; + } + values [ins->dreg] = LLVMBuildShuffleVector (builder, lhs, rhs, create_const_vector_i32 (mask, src_elems), "arm64_zip"); + break; + } + case OP_ARM64_ABSCOMPARE: { + IntrinsicId iid = (IntrinsicId) ins->inst_c0; + gboolean scalar = ins->inst_c1; + LLVMTypeRef ret_t = simd_class_to_llvm_type (ctx, ins->klass); + LLVMTypeRef elem_t = LLVMGetElementType (ret_t); + llvm_ovr_tag_t ovr_tag = ovr_tag_from_mono_vector_class (ins->klass); + ovr_tag = ovr_tag_corresponding_integer (ovr_tag); + LLVMValueRef args [] = { lhs, rhs }; + LLVMTypeRef result_t = ret_t; + if (scalar) { + ovr_tag = ovr_tag_force_scalar (ovr_tag); + result_t = elem_t; + for (int i = 0; i < 2; ++i) + args [i] = scalar_from_vector (ctx, args [i]); + } + LLVMValueRef result = call_overloaded_intrins (ctx, iid, ovr_tag, args, ""); + result = LLVMBuildBitCast (builder, result, result_t, ""); + if (scalar) + result = vector_from_scalar (ctx, ret_t, result); + values [ins->dreg] = result; + break; + } + case OP_XOP_OVR_X_X: { + IntrinsicId iid = (IntrinsicId) ins->inst_c0; + llvm_ovr_tag_t ovr_tag = ovr_tag_from_mono_vector_class (ins->klass); + values [ins->dreg] = call_overloaded_intrins (ctx, iid, ovr_tag, &lhs, ""); + break; + } + case OP_XOP_OVR_X_X_X: { + IntrinsicId iid = (IntrinsicId) ins->inst_c0; + llvm_ovr_tag_t ovr_tag = ovr_tag_from_mono_vector_class (ins->klass); + LLVMValueRef args [] = { lhs, rhs }; + values [ins->dreg] = call_overloaded_intrins (ctx, iid, ovr_tag, args, ""); + break; + } + case OP_XOP_OVR_X_X_X_X: { + IntrinsicId iid = (IntrinsicId) ins->inst_c0; + llvm_ovr_tag_t ovr_tag = ovr_tag_from_mono_vector_class (ins->klass); + LLVMValueRef args [] = { lhs, rhs, arg3 }; + values [ins->dreg] = call_overloaded_intrins (ctx, iid, ovr_tag, args, ""); + break; + } + case OP_XOP_OVR_BYSCALAR_X_X_X: { + IntrinsicId iid = (IntrinsicId) ins->inst_c0; + llvm_ovr_tag_t ovr_tag = ovr_tag_from_mono_vector_class (ins->klass); + LLVMTypeRef t = LLVMTypeOf (lhs); + unsigned int elems = LLVMGetVectorSize (t); + LLVMValueRef arg2 = broadcast_element (ctx, scalar_from_vector (ctx, rhs), elems); + LLVMValueRef args [] = { lhs, arg2 }; + values [ins->dreg] = call_overloaded_intrins (ctx, iid, ovr_tag, args, ""); + break; + } + case OP_XOP_OVR_SCALAR_X_X: + case OP_XOP_OVR_SCALAR_X_X_X: + case OP_XOP_OVR_SCALAR_X_X_X_X: { + int num_args = 0; + IntrinsicId iid = (IntrinsicId) ins->inst_c0; + LLVMTypeRef ret_t = simd_class_to_llvm_type (ctx, ins->klass); + switch (ins->opcode) { + case OP_XOP_OVR_SCALAR_X_X: num_args = 1; break; + case OP_XOP_OVR_SCALAR_X_X_X: num_args = 2; break; + case OP_XOP_OVR_SCALAR_X_X_X_X: num_args = 3; break; + } + /* LLVM 9 NEON intrinsic functions have scalar overloads. Unfortunately + * only overloads for 32 and 64-bit integers and floating point types are + * supported. 8 and 16-bit integers are unsupported, and will fail during + * instruction selection. This is worked around by using a vector + * operation and then explicitly clearing the upper bits of the register. + */ + ScalarOpFromVectorOpCtx sctx = scalar_op_from_vector_op (ctx, ret_t, ins); + LLVMValueRef args [3] = { lhs, rhs, arg3 }; + scalar_op_from_vector_op_process_args (&sctx, args, num_args); + LLVMValueRef result = call_overloaded_intrins (ctx, iid, sctx.ovr_tag, args, ""); + result = scalar_op_from_vector_op_process_result (&sctx, result); + values [ins->dreg] = result; + break; + } #endif case OP_DUMMY_USE: @@ -9603,8 +11342,6 @@ mono_llvm_emit_method (MonoCompile *cfg) ctx->lmodule = ctx->module->lmodule; } else { ctx->lmodule = LLVMModuleCreateWithName (g_strdup_printf ("jit-module-%s", cfg->method->name)); - /* Reset this as it contains values from lmodule */ - memset (ctx->module->intrins_by_id, 0, sizeof (LLVMValueRef) * INTRINS_NUM); } ctx->llvm_only = ctx->module->llvm_only; #ifdef TARGET_WASM @@ -10512,12 +12249,67 @@ add_intrinsic (LLVMModuleRef module, int id) return; } + if (intrin_arm64_ovr [id] != 0) { + llvm_ovr_tag_t spec = intrin_arm64_ovr [id]; + for (int vw = 0; vw < INTRIN_vectorwidths; ++vw) { + for (int ew = 0; ew < INTRIN_elementwidths; ++ew) { + llvm_ovr_tag_t vec_bit = INTRIN_vector128 >> ((INTRIN_vectorwidths - 1) - vw); + llvm_ovr_tag_t elem_bit = INTRIN_int8 << ew; + llvm_ovr_tag_t test = vec_bit | elem_bit; + if ((spec & test) == test) { + uint8_t kind = intrin_kind [id]; + LLVMTypeRef distinguishing_type = intrin_types [vw][ew]; + if (kind == INTRIN_kind_ftoi && (elem_bit & (INTRIN_int32 | INTRIN_int64))) { + /* + * @llvm.aarch64.neon.fcvtas.v4i32.v4f32 + * @llvm.aarch64.neon.fcvtas.v2i64.v2f64 + */ + intrins = add_intrins2 (module, id, distinguishing_type, intrin_types [vw][ew + 2]); + } else if (kind == INTRIN_kind_widen) { + /* + * @llvm.aarch64.neon.saddlp.v2i64.v4i32 + * @llvm.aarch64.neon.saddlp.v4i16.v8i8 + */ + intrins = add_intrins2 (module, id, distinguishing_type, intrin_types [vw][ew - 1]); + } else if (kind == INTRIN_kind_widen_across) { + /* + * @llvm.aarch64.neon.saddlv.i64.v4i32 + * @llvm.aarch64.neon.saddlv.i32.v8i16 + * @llvm.aarch64.neon.saddlv.i32.v16i8 + * i8/i16 return types for NEON intrinsics will make isel fail as of LLVM 9. + */ + int associated_prim = MAX(ew + 1, 2); + LLVMTypeRef associated_scalar_type = intrin_types [0][associated_prim]; + intrins = add_intrins2 (module, id, associated_scalar_type, distinguishing_type); + } else if (kind == INTRIN_kind_across) { + /* + * @llvm.aarch64.neon.uaddv.i64.v4i64 + * @llvm.aarch64.neon.uaddv.i32.v4i32 + * @llvm.aarch64.neon.uaddv.i32.v8i16 + * @llvm.aarch64.neon.uaddv.i32.v16i8 + * i8/i16 return types for NEON intrinsics will make isel fail as of LLVM 9. + */ + int associated_prim = MAX(ew, 2); + LLVMTypeRef associated_scalar_type = intrin_types [0][associated_prim]; + intrins = add_intrins2 (module, id, associated_scalar_type, distinguishing_type); + } else + intrins = add_intrins1 (module, id, distinguishing_type); + int key = key_from_id_and_tag (id, test); + g_hash_table_insert (intrins_id_to_intrins, GINT_TO_POINTER (key), intrins); + } + } + } + return; + } + /* Register overloaded intrinsics */ switch (id) { #define INTRINS(intrin_name, llvm_id) #define INTRINS_OVR(intrin_name, llvm_id, llvm_type) case INTRINS_ ## intrin_name: intrins = add_intrins1(module, id, llvm_type); break; #define INTRINS_OVR_2_ARG(intrin_name, llvm_id, llvm_type1, llvm_type2) case INTRINS_ ## intrin_name: intrins = add_intrins2(module, id, llvm_type1, llvm_type2); break; #define INTRINS_OVR_3_ARG(intrin_name, llvm_id, llvm_type1, llvm_type2, llvm_type3) case INTRINS_ ## intrin_name: intrins = add_intrins3(module, id, llvm_type1, llvm_type2, llvm_type3); break; + #define INTRINS_OVR_TAG(...) + #define INTRINS_OVR_TAG_KIND(...) #include "llvm-intrinsics.h" default: @@ -10541,19 +12333,7 @@ get_intrins_from_module (LLVMModuleRef lmodule, int id) static LLVMValueRef get_intrins (EmitContext *ctx, int id) { - MonoLLVMModule *module = ctx->module; - LLVMValueRef res; - - /* - * Every method is emitted into its own module so - * we can add intrinsics on demand. - */ - res = module->intrins_by_id [id]; - if (!res) { - res = get_intrins_from_module (ctx->lmodule, id); - module->intrins_by_id [id] = res; - } - return res; + return get_intrins_from_module (ctx->lmodule, id); } static void @@ -10583,12 +12363,26 @@ add_types (MonoLLVMModule *module) void mono_llvm_init (gboolean enable_jit) { - sse_i1_t = type_to_sse_type (MONO_TYPE_I1); - sse_i2_t = type_to_sse_type (MONO_TYPE_I2); - sse_i4_t = type_to_sse_type (MONO_TYPE_I4); - sse_i8_t = type_to_sse_type (MONO_TYPE_I8); - sse_r4_t = type_to_sse_type (MONO_TYPE_R4); - sse_r8_t = type_to_sse_type (MONO_TYPE_R8); + intrin_types [0][0] = i1_t = LLVMInt8Type (); + intrin_types [0][1] = i2_t = LLVMInt16Type (); + intrin_types [0][2] = i4_t = LLVMInt32Type (); + intrin_types [0][3] = i8_t = LLVMInt64Type (); + intrin_types [0][4] = r4_t = LLVMFloatType (); + intrin_types [0][5] = r8_t = LLVMDoubleType (); + + intrin_types [1][0] = v64_i1_t = LLVMVectorType (LLVMInt8Type (), 8); + intrin_types [1][1] = v64_i2_t = LLVMVectorType (LLVMInt16Type (), 4); + intrin_types [1][2] = v64_i4_t = LLVMVectorType (LLVMInt32Type (), 2); + intrin_types [1][3] = v64_i8_t = LLVMVectorType (LLVMInt64Type (), 1); + intrin_types [1][4] = v64_r4_t = LLVMVectorType (LLVMFloatType (), 2); + intrin_types [1][5] = v64_r8_t = LLVMVectorType (LLVMDoubleType (), 1); + + intrin_types [2][0] = v128_i1_t = sse_i1_t = type_to_sse_type (MONO_TYPE_I1); + intrin_types [2][1] = v128_i2_t = sse_i2_t = type_to_sse_type (MONO_TYPE_I2); + intrin_types [2][2] = v128_i4_t = sse_i4_t = type_to_sse_type (MONO_TYPE_I4); + intrin_types [2][3] = v128_i8_t = sse_i8_t = type_to_sse_type (MONO_TYPE_I8); + intrin_types [2][4] = v128_r4_t = sse_r4_t = type_to_sse_type (MONO_TYPE_R4); + intrin_types [2][5] = v128_r8_t = sse_r8_t = type_to_sse_type (MONO_TYPE_R8); intrins_id_to_intrins = g_hash_table_new (NULL, NULL); @@ -10668,7 +12462,6 @@ mono_llvm_create_aot_module (MonoAssembly *assembly, const char *global_prefix, module->max_got_offset = initial_got_size; module->context = LLVMGetGlobalContext (); module->cfgs = g_ptr_array_new (); - module->intrins_by_id = g_new0 (LLVMValueRef, INTRINS_NUM); module->aotconst_vars = g_hash_table_new (NULL, NULL); module->llvm_types = g_hash_table_new (NULL, NULL); module->plt_entries = g_hash_table_new (g_str_hash, g_str_equal); @@ -11787,7 +13580,6 @@ init_jit_module (void) module = g_new0 (MonoLLVMModule, 1); module->context = LLVMGetGlobalContext (); - module->intrins_by_id = g_new0 (LLVMValueRef, INTRINS_NUM); module->mono_ee = (MonoEERef*)mono_llvm_create_ee (&module->ee); diff --git a/src/mono/mono/mini/mini-ops.h b/src/mono/mono/mini/mini-ops.h index 93036193ce11bd..f11da0f5ebf390 100644 --- a/src/mono/mono/mini/mini-ops.h +++ b/src/mono/mono/mini/mini-ops.h @@ -1529,12 +1529,17 @@ MINI_OP(OP_FILL_PROF_CALL_CTX, "fill_prof_call_ctx", NONE, IREG, NONE) MINI_OP(OP_XEQUAL, "xequal", IREG, XREG, XREG) /* Per element compate, inst_c0 contains a CompRelation */ MINI_OP(OP_XCOMPARE, "xcompare", XREG, XREG, XREG) +MINI_OP(OP_XCOMPARE_SCALAR, "xcompare_scalar", XREG, XREG, XREG) MINI_OP(OP_XCOMPARE_FP, "xcompare_fp", XREG, XREG, XREG) +MINI_OP(OP_XCOMPARE_FP_SCALAR, "xcompare_fp_scalar", XREG, XREG, XREG) /* * Generic SIMD operations, the rest of the JIT doesn't care about the exact operation. */ MINI_OP(OP_XBINOP, "xbinop", XREG, XREG, XREG) +MINI_OP(OP_XBINOP_FORCEINT, "xbinop_forceint", XREG, XREG, XREG) +MINI_OP(OP_XBINOP_SCALAR, "xbinop_scalar", XREG, XREG, XREG) +MINI_OP(OP_XBINOP_BYSCALAR, "xbinop_byscalar", XREG, XREG, XREG) /* inst_c0 contains a SimdOp, inst_c1 might contain additional data */ MINI_OP(OP_XOP, "xop", NONE, NONE, NONE) MINI_OP(OP_XOP_X_I, "xop_x_i", XREG, IREG, NONE) @@ -1552,6 +1557,16 @@ MINI_OP(OP_XOP_I4_I4_I4, "xop_i4_i4_i4", IREG, IREG, IREG) MINI_OP(OP_XOP_I4_I4_I8, "xop_i4_i4_i8", IREG, IREG, LREG) MINI_OP3(OP_XOP_X_X_X_X, "xop_x_x_x_x", XREG, XREG, XREG, XREG) +MINI_OP(OP_XOP_OVR_X_X, "xop_ovr_x_x", XREG, XREG, NONE) +MINI_OP(OP_XOP_OVR_X_X_X, "xop_ovr_x_x_x", XREG, XREG, XREG) +MINI_OP3(OP_XOP_OVR_X_X_X_X, "xop_ovr_x_x_x_x", XREG, XREG, XREG, XREG) +MINI_OP(OP_XOP_OVR_SCALAR_X_X, "xop_ovr_scalar_x_x", XREG, XREG, NONE) +MINI_OP(OP_XOP_OVR_SCALAR_X_X_X, "xop_ovr_scalar_x_x_x", XREG, XREG, XREG) +MINI_OP3(OP_XOP_OVR_SCALAR_X_X_X_X, "xop_ovr_scalar_x_x_x_x", XREG, XREG, XREG, XREG) +MINI_OP(OP_XOP_OVR_BYSCALAR_X_X_X, "xop_ovr_byscalar_x_x_x", XREG, XREG, XREG) + +MINI_OP(OP_XCONCAT, "xconcat", XREG, XREG, XREG) + MINI_OP(OP_XCAST, "xcast", XREG, XREG, NONE) /* Extract element of vector */ /* The index is assumed to be in range */ @@ -1579,6 +1594,223 @@ MINI_OP(OP_POPCNT64, "popcnt64", LREG, LREG, NONE) #ifdef TARGET_ARM64 MINI_OP(OP_LSCNT32, "lscnt32", IREG, IREG, NONE) MINI_OP(OP_LSCNT64, "lscnt64", LREG, LREG, NONE) + +MINI_OP(OP_ARM64_CLZ, "arm64_clz", XREG, XREG, NONE) + +MINI_OP3(OP_ARM64_LD1_INSERT, "arm64_ld1_insert", XREG, IREG, XREG, IREG) +MINI_OP(OP_ARM64_LD1, "arm64_ld1", XREG, IREG, NONE) +MINI_OP(OP_ARM64_LD1R, "arm64_ld1r", XREG, IREG, NONE) + +MINI_OP(OP_ARM64_ST1, "arm64_st1", NONE, IREG, XREG) +MINI_OP(OP_ARM64_SXTL, "arm64_sxtl", XREG, XREG, NONE) +MINI_OP(OP_ARM64_SXTL2, "arm64_sxtl2", XREG, XREG, NONE) + MINI_OP(OP_ARM64_SMULH, "arm64_smulh", LREG, LREG, LREG) +MINI_OP(OP_ARM64_SQRT_SCALAR, "arm64_sqrt_scalar", XREG, XREG, NONE) +MINI_OP(OP_ARM64_TRN1, "arm64_trn1", XREG, XREG, XREG) +MINI_OP(OP_ARM64_TRN2, "arm64_trn2", XREG, XREG, XREG) MINI_OP(OP_ARM64_UMULH, "arm64_umulh", LREG, LREG, LREG) +MINI_OP(OP_ARM64_UXTL, "arm64_uxtl", XREG, XREG, NONE) +MINI_OP(OP_ARM64_UXTL2, "arm64_uxtl2", XREG, XREG, NONE) +MINI_OP(OP_ARM64_UZP1, "arm64_uzp1", XREG, XREG, XREG) +MINI_OP(OP_ARM64_UZP2, "arm64_uzp2", XREG, XREG, XREG) +MINI_OP(OP_ARM64_ZIP1, "arm64_zip1", XREG, XREG, XREG) +MINI_OP(OP_ARM64_ZIP2, "arm64_zip2", XREG, XREG, XREG) +MINI_OP3(OP_ARM64_ST1_SCALAR, "arm64_st1_scalar", NONE, IREG, XREG, IREG) +MINI_OP3(OP_ARM64_STNP, "arm64_stnp", NONE, IREG, XREG, XREG) +MINI_OP3(OP_ARM64_STNP_SCALAR, "arm64_stnp_scalar", NONE, IREG, XREG, XREG) +MINI_OP3(OP_ARM64_STP, "arm64_stp", NONE, IREG, XREG, XREG) +MINI_OP3(OP_ARM64_STP_SCALAR, "arm64_stp_scalar", NONE, IREG, XREG, XREG) + +MINI_OP(OP_ARM64_ADDHN, "arm64_addhn", XREG, XREG, XREG) +MINI_OP3(OP_ARM64_ADDHN2, "arm64_addhn2", XREG, XREG, XREG, XREG) +MINI_OP(OP_ARM64_SUBHN, "arm64_subhn", XREG, XREG, XREG) +MINI_OP3(OP_ARM64_SUBHN2, "arm64_subhn2", XREG, XREG, XREG, XREG) + +MINI_OP(OP_ARM64_RADDHN, "arm64_raddhn", XREG, XREG, XREG) +MINI_OP3(OP_ARM64_RADDHN2, "arm64_raddhn2", XREG, XREG, XREG, XREG) +MINI_OP(OP_ARM64_RSUBHN, "arm64_rsubhn", XREG, XREG, XREG) +MINI_OP3(OP_ARM64_RSUBHN2, "arm64_rsubhn2", XREG, XREG, XREG, XREG) + +MINI_OP(OP_ARM64_SHRN, "arm64_shrn", XREG, XREG, IREG) +MINI_OP3(OP_ARM64_SHRN2, "arm64_shrn2", XREG, XREG, XREG, IREG) + +MINI_OP3(OP_ARM64_SLI, "arm64_sli", XREG, XREG, XREG, IREG) +MINI_OP3(OP_ARM64_SRI, "arm64_sri", XREG, XREG, XREG, IREG) + +MINI_OP(OP_ARM64_SRSHR, "arm64_srshr", XREG, XREG, IREG) +MINI_OP(OP_ARM64_URSHR, "arm64_urshr", XREG, XREG, IREG) +MINI_OP3(OP_ARM64_SRSRA, "arm64_srsra", XREG, XREG, XREG, IREG) +MINI_OP3(OP_ARM64_URSRA, "arm64_ursra", XREG, XREG, XREG, IREG) + +MINI_OP(OP_ARM64_SHL, "arm64_shl", XREG, XREG, IREG) +MINI_OP(OP_ARM64_SSHR, "arm64_sshr", XREG, XREG, IREG) +MINI_OP(OP_ARM64_USHR, "arm64_ushr", XREG, XREG, IREG) +MINI_OP3(OP_ARM64_USRA, "arm64_usra", XREG, XREG, XREG, IREG) +MINI_OP3(OP_ARM64_SSRA, "arm64_ssra", XREG, XREG, XREG, IREG) + +MINI_OP(OP_ARM64_USHLL, "arm64_ushll", XREG, XREG, IREG) +MINI_OP(OP_ARM64_USHLL2, "arm64_ushll2", XREG, XREG, IREG) +MINI_OP(OP_ARM64_SSHLL, "arm64_sshll", XREG, XREG, IREG) +MINI_OP(OP_ARM64_SSHLL2, "arm64_sshll2", XREG, XREG, IREG) + +/* Narrowing arm64 shifts that aren't decomposed into urshl or srshl. */ +MINI_OP(OP_ARM64_XNSHIFT_SCALAR, "arm64_xrshift_scalar", XREG, XREG, IREG) +MINI_OP(OP_ARM64_XNSHIFT, "arm64_xnshift", XREG, XREG, IREG) +MINI_OP3(OP_ARM64_XNSHIFT2, "arm64_xnshift2", XREG, XREG, XREG, IREG) + +MINI_OP(OP_ARM64_SQSHLU, "arm64_sqshlu", XREG, XREG, IREG) +MINI_OP(OP_ARM64_SQSHLU_SCALAR, "arm64_sqshlu_scalar", XREG, XREG, IREG) + +MINI_OP(OP_ARM64_REVN, "arm64_revn", XREG, XREG, NONE) + +MINI_OP(OP_ARM64_PMULL, "arm64_pmull", XREG, XREG, XREG) +MINI_OP(OP_ARM64_PMULL2, "arm64_pmull2", XREG, XREG, XREG) + +MINI_OP(OP_ARM64_XNEG, "arm64_xneg", XREG, XREG, NONE) +MINI_OP(OP_ARM64_XNEG_SCALAR, "arm64_xneg_scalar", XREG, XREG, NONE) + +MINI_OP(OP_ARM64_SMULL, "arm64_smull", XREG, XREG, XREG) +MINI_OP(OP_ARM64_SMULL_SCALAR, "arm64_smull_scalar", XREG, XREG, XREG) +MINI_OP(OP_ARM64_SMULL2, "arm64_smull2", XREG, XREG, XREG) +MINI_OP(OP_ARM64_SMULL2_SCALAR, "arm64_smull2_scalar", XREG, XREG, XREG) +MINI_OP(OP_ARM64_UMULL, "arm64_umull", XREG, XREG, XREG) +MINI_OP(OP_ARM64_UMULL_SCALAR, "arm64_umull_scalar", XREG, XREG, XREG) +MINI_OP(OP_ARM64_UMULL2, "arm64_umull2", XREG, XREG, XREG) +MINI_OP(OP_ARM64_UMULL2_SCALAR, "arm64_umull2_scalar", XREG, XREG, XREG) +MINI_OP3(OP_ARM64_SMLAL, "arm64_smlal", XREG, XREG, XREG, XREG) +MINI_OP3(OP_ARM64_SMLAL_SCALAR, "arm64_smlal_scalar", XREG, XREG, XREG, XREG) +MINI_OP3(OP_ARM64_SMLAL2, "arm64_smlal2", XREG, XREG, XREG, XREG) +MINI_OP3(OP_ARM64_SMLAL2_SCALAR, "arm64_smlal2_scalar", XREG, XREG, XREG, XREG) +MINI_OP3(OP_ARM64_UMLAL, "arm64_umlal", XREG, XREG, XREG, XREG) +MINI_OP3(OP_ARM64_UMLAL_SCALAR, "arm64_umlal_scalar", XREG, XREG, XREG, XREG) +MINI_OP3(OP_ARM64_UMLAL2, "arm64_umlal2", XREG, XREG, XREG, XREG) +MINI_OP3(OP_ARM64_UMLAL2_SCALAR, "arm64_umlal2_scalar", XREG, XREG, XREG, XREG) +MINI_OP3(OP_ARM64_SMLSL, "arm64_smlsl", XREG, XREG, XREG, XREG) +MINI_OP3(OP_ARM64_SMLSL_SCALAR, "arm64_smlsl_scalar", XREG, XREG, XREG, XREG) +MINI_OP3(OP_ARM64_SMLSL2, "arm64_smlsl2", XREG, XREG, XREG, XREG) +MINI_OP3(OP_ARM64_SMLSL2_SCALAR, "arm64_smlsl2_scalar", XREG, XREG, XREG, XREG) +MINI_OP3(OP_ARM64_UMLSL, "arm64_umlsl", XREG, XREG, XREG, XREG) +MINI_OP3(OP_ARM64_UMLSL_SCALAR, "arm64_umlsl_scalar", XREG, XREG, XREG, XREG) +MINI_OP3(OP_ARM64_UMLSL2, "arm64_umlsl2", XREG, XREG, XREG, XREG) +MINI_OP3(OP_ARM64_UMLSL2_SCALAR, "arm64_umlsl2_scalar", XREG, XREG, XREG, XREG) + +MINI_OP3(OP_ARM64_MLS, "arm64_mls", XREG, XREG, XREG, XREG) +MINI_OP3(OP_ARM64_MLS_SCALAR, "arm64_mls_scalar", XREG, XREG, XREG, XREG) +MINI_OP3(OP_ARM64_MLA, "arm64_mla", XREG, XREG, XREG, XREG) +MINI_OP3(OP_ARM64_MLA_SCALAR, "arm64_mla_scalar", XREG, XREG, XREG, XREG) + +MINI_OP3(OP_ARM64_FMUL_SEL, "arm64_fmul_sel", XREG, XREG, XREG, IREG) + +MINI_OP(OP_ARM64_SQDMULL_SCALAR, "arm64_sqdmull_scalar", XREG, XREG, XREG) +MINI_OP3(OP_ARM64_SQDMLSL_SCALAR, "arm64_sqdmlsl_scalar", XREG, XREG, XREG, XREG) +MINI_OP3(OP_ARM64_SQDMLAL_SCALAR, "arm64_sqdmlal_scalar", XREG, XREG, XREG, XREG) + +MINI_OP(OP_ARM64_SQDMULL, "arm64_sqdmull", XREG, XREG, XREG) +MINI_OP(OP_ARM64_SQDMULL_BYSCALAR, "arm64_sqdmull", XREG, XREG, XREG) +MINI_OP(OP_ARM64_SQDMULL2, "arm64_sqdmull2", XREG, XREG, XREG) +MINI_OP(OP_ARM64_SQDMULL2_BYSCALAR, "arm64_sqdmull2", XREG, XREG, XREG) +MINI_OP3(OP_ARM64_SQDMLSL, "arm64_sqdmlsl", XREG, XREG, XREG, XREG) +MINI_OP3(OP_ARM64_SQDMLSL_BYSCALAR, "arm64_sqdmlsl", XREG, XREG, XREG, XREG) +MINI_OP3(OP_ARM64_SQDMLAL, "arm64_sqdmlal", XREG, XREG, XREG, XREG) +MINI_OP3(OP_ARM64_SQDMLAL_BYSCALAR, "arm64_sqdmlal", XREG, XREG, XREG, XREG) +MINI_OP3(OP_ARM64_SQDMLSL2, "arm64_sqdmlsl2", XREG, XREG, XREG, XREG) +MINI_OP3(OP_ARM64_SQDMLSL2_BYSCALAR, "arm64_sqdmlsl2", XREG, XREG, XREG, XREG) +MINI_OP3(OP_ARM64_SQDMLAL2, "arm64_sqdmlal2", XREG, XREG, XREG, XREG) +MINI_OP3(OP_ARM64_SQDMLAL2_BYSCALAR, "arm64_sqdmlal2", XREG, XREG, XREG, XREG) + +MINI_OP3(OP_ARM64_FMSUB, "arm64_fmsub", XREG, XREG, XREG, XREG) +MINI_OP3(OP_ARM64_FMSUB_BYSCALAR, "arm64_fmsub_byscalar", XREG, XREG, XREG, XREG) +MINI_OP3(OP_ARM64_FMSUB_SCALAR, "arm64_fmsub_scalar", XREG, XREG, XREG, XREG) +MINI_OP3(OP_ARM64_FNMSUB_SCALAR, "arm64_fnmsub_scalar", XREG, XREG, XREG, XREG) + +MINI_OP3(OP_ARM64_FMADD, "arm64_fmadd", XREG, XREG, XREG, XREG) +MINI_OP3(OP_ARM64_FMADD_BYSCALAR, "arm64_fmadd_byscalar", XREG, XREG, XREG, XREG) +MINI_OP3(OP_ARM64_FMADD_SCALAR, "arm64_fmadd_scalar", XREG, XREG, XREG, XREG) +MINI_OP3(OP_ARM64_FNMADD_SCALAR, "arm64_fnmadd_scalar", XREG, XREG, XREG, XREG) + +MINI_OP(OP_ARM64_XTN, "arm64_xtn", XREG, XREG, NONE) +MINI_OP(OP_ARM64_XTN2, "arm64_xtn2", XREG, XREG, XREG) + +MINI_OP(OP_ARM64_SQXTN2, "arm64_sqxtn2", XREG, XREG, XREG) +MINI_OP(OP_ARM64_UQXTN2, "arm64_uqxtn2", XREG, XREG, XREG) +MINI_OP(OP_ARM64_SQXTUN2, "arm64_sqxtun2", XREG, XREG, XREG) + +MINI_OP(OP_ARM64_SELECT_SCALAR, "arm64_select_scalar", XREG, XREG, IREG) + +MINI_OP(OP_EXTRACT_VAR_I1, "extract_var_i1", IREG, XREG, IREG) +MINI_OP(OP_EXTRACT_VAR_U1, "extract_var_u1", IREG, XREG, IREG) +MINI_OP(OP_EXTRACT_VAR_I2, "extract_var_i2", IREG, XREG, IREG) +MINI_OP(OP_EXTRACT_VAR_U2, "extract_var_u2", IREG, XREG, IREG) +MINI_OP(OP_EXTRACT_VAR_I4, "extract_var_i4", IREG, XREG, IREG) +MINI_OP(OP_EXTRACT_VAR_R4, "extract_var_r4", FREG, XREG, IREG) +MINI_OP(OP_EXTRACT_VAR_R8, "extract_var_r8", FREG, XREG, IREG) +MINI_OP(OP_EXTRACT_VAR_I8, "extract_var_i8", LREG, XREG, IREG) + +MINI_OP(OP_ARM64_FCVTZU, "arm64_fcvtzu", XREG, XREG, NONE) +MINI_OP(OP_ARM64_FCVTZS, "arm64_fcvtzs", XREG, XREG, NONE) +MINI_OP(OP_ARM64_FCVTZU_SCALAR, "arm64_fcvtzu_scalar", XREG, XREG, NONE) +MINI_OP(OP_ARM64_FCVTZS_SCALAR, "arm64_fcvtzs_scalar", XREG, XREG, NONE) + +MINI_OP(OP_ARM64_UCVTF, "arm64_ucvtf", XREG, XREG, NONE) +MINI_OP(OP_ARM64_SCVTF, "arm64_scvtf", XREG, XREG, NONE) +MINI_OP(OP_ARM64_UCVTF_SCALAR, "arm64_ucvtf_scalar", XREG, XREG, NONE) +MINI_OP(OP_ARM64_SCVTF_SCALAR, "arm64_scvtf_scalar", XREG, XREG, NONE) + +MINI_OP(OP_ARM64_FCVTN, "arm64_fcvtn", XREG, XREG, NONE) +MINI_OP(OP_ARM64_FCVTN2, "arm64_fcvtn2", XREG, XREG, XREG) +MINI_OP(OP_ARM64_FCVTXN, "arm64_fcvtxn", XREG, XREG, NONE) +MINI_OP(OP_ARM64_FCVTXN2, "arm64_fcvtxn2", XREG, XREG, XREG) + +MINI_OP(OP_ARM64_FCVTL, "arm64_fcvtl", XREG, XREG, NONE) +MINI_OP(OP_ARM64_FCVTL2, "arm64_fcvtl2", XREG, XREG, NONE) + +MINI_OP(OP_ARM64_CMTST, "arm64_cmtst", XREG, XREG, XREG) + +MINI_OP3(OP_ARM64_BSL, "arm64_bsl", XREG, XREG, XREG, XREG) +MINI_OP(OP_ARM64_BIC, "arm64_bic", XREG, XREG, XREG) + +MINI_OP(OP_ARM64_MVN, "arm64_mvn", XREG, XREG, NONE) + +MINI_OP(OP_ARM64_SADD, "arm64_sadd", XREG, XREG, XREG) +MINI_OP(OP_ARM64_SADD2, "arm64_sadd2", XREG, XREG, XREG) +MINI_OP(OP_ARM64_UADD, "arm64_uadd", XREG, XREG, XREG) +MINI_OP(OP_ARM64_UADD2, "arm64_uadd2", XREG, XREG, XREG) +MINI_OP(OP_ARM64_SSUB, "arm64_ssub", XREG, XREG, XREG) +MINI_OP(OP_ARM64_SSUB2, "arm64_ssub2", XREG, XREG, XREG) +MINI_OP(OP_ARM64_USUB, "arm64_usub", XREG, XREG, XREG) +MINI_OP(OP_ARM64_USUB2, "arm64_usub2", XREG, XREG, XREG) + +MINI_OP(OP_ARM64_ADDP_SCALAR, "arm64_addp_scalar", XREG, XREG, NONE) +MINI_OP(OP_ARM64_FADDP_SCALAR, "arm64_faddp_scalar", XREG, XREG, NONE) + +MINI_OP(OP_ARM64_SADALP, "arm64_sadalp_scalar", XREG, XREG, XREG) +MINI_OP(OP_ARM64_UADALP, "arm64_uadalp_scalar", XREG, XREG, XREG) + +MINI_OP(OP_ARM64_SADDLV, "arm64_saddlv", XREG, XREG, NONE) +MINI_OP(OP_ARM64_UADDLV, "arm64_uaddlv", XREG, XREG, NONE) + +MINI_OP(OP_ARM64_XHORIZ, "arm64_xhoriz", XREG, XREG, NONE) + +MINI_OP3(OP_ARM64_SABAL, "arm64_sabal", XREG, XREG, XREG, XREG) +MINI_OP3(OP_ARM64_SABAL2, "arm64_sabal2", XREG, XREG, XREG, XREG) +MINI_OP3(OP_ARM64_UABAL, "arm64_uabal", XREG, XREG, XREG, XREG) +MINI_OP3(OP_ARM64_UABAL2, "arm64_uabal2", XREG, XREG, XREG, XREG) + +MINI_OP(OP_ARM64_SABDL, "arm64_sabdl", XREG, XREG, XREG) +MINI_OP(OP_ARM64_SABDL2, "arm64_sabdl2", XREG, XREG, XREG) +MINI_OP(OP_ARM64_UABDL, "arm64_uabdl", XREG, XREG, XREG) +MINI_OP(OP_ARM64_UABDL2, "arm64_uabdl2", XREG, XREG, XREG) + +MINI_OP3(OP_ARM64_SABA, "arm64_saba", XREG, XREG, XREG, XREG) +MINI_OP3(OP_ARM64_UABA, "arm64_uaba", XREG, XREG, XREG, XREG) +MINI_OP(OP_ARM64_SABD, "arm64_sabd", XREG, XREG, XREG) +MINI_OP(OP_ARM64_UABD, "arm64_uabd", XREG, XREG, XREG) + +MINI_OP(OP_ARM64_ABSCOMPARE, "arm64_abscompare", XREG, XREG, XREG) + +MINI_OP(OP_ARM64_XNARROW_SCALAR, "arm64_xnarrow_scalar", XREG, XREG, NONE) + +MINI_OP3(OP_ARM64_EXT, "arm64_ext", XREG, XREG, XREG, IREG) + #endif // TARGET_ARM64 diff --git a/src/mono/mono/mini/mini.h b/src/mono/mono/mini/mini.h index e5fdbde9c441c2..6d2f0fcf819aef 100644 --- a/src/mono/mono/mini/mini.h +++ b/src/mono/mono/mini/mini.h @@ -2877,24 +2877,6 @@ enum { /* SIMD operations */ typedef enum { - SIMD_OP_ARM64_FABS, - SIMD_OP_ARM64_DABS, - SIMD_OP_ARM64_I8ABS, - SIMD_OP_ARM64_I16ABS, - SIMD_OP_ARM64_I32ABS, - SIMD_OP_ARM64_I64ABS, - SIMD_OP_ARM64_I8ABS_SATURATE, - SIMD_OP_ARM64_I16ABS_SATURATE, - SIMD_OP_ARM64_I32ABS_SATURATE, - SIMD_OP_ARM64_I64ABS_SATURATE, - SIMD_OP_ARM64_FABSOLUTE_COMPARE_GREATER_THAN, - SIMD_OP_ARM64_DABSOLUTE_COMPARE_GREATER_THAN, - SIMD_OP_ARM64_FABSOLUTE_COMPARE_GREATER_THAN_OR_EQUAL, - SIMD_OP_ARM64_DABSOLUTE_COMPARE_GREATER_THAN_OR_EQUAL, - SIMD_OP_ARM64_FABSOLUTE_COMPARE_LESS_THAN, - SIMD_OP_ARM64_DABSOLUTE_COMPARE_LESS_THAN, - SIMD_OP_ARM64_FABSOLUTE_COMPARE_LESS_THAN_OR_EQUAL, - SIMD_OP_ARM64_DABSOLUTE_COMPARE_LESS_THAN_OR_EQUAL, SIMD_OP_SSE_CVTSS2SI, SIMD_OP_SSE_CVTTSS2SI, SIMD_OP_SSE_CVTSS2SI64, @@ -2996,7 +2978,7 @@ typedef enum { SIMD_OP_ARM64_SHA256SU0, SIMD_OP_ARM64_SHA256SU1, SIMD_OP_ARM64_PMULL64_LOWER, - SIMD_OP_ARM64_PMULL64_UPPER + SIMD_OP_ARM64_PMULL64_UPPER, } SimdOp; const char *mono_arch_xregname (int reg); diff --git a/src/mono/mono/mini/simd-intrinsics.c b/src/mono/mono/mini/simd-intrinsics.c index 2318b9f4be936f..04b9d903a20c43 100644 --- a/src/mono/mono/mini/simd-intrinsics.c +++ b/src/mono/mono/mini/simd-intrinsics.c @@ -9,8 +9,10 @@ #include "mini.h" #include "mini-runtime.h" #include "ir-emit.h" +#include "llvm-intrinsics-types.h" #ifdef ENABLE_LLVM #include "mini-llvm.h" +#include "mini-llvm-cpp.h" #endif #include "mono/utils/bsearch.h" #include @@ -53,13 +55,16 @@ enum { static int register_size; +#define None 0 + typedef struct { - // One of the SN_ constants - guint16 id; - // ins->opcode - int op; - // ins->inst_c0 - int instc0; + uint16_t id; // One of the SN_ constants + uint16_t default_op; // ins->opcode + uint16_t default_instc0; // ins->inst_c0 + uint16_t unsigned_op; + uint16_t unsigned_instc0; + uint16_t floating_op; + uint16_t floating_instc0; } SimdIntrinsic; static const SimdIntrinsic unsupported [] = { {SN_get_IsSupported} }; @@ -99,7 +104,7 @@ static int lookup_intrins (guint16 *intrinsics, int size, MonoMethod *cmethod) { const guint16 *result = (const guint16 *)mono_binary_search (cmethod->name, intrinsics, size / sizeof (guint16), sizeof (guint16), &simd_intrinsic_compare_by_name); - + if (result == NULL) return -1; else @@ -125,7 +130,6 @@ lookup_intrins_info (SimdIntrinsic *intrinsics, int size, MonoMethod *cmethod) } } #endif - return (SimdIntrinsic *)mono_binary_search (cmethod->name, intrinsics, size / sizeof (SimdIntrinsic), sizeof (SimdIntrinsic), &simd_intrinsic_info_compare_by_name); } @@ -253,6 +257,15 @@ emit_xcompare (MonoCompile *cfg, MonoClass *klass, MonoTypeEnum etype, MonoInst return ins; } +static gboolean +is_intrinsics_vector_type (MonoType *vector_type) +{ + if (vector_type->type != MONO_TYPE_GENERICINST) return FALSE; + MonoClass *klass = mono_class_from_mono_type_internal (vector_type); + const char *name = m_class_get_name (klass); + return !strcmp (name, "Vector64`1") || !strcmp (name, "Vector128`1") || !strcmp (name, "Vector256`1"); +} + static MonoType* get_vector_t_elem_type (MonoType *vector_type) { @@ -270,6 +283,32 @@ get_vector_t_elem_type (MonoType *vector_type) return etype; } +static gboolean +type_is_unsigned (MonoType *type) { + MonoClass *klass = mono_class_from_mono_type_internal (type); + MonoType *etype = mono_class_get_context (klass)->class_inst->type_argv [0]; + switch (etype->type) { + case MONO_TYPE_U1: + case MONO_TYPE_U2: + case MONO_TYPE_U4: + case MONO_TYPE_U8: + return TRUE; + } + return FALSE; +} + +static gboolean +type_is_float (MonoType *type) { + MonoClass *klass = mono_class_from_mono_type_internal (type); + MonoType *etype = mono_class_get_context (klass)->class_inst->type_argv [0]; + switch (etype->type) { + case MONO_TYPE_R4: + case MONO_TYPE_R8: + return TRUE; + } + return FALSE; +} + static int type_to_expand_op (MonoType *type) { @@ -357,6 +396,8 @@ emit_hardware_intrinsics ( MonoInst *ins = NULL; gboolean supported = FALSE; MonoTypeEnum arg0_type = fsig->param_count > 0 ? get_underlying_type (fsig->params [0]) : MONO_TYPE_VOID; + uint16_t op = 0; + uint16_t c0 = 0; if (intrin_group) { const SimdIntrinsic *intrinsics = intrin_group->intrinsics; int intrinsics_size = intrin_group->intrinsics_size; @@ -375,15 +416,32 @@ emit_hardware_intrinsics ( else supported = TRUE; -#if defined(TARGET_ARM64) - // HACK: Mark AdvSimd as unsupported until it's completely implemented - if (feature == MONO_CPU_ARM64_NEON) supported = FALSE; -#endif - id = info->id; - if (info->op != 0) - return emit_simd_ins_for_sig (cfg, klass, info->op, info->instc0, arg0_type, fsig, args); + op = info->default_op; + c0 = info->default_instc0; + gboolean is_unsigned = FALSE; + gboolean is_float = FALSE; + switch (arg0_type) { + case MONO_TYPE_U1: + case MONO_TYPE_U2: + case MONO_TYPE_U4: + case MONO_TYPE_U8: + is_unsigned = TRUE; + break; + case MONO_TYPE_R4: + case MONO_TYPE_R8: + is_float = TRUE; + break; + } + if (is_unsigned && info->unsigned_op != 0) { + op = info->unsigned_op; + c0 = info->unsigned_instc0; + } else if (is_float && info->floating_op != 0) { + op = info->floating_op; + c0 = info->floating_instc0; + } + } support_probe_complete: if (id == SN_get_IsSupported) { @@ -401,6 +459,8 @@ emit_hardware_intrinsics ( return NULL; } } + if (op != 0) + return emit_simd_ins_for_sig (cfg, klass, op, c0, arg0_type, fsig, args); return custom_emit (cfg, fsig, args, klass, intrin_group, info, id, arg0_type, is_64bit); } @@ -436,6 +496,28 @@ static guint16 sri_vector_methods [] = { SN_CreateScalarUnsafe, }; +static gboolean +is_elementwise_create_overload (MonoMethodSignature *fsig, MonoType *ret_type) +{ + uint16_t param_count = fsig->param_count; + if (param_count < 1) return FALSE; + MonoType *type = fsig->params [0]; + gboolean is_vector_primitive = MONO_TYPE_IS_PRIMITIVE (type) && (type->type >= MONO_TYPE_I1 && type->type <= MONO_TYPE_R8); + if (!is_vector_primitive) return FALSE; + if (!mono_metadata_type_equal (ret_type, type)) return FALSE; + for (uint16_t i = 1; i < param_count; ++i) + if (!mono_metadata_type_equal (type, fsig->params [i])) return FALSE; + return TRUE; +} + +static gboolean +is_create_from_half_vectors_overload (MonoMethodSignature *fsig) +{ + if (fsig->param_count != 2) return FALSE; + if (!is_intrinsics_vector_type (fsig->params [0])) return FALSE; + return mono_metadata_type_equal (fsig->params [0], fsig->params [1]); +} + static MonoInst* emit_sri_vector (MonoCompile *cfg, MonoMethod *cmethod, MonoMethodSignature *fsig, MonoInst **args) { @@ -468,8 +550,11 @@ emit_sri_vector (MonoCompile *cfg, MonoMethod *cmethod, MonoMethodSignature *fsi MonoType *etype = get_vector_t_elem_type (fsig->ret); if (fsig->param_count == 1 && mono_metadata_type_equal (fsig->params [0], etype)) return emit_simd_ins (cfg, klass, type_to_expand_op (etype), args [0]->dreg, -1); - else + else if (is_create_from_half_vectors_overload (fsig)) + return emit_simd_ins (cfg, klass, OP_XCONCAT, args [0]->dreg, args [1]->dreg); + else if (is_elementwise_create_overload (fsig, etype)) return emit_vector_create_elementwise (cfg, fsig, fsig->ret, etype, args); + break; } case SN_CreateScalarUnsafe: return emit_simd_ins_for_sig (cfg, klass, OP_CREATE_SCALAR_UNSAFE, -1, arg0_type, fsig, args); @@ -904,13 +989,28 @@ emit_invalid_operation (MonoCompile *cfg, const char* message) #ifdef TARGET_ARM64 +static int +type_to_extract_var_op (MonoTypeEnum type) +{ + switch (type) { + case MONO_TYPE_I1: return OP_EXTRACT_VAR_U1; + case MONO_TYPE_U1: return OP_EXTRACT_VAR_I1; + case MONO_TYPE_I2: return OP_EXTRACT_VAR_U2; + case MONO_TYPE_U2: return OP_EXTRACT_VAR_I2; + case MONO_TYPE_I4: case MONO_TYPE_U4: return OP_EXTRACT_VAR_I4; + case MONO_TYPE_I8: case MONO_TYPE_U8: return OP_EXTRACT_VAR_I8; + case MONO_TYPE_R4: return OP_EXTRACT_VAR_R4; + case MONO_TYPE_R8: return OP_EXTRACT_VAR_R8; + default: g_assert_not_reached (); + } +} static SimdIntrinsic armbase_methods [] = { {SN_LeadingSignCount}, {SN_LeadingZeroCount}, {SN_MultiplyHigh}, {SN_ReverseElementBits}, - {SN_get_IsSupported} + {SN_get_IsSupported}, }; static SimdIntrinsic crc32_methods [] = { @@ -947,36 +1047,376 @@ static SimdIntrinsic sha256_methods [] = { {SN_get_IsSupported} }; +// This table must be kept in sorted order. ASCII } is sorted after alphanumeric +// characters, so blind use of your editor's "sort lines" facility will +// mis-order the lines. +// +// In Vim you can use `sort /.*{[0-9A-z]*/ r` to sort this table. + static SimdIntrinsic advsimd_methods [] = { - {SN_Abs}, - {SN_AbsSaturate}, - {SN_AbsScalar}, + {SN_Abs, OP_XOP_OVR_X_X, INTRINS_AARCH64_ADV_SIMD_ABS, None, None, OP_XOP_OVR_X_X, INTRINS_AARCH64_ADV_SIMD_FABS}, + {SN_AbsSaturate, OP_XOP_OVR_X_X, INTRINS_AARCH64_ADV_SIMD_SQABS}, + {SN_AbsSaturateScalar, OP_XOP_OVR_SCALAR_X_X, INTRINS_AARCH64_ADV_SIMD_SQABS}, + {SN_AbsScalar, OP_XOP_OVR_SCALAR_X_X, INTRINS_AARCH64_ADV_SIMD_ABS, None, None, OP_XOP_OVR_SCALAR_X_X, INTRINS_AARCH64_ADV_SIMD_FABS}, {SN_AbsoluteCompareGreaterThan}, {SN_AbsoluteCompareGreaterThanOrEqual}, + {SN_AbsoluteCompareGreaterThanOrEqualScalar}, + {SN_AbsoluteCompareGreaterThanScalar}, {SN_AbsoluteCompareLessThan}, {SN_AbsoluteCompareLessThanOrEqual}, + {SN_AbsoluteCompareLessThanOrEqualScalar}, + {SN_AbsoluteCompareLessThanScalar}, + {SN_AbsoluteDifference, OP_ARM64_SABD, None, OP_ARM64_UABD, None, OP_XOP_OVR_X_X_X, INTRINS_AARCH64_ADV_SIMD_FABD}, + {SN_AbsoluteDifferenceAdd, OP_ARM64_SABA, None, OP_ARM64_UABA}, + {SN_AbsoluteDifferenceScalar, OP_XOP_OVR_SCALAR_X_X_X, INTRINS_AARCH64_ADV_SIMD_FABD_SCALAR}, + {SN_AbsoluteDifferenceWideningLower, OP_ARM64_SABDL, None, OP_ARM64_UABDL}, + {SN_AbsoluteDifferenceWideningLowerAndAdd, OP_ARM64_SABAL, None, OP_ARM64_UABAL}, + {SN_AbsoluteDifferenceWideningUpper, OP_ARM64_SABDL2, None, OP_ARM64_UABDL2}, + {SN_AbsoluteDifferenceWideningUpperAndAdd, OP_ARM64_SABAL2, None, OP_ARM64_UABAL2}, + {SN_Add, OP_XBINOP, OP_IADD, None, None, OP_XBINOP, OP_FADD}, + {SN_AddAcross, OP_ARM64_XHORIZ, INTRINS_AARCH64_ADV_SIMD_SADDV, OP_ARM64_XHORIZ, INTRINS_AARCH64_ADV_SIMD_UADDV}, + {SN_AddAcrossWidening, OP_ARM64_SADDLV, None, OP_ARM64_UADDLV}, + {SN_AddHighNarrowingLower, OP_ARM64_ADDHN}, + {SN_AddHighNarrowingUpper, OP_ARM64_ADDHN2}, + {SN_AddPairwise, OP_XOP_OVR_X_X_X, INTRINS_AARCH64_ADV_SIMD_ADDP, None, None, OP_XOP_OVR_X_X_X, INTRINS_AARCH64_ADV_SIMD_FADDP}, + {SN_AddPairwiseScalar, OP_ARM64_ADDP_SCALAR, None, None, None, OP_ARM64_FADDP_SCALAR}, + {SN_AddPairwiseWidening, OP_XOP_OVR_X_X, INTRINS_AARCH64_ADV_SIMD_SADDLP, OP_XOP_OVR_X_X, INTRINS_AARCH64_ADV_SIMD_UADDLP}, + {SN_AddPairwiseWideningAndAdd, OP_ARM64_SADALP, None, OP_ARM64_UADALP}, + {SN_AddPairwiseWideningAndAddScalar, OP_ARM64_SADALP, None, OP_ARM64_UADALP}, + {SN_AddPairwiseWideningScalar, OP_XOP_OVR_X_X, INTRINS_AARCH64_ADV_SIMD_SADDLP, OP_XOP_OVR_X_X, INTRINS_AARCH64_ADV_SIMD_UADDLP}, + {SN_AddRoundedHighNarrowingLower, OP_ARM64_RADDHN}, + {SN_AddRoundedHighNarrowingUpper, OP_ARM64_RADDHN2}, + {SN_AddSaturate}, + {SN_AddSaturateScalar}, + {SN_AddScalar, OP_XBINOP_SCALAR, OP_IADD, None, None, OP_XBINOP_SCALAR, OP_FADD}, + {SN_AddWideningLower, OP_ARM64_SADD, None, OP_ARM64_UADD}, + {SN_AddWideningUpper, OP_ARM64_SADD2, None, OP_ARM64_UADD2}, + {SN_And, OP_XBINOP_FORCEINT, XBINOP_FORCEINT_and}, + {SN_BitwiseClear, OP_ARM64_BIC}, + {SN_BitwiseSelect, OP_ARM64_BSL}, + {SN_Ceiling, OP_XOP_OVR_X_X, INTRINS_AARCH64_ADV_SIMD_FRINTP}, + {SN_CeilingScalar, OP_XOP_OVR_SCALAR_X_X, INTRINS_AARCH64_ADV_SIMD_FRINTP}, + {SN_CompareEqual, OP_XCOMPARE, CMP_EQ, OP_XCOMPARE, CMP_EQ, OP_XCOMPARE_FP, CMP_EQ}, + {SN_CompareEqualScalar, OP_XCOMPARE_SCALAR, CMP_EQ, OP_XCOMPARE_SCALAR, CMP_EQ, OP_XCOMPARE_FP_SCALAR, CMP_EQ}, + {SN_CompareGreaterThan, OP_XCOMPARE, CMP_GT, OP_XCOMPARE, CMP_GT_UN, OP_XCOMPARE_FP, CMP_GT}, + {SN_CompareGreaterThanOrEqual, OP_XCOMPARE, CMP_GE, OP_XCOMPARE, CMP_GE_UN, OP_XCOMPARE_FP, CMP_GE}, + {SN_CompareGreaterThanOrEqualScalar, OP_XCOMPARE_SCALAR, CMP_GE, OP_XCOMPARE_SCALAR, CMP_GE_UN, OP_XCOMPARE_FP_SCALAR, CMP_GE}, + {SN_CompareGreaterThanScalar, OP_XCOMPARE_SCALAR, CMP_GT, OP_XCOMPARE_SCALAR, CMP_GT_UN, OP_XCOMPARE_FP_SCALAR, CMP_GT}, + {SN_CompareLessThan, OP_XCOMPARE, CMP_LT, OP_XCOMPARE, CMP_LT_UN, OP_XCOMPARE_FP, CMP_LT}, + {SN_CompareLessThanOrEqual, OP_XCOMPARE, CMP_LE, OP_XCOMPARE, CMP_LE_UN, OP_XCOMPARE_FP, CMP_LE}, + {SN_CompareLessThanOrEqualScalar, OP_XCOMPARE_SCALAR, CMP_LE, OP_XCOMPARE_SCALAR, CMP_LE_UN, OP_XCOMPARE_FP_SCALAR, CMP_LE}, + {SN_CompareLessThanScalar, OP_XCOMPARE_SCALAR, CMP_LT, OP_XCOMPARE_SCALAR, CMP_LT_UN, OP_XCOMPARE_FP_SCALAR, CMP_LT}, + {SN_CompareTest, OP_ARM64_CMTST}, + {SN_CompareTestScalar, OP_ARM64_CMTST}, + {SN_ConvertToDouble, OP_ARM64_SCVTF, None, OP_ARM64_UCVTF, None, OP_ARM64_FCVTL}, + {SN_ConvertToDoubleScalar, OP_ARM64_SCVTF_SCALAR, None, OP_ARM64_UCVTF_SCALAR}, + {SN_ConvertToDoubleUpper, OP_ARM64_FCVTL2}, + {SN_ConvertToInt32RoundAwayFromZero, OP_XOP_OVR_X_X, INTRINS_AARCH64_ADV_SIMD_FCVTAS}, + {SN_ConvertToInt32RoundAwayFromZeroScalar, OP_XOP_OVR_SCALAR_X_X, INTRINS_AARCH64_ADV_SIMD_FCVTAS}, + {SN_ConvertToInt32RoundToEven, OP_XOP_OVR_X_X, INTRINS_AARCH64_ADV_SIMD_FCVTNS}, + {SN_ConvertToInt32RoundToEvenScalar, OP_XOP_OVR_SCALAR_X_X, INTRINS_AARCH64_ADV_SIMD_FCVTNS}, + {SN_ConvertToInt32RoundToNegativeInfinity, OP_XOP_OVR_X_X, INTRINS_AARCH64_ADV_SIMD_FCVTMS}, + {SN_ConvertToInt32RoundToNegativeInfinityScalar, OP_XOP_OVR_SCALAR_X_X, INTRINS_AARCH64_ADV_SIMD_FCVTMS}, + {SN_ConvertToInt32RoundToPositiveInfinity, OP_XOP_OVR_X_X, INTRINS_AARCH64_ADV_SIMD_FCVTPS}, + {SN_ConvertToInt32RoundToPositiveInfinityScalar, OP_XOP_OVR_SCALAR_X_X, INTRINS_AARCH64_ADV_SIMD_FCVTPS}, + {SN_ConvertToInt32RoundToZero, OP_ARM64_FCVTZS}, + {SN_ConvertToInt32RoundToZeroScalar, OP_ARM64_FCVTZS_SCALAR}, + {SN_ConvertToInt64RoundAwayFromZero, OP_XOP_OVR_X_X, INTRINS_AARCH64_ADV_SIMD_FCVTAS}, + {SN_ConvertToInt64RoundAwayFromZeroScalar, OP_XOP_OVR_SCALAR_X_X, INTRINS_AARCH64_ADV_SIMD_FCVTAS}, + {SN_ConvertToInt64RoundToEven, OP_XOP_OVR_X_X, INTRINS_AARCH64_ADV_SIMD_FCVTNS}, + {SN_ConvertToInt64RoundToEvenScalar, OP_XOP_OVR_SCALAR_X_X, INTRINS_AARCH64_ADV_SIMD_FCVTNS}, + {SN_ConvertToInt64RoundToNegativeInfinity, OP_XOP_OVR_X_X, INTRINS_AARCH64_ADV_SIMD_FCVTMS}, + {SN_ConvertToInt64RoundToNegativeInfinityScalar, OP_XOP_OVR_SCALAR_X_X, INTRINS_AARCH64_ADV_SIMD_FCVTMS}, + {SN_ConvertToInt64RoundToPositiveInfinity, OP_XOP_OVR_X_X, INTRINS_AARCH64_ADV_SIMD_FCVTPS}, + {SN_ConvertToInt64RoundToPositiveInfinityScalar, OP_XOP_OVR_SCALAR_X_X, INTRINS_AARCH64_ADV_SIMD_FCVTPS}, + {SN_ConvertToInt64RoundToZero, OP_ARM64_FCVTZS}, + {SN_ConvertToInt64RoundToZeroScalar, OP_ARM64_FCVTZS_SCALAR}, + {SN_ConvertToSingle, OP_ARM64_SCVTF, None, OP_ARM64_UCVTF}, + {SN_ConvertToSingleLower, OP_ARM64_FCVTN}, + {SN_ConvertToSingleRoundToOddLower, OP_ARM64_FCVTXN}, + {SN_ConvertToSingleRoundToOddUpper, OP_ARM64_FCVTXN2}, + {SN_ConvertToSingleScalar, OP_ARM64_SCVTF_SCALAR, None, OP_ARM64_UCVTF_SCALAR}, + {SN_ConvertToSingleUpper, OP_ARM64_FCVTN2}, + {SN_ConvertToUInt32RoundAwayFromZero, OP_XOP_OVR_X_X, INTRINS_AARCH64_ADV_SIMD_FCVTAU}, + {SN_ConvertToUInt32RoundAwayFromZeroScalar, OP_XOP_OVR_SCALAR_X_X, INTRINS_AARCH64_ADV_SIMD_FCVTAU}, + {SN_ConvertToUInt32RoundToEven, OP_XOP_OVR_X_X, INTRINS_AARCH64_ADV_SIMD_FCVTNU}, + {SN_ConvertToUInt32RoundToEvenScalar, OP_XOP_OVR_SCALAR_X_X, INTRINS_AARCH64_ADV_SIMD_FCVTNU}, + {SN_ConvertToUInt32RoundToNegativeInfinity, OP_XOP_OVR_X_X, INTRINS_AARCH64_ADV_SIMD_FCVTMU}, + {SN_ConvertToUInt32RoundToNegativeInfinityScalar, OP_XOP_OVR_SCALAR_X_X, INTRINS_AARCH64_ADV_SIMD_FCVTMU}, + {SN_ConvertToUInt32RoundToPositiveInfinity, OP_XOP_OVR_X_X, INTRINS_AARCH64_ADV_SIMD_FCVTPU}, + {SN_ConvertToUInt32RoundToPositiveInfinityScalar, OP_XOP_OVR_SCALAR_X_X, INTRINS_AARCH64_ADV_SIMD_FCVTPU}, + {SN_ConvertToUInt32RoundToZero, OP_ARM64_FCVTZU}, + {SN_ConvertToUInt32RoundToZeroScalar, OP_ARM64_FCVTZU_SCALAR}, + {SN_ConvertToUInt64RoundAwayFromZero, OP_XOP_OVR_X_X, INTRINS_AARCH64_ADV_SIMD_FCVTAU}, + {SN_ConvertToUInt64RoundAwayFromZeroScalar, OP_XOP_OVR_SCALAR_X_X, INTRINS_AARCH64_ADV_SIMD_FCVTAU}, + {SN_ConvertToUInt64RoundToEven, OP_XOP_OVR_X_X, INTRINS_AARCH64_ADV_SIMD_FCVTNU}, + {SN_ConvertToUInt64RoundToEvenScalar, OP_XOP_OVR_SCALAR_X_X, INTRINS_AARCH64_ADV_SIMD_FCVTNU}, + {SN_ConvertToUInt64RoundToNegativeInfinity, OP_XOP_OVR_X_X, INTRINS_AARCH64_ADV_SIMD_FCVTMU}, + {SN_ConvertToUInt64RoundToNegativeInfinityScalar, OP_XOP_OVR_SCALAR_X_X, INTRINS_AARCH64_ADV_SIMD_FCVTMU}, + {SN_ConvertToUInt64RoundToPositiveInfinity, OP_XOP_OVR_X_X, INTRINS_AARCH64_ADV_SIMD_FCVTPU}, + {SN_ConvertToUInt64RoundToPositiveInfinityScalar, OP_XOP_OVR_SCALAR_X_X, INTRINS_AARCH64_ADV_SIMD_FCVTPU}, + {SN_ConvertToUInt64RoundToZero, OP_ARM64_FCVTZU}, + {SN_ConvertToUInt64RoundToZeroScalar, OP_ARM64_FCVTZU_SCALAR}, + {SN_Divide, OP_XBINOP, OP_FDIV}, + {SN_DivideScalar, OP_XBINOP_SCALAR, OP_FDIV}, + {SN_DuplicateSelectedScalarToVector128}, + {SN_DuplicateSelectedScalarToVector64}, + {SN_DuplicateToVector128}, + {SN_DuplicateToVector64}, + {SN_Extract}, + {SN_ExtractNarrowingLower, OP_ARM64_XTN}, + {SN_ExtractNarrowingSaturateLower, OP_XOP_OVR_X_X, INTRINS_AARCH64_ADV_SIMD_SQXTN, OP_XOP_OVR_X_X, INTRINS_AARCH64_ADV_SIMD_UQXTN}, + {SN_ExtractNarrowingSaturateScalar, OP_ARM64_XNARROW_SCALAR, INTRINS_AARCH64_ADV_SIMD_SQXTN, OP_ARM64_XNARROW_SCALAR, INTRINS_AARCH64_ADV_SIMD_UQXTN}, + {SN_ExtractNarrowingSaturateUnsignedLower, OP_XOP_OVR_X_X, INTRINS_AARCH64_ADV_SIMD_SQXTUN}, + {SN_ExtractNarrowingSaturateUnsignedScalar, OP_ARM64_XNARROW_SCALAR, INTRINS_AARCH64_ADV_SIMD_SQXTUN}, + {SN_ExtractNarrowingSaturateUnsignedUpper, OP_ARM64_SQXTUN2}, + {SN_ExtractNarrowingSaturateUpper, OP_ARM64_SQXTN2, None, OP_ARM64_UQXTN2}, + {SN_ExtractNarrowingUpper, OP_ARM64_XTN2}, + {SN_ExtractVector128, OP_ARM64_EXT}, + {SN_ExtractVector64, OP_ARM64_EXT}, + {SN_Floor, OP_XOP_OVR_X_X, INTRINS_AARCH64_ADV_SIMD_FRINTM}, + {SN_FloorScalar, OP_XOP_OVR_SCALAR_X_X, INTRINS_AARCH64_ADV_SIMD_FRINTM}, + {SN_FusedAddHalving, OP_XOP_OVR_X_X_X, INTRINS_AARCH64_ADV_SIMD_SHADD, OP_XOP_OVR_X_X_X, INTRINS_AARCH64_ADV_SIMD_UHADD}, + {SN_FusedAddRoundedHalving, OP_XOP_OVR_X_X_X, INTRINS_AARCH64_ADV_SIMD_SRHADD, OP_XOP_OVR_X_X_X, INTRINS_AARCH64_ADV_SIMD_URHADD}, + {SN_FusedMultiplyAdd, OP_ARM64_FMADD}, + {SN_FusedMultiplyAddByScalar, OP_ARM64_FMADD_BYSCALAR}, + {SN_FusedMultiplyAddBySelectedScalar}, + {SN_FusedMultiplyAddNegatedScalar, OP_ARM64_FNMADD_SCALAR}, + {SN_FusedMultiplyAddScalar, OP_ARM64_FMADD_SCALAR}, + {SN_FusedMultiplyAddScalarBySelectedScalar}, + {SN_FusedMultiplySubtract, OP_ARM64_FMSUB}, + {SN_FusedMultiplySubtractByScalar, OP_ARM64_FMSUB_BYSCALAR}, + {SN_FusedMultiplySubtractBySelectedScalar}, + {SN_FusedMultiplySubtractNegatedScalar, OP_ARM64_FNMSUB_SCALAR}, + {SN_FusedMultiplySubtractScalar, OP_ARM64_FMSUB_SCALAR}, + {SN_FusedMultiplySubtractScalarBySelectedScalar}, + {SN_FusedSubtractHalving, OP_XOP_OVR_X_X_X, INTRINS_AARCH64_ADV_SIMD_SHSUB, OP_XOP_OVR_X_X_X, INTRINS_AARCH64_ADV_SIMD_UHSUB}, + {SN_Insert}, + {SN_InsertScalar}, + {SN_InsertSelectedScalar}, + {SN_LeadingSignCount, OP_XOP_OVR_X_X, INTRINS_AARCH64_ADV_SIMD_CLS}, + {SN_LeadingZeroCount, OP_ARM64_CLZ}, + {SN_LoadAndInsertScalar, OP_ARM64_LD1_INSERT}, + {SN_LoadAndReplicateToVector128, OP_ARM64_LD1R}, + {SN_LoadAndReplicateToVector64, OP_ARM64_LD1R}, + {SN_LoadVector128, OP_ARM64_LD1}, + {SN_LoadVector64, OP_ARM64_LD1}, + {SN_Max, OP_XOP_OVR_X_X_X, INTRINS_AARCH64_ADV_SIMD_SMAX, OP_XOP_OVR_X_X_X, INTRINS_AARCH64_ADV_SIMD_UMAX, OP_XOP_OVR_X_X_X, INTRINS_AARCH64_ADV_SIMD_FMAX}, + {SN_MaxAcross, OP_ARM64_XHORIZ, INTRINS_AARCH64_ADV_SIMD_SMAXV, OP_ARM64_XHORIZ, INTRINS_AARCH64_ADV_SIMD_UMAXV, OP_ARM64_XHORIZ, INTRINS_AARCH64_ADV_SIMD_FMAXV}, + {SN_MaxNumber, OP_XOP_OVR_X_X_X, INTRINS_AARCH64_ADV_SIMD_FMAXNM}, + {SN_MaxNumberAcross, OP_ARM64_XHORIZ, INTRINS_AARCH64_ADV_SIMD_FMAXNMV}, + {SN_MaxNumberPairwise, OP_XOP_OVR_X_X_X, INTRINS_AARCH64_ADV_SIMD_FMAXNMP}, + {SN_MaxNumberPairwiseScalar, OP_ARM64_XHORIZ, INTRINS_AARCH64_ADV_SIMD_FMAXNMV}, + {SN_MaxNumberScalar, OP_XOP_OVR_SCALAR_X_X_X, INTRINS_AARCH64_ADV_SIMD_FMAXNM}, + {SN_MaxPairwise, OP_XOP_OVR_X_X_X, INTRINS_AARCH64_ADV_SIMD_SMAXP, OP_XOP_OVR_X_X_X, INTRINS_AARCH64_ADV_SIMD_UMAXP, OP_XOP_OVR_X_X_X, INTRINS_AARCH64_ADV_SIMD_FMAXP}, + {SN_MaxPairwiseScalar, OP_ARM64_XHORIZ, INTRINS_AARCH64_ADV_SIMD_FMAXV}, + {SN_MaxScalar, OP_XOP_OVR_SCALAR_X_X_X, INTRINS_AARCH64_ADV_SIMD_FMAX}, + {SN_Min, OP_XOP_OVR_X_X_X, INTRINS_AARCH64_ADV_SIMD_SMIN, OP_XOP_OVR_X_X_X, INTRINS_AARCH64_ADV_SIMD_UMIN, OP_XOP_OVR_X_X_X, INTRINS_AARCH64_ADV_SIMD_FMIN}, + {SN_MinAcross, OP_ARM64_XHORIZ, INTRINS_AARCH64_ADV_SIMD_SMINV, OP_ARM64_XHORIZ, INTRINS_AARCH64_ADV_SIMD_UMINV, OP_ARM64_XHORIZ, INTRINS_AARCH64_ADV_SIMD_FMINV}, + {SN_MinNumber, OP_XOP_OVR_X_X_X, INTRINS_AARCH64_ADV_SIMD_FMINNM}, + {SN_MinNumberAcross, OP_ARM64_XHORIZ, INTRINS_AARCH64_ADV_SIMD_FMINNMV}, + {SN_MinNumberPairwise, OP_XOP_OVR_X_X_X, INTRINS_AARCH64_ADV_SIMD_FMINNMP}, + {SN_MinNumberPairwiseScalar, OP_ARM64_XHORIZ, INTRINS_AARCH64_ADV_SIMD_FMINNMV}, + {SN_MinNumberScalar, OP_XOP_OVR_SCALAR_X_X_X, INTRINS_AARCH64_ADV_SIMD_FMINNM}, + {SN_MinPairwise, OP_XOP_OVR_X_X_X, INTRINS_AARCH64_ADV_SIMD_SMINP, OP_XOP_OVR_X_X_X, INTRINS_AARCH64_ADV_SIMD_UMINP, OP_XOP_OVR_X_X_X, INTRINS_AARCH64_ADV_SIMD_FMINP}, + {SN_MinPairwiseScalar, OP_ARM64_XHORIZ, INTRINS_AARCH64_ADV_SIMD_FMINV}, + {SN_MinScalar, OP_XOP_OVR_SCALAR_X_X_X, INTRINS_AARCH64_ADV_SIMD_FMIN}, + {SN_Multiply, OP_XBINOP, OP_IMUL, None, None, OP_XBINOP, OP_FMUL}, + {SN_MultiplyAdd, OP_ARM64_MLA}, + {SN_MultiplyAddByScalar, OP_ARM64_MLA_SCALAR}, + {SN_MultiplyAddBySelectedScalar}, + {SN_MultiplyByScalar, OP_XBINOP_BYSCALAR, OP_IMUL, None, None, OP_XBINOP_BYSCALAR, OP_FMUL}, + {SN_MultiplyBySelectedScalar}, + {SN_MultiplyBySelectedScalarWideningLower}, + {SN_MultiplyBySelectedScalarWideningLowerAndAdd}, + {SN_MultiplyBySelectedScalarWideningLowerAndSubtract}, + {SN_MultiplyBySelectedScalarWideningUpper}, + {SN_MultiplyBySelectedScalarWideningUpperAndAdd}, + {SN_MultiplyBySelectedScalarWideningUpperAndSubtract}, + {SN_MultiplyDoublingByScalarSaturateHigh, OP_XOP_OVR_BYSCALAR_X_X_X, INTRINS_AARCH64_ADV_SIMD_SQDMULH}, + {SN_MultiplyDoublingBySelectedScalarSaturateHigh}, + {SN_MultiplyDoublingSaturateHigh, OP_XOP_OVR_X_X_X, INTRINS_AARCH64_ADV_SIMD_SQDMULH}, + {SN_MultiplyDoublingSaturateHighScalar, OP_XOP_OVR_SCALAR_X_X_X, INTRINS_AARCH64_ADV_SIMD_SQDMULH}, + {SN_MultiplyDoublingScalarBySelectedScalarSaturateHigh}, + {SN_MultiplyDoublingWideningAndAddSaturateScalar, OP_ARM64_SQDMLAL_SCALAR}, + {SN_MultiplyDoublingWideningAndSubtractSaturateScalar, OP_ARM64_SQDMLSL_SCALAR}, + {SN_MultiplyDoublingWideningLowerAndAddSaturate, OP_ARM64_SQDMLAL}, + {SN_MultiplyDoublingWideningLowerAndSubtractSaturate, OP_ARM64_SQDMLSL}, + {SN_MultiplyDoublingWideningLowerByScalarAndAddSaturate, OP_ARM64_SQDMLAL_BYSCALAR}, + {SN_MultiplyDoublingWideningLowerByScalarAndSubtractSaturate, OP_ARM64_SQDMLSL_BYSCALAR}, + {SN_MultiplyDoublingWideningLowerBySelectedScalarAndAddSaturate}, + {SN_MultiplyDoublingWideningLowerBySelectedScalarAndSubtractSaturate}, + {SN_MultiplyDoublingWideningSaturateLower, OP_ARM64_SQDMULL}, + {SN_MultiplyDoublingWideningSaturateLowerByScalar, OP_ARM64_SQDMULL_BYSCALAR}, + {SN_MultiplyDoublingWideningSaturateLowerBySelectedScalar}, + {SN_MultiplyDoublingWideningSaturateScalar, OP_ARM64_SQDMULL_SCALAR}, + {SN_MultiplyDoublingWideningSaturateScalarBySelectedScalar}, + {SN_MultiplyDoublingWideningSaturateUpper, OP_ARM64_SQDMULL2}, + {SN_MultiplyDoublingWideningSaturateUpperByScalar, OP_ARM64_SQDMULL2_BYSCALAR}, + {SN_MultiplyDoublingWideningSaturateUpperBySelectedScalar}, + {SN_MultiplyDoublingWideningScalarBySelectedScalarAndAddSaturate}, + {SN_MultiplyDoublingWideningScalarBySelectedScalarAndSubtractSaturate}, + {SN_MultiplyDoublingWideningUpperAndAddSaturate, OP_ARM64_SQDMLAL2}, + {SN_MultiplyDoublingWideningUpperAndSubtractSaturate, OP_ARM64_SQDMLSL2}, + {SN_MultiplyDoublingWideningUpperByScalarAndAddSaturate, OP_ARM64_SQDMLAL2_BYSCALAR}, + {SN_MultiplyDoublingWideningUpperByScalarAndSubtractSaturate, OP_ARM64_SQDMLSL2_BYSCALAR}, + {SN_MultiplyDoublingWideningUpperBySelectedScalarAndAddSaturate}, + {SN_MultiplyDoublingWideningUpperBySelectedScalarAndSubtractSaturate}, + {SN_MultiplyExtended, OP_XOP_OVR_X_X_X, INTRINS_AARCH64_ADV_SIMD_FMULX}, + {SN_MultiplyExtendedByScalar, OP_XOP_OVR_BYSCALAR_X_X_X, INTRINS_AARCH64_ADV_SIMD_FMULX}, + {SN_MultiplyExtendedBySelectedScalar}, + {SN_MultiplyExtendedScalar, OP_XOP_OVR_SCALAR_X_X_X, INTRINS_AARCH64_ADV_SIMD_FMULX}, + {SN_MultiplyExtendedScalarBySelectedScalar}, + {SN_MultiplyRoundedDoublingByScalarSaturateHigh, OP_XOP_OVR_BYSCALAR_X_X_X, INTRINS_AARCH64_ADV_SIMD_SQRDMULH}, + {SN_MultiplyRoundedDoublingBySelectedScalarSaturateHigh}, + {SN_MultiplyRoundedDoublingSaturateHigh, OP_XOP_OVR_X_X_X, INTRINS_AARCH64_ADV_SIMD_SQRDMULH}, + {SN_MultiplyRoundedDoublingSaturateHighScalar, OP_XOP_OVR_SCALAR_X_X_X, INTRINS_AARCH64_ADV_SIMD_SQRDMULH}, + {SN_MultiplyRoundedDoublingScalarBySelectedScalarSaturateHigh}, + {SN_MultiplyScalar, OP_XBINOP_SCALAR, OP_FMUL}, + {SN_MultiplyScalarBySelectedScalar, OP_ARM64_FMUL_SEL}, + {SN_MultiplySubtract, OP_ARM64_MLS}, + {SN_MultiplySubtractByScalar, OP_ARM64_MLS_SCALAR}, + {SN_MultiplySubtractBySelectedScalar}, + {SN_MultiplyWideningLower, OP_ARM64_SMULL, None, OP_ARM64_UMULL}, + {SN_MultiplyWideningLowerAndAdd, OP_ARM64_SMLAL, None, OP_ARM64_UMLAL}, + {SN_MultiplyWideningLowerAndSubtract, OP_ARM64_SMLSL, None, OP_ARM64_UMLSL}, + {SN_MultiplyWideningUpper, OP_ARM64_SMULL2, None, OP_ARM64_UMULL2}, + {SN_MultiplyWideningUpperAndAdd, OP_ARM64_SMLAL2, None, OP_ARM64_UMLAL2}, + {SN_MultiplyWideningUpperAndSubtract, OP_ARM64_SMLSL2, None, OP_ARM64_UMLSL2}, + {SN_Negate, OP_ARM64_XNEG}, + {SN_NegateSaturate, OP_XOP_OVR_X_X, INTRINS_AARCH64_ADV_SIMD_SQNEG}, + {SN_NegateSaturateScalar, OP_XOP_OVR_SCALAR_X_X, INTRINS_AARCH64_ADV_SIMD_SQNEG}, + {SN_NegateScalar, OP_ARM64_XNEG_SCALAR}, + {SN_Not, OP_ARM64_MVN}, + {SN_Or, OP_XBINOP_FORCEINT, XBINOP_FORCEINT_or}, + {SN_OrNot, OP_XBINOP_FORCEINT, XBINOP_FORCEINT_ornot}, + {SN_PolynomialMultiply, OP_XOP_OVR_X_X_X, INTRINS_AARCH64_ADV_SIMD_PMUL}, + {SN_PolynomialMultiplyWideningLower, OP_ARM64_PMULL}, + {SN_PolynomialMultiplyWideningUpper, OP_ARM64_PMULL2}, + {SN_PopCount, OP_XOP_OVR_X_X, INTRINS_AARCH64_ADV_SIMD_CNT}, + {SN_ReciprocalEstimate, None, None, OP_XOP_OVR_X_X, INTRINS_AARCH64_ADV_SIMD_URECPE, OP_XOP_OVR_X_X, INTRINS_AARCH64_ADV_SIMD_FRECPE}, + {SN_ReciprocalEstimateScalar, OP_XOP_OVR_SCALAR_X_X, INTRINS_AARCH64_ADV_SIMD_FRECPE}, + {SN_ReciprocalExponentScalar, OP_XOP_OVR_SCALAR_X_X, INTRINS_AARCH64_ADV_SIMD_FRECPX}, + {SN_ReciprocalSquareRootEstimate, None, None, OP_XOP_OVR_X_X, INTRINS_AARCH64_ADV_SIMD_URSQRTE, OP_XOP_OVR_X_X, INTRINS_AARCH64_ADV_SIMD_FRSQRTE}, + {SN_ReciprocalSquareRootEstimateScalar, OP_XOP_OVR_SCALAR_X_X, INTRINS_AARCH64_ADV_SIMD_FRSQRTE}, + {SN_ReciprocalSquareRootStep, OP_XOP_OVR_X_X_X, INTRINS_AARCH64_ADV_SIMD_FRSQRTS}, + {SN_ReciprocalSquareRootStepScalar, OP_XOP_OVR_SCALAR_X_X_X, INTRINS_AARCH64_ADV_SIMD_FRSQRTS}, + {SN_ReciprocalStep, OP_XOP_OVR_X_X_X, INTRINS_AARCH64_ADV_SIMD_FRECPS}, + {SN_ReciprocalStepScalar, OP_XOP_OVR_SCALAR_X_X_X, INTRINS_AARCH64_ADV_SIMD_FRECPS}, + {SN_ReverseElement16, OP_ARM64_REVN, 16}, + {SN_ReverseElement32, OP_ARM64_REVN, 32}, + {SN_ReverseElement8, OP_ARM64_REVN, 8}, + {SN_ReverseElementBits, OP_XOP_OVR_X_X, INTRINS_AARCH64_ADV_SIMD_RBIT}, + {SN_RoundAwayFromZero, OP_XOP_OVR_X_X, INTRINS_AARCH64_ADV_SIMD_FRINTA}, + {SN_RoundAwayFromZeroScalar, OP_XOP_OVR_SCALAR_X_X, INTRINS_AARCH64_ADV_SIMD_FRINTA}, + {SN_RoundToNearest, OP_XOP_OVR_X_X, INTRINS_AARCH64_ADV_SIMD_FRINTN}, + {SN_RoundToNearestScalar, OP_XOP_OVR_SCALAR_X_X, INTRINS_AARCH64_ADV_SIMD_FRINTN}, + {SN_RoundToNegativeInfinity, OP_XOP_OVR_X_X, INTRINS_AARCH64_ADV_SIMD_FRINTM}, + {SN_RoundToNegativeInfinityScalar, OP_XOP_OVR_SCALAR_X_X, INTRINS_AARCH64_ADV_SIMD_FRINTM}, + {SN_RoundToPositiveInfinity, OP_XOP_OVR_X_X, INTRINS_AARCH64_ADV_SIMD_FRINTP}, + {SN_RoundToPositiveInfinityScalar, OP_XOP_OVR_SCALAR_X_X, INTRINS_AARCH64_ADV_SIMD_FRINTP}, + {SN_RoundToZero, OP_XOP_OVR_X_X, INTRINS_AARCH64_ADV_SIMD_FRINTZ}, + {SN_RoundToZeroScalar, OP_XOP_OVR_SCALAR_X_X, INTRINS_AARCH64_ADV_SIMD_FRINTZ}, + {SN_ShiftArithmetic, OP_XOP_OVR_X_X_X, INTRINS_AARCH64_ADV_SIMD_SSHL}, + {SN_ShiftArithmeticRounded, OP_XOP_OVR_X_X_X, INTRINS_AARCH64_ADV_SIMD_SRSHL}, + {SN_ShiftArithmeticRoundedSaturate, OP_XOP_OVR_X_X_X, INTRINS_AARCH64_ADV_SIMD_SQRSHL}, + {SN_ShiftArithmeticRoundedSaturateScalar, OP_XOP_OVR_SCALAR_X_X_X, INTRINS_AARCH64_ADV_SIMD_SQRSHL}, + {SN_ShiftArithmeticRoundedScalar, OP_XOP_OVR_X_X_X, INTRINS_AARCH64_ADV_SIMD_SRSHL}, + {SN_ShiftArithmeticSaturate, OP_XOP_OVR_X_X_X, INTRINS_AARCH64_ADV_SIMD_SQSHL}, + {SN_ShiftArithmeticSaturateScalar, OP_XOP_OVR_SCALAR_X_X_X, INTRINS_AARCH64_ADV_SIMD_SQSHL}, + {SN_ShiftArithmeticScalar, OP_XOP_OVR_X_X_X, INTRINS_AARCH64_ADV_SIMD_SSHL}, + {SN_ShiftLeftAndInsert, OP_ARM64_SLI}, + {SN_ShiftLeftAndInsertScalar, OP_ARM64_SLI}, + {SN_ShiftLeftLogical, OP_ARM64_SHL}, + {SN_ShiftLeftLogicalSaturate}, + {SN_ShiftLeftLogicalSaturateScalar}, + {SN_ShiftLeftLogicalSaturateUnsigned, OP_ARM64_SQSHLU}, + {SN_ShiftLeftLogicalSaturateUnsignedScalar, OP_ARM64_SQSHLU_SCALAR}, + {SN_ShiftLeftLogicalScalar, OP_ARM64_SHL}, + {SN_ShiftLeftLogicalWideningLower, OP_ARM64_SSHLL, None, OP_ARM64_USHLL}, + {SN_ShiftLeftLogicalWideningUpper, OP_ARM64_SSHLL2, None, OP_ARM64_USHLL2}, + {SN_ShiftLogical, OP_XOP_OVR_X_X_X, INTRINS_AARCH64_ADV_SIMD_USHL}, + {SN_ShiftLogicalRounded, OP_XOP_OVR_X_X_X, INTRINS_AARCH64_ADV_SIMD_URSHL}, + {SN_ShiftLogicalRoundedSaturate, OP_XOP_OVR_X_X_X, INTRINS_AARCH64_ADV_SIMD_UQRSHL}, + {SN_ShiftLogicalRoundedSaturateScalar, OP_XOP_OVR_SCALAR_X_X_X, INTRINS_AARCH64_ADV_SIMD_UQRSHL}, + {SN_ShiftLogicalRoundedScalar, OP_XOP_OVR_X_X_X, INTRINS_AARCH64_ADV_SIMD_URSHL}, + {SN_ShiftLogicalSaturate, OP_XOP_OVR_X_X_X, INTRINS_AARCH64_ADV_SIMD_UQSHL}, + {SN_ShiftLogicalSaturateScalar, OP_XOP_OVR_SCALAR_X_X_X, INTRINS_AARCH64_ADV_SIMD_UQSHL}, + {SN_ShiftLogicalScalar, OP_XOP_OVR_X_X_X, INTRINS_AARCH64_ADV_SIMD_USHL}, + {SN_ShiftRightAndInsert, OP_ARM64_SRI}, + {SN_ShiftRightAndInsertScalar, OP_ARM64_SRI}, + {SN_ShiftRightArithmetic, OP_ARM64_SSHR}, + {SN_ShiftRightArithmeticAdd, OP_ARM64_SSRA}, + {SN_ShiftRightArithmeticAddScalar, OP_ARM64_SSRA}, + {SN_ShiftRightArithmeticNarrowingSaturateLower, OP_ARM64_XNSHIFT, INTRINS_AARCH64_ADV_SIMD_SQSHRN}, + {SN_ShiftRightArithmeticNarrowingSaturateScalar, OP_ARM64_XNSHIFT_SCALAR, INTRINS_AARCH64_ADV_SIMD_SQSHRN}, + {SN_ShiftRightArithmeticNarrowingSaturateUnsignedLower, OP_ARM64_XNSHIFT, INTRINS_AARCH64_ADV_SIMD_SQSHRUN}, + {SN_ShiftRightArithmeticNarrowingSaturateUnsignedScalar, OP_ARM64_XNSHIFT_SCALAR, INTRINS_AARCH64_ADV_SIMD_SQSHRUN}, + {SN_ShiftRightArithmeticNarrowingSaturateUnsignedUpper, OP_ARM64_XNSHIFT2, INTRINS_AARCH64_ADV_SIMD_SQSHRUN}, + {SN_ShiftRightArithmeticNarrowingSaturateUpper, OP_ARM64_XNSHIFT2, INTRINS_AARCH64_ADV_SIMD_SQSHRN}, + {SN_ShiftRightArithmeticRounded, OP_ARM64_SRSHR}, + {SN_ShiftRightArithmeticRoundedAdd, OP_ARM64_SRSRA}, + {SN_ShiftRightArithmeticRoundedAddScalar, OP_ARM64_SRSRA}, + {SN_ShiftRightArithmeticRoundedNarrowingSaturateLower, OP_ARM64_XNSHIFT, INTRINS_AARCH64_ADV_SIMD_SQRSHRN}, + {SN_ShiftRightArithmeticRoundedNarrowingSaturateScalar, OP_ARM64_XNSHIFT_SCALAR, INTRINS_AARCH64_ADV_SIMD_SQRSHRN}, + {SN_ShiftRightArithmeticRoundedNarrowingSaturateUnsignedLower, OP_ARM64_XNSHIFT, INTRINS_AARCH64_ADV_SIMD_SQRSHRUN}, + {SN_ShiftRightArithmeticRoundedNarrowingSaturateUnsignedScalar, OP_ARM64_XNSHIFT_SCALAR, INTRINS_AARCH64_ADV_SIMD_SQRSHRUN}, + {SN_ShiftRightArithmeticRoundedNarrowingSaturateUnsignedUpper, OP_ARM64_XNSHIFT2, INTRINS_AARCH64_ADV_SIMD_SQRSHRUN}, + {SN_ShiftRightArithmeticRoundedNarrowingSaturateUpper, OP_ARM64_XNSHIFT2, INTRINS_AARCH64_ADV_SIMD_SQRSHRN}, + {SN_ShiftRightArithmeticRoundedScalar, OP_ARM64_SRSHR}, + {SN_ShiftRightArithmeticScalar, OP_ARM64_SSHR}, + {SN_ShiftRightLogical, OP_ARM64_USHR}, + {SN_ShiftRightLogicalAdd, OP_ARM64_USRA}, + {SN_ShiftRightLogicalAddScalar, OP_ARM64_USRA}, + {SN_ShiftRightLogicalNarrowingLower, OP_ARM64_SHRN}, + {SN_ShiftRightLogicalNarrowingSaturateLower, OP_ARM64_XNSHIFT, INTRINS_AARCH64_ADV_SIMD_UQSHRN}, + {SN_ShiftRightLogicalNarrowingSaturateScalar, OP_ARM64_XNSHIFT_SCALAR, INTRINS_AARCH64_ADV_SIMD_UQSHRN}, + {SN_ShiftRightLogicalNarrowingSaturateUpper, OP_ARM64_XNSHIFT2, INTRINS_AARCH64_ADV_SIMD_UQSHRN}, + {SN_ShiftRightLogicalNarrowingUpper, OP_ARM64_SHRN2}, + {SN_ShiftRightLogicalRounded, OP_ARM64_URSHR}, + {SN_ShiftRightLogicalRoundedAdd, OP_ARM64_URSRA}, + {SN_ShiftRightLogicalRoundedAddScalar, OP_ARM64_URSRA}, + {SN_ShiftRightLogicalRoundedNarrowingLower, OP_ARM64_XNSHIFT, INTRINS_AARCH64_ADV_SIMD_RSHRN}, + {SN_ShiftRightLogicalRoundedNarrowingSaturateLower, OP_ARM64_XNSHIFT, INTRINS_AARCH64_ADV_SIMD_UQRSHRN}, + {SN_ShiftRightLogicalRoundedNarrowingSaturateScalar, OP_ARM64_XNSHIFT_SCALAR, INTRINS_AARCH64_ADV_SIMD_UQRSHRN}, + {SN_ShiftRightLogicalRoundedNarrowingSaturateUpper, OP_ARM64_XNSHIFT2, INTRINS_AARCH64_ADV_SIMD_UQRSHRN}, + {SN_ShiftRightLogicalRoundedNarrowingUpper, OP_ARM64_XNSHIFT2, INTRINS_AARCH64_ADV_SIMD_RSHRN}, + {SN_ShiftRightLogicalRoundedScalar, OP_ARM64_URSHR}, + {SN_ShiftRightLogicalScalar, OP_ARM64_USHR}, + {SN_SignExtendWideningLower, OP_ARM64_SXTL}, + {SN_SignExtendWideningUpper, OP_ARM64_SXTL2}, + {SN_Sqrt, OP_XOP_OVR_X_X, INTRINS_AARCH64_ADV_SIMD_FSQRT}, + {SN_SqrtScalar, OP_XOP_OVR_SCALAR_X_X, INTRINS_AARCH64_ADV_SIMD_FSQRT}, + {SN_Store, OP_ARM64_ST1}, + {SN_StorePair, OP_ARM64_STP}, + {SN_StorePairNonTemporal, OP_ARM64_STNP}, + {SN_StorePairScalar, OP_ARM64_STP_SCALAR}, + {SN_StorePairScalarNonTemporal, OP_ARM64_STNP_SCALAR}, + {SN_StoreSelectedScalar, OP_ARM64_ST1_SCALAR}, + {SN_Subtract, OP_XBINOP, OP_ISUB, None, None, OP_XBINOP, OP_FSUB}, + {SN_SubtractHighNarrowingLower, OP_ARM64_SUBHN}, + {SN_SubtractHighNarrowingUpper, OP_ARM64_SUBHN2}, + {SN_SubtractRoundedHighNarrowingLower, OP_ARM64_RSUBHN}, + {SN_SubtractRoundedHighNarrowingUpper, OP_ARM64_RSUBHN2}, + {SN_SubtractSaturate, OP_XOP_OVR_X_X_X, INTRINS_AARCH64_ADV_SIMD_SQSUB, OP_XOP_OVR_X_X_X, INTRINS_AARCH64_ADV_SIMD_UQSUB}, + {SN_SubtractSaturateScalar, OP_XOP_OVR_SCALAR_X_X_X, INTRINS_AARCH64_ADV_SIMD_SQSUB, OP_XOP_OVR_SCALAR_X_X_X, INTRINS_AARCH64_ADV_SIMD_UQSUB}, + {SN_SubtractScalar, OP_XBINOP_SCALAR, OP_ISUB, None, None, OP_XBINOP_SCALAR, OP_FSUB}, + {SN_SubtractWideningLower, OP_ARM64_SSUB, None, OP_ARM64_USUB}, + {SN_SubtractWideningUpper, OP_ARM64_SSUB2, None, OP_ARM64_USUB2}, + {SN_TransposeEven, OP_ARM64_TRN1}, + {SN_TransposeOdd, OP_ARM64_TRN2}, + {SN_UnzipEven, OP_ARM64_UZP1}, + {SN_UnzipOdd, OP_ARM64_UZP2}, + {SN_VectorTableLookup, OP_XOP_OVR_X_X_X, INTRINS_AARCH64_ADV_SIMD_TBL1}, + {SN_VectorTableLookupExtension, OP_XOP_OVR_X_X_X_X, INTRINS_AARCH64_ADV_SIMD_TBX1}, + {SN_Xor, OP_XBINOP_FORCEINT, XBINOP_FORCEINT_xor}, + {SN_ZeroExtendWideningLower, OP_ARM64_UXTL}, + {SN_ZeroExtendWideningUpper, OP_ARM64_UXTL2}, + {SN_ZipHigh, OP_ARM64_ZIP2}, + {SN_ZipLow, OP_ARM64_ZIP1}, {SN_get_IsSupported}, }; -static -MonoInst *emit_absolute_compare (MonoCompile *cfg, MonoClass *klass, MonoMethodSignature *fsig, MonoTypeEnum arg0_type, MonoInst **args, SimdOp op_for_r4, SimdOp op_for_r8) -{ - SimdOp op = (SimdOp)0; - - switch (get_underlying_type (fsig->params [0])) { - case MONO_TYPE_R4: - op = op_for_r4; - break; - case MONO_TYPE_R8: - op = op_for_r8; - break; - default: - g_assert_not_reached(); - } - - return emit_simd_ins_for_sig (cfg, klass, OP_XOP_X_X_X, op, arg0_type, fsig, args); -} - static const IntrinGroup supported_arm_intrinsics [] = { { "AdvSimd", MONO_CPU_ARM64_NEON, advsimd_methods, sizeof (advsimd_methods) }, { "Aes", MONO_CPU_ARM64_CRYPTO, crypto_aes_methods, sizeof (crypto_aes_methods) }, @@ -1040,100 +1480,225 @@ emit_arm64_intrinsics ( if (feature == MONO_CPU_ARM64_NEON) { switch (id) { - case SN_Abs: { - SimdOp op = (SimdOp)0; + case SN_AbsoluteCompareGreaterThan: + case SN_AbsoluteCompareGreaterThanOrEqual: + case SN_AbsoluteCompareLessThan: + case SN_AbsoluteCompareLessThanOrEqual: + case SN_AbsoluteCompareGreaterThanScalar: + case SN_AbsoluteCompareGreaterThanOrEqualScalar: + case SN_AbsoluteCompareLessThanScalar: + case SN_AbsoluteCompareLessThanOrEqualScalar: { + gboolean reverse_args = FALSE; + gboolean use_geq = FALSE; + gboolean scalar = FALSE; + MonoInst *cmp_args [] = { args [0], args [1] }; + switch (id) { + case SN_AbsoluteCompareGreaterThanScalar: scalar = TRUE; + case SN_AbsoluteCompareGreaterThan: break; - // HACK: Temporary, while Vector64 support is completed - MonoClass *arg0_klass = mono_class_from_mono_type_internal (fsig->params [0]); - if (m_class_get_name (arg0_klass), "Vector64`1") - mono_emit_jit_icall (cfg, mono_throw_platform_not_supported, NULL); + case SN_AbsoluteCompareGreaterThanOrEqualScalar: scalar = TRUE; + case SN_AbsoluteCompareGreaterThanOrEqual: use_geq = TRUE; break; - switch (get_underlying_type (fsig->params [0])) { - case MONO_TYPE_R8: - op = SIMD_OP_ARM64_DABS; - break; - case MONO_TYPE_R4: - op = SIMD_OP_ARM64_FABS; - break; - case MONO_TYPE_I1: - op = SIMD_OP_ARM64_I8ABS; - break; - case MONO_TYPE_I2: - op = SIMD_OP_ARM64_I16ABS; - break; - case MONO_TYPE_I4: - op = SIMD_OP_ARM64_I32ABS; - break; - case MONO_TYPE_I8: - op = SIMD_OP_ARM64_I64ABS; - break; - } - } + case SN_AbsoluteCompareLessThanScalar: scalar = TRUE; + case SN_AbsoluteCompareLessThan: reverse_args = TRUE; break; - case SN_AbsoluteCompareGreaterThan: { - return emit_absolute_compare (cfg, klass, fsig, arg0_type, args, SIMD_OP_ARM64_FABSOLUTE_COMPARE_GREATER_THAN, SIMD_OP_ARM64_DABSOLUTE_COMPARE_GREATER_THAN); + case SN_AbsoluteCompareLessThanOrEqualScalar: scalar = TRUE; + case SN_AbsoluteCompareLessThanOrEqual: reverse_args = TRUE; use_geq = TRUE; break; + } + if (reverse_args) { + cmp_args [0] = args [1]; + cmp_args [1] = args [0]; + } + int iid = use_geq ? INTRINS_AARCH64_ADV_SIMD_FACGE : INTRINS_AARCH64_ADV_SIMD_FACGT; + return emit_simd_ins_for_sig (cfg, klass, OP_ARM64_ABSCOMPARE, iid, scalar, fsig, cmp_args); } - - case SN_AbsoluteCompareGreaterThanOrEqual: { - return emit_absolute_compare (cfg, klass, fsig, arg0_type, args, SIMD_OP_ARM64_FABSOLUTE_COMPARE_GREATER_THAN_OR_EQUAL, SIMD_OP_ARM64_DABSOLUTE_COMPARE_GREATER_THAN_OR_EQUAL); + case SN_AddSaturate: + case SN_AddSaturateScalar: { + gboolean arg0_unsigned = type_is_unsigned (fsig->params [0]); + gboolean arg1_unsigned = type_is_unsigned (fsig->params [1]); + int iid = 0; + if (arg0_unsigned && arg1_unsigned) + iid = INTRINS_AARCH64_ADV_SIMD_UQADD; + else if (arg0_unsigned && !arg1_unsigned) + iid = INTRINS_AARCH64_ADV_SIMD_USQADD; + else if (!arg0_unsigned && arg1_unsigned) + iid = INTRINS_AARCH64_ADV_SIMD_SUQADD; + else + iid = INTRINS_AARCH64_ADV_SIMD_SQADD; + int op = id == SN_AddSaturateScalar ? OP_XOP_OVR_SCALAR_X_X_X : OP_XOP_OVR_X_X_X; + return emit_simd_ins_for_sig (cfg, klass, op, iid, arg0_type, fsig, args); } - - case SN_AbsoluteCompareLessThan: { - // Compare less than uses the same instructions as greater than, with arguments swapped. - MonoInst *temp_for_swap = args [0]; - args [0] = args [1]; - args [1] = temp_for_swap; - - return emit_absolute_compare (cfg, klass, fsig, arg0_type, args, SIMD_OP_ARM64_FABSOLUTE_COMPARE_LESS_THAN, SIMD_OP_ARM64_DABSOLUTE_COMPARE_LESS_THAN); + case SN_DuplicateSelectedScalarToVector128: + case SN_DuplicateSelectedScalarToVector64: + case SN_DuplicateToVector64: + case SN_DuplicateToVector128: { + MonoClass *ret_klass = mono_class_from_mono_type_internal (fsig->ret); + MonoType *rtype = get_vector_t_elem_type (fsig->ret); + int scalar_src_reg = args [0]->dreg; + switch (id) { + case SN_DuplicateSelectedScalarToVector128: + case SN_DuplicateSelectedScalarToVector64: { + MonoInst *ins = emit_simd_ins (cfg, ret_klass, type_to_extract_var_op (rtype->type), args [0]->dreg, args [1]->dreg); + scalar_src_reg = ins->dreg; + break; + } + } + return emit_simd_ins (cfg, ret_klass, type_to_expand_op (rtype), scalar_src_reg, -1); } - - case SN_AbsoluteCompareLessThanOrEqual: { - // Compare less than uses the same instructions as greater than, with arguments swapped. - MonoInst *temp_for_swap = args [0]; - args [0] = args [1]; - args [1] = temp_for_swap; - - return emit_absolute_compare (cfg, klass, fsig, arg0_type, args, SIMD_OP_ARM64_FABSOLUTE_COMPARE_LESS_THAN_OR_EQUAL, SIMD_OP_ARM64_DABSOLUTE_COMPARE_LESS_THAN_OR_EQUAL); + case SN_Extract: { + int extract_op = type_to_extract_var_op (arg0_type); + MonoInst *ins = emit_simd_ins (cfg, klass, extract_op, args [0]->dreg, args [1]->dreg); + ins->inst_c1 = arg0_type; + return ins; } - - case SN_AbsSaturate: { - SimdOp op = (SimdOp)0; - switch (get_underlying_type (fsig->params [0])) { - case MONO_TYPE_I1: - op = SIMD_OP_ARM64_I8ABS_SATURATE; - break; - case MONO_TYPE_I2: - op = SIMD_OP_ARM64_I16ABS_SATURATE; - break; - case MONO_TYPE_I4: - op = SIMD_OP_ARM64_I32ABS_SATURATE; - break; - case MONO_TYPE_I8: - op = SIMD_OP_ARM64_I64ABS_SATURATE; + case SN_InsertSelectedScalar: + case SN_InsertScalar: + case SN_Insert: { + int insert_op = 0; + int extract_op = 0; + switch (arg0_type) { + case MONO_TYPE_I1: insert_op = OP_XINSERT_I1; extract_op = OP_EXTRACT_U1; break; + case MONO_TYPE_U1: insert_op = OP_XINSERT_I1; extract_op = OP_EXTRACT_I1; break; + case MONO_TYPE_I2: insert_op = OP_XINSERT_I2; extract_op = OP_EXTRACT_U2; break; + case MONO_TYPE_U2: insert_op = OP_XINSERT_I2; extract_op = OP_EXTRACT_I2; break; + case MONO_TYPE_I4: case MONO_TYPE_U4: insert_op = OP_XINSERT_I4; extract_op = OP_EXTRACT_I4; break; + case MONO_TYPE_I8: case MONO_TYPE_U8: insert_op = OP_XINSERT_I8; extract_op = OP_EXTRACT_I8; break; + case MONO_TYPE_R4: insert_op = OP_XINSERT_R4; extract_op = OP_EXTRACT_R4; break; + case MONO_TYPE_R8: insert_op = OP_XINSERT_R8; extract_op = OP_EXTRACT_R8; break; + default: g_assert_not_reached (); + } + int val_src_reg = args [2]->dreg; + switch (id) { + case SN_InsertSelectedScalar: { + MonoInst *scalar = emit_simd_ins (cfg, klass, OP_ARM64_SELECT_SCALAR, args [2]->dreg, args [3]->dreg); + val_src_reg = scalar->dreg; + // fallthrough + } + case SN_InsertScalar: { + MonoInst *ins = emit_simd_ins (cfg, klass, extract_op, val_src_reg, -1); + ins->inst_c0 = 0; + val_src_reg = ins->dreg; break; } - - return emit_simd_ins_for_sig (cfg, klass, OP_XOP_X_X, op, arg0_type, fsig, args); + } + MonoInst *ins = emit_simd_ins (cfg, klass, insert_op, args [0]->dreg, val_src_reg); + ins->sreg3 = args [1]->dreg; + ins->inst_c1 = arg0_type; + return ins; } - - case SN_AbsScalar: { - SimdOp op = (SimdOp)0; - switch (get_underlying_type (fsig->params [0])) { - case MONO_TYPE_I1: - op = SIMD_OP_ARM64_I8ABS_SATURATE; - break; - case MONO_TYPE_I2: - op = SIMD_OP_ARM64_I16ABS_SATURATE; - break; - case MONO_TYPE_I4: - op = SIMD_OP_ARM64_I32ABS_SATURATE; - break; - case MONO_TYPE_I8: - op = SIMD_OP_ARM64_I64ABS_SATURATE; - break; + case SN_ShiftLeftLogicalSaturate: + case SN_ShiftLeftLogicalSaturateScalar: { + MonoClass *ret_klass = mono_class_from_mono_type_internal (fsig->ret); + MonoType *etype = get_vector_t_elem_type (fsig->ret); + gboolean is_unsigned = type_is_unsigned (fsig->ret); + gboolean scalar = id == SN_ShiftLeftLogicalSaturateScalar; + int s2v = scalar ? OP_CREATE_SCALAR_UNSAFE : type_to_expand_op (etype); + int xop = scalar ? OP_XOP_OVR_SCALAR_X_X_X : OP_XOP_OVR_X_X_X; + int iid = is_unsigned ? INTRINS_AARCH64_ADV_SIMD_UQSHL : INTRINS_AARCH64_ADV_SIMD_SQSHL; + MonoInst *shift_vector = emit_simd_ins (cfg, ret_klass, s2v, args [1]->dreg, -1); + shift_vector->inst_c1 = etype->type; + MonoInst *ret = emit_simd_ins (cfg, ret_klass, xop, args [0]->dreg, shift_vector->dreg); + ret->inst_c0 = iid; + ret->inst_c1 = etype->type; + return ret; + } + case SN_MultiplyRoundedDoublingBySelectedScalarSaturateHigh: + case SN_MultiplyRoundedDoublingScalarBySelectedScalarSaturateHigh: + case SN_MultiplyDoublingScalarBySelectedScalarSaturateHigh: + case SN_MultiplyDoublingWideningSaturateScalarBySelectedScalar: + case SN_MultiplyExtendedBySelectedScalar: + case SN_MultiplyExtendedScalarBySelectedScalar: + case SN_MultiplyBySelectedScalar: + case SN_MultiplyBySelectedScalarWideningLower: + case SN_MultiplyBySelectedScalarWideningUpper: + case SN_MultiplyDoublingBySelectedScalarSaturateHigh: + case SN_MultiplyDoublingWideningSaturateLowerBySelectedScalar: + case SN_MultiplyDoublingWideningSaturateUpperBySelectedScalar: { + MonoClass *ret_klass = mono_class_from_mono_type_internal (fsig->ret); + gboolean is_unsigned = type_is_unsigned (fsig->ret); + gboolean is_float = type_is_float (fsig->ret); + int opcode = 0; + int c0 = 0; + switch (id) { + case SN_MultiplyRoundedDoublingBySelectedScalarSaturateHigh: opcode = OP_XOP_OVR_BYSCALAR_X_X_X; c0 = INTRINS_AARCH64_ADV_SIMD_SQRDMULH; break; + case SN_MultiplyRoundedDoublingScalarBySelectedScalarSaturateHigh: opcode = OP_XOP_OVR_SCALAR_X_X_X; c0 = INTRINS_AARCH64_ADV_SIMD_SQRDMULH; break; + case SN_MultiplyDoublingScalarBySelectedScalarSaturateHigh: opcode = OP_XOP_OVR_SCALAR_X_X_X; c0 = INTRINS_AARCH64_ADV_SIMD_SQDMULH; break; + case SN_MultiplyDoublingWideningSaturateScalarBySelectedScalar: opcode = OP_ARM64_SQDMULL_SCALAR; break; + case SN_MultiplyExtendedBySelectedScalar: opcode = OP_XOP_OVR_BYSCALAR_X_X_X; c0 = INTRINS_AARCH64_ADV_SIMD_FMULX; break; + case SN_MultiplyExtendedScalarBySelectedScalar: opcode = OP_XOP_OVR_SCALAR_X_X_X; c0 = INTRINS_AARCH64_ADV_SIMD_FMULX; break; + case SN_MultiplyBySelectedScalar: opcode = OP_XBINOP_BYSCALAR; c0 = OP_IMUL; break; + case SN_MultiplyBySelectedScalarWideningLower: opcode = OP_ARM64_SMULL_SCALAR; break; + case SN_MultiplyBySelectedScalarWideningUpper: opcode = OP_ARM64_SMULL2_SCALAR; break; + case SN_MultiplyDoublingBySelectedScalarSaturateHigh: opcode = OP_XOP_OVR_BYSCALAR_X_X_X; c0 = INTRINS_AARCH64_ADV_SIMD_SQDMULH; break; + case SN_MultiplyDoublingWideningSaturateLowerBySelectedScalar: opcode = OP_ARM64_SQDMULL_BYSCALAR; break; + case SN_MultiplyDoublingWideningSaturateUpperBySelectedScalar: opcode = OP_ARM64_SQDMULL2_BYSCALAR; break; + default: g_assert_not_reached(); } - return emit_simd_ins_for_sig (cfg, klass, OP_XOP_X_X, op, arg0_type, fsig, args); - } + if (is_unsigned) + switch (opcode) { + case OP_ARM64_SMULL_SCALAR: opcode = OP_ARM64_UMULL_SCALAR; break; + case OP_ARM64_SMULL2_SCALAR: opcode = OP_ARM64_UMULL2_SCALAR; break; + } + if (is_float) + switch (opcode) { + case OP_XBINOP_BYSCALAR: c0 = OP_FMUL; + } + MonoInst *scalar = emit_simd_ins (cfg, ret_klass, OP_ARM64_SELECT_SCALAR, args [1]->dreg, args [2]->dreg); + MonoInst *ret = emit_simd_ins (cfg, ret_klass, opcode, args [0]->dreg, scalar->dreg); + ret->inst_c0 = c0; + ret->inst_c1 = arg0_type; + return ret; + } + case SN_FusedMultiplyAddBySelectedScalar: + case SN_FusedMultiplyAddScalarBySelectedScalar: + case SN_FusedMultiplySubtractBySelectedScalar: + case SN_FusedMultiplySubtractScalarBySelectedScalar: + case SN_MultiplyDoublingWideningScalarBySelectedScalarAndAddSaturate: + case SN_MultiplyDoublingWideningScalarBySelectedScalarAndSubtractSaturate: + case SN_MultiplyAddBySelectedScalar: + case SN_MultiplySubtractBySelectedScalar: + case SN_MultiplyBySelectedScalarWideningLowerAndAdd: + case SN_MultiplyBySelectedScalarWideningLowerAndSubtract: + case SN_MultiplyBySelectedScalarWideningUpperAndAdd: + case SN_MultiplyBySelectedScalarWideningUpperAndSubtract: + case SN_MultiplyDoublingWideningLowerBySelectedScalarAndAddSaturate: + case SN_MultiplyDoublingWideningLowerBySelectedScalarAndSubtractSaturate: + case SN_MultiplyDoublingWideningUpperBySelectedScalarAndAddSaturate: + case SN_MultiplyDoublingWideningUpperBySelectedScalarAndSubtractSaturate: { + MonoClass *ret_klass = mono_class_from_mono_type_internal (fsig->ret); + gboolean is_unsigned = type_is_unsigned (fsig->ret); + int opcode = 0; + switch (id) { + case SN_FusedMultiplyAddBySelectedScalar: opcode = OP_ARM64_FMADD_BYSCALAR; break; + case SN_FusedMultiplyAddScalarBySelectedScalar: opcode = OP_ARM64_FMADD_SCALAR; break; + case SN_FusedMultiplySubtractBySelectedScalar: opcode = OP_ARM64_FMSUB_BYSCALAR; break; + case SN_FusedMultiplySubtractScalarBySelectedScalar: opcode = OP_ARM64_FMSUB_SCALAR; break; + case SN_MultiplyDoublingWideningScalarBySelectedScalarAndAddSaturate: opcode = OP_ARM64_SQDMLAL_SCALAR; break; + case SN_MultiplyDoublingWideningScalarBySelectedScalarAndSubtractSaturate: opcode = OP_ARM64_SQDMLSL_SCALAR; break; + case SN_MultiplyAddBySelectedScalar: opcode = OP_ARM64_MLA_SCALAR; break; + case SN_MultiplySubtractBySelectedScalar: opcode = OP_ARM64_MLS_SCALAR; break; + case SN_MultiplyBySelectedScalarWideningLowerAndAdd: opcode = OP_ARM64_SMLAL_SCALAR; break; + case SN_MultiplyBySelectedScalarWideningLowerAndSubtract: opcode = OP_ARM64_SMLSL_SCALAR; break; + case SN_MultiplyBySelectedScalarWideningUpperAndAdd: opcode = OP_ARM64_SMLAL2_SCALAR; break; + case SN_MultiplyBySelectedScalarWideningUpperAndSubtract: opcode = OP_ARM64_SMLSL2_SCALAR; break; + case SN_MultiplyDoublingWideningLowerBySelectedScalarAndAddSaturate: opcode = OP_ARM64_SQDMLAL_BYSCALAR; break; + case SN_MultiplyDoublingWideningLowerBySelectedScalarAndSubtractSaturate: opcode = OP_ARM64_SQDMLSL_BYSCALAR; break; + case SN_MultiplyDoublingWideningUpperBySelectedScalarAndAddSaturate: opcode = OP_ARM64_SQDMLAL2_BYSCALAR; break; + case SN_MultiplyDoublingWideningUpperBySelectedScalarAndSubtractSaturate: opcode = OP_ARM64_SQDMLSL2_BYSCALAR; break; + default: g_assert_not_reached(); + } + if (is_unsigned) + switch (opcode) { + case OP_ARM64_SMLAL_SCALAR: opcode = OP_ARM64_UMLAL_SCALAR; break; + case OP_ARM64_SMLSL_SCALAR: opcode = OP_ARM64_UMLSL_SCALAR; break; + case OP_ARM64_SMLAL2_SCALAR: opcode = OP_ARM64_UMLAL2_SCALAR; break; + case OP_ARM64_SMLSL2_SCALAR: opcode = OP_ARM64_UMLSL2_SCALAR; break; + } + MonoInst *scalar = emit_simd_ins (cfg, ret_klass, OP_ARM64_SELECT_SCALAR, args [2]->dreg, args [3]->dreg); + MonoInst *ret = emit_simd_ins (cfg, ret_klass, opcode, args [0]->dreg, args [1]->dreg); + ret->sreg3 = scalar->dreg; + return ret; + } default: g_assert_not_reached (); } @@ -1885,10 +2450,10 @@ emit_x86_intrinsics ( case SN_RoundToPositiveInfinityScalar: case SN_RoundToZeroScalar: if (fsig->param_count == 2) { - return emit_simd_ins_for_sig (cfg, klass, OP_SSE41_ROUNDS, info->instc0, arg0_type, fsig, args); + return emit_simd_ins_for_sig (cfg, klass, OP_SSE41_ROUNDS, info->default_instc0, arg0_type, fsig, args); } else { MonoInst* ins = emit_simd_ins (cfg, klass, OP_SSE41_ROUNDS, args [0]->dreg, args [0]->dreg); - ins->inst_c0 = info->instc0; + ins->inst_c0 = info->default_instc0; ins->inst_c1 = arg0_type; return ins; } diff --git a/src/mono/mono/mini/simd-methods.h b/src/mono/mono/mini/simd-methods.h index 6b529cb95480b5..673543d4389554 100644 --- a/src/mono/mono/mini/simd-methods.h +++ b/src/mono/mono/mini/simd-methods.h @@ -250,8 +250,312 @@ METHOD(ScheduleUpdate1) METHOD(MixColumns) //AdvSimd METHOD(AbsSaturate) +METHOD(AbsSaturateScalar) METHOD(AbsScalar) METHOD(AbsoluteCompareGreaterThan) METHOD(AbsoluteCompareGreaterThanOrEqual) +METHOD(AbsoluteCompareGreaterThanOrEqualScalar) +METHOD(AbsoluteCompareGreaterThanScalar) METHOD(AbsoluteCompareLessThan) METHOD(AbsoluteCompareLessThanOrEqual) +METHOD(AbsoluteCompareLessThanOrEqualScalar) +METHOD(AbsoluteCompareLessThanScalar) +METHOD(AbsoluteDifference) +METHOD(AbsoluteDifferenceAdd) +METHOD(AbsoluteDifferenceScalar) +METHOD(AbsoluteDifferenceWideningLower) +METHOD(AbsoluteDifferenceWideningLowerAndAdd) +METHOD(AbsoluteDifferenceWideningUpper) +METHOD(AbsoluteDifferenceWideningUpperAndAdd) +METHOD(AddAcross) +METHOD(AddAcrossWidening) +METHOD(AddHighNarrowingLower) +METHOD(AddHighNarrowingUpper) +METHOD(AddPairwise) +METHOD(AddPairwiseScalar) +METHOD(AddPairwiseWidening) +METHOD(AddPairwiseWideningAndAdd) +METHOD(AddPairwiseWideningAndAddScalar) +METHOD(AddPairwiseWideningScalar) +METHOD(AddRoundedHighNarrowingLower) +METHOD(AddRoundedHighNarrowingUpper) +METHOD(AddSaturateScalar) +METHOD(AddWideningLower) +METHOD(AddWideningUpper) +METHOD(BitwiseClear) +METHOD(BitwiseSelect) +METHOD(CompareEqualScalar) +METHOD(CompareGreaterThanOrEqualScalar) +METHOD(CompareGreaterThanScalar) +METHOD(CompareLessThanOrEqualScalar) +METHOD(CompareLessThanScalar) +METHOD(CompareTest) +METHOD(CompareTestScalar) +METHOD(ConvertToDoubleScalar) +METHOD(ConvertToDoubleUpper) +METHOD(ConvertToInt32RoundAwayFromZero) +METHOD(ConvertToInt32RoundAwayFromZeroScalar) +METHOD(ConvertToInt32RoundToEven) +METHOD(ConvertToInt32RoundToEvenScalar) +METHOD(ConvertToInt32RoundToNegativeInfinity) +METHOD(ConvertToInt32RoundToNegativeInfinityScalar) +METHOD(ConvertToInt32RoundToPositiveInfinity) +METHOD(ConvertToInt32RoundToPositiveInfinityScalar) +METHOD(ConvertToInt32RoundToZero) +METHOD(ConvertToInt32RoundToZeroScalar) +METHOD(ConvertToInt64RoundAwayFromZero) +METHOD(ConvertToInt64RoundAwayFromZeroScalar) +METHOD(ConvertToInt64RoundToEven) +METHOD(ConvertToInt64RoundToEvenScalar) +METHOD(ConvertToInt64RoundToNegativeInfinity) +METHOD(ConvertToInt64RoundToNegativeInfinityScalar) +METHOD(ConvertToInt64RoundToPositiveInfinity) +METHOD(ConvertToInt64RoundToPositiveInfinityScalar) +METHOD(ConvertToInt64RoundToZero) +METHOD(ConvertToInt64RoundToZeroScalar) +METHOD(ConvertToSingleLower) +METHOD(ConvertToSingleRoundToOddLower) +METHOD(ConvertToSingleRoundToOddUpper) +METHOD(ConvertToSingleScalar) +METHOD(ConvertToSingleUpper) +METHOD(ConvertToUInt32RoundAwayFromZero) +METHOD(ConvertToUInt32RoundAwayFromZeroScalar) +METHOD(ConvertToUInt32RoundToEven) +METHOD(ConvertToUInt32RoundToEvenScalar) +METHOD(ConvertToUInt32RoundToNegativeInfinity) +METHOD(ConvertToUInt32RoundToNegativeInfinityScalar) +METHOD(ConvertToUInt32RoundToPositiveInfinity) +METHOD(ConvertToUInt32RoundToPositiveInfinityScalar) +METHOD(ConvertToUInt32RoundToZero) +METHOD(ConvertToUInt32RoundToZeroScalar) +METHOD(ConvertToUInt64RoundAwayFromZero) +METHOD(ConvertToUInt64RoundAwayFromZeroScalar) +METHOD(ConvertToUInt64RoundToEven) +METHOD(ConvertToUInt64RoundToEvenScalar) +METHOD(ConvertToUInt64RoundToNegativeInfinity) +METHOD(ConvertToUInt64RoundToNegativeInfinityScalar) +METHOD(ConvertToUInt64RoundToPositiveInfinity) +METHOD(ConvertToUInt64RoundToPositiveInfinityScalar) +METHOD(ConvertToUInt64RoundToZero) +METHOD(ConvertToUInt64RoundToZeroScalar) +METHOD(DuplicateSelectedScalarToVector128) +METHOD(DuplicateSelectedScalarToVector64) +METHOD(DuplicateToVector128) +METHOD(DuplicateToVector64) +METHOD(ExtractNarrowingLower) +METHOD(ExtractNarrowingSaturateLower) +METHOD(ExtractNarrowingSaturateScalar) +METHOD(ExtractNarrowingSaturateUnsignedLower) +METHOD(ExtractNarrowingSaturateUnsignedScalar) +METHOD(ExtractNarrowingSaturateUnsignedUpper) +METHOD(ExtractNarrowingSaturateUpper) +METHOD(ExtractNarrowingUpper) +METHOD(ExtractVector128) +METHOD(ExtractVector64) +METHOD(FusedAddHalving) +METHOD(FusedAddRoundedHalving) +METHOD(FusedMultiplyAdd) +METHOD(FusedMultiplyAddByScalar) +METHOD(FusedMultiplyAddBySelectedScalar) +METHOD(FusedMultiplyAddNegatedScalar) +METHOD(FusedMultiplyAddScalar) +METHOD(FusedMultiplyAddScalarBySelectedScalar) +METHOD(FusedMultiplySubtract) +METHOD(FusedMultiplySubtractByScalar) +METHOD(FusedMultiplySubtractBySelectedScalar) +METHOD(FusedMultiplySubtractNegatedScalar) +METHOD(FusedMultiplySubtractScalar) +METHOD(FusedMultiplySubtractScalarBySelectedScalar) +METHOD(FusedSubtractHalving) +METHOD(InsertScalar) +METHOD(InsertSelectedScalar) +METHOD(LoadAndInsertScalar) +METHOD(LoadAndReplicateToVector128) +METHOD(LoadAndReplicateToVector64) +METHOD(LoadVector64) +METHOD(MaxAcross) +METHOD(MaxNumber) +METHOD(MaxNumberAcross) +METHOD(MaxNumberPairwise) +METHOD(MaxNumberPairwiseScalar) +METHOD(MaxNumberScalar) +METHOD(MaxPairwise) +METHOD(MaxPairwiseScalar) +METHOD(MinAcross) +METHOD(MinNumber) +METHOD(MinNumberAcross) +METHOD(MinNumberPairwise) +METHOD(MinNumberPairwiseScalar) +METHOD(MinNumberScalar) +METHOD(MinPairwise) +METHOD(MinPairwiseScalar) +METHOD(MultiplyAdd) +METHOD(MultiplyAddByScalar) +METHOD(MultiplyAddBySelectedScalar) +METHOD(MultiplyByScalar) +METHOD(MultiplyBySelectedScalar) +METHOD(MultiplyBySelectedScalarWideningLower) +METHOD(MultiplyBySelectedScalarWideningLowerAndAdd) +METHOD(MultiplyBySelectedScalarWideningLowerAndSubtract) +METHOD(MultiplyBySelectedScalarWideningUpper) +METHOD(MultiplyBySelectedScalarWideningUpperAndAdd) +METHOD(MultiplyBySelectedScalarWideningUpperAndSubtract) +METHOD(MultiplyDoublingByScalarSaturateHigh) +METHOD(MultiplyDoublingBySelectedScalarSaturateHigh) +METHOD(MultiplyDoublingSaturateHigh) +METHOD(MultiplyDoublingSaturateHighScalar) +METHOD(MultiplyDoublingScalarBySelectedScalarSaturateHigh) +METHOD(MultiplyDoublingWideningAndAddSaturateScalar) +METHOD(MultiplyDoublingWideningAndSubtractSaturateScalar) +METHOD(MultiplyDoublingWideningLowerAndAddSaturate) +METHOD(MultiplyDoublingWideningLowerAndSubtractSaturate) +METHOD(MultiplyDoublingWideningLowerByScalarAndAddSaturate) +METHOD(MultiplyDoublingWideningLowerByScalarAndSubtractSaturate) +METHOD(MultiplyDoublingWideningLowerBySelectedScalarAndAddSaturate) +METHOD(MultiplyDoublingWideningLowerBySelectedScalarAndSubtractSaturate) +METHOD(MultiplyDoublingWideningSaturateLower) +METHOD(MultiplyDoublingWideningSaturateLowerByScalar) +METHOD(MultiplyDoublingWideningSaturateLowerBySelectedScalar) +METHOD(MultiplyDoublingWideningSaturateScalar) +METHOD(MultiplyDoublingWideningSaturateScalarBySelectedScalar) +METHOD(MultiplyDoublingWideningSaturateUpper) +METHOD(MultiplyDoublingWideningSaturateUpperByScalar) +METHOD(MultiplyDoublingWideningSaturateUpperBySelectedScalar) +METHOD(MultiplyDoublingWideningScalarBySelectedScalarAndAddSaturate) +METHOD(MultiplyDoublingWideningScalarBySelectedScalarAndSubtractSaturate) +METHOD(MultiplyDoublingWideningUpperAndAddSaturate) +METHOD(MultiplyDoublingWideningUpperAndSubtractSaturate) +METHOD(MultiplyDoublingWideningUpperByScalarAndAddSaturate) +METHOD(MultiplyDoublingWideningUpperByScalarAndSubtractSaturate) +METHOD(MultiplyDoublingWideningUpperBySelectedScalarAndAddSaturate) +METHOD(MultiplyDoublingWideningUpperBySelectedScalarAndSubtractSaturate) +METHOD(MultiplyExtended) +METHOD(MultiplyExtendedByScalar) +METHOD(MultiplyExtendedBySelectedScalar) +METHOD(MultiplyExtendedScalar) +METHOD(MultiplyExtendedScalarBySelectedScalar) +METHOD(MultiplyRoundedDoublingByScalarSaturateHigh) +METHOD(MultiplyRoundedDoublingBySelectedScalarSaturateHigh) +METHOD(MultiplyRoundedDoublingSaturateHigh) +METHOD(MultiplyRoundedDoublingSaturateHighScalar) +METHOD(MultiplyRoundedDoublingScalarBySelectedScalarSaturateHigh) +METHOD(MultiplyScalarBySelectedScalar) +METHOD(MultiplySubtract) +METHOD(MultiplySubtractByScalar) +METHOD(MultiplySubtractBySelectedScalar) +METHOD(MultiplyWideningLower) +METHOD(MultiplyWideningLowerAndAdd) +METHOD(MultiplyWideningLowerAndSubtract) +METHOD(MultiplyWideningUpper) +METHOD(MultiplyWideningUpperAndAdd) +METHOD(MultiplyWideningUpperAndSubtract) +METHOD(Negate) +METHOD(NegateSaturate) +METHOD(NegateSaturateScalar) +METHOD(NegateScalar) +METHOD(Not) +METHOD(OrNot) +METHOD(PolynomialMultiply) +METHOD(ReciprocalEstimate) +METHOD(ReciprocalEstimateScalar) +METHOD(ReciprocalExponentScalar) +METHOD(ReciprocalSquareRootEstimate) +METHOD(ReciprocalSquareRootEstimateScalar) +METHOD(ReciprocalSquareRootStep) +METHOD(ReciprocalSquareRootStepScalar) +METHOD(ReciprocalStep) +METHOD(ReciprocalStepScalar) +METHOD(ReverseElement16) +METHOD(ReverseElement32) +METHOD(ReverseElement8) +METHOD(RoundAwayFromZero) +METHOD(RoundAwayFromZeroScalar) +METHOD(RoundToNearest) +METHOD(RoundToNearestScalar) +METHOD(ShiftArithmetic) +METHOD(ShiftArithmeticRounded) +METHOD(ShiftArithmeticRoundedSaturate) +METHOD(ShiftArithmeticRoundedSaturateScalar) +METHOD(ShiftArithmeticRoundedScalar) +METHOD(ShiftArithmeticSaturate) +METHOD(ShiftArithmeticSaturateScalar) +METHOD(ShiftArithmeticScalar) +METHOD(ShiftLeftAndInsert) +METHOD(ShiftLeftAndInsertScalar) +METHOD(ShiftLeftLogicalSaturate) +METHOD(ShiftLeftLogicalSaturateScalar) +METHOD(ShiftLeftLogicalSaturateUnsigned) +METHOD(ShiftLeftLogicalSaturateUnsignedScalar) +METHOD(ShiftLeftLogicalScalar) +METHOD(ShiftLeftLogicalWideningLower) +METHOD(ShiftLeftLogicalWideningUpper) +METHOD(ShiftLogical) +METHOD(ShiftLogicalRounded) +METHOD(ShiftLogicalRoundedSaturate) +METHOD(ShiftLogicalRoundedSaturateScalar) +METHOD(ShiftLogicalRoundedScalar) +METHOD(ShiftLogicalSaturate) +METHOD(ShiftLogicalSaturateScalar) +METHOD(ShiftLogicalScalar) +METHOD(ShiftRightAndInsert) +METHOD(ShiftRightAndInsertScalar) +METHOD(ShiftRightArithmeticAdd) +METHOD(ShiftRightArithmeticAddScalar) +METHOD(ShiftRightArithmeticNarrowingSaturateLower) +METHOD(ShiftRightArithmeticNarrowingSaturateScalar) +METHOD(ShiftRightArithmeticNarrowingSaturateUnsignedLower) +METHOD(ShiftRightArithmeticNarrowingSaturateUnsignedScalar) +METHOD(ShiftRightArithmeticNarrowingSaturateUnsignedUpper) +METHOD(ShiftRightArithmeticNarrowingSaturateUpper) +METHOD(ShiftRightArithmeticRounded) +METHOD(ShiftRightArithmeticRoundedAdd) +METHOD(ShiftRightArithmeticRoundedAddScalar) +METHOD(ShiftRightArithmeticRoundedNarrowingSaturateLower) +METHOD(ShiftRightArithmeticRoundedNarrowingSaturateScalar) +METHOD(ShiftRightArithmeticRoundedNarrowingSaturateUnsignedLower) +METHOD(ShiftRightArithmeticRoundedNarrowingSaturateUnsignedScalar) +METHOD(ShiftRightArithmeticRoundedNarrowingSaturateUnsignedUpper) +METHOD(ShiftRightArithmeticRoundedNarrowingSaturateUpper) +METHOD(ShiftRightArithmeticRoundedScalar) +METHOD(ShiftRightArithmeticScalar) +METHOD(ShiftRightLogicalAdd) +METHOD(ShiftRightLogicalAddScalar) +METHOD(ShiftRightLogicalNarrowingLower) +METHOD(ShiftRightLogicalNarrowingSaturateLower) +METHOD(ShiftRightLogicalNarrowingSaturateScalar) +METHOD(ShiftRightLogicalNarrowingSaturateUpper) +METHOD(ShiftRightLogicalNarrowingUpper) +METHOD(ShiftRightLogicalRounded) +METHOD(ShiftRightLogicalRoundedAdd) +METHOD(ShiftRightLogicalRoundedAddScalar) +METHOD(ShiftRightLogicalRoundedNarrowingLower) +METHOD(ShiftRightLogicalRoundedNarrowingSaturateLower) +METHOD(ShiftRightLogicalRoundedNarrowingSaturateScalar) +METHOD(ShiftRightLogicalRoundedNarrowingSaturateUpper) +METHOD(ShiftRightLogicalRoundedNarrowingUpper) +METHOD(ShiftRightLogicalRoundedScalar) +METHOD(ShiftRightLogicalScalar) +METHOD(SignExtendWideningLower) +METHOD(SignExtendWideningUpper) +METHOD(StorePair) +METHOD(StorePairNonTemporal) +METHOD(StorePairScalar) +METHOD(StorePairScalarNonTemporal) +METHOD(StoreSelectedScalar) +METHOD(SubtractHighNarrowingLower) +METHOD(SubtractHighNarrowingUpper) +METHOD(SubtractRoundedHighNarrowingLower) +METHOD(SubtractRoundedHighNarrowingUpper) +METHOD(SubtractSaturateScalar) +METHOD(SubtractWideningLower) +METHOD(SubtractWideningUpper) +METHOD(TransposeEven) +METHOD(TransposeOdd) +METHOD(UnzipEven) +METHOD(UnzipOdd) +METHOD(VectorTableLookup) +METHOD(VectorTableLookupExtension) +METHOD(ZeroExtendWideningLower) +METHOD(ZeroExtendWideningUpper) +METHOD(ZipHigh) +METHOD(ZipLow)