From 0848153637f4d94b4a63732200940906ffd12f14 Mon Sep 17 00:00:00 2001 From: Imran Hameed Date: Wed, 17 Feb 2021 00:43:03 -0800 Subject: [PATCH 01/58] Checkpoint (Insert meaningful description here) --- src/mono/mono/mini/llvm-intrinsics.h | 29 +- src/mono/mono/mini/mini-llvm-cpp.cpp | 37 +- src/mono/mono/mini/mini-llvm-cpp.h | 29 ++ src/mono/mono/mini/mini-llvm.c | 483 ++++++++++++++++++++++----- src/mono/mono/mini/mini-ops.h | 32 +- src/mono/mono/mini/mini.h | 4 +- src/mono/mono/mini/simd-intrinsics.c | 146 ++++++-- src/mono/mono/mini/simd-methods.h | 24 ++ 8 files changed, 650 insertions(+), 134 deletions(-) diff --git a/src/mono/mono/mini/llvm-intrinsics.h b/src/mono/mono/mini/llvm-intrinsics.h index ffbe23dcfa5fe1..a6073cc6c61f87 100644 --- a/src/mono/mono/mini/llvm-intrinsics.h +++ b/src/mono/mono/mini/llvm-intrinsics.h @@ -12,6 +12,15 @@ * To define an overloaded intrinsic with three arguments */ +#define V64 LLVM_Vector64 +#define V128 LLVM_Vector128 +#define I1 LLVM_Int8 +#define I2 LLVM_Int16 +#define I4 LLVM_Int32 +#define I8 LLVM_Int64 +#define R4 LLVM_Float32 +#define R8 LLVM_Float64 + INTRINS_OVR_2_ARG(MEMSET, memset, LLVMPointerType (LLVMInt8Type (), 0), LLVMInt32Type ()) INTRINS_OVR_3_ARG(MEMCPY, memcpy, LLVMPointerType (LLVMInt8Type (), 0), LLVMPointerType (LLVMInt8Type (), 0), LLVMInt32Type () ) INTRINS_OVR_3_ARG(MEMMOVE, memmove, LLVMPointerType (LLVMInt8Type (), 0), LLVMPointerType (LLVMInt8Type (), 0), LLVMInt64Type ()) @@ -296,10 +305,28 @@ INTRINS_OVR_2_ARG(AARCH64_ADV_SIMD_ABS_COMPARE_LT_FLOAT, aarch64_neon_facgt, sse INTRINS_OVR_2_ARG(AARCH64_ADV_SIMD_ABS_COMPARE_LT_DOUBLE, aarch64_neon_facgt, sse_i4_t, sse_r8_t) INTRINS_OVR_2_ARG(AARCH64_ADV_SIMD_ABS_COMPARE_LTE_FLOAT, aarch64_neon_facge, sse_i4_t, sse_r4_t) INTRINS_OVR_2_ARG(AARCH64_ADV_SIMD_ABS_COMPARE_LTE_DOUBLE, aarch64_neon_facge, sse_i4_t, sse_r8_t) +INTRINS_OVR(AARCH64_ADV_SIMD_TBX64, aarch64_neon_tbx1, LLVMVectorType (LLVMInt8Type (), 8)) +INTRINS_OVR(AARCH64_ADV_SIMD_TBX128, aarch64_neon_tbx1, sse_i1_t) +INTRINS_OVR(AARCH64_ADV_SIMD_TBL64, aarch64_neon_tbl1, LLVMVectorType (LLVMInt8Type (), 8)) +INTRINS_OVR(AARCH64_ADV_SIMD_TBL128, aarch64_neon_tbl1, sse_i1_t) + +INTRINS_OVR_TAG(AARCH64_ADV_SIMD_UQSUB, aarch64_neon_uqsub, V64 | V128 | I1 | I2 | I4 | I8) +INTRINS_OVR_TAG(AARCH64_ADV_SIMD_SQSUB, aarch64_neon_sqsub, V64 | V128 | I1 | I2 | I4 | I8) +INTRINS_OVR_TAG(AARCH64_ADV_SIMD_RSUBHN, aarch64_neon_rsubhn, V64 | I1 | I2 | I4) +INTRINS_OVR_TAG(AARCH64_ADV_SIMD_FSQRT, sqrt, V64 | V128 | R4 | R8) + #endif #undef INTRINS #undef INTRINS_OVR #undef INTRINS_OVR_2_ARG #undef INTRINS_OVR_3_ARG - +#undef INTRINS_OVR_TAG +#undef V64 +#undef V128 +#undef I1 +#undef I2 +#undef I4 +#undef I8 +#undef R4 +#undef R8 diff --git a/src/mono/mono/mini/mini-llvm-cpp.cpp b/src/mono/mono/mini/mini-llvm-cpp.cpp index fa1dea68c6a2e4..f37703def760d1 100644 --- a/src/mono/mono/mini/mini-llvm-cpp.cpp +++ b/src/mono/mono/mini/mini-llvm-cpp.cpp @@ -636,6 +636,7 @@ get_intrins_id (IntrinsicId id) #define INTRINS_OVR(id, llvm_id, ty) case INTRINS_ ## id: intrins_id = Intrinsic::ID::llvm_id; break; #define INTRINS_OVR_2_ARG(id, llvm_id, ty1, ty2) case INTRINS_ ## id: intrins_id = Intrinsic::ID::llvm_id; break; #define INTRINS_OVR_3_ARG(id, llvm_id, ty1, ty2, ty3) case INTRINS_ ## id: intrins_id = Intrinsic::ID::llvm_id; break; +#define INTRINS_OVR_TAG(id, llvm_id, ...) case INTRINS_ ## id: intrins_id = Intrinsic::ID::llvm_id; break; #include "llvm-intrinsics.h" default: break; @@ -651,6 +652,7 @@ is_overloaded_intrins (IntrinsicId id) #define INTRINS_OVR(id, llvm_id, ty) case INTRINS_ ## id: return true; #define INTRINS_OVR_2_ARG(id, llvm_id, ty1, ty2) case INTRINS_ ## id: return true; #define INTRINS_OVR_3_ARG(id, llvm_id, ty1, ty2, ty3) case INTRINS_ ## id: return true; +#define INTRINS_OVR_TAG(id, llvm_id, ...) case INTRINS_ ## id: return true; #include "llvm-intrinsics.h" default: break; @@ -658,6 +660,33 @@ is_overloaded_intrins (IntrinsicId id) return false; } +llvm_ovr_tag_t +ovr_tag_from_mono_vector_class (MonoClass *klass) { + int size = mono_class_value_size (klass, NULL); + llvm_ovr_tag_t ret = 0; + switch (size) { + case 8: ret |= LLVM_Vector64; break; + case 16: ret |= LLVM_Vector128; break; + } + MonoType *etype = mono_class_get_context (klass)->class_inst->type_argv [0]; + switch (etype->type) { + case MONO_TYPE_I1: case MONO_TYPE_U1: ret |= LLVM_Int8; break; + case MONO_TYPE_I2: case MONO_TYPE_U2: ret |= LLVM_Int16; break; + case MONO_TYPE_I4: case MONO_TYPE_U4: ret |= LLVM_Int32; break; + case MONO_TYPE_I8: case MONO_TYPE_U8: ret |= LLVM_Int64; break; + case MONO_TYPE_R4: ret |= LLVM_Float32; break; + case MONO_TYPE_R8: ret |= LLVM_Float64; break; + } + return ret; +} + +llvm_ovr_tag_t +ovr_tag_from_mono_vector_type (MonoType *type) +{ + MonoClass *klass = mono_class_from_mono_type_internal (type); + return ovr_tag_from_mono_vector_class (klass); +} + /* * mono_llvm_register_intrinsic: * @@ -694,11 +723,11 @@ mono_llvm_register_overloaded_intrinsic (LLVMModuleRef module, IntrinsicId id, L const int max_types = 5; g_assert (ntypes <= max_types); - Type *arr [max_types]; - for (int i = 0; i < ntypes; ++i) + Type *arr [max_types]; + for (int i = 0; i < ntypes; ++i) arr [i] = unwrap (types [i]); - auto f = Intrinsic::getDeclaration (unwrap (module), intrins_id, { arr, (size_t)ntypes }); - return wrap (f); + auto f = Intrinsic::getDeclaration (unwrap (module), intrins_id, { arr, (size_t)ntypes }); + return wrap (f); } unsigned int diff --git a/src/mono/mono/mini/mini-llvm-cpp.h b/src/mono/mono/mini/mini-llvm-cpp.h index 72cf7bfd63e7cd..b50e5aa8a0aa31 100644 --- a/src/mono/mono/mini/mini-llvm-cpp.h +++ b/src/mono/mono/mini/mini-llvm-cpp.h @@ -16,21 +16,50 @@ #include "llvm-c/Core.h" #include "llvm-c/ExecutionEngine.h" +#include "mini-llvm.h" + #ifdef HAVE_UNWIND_H #include #endif G_BEGIN_DECLS + +/* An intrinsic id. The lower 24 bits are used to store a mono-specific ID. The + * next 8 bits store arm64 overload tag bits. In the configuration of LLVM 9 we + * use, there are 7017 total intrinsics defined in IntrinsicEnums.inc. + */ typedef enum { #define INTRINS(id, llvm_id) INTRINS_ ## id, #define INTRINS_OVR(id, llvm_id, ty) INTRINS_ ## id, #define INTRINS_OVR_2_ARG(id, llvm_id, ty1, ty2) INTRINS_ ## id, #define INTRINS_OVR_3_ARG(id, llvm_id, ty1, ty2, ty3) INTRINS_ ## id, +#define INTRINS_OVR_TAG(id, ...) INTRINS_ ## id, #include "llvm-intrinsics.h" INTRINS_NUM } IntrinsicId; +enum { + LLVM_Vector64 = 1 << 0, + LLVM_Vector128 = 1 << 1, + LLVM_VectorWidths = 2, + + LLVM_Int8 = 1 << 2, + LLVM_Int16 = 1 << 3, + LLVM_Int32 = 1 << 4, + LLVM_Int64 = 1 << 5, + LLVM_Float32 = 1 << 6, + LLVM_Float64 = 1 << 7, + LLVM_ElementWidths = 6, +}; + +typedef uint8_t llvm_ovr_tag_t; + +llvm_ovr_tag_t +ovr_tag_from_mono_vector_class (MonoClass *klass); + +llvm_ovr_tag_t +ovr_tag_from_mono_vector_type (MonoType *type); /* * Keep in sync with the enum in utils/mono-memory-model.h. diff --git a/src/mono/mono/mini/mini-llvm.c b/src/mono/mono/mini/mini-llvm.c index 7dcc4b71a810db..9d452ee929cb70 100644 --- a/src/mono/mono/mini/mini-llvm.c +++ b/src/mono/mono/mini/mini-llvm.c @@ -51,6 +51,17 @@ #error "The version of the mono llvm repository is too old." #endif +static LLVMTypeRef intrin_types [LLVM_VectorWidths][LLVM_ElementWidths]; + +static const llvm_ovr_tag_t intrin_arm64_ovr [] = { + #define INTRINS(sym, ...) 0, + #define INTRINS_OVR(sym, ...) 0, + #define INTRINS_OVR_2_ARG(sym, ...) 0, + #define INTRINS_OVR_3_ARG(sym, ...) 0, + #define INTRINS_OVR_TAG(sym, _, spec) spec, + #include "llvm-intrinsics.h" +}; + /* * Information associated by mono with LLVM modules. */ @@ -254,6 +265,10 @@ mini_llvm_ins_info[] = { #define ctx_ok(ctx) (!(ctx)->cfg->disable_llvm) +enum { + MAX_VECTOR_ELEMS = 32, // 2 vectors * 128 bits per vector / 8 bits per element +}; + static LLVMIntPredicate cond_to_llvm_cond [] = { LLVMIntEQ, LLVMIntNE, @@ -286,6 +301,8 @@ static MonoLLVMModule aot_module; static GHashTable *intrins_id_to_intrins; static LLVMTypeRef sse_i1_t, sse_i2_t, sse_i4_t, sse_i8_t, sse_r4_t, sse_r8_t; +static LLVMTypeRef v64_i1_t, v64_i2_t, v64_i4_t, v64_i8_t, v64_r4_t, v64_r8_t; +static LLVMTypeRef v128_i1_t, v128_i2_t, v128_i4_t, v128_i8_t, v128_r4_t, v128_r8_t; static MonoLLVMModule *init_jit_module (void); @@ -4668,8 +4685,8 @@ emit_landing_pad (EmitContext *ctx, int group_index, int group_size) static LLVMValueRef create_const_vector (LLVMTypeRef t, const int *vals, int count) { - g_assert (count <= 16); - LLVMValueRef llvm_vals [16]; + g_assert (count <= MAX_VECTOR_ELEMS); + LLVMValueRef llvm_vals [MAX_VECTOR_ELEMS]; for (int i = 0; i < count; i++) llvm_vals [i] = LLVMConstInt (t, vals [i], FALSE); return LLVMConstVector (llvm_vals, count); @@ -4701,6 +4718,41 @@ create_const_vector_2_i32 (int v0, int v1) return LLVMConstVector (mask, 2); } +static LLVMValueRef +extract_high_elements (EmitContext *ctx, LLVMValueRef src_vec) +{ + LLVMTypeRef src_t = LLVMTypeOf (src_vec); + unsigned int src_elems = LLVMGetVectorSize (src_t); + unsigned int dst_elems = src_elems / 2; + int mask [MAX_VECTOR_ELEMS] = { 0 }; + for (int i = 0; i < dst_elems; ++i) + mask [i] = dst_elems + i; + return LLVMBuildShuffleVector (ctx->builder, src_vec, LLVMGetUndef (src_t), create_const_vector_i32 (mask, dst_elems), ""); +} + +static LLVMValueRef +select_lowest_element (EmitContext *ctx, LLVMValueRef vec) +{ + int mask [MAX_VECTOR_ELEMS] = { 0 }; + LLVMTypeRef t = LLVMTypeOf (vec); + unsigned int elems = LLVMGetVectorSize (t); + mask [0] = 0; + for (unsigned int i = 1; i < elems; ++i) + mask [i] = elems + i; + return LLVMBuildShuffleVector (ctx->builder, vec, LLVMConstNull (t), create_const_vector_i32 (mask, elems), ""); +} + +static LLVMValueRef +concatenate_vectors (EmitContext *ctx, LLVMValueRef xs, LLVMValueRef ys) +{ + LLVMTypeRef t = LLVMTypeOf (xs); + unsigned int elems = LLVMGetVectorSize (t) * 2; + int mask [MAX_VECTOR_ELEMS] = { 0 }; + for (int i = 0; i < elems; ++i) + mask [i] = i; + return LLVMBuildShuffleVector (ctx->builder, xs, ys, create_const_vector_i32 (mask, elems), ""); +} + static void emit_llvmonly_handler_start (EmitContext *ctx, MonoBasicBlock *bb, LLVMBasicBlockRef cbb) { @@ -4901,22 +4953,33 @@ get_float_const (MonoCompile *cfg, float val) return LLVMConstFPExt (LLVMConstReal (LLVMFloatType (), val), LLVMDoubleType ()); } +static int +int_from_id_and_ovr_tag (int id, llvm_ovr_tag_t ovr_tag) +{ + return (((int) ovr_tag) << 24) | id; +} + static LLVMValueRef -call_intrins (EmitContext *ctx, int id, LLVMValueRef *args, const char *name) +call_overloaded_intrins (EmitContext *ctx, int id, llvm_ovr_tag_t ovr_tag, LLVMValueRef *args, const char *name) { - LLVMValueRef intrins = get_intrins (ctx, id); + int ovr_id = int_from_id_and_ovr_tag (id, ovr_tag); + LLVMValueRef intrins = get_intrins (ctx, ovr_id); int nargs = LLVMCountParamTypes (LLVMGetElementType (LLVMTypeOf (intrins))); - for (int i = 0; i < nargs; ++i) { LLVMTypeRef t1 = LLVMTypeOf (args [i]); LLVMTypeRef t2 = LLVMTypeOf (LLVMGetParam (intrins, i)); if (t1 != t2) args [i] = convert (ctx, args [i], t2); } - return LLVMBuildCall (ctx->builder, intrins, args, nargs, name); } +static LLVMValueRef +call_intrins (EmitContext *ctx, int id, LLVMValueRef *args, const char *name) +{ + return call_overloaded_intrins (ctx, id, 0, args, name); +} + static void process_bb (EmitContext *ctx, MonoBasicBlock *bb) { @@ -6967,9 +7030,6 @@ process_bb (EmitContext *ctx, MonoBasicBlock *bb) break; } - /* - * SIMD - */ #if defined(TARGET_X86) || defined(TARGET_AMD64) || defined(TARGET_ARM64) || defined(TARGET_WASM) case OP_EXPAND_I1: case OP_EXPAND_I2: @@ -6978,11 +7038,11 @@ process_bb (EmitContext *ctx, MonoBasicBlock *bb) case OP_EXPAND_R4: case OP_EXPAND_R8: { LLVMTypeRef t; - LLVMValueRef mask [32], v; + LLVMValueRef mask [MAX_VECTOR_ELEMS], v; int i; t = simd_class_to_llvm_type (ctx, ins->klass); - for (i = 0; i < 32; ++i) + for (i = 0; i < MAX_VECTOR_ELEMS; ++i) mask [i] = LLVMConstInt (LLVMInt32Type (), 0, FALSE); v = convert (ctx, values [ins->sreg1], LLVMGetElementType (t)); @@ -7011,6 +7071,78 @@ process_bb (EmitContext *ctx, MonoBasicBlock *bb) mono_llvm_build_aligned_store (builder, values [ins->sreg1], dest, FALSE, 1); break; } + case OP_XXOR: { + LLVMTypeRef t = LLVMTypeOf (lhs); + unsigned int bit_width = mono_llvm_get_prim_size_bits (t); + LLVMTypeRef intermediate_t = LLVMVectorType (LLVMInt8Type (), bit_width / 8); + LLVMValueRef lhs_i8 = convert (ctx, lhs, intermediate_t); + LLVMValueRef rhs_i8 = convert (ctx, rhs, intermediate_t); + LLVMValueRef result = LLVMBuildXor (builder, lhs_i8, rhs_i8, ""); + values [ins->dreg] = LLVMBuildBitCast (builder, result, t, ""); + break; + } + case OP_XBINOP: { + switch (ins->inst_c0) { + case OP_IADD: + values [ins->dreg] = LLVMBuildAdd (builder, lhs, rhs, ""); + break; + case OP_ISUB: + values [ins->dreg] = LLVMBuildSub (builder, lhs, rhs, ""); + break; + case OP_IAND: + values [ins->dreg] = LLVMBuildAnd (builder, lhs, rhs, ""); + break; + case OP_IOR: + values [ins->dreg] = LLVMBuildOr (builder, lhs, rhs, ""); + break; + case OP_IXOR: + values [ins->dreg] = LLVMBuildXor (builder, lhs, rhs, ""); + break; + case OP_FADD: + values [ins->dreg] = LLVMBuildFAdd (builder, lhs, rhs, ""); + break; + case OP_FSUB: + values [ins->dreg] = LLVMBuildFSub (builder, lhs, rhs, ""); + break; + case OP_FMUL: + values [ins->dreg] = LLVMBuildFMul (builder, lhs, rhs, ""); + break; + case OP_FDIV: + values [ins->dreg] = LLVMBuildFDiv (builder, lhs, rhs, ""); + break; + case OP_FMAX: + case OP_FMIN: { +#if defined(TARGET_X86) || defined(TARGET_AMD64) + LLVMValueRef args [] = { lhs, rhs }; + + gboolean is_r4 = ins->inst_c1 == MONO_TYPE_R4; + if (ins->inst_c0 == OP_FMAX) + values [ins->dreg] = call_intrins (ctx, is_r4 ? INTRINS_SSE_MAXPS : INTRINS_SSE_MAXPD, args, dname); + else + values [ins->dreg] = call_intrins (ctx, is_r4 ? INTRINS_SSE_MINPS : INTRINS_SSE_MINPD, args, dname); +#else + NOT_IMPLEMENTED; +#endif + break; + } + case OP_IMAX: { + gboolean is_unsigned = ins->inst_c1 == MONO_TYPE_U1 || ins->inst_c1 == MONO_TYPE_U2 || ins->inst_c1 == MONO_TYPE_U4 || ins->inst_c1 == MONO_TYPE_U8; + LLVMValueRef cmp = LLVMBuildICmp (builder, is_unsigned ? LLVMIntUGT : LLVMIntSGT, lhs, rhs, ""); + values [ins->dreg] = LLVMBuildSelect (builder, cmp, lhs, rhs, ""); + break; + } + case OP_IMIN: { + gboolean is_unsigned = ins->inst_c1 == MONO_TYPE_U1 || ins->inst_c1 == MONO_TYPE_U2 || ins->inst_c1 == MONO_TYPE_U4 || ins->inst_c1 == MONO_TYPE_U8; + LLVMValueRef cmp = LLVMBuildICmp (builder, is_unsigned ? LLVMIntULT : LLVMIntSLT, lhs, rhs, ""); + values [ins->dreg] = LLVMBuildSelect (builder, cmp, lhs, rhs, ""); + } + break; + + default: + g_assert_not_reached (); + } + break; + } #endif // defined(TARGET_X86) || defined(TARGET_AMD64) || defined(TARGET_ARM64) || defined(TARGET_WASM) #if defined(TARGET_X86) || defined(TARGET_AMD64) || defined(TARGET_WASM) @@ -7161,11 +7293,11 @@ process_bb (EmitContext *ctx, MonoBasicBlock *bb) case OP_PAVGB_UN: case OP_PAVGW_UN: { LLVMValueRef ones_vec; - LLVMValueRef ones [32]; + LLVMValueRef ones [MAX_VECTOR_ELEMS]; int vector_size = LLVMGetVectorSize (LLVMTypeOf (lhs)); LLVMTypeRef ext_elem_type = vector_size == 16 ? LLVMInt16Type () : LLVMInt32Type (); - for (int i = 0; i < 32; ++i) + for (int i = 0; i < MAX_VECTOR_ELEMS; ++i) ones [i] = LLVMConstInt (ext_elem_type, 1, FALSE); ones_vec = LLVMConstVector (ones, vector_size); @@ -7910,14 +8042,6 @@ process_bb (EmitContext *ctx, MonoBasicBlock *bb) break; } - case OP_SSE_XOR: { - LLVMValueRef vec_lhs_i64 = convert (ctx, lhs, sse_i8_t); - LLVMValueRef vec_rhs_i64 = convert (ctx, rhs, sse_i8_t); - LLVMValueRef vec_and = LLVMBuildXor (builder, vec_lhs_i64, vec_rhs_i64, ""); - values [ins->dreg] = LLVMBuildBitCast (builder, vec_and, type_to_sse_type (ins->inst_c1), ""); - break; - } - case OP_SSE_AND: { LLVMValueRef vec_lhs_i64 = convert (ctx, lhs, sse_i8_t); LLVMValueRef vec_rhs_i64 = convert (ctx, rhs, sse_i8_t); @@ -8811,7 +8935,7 @@ process_bb (EmitContext *ctx, MonoBasicBlock *bb) } case OP_XEQUAL: { LLVMTypeRef t; - LLVMValueRef cmp, mask [32], shuffle; + LLVMValueRef cmp, mask [MAX_VECTOR_ELEMS], shuffle; int nelems; #if defined(TARGET_WASM) && LLVM_API_VERSION >= 800 @@ -8882,68 +9006,6 @@ process_bb (EmitContext *ctx, MonoBasicBlock *bb) values [ins->dreg] = LLVMBuildZExt (builder, cmp_zero, LLVMInt8Type (), ""); break; } - case OP_XBINOP: { - switch (ins->inst_c0) { - case OP_IADD: - values [ins->dreg] = LLVMBuildAdd (builder, lhs, rhs, ""); - break; - case OP_ISUB: - values [ins->dreg] = LLVMBuildSub (builder, lhs, rhs, ""); - break; - case OP_IAND: - values [ins->dreg] = LLVMBuildAnd (builder, lhs, rhs, ""); - break; - case OP_IOR: - values [ins->dreg] = LLVMBuildOr (builder, lhs, rhs, ""); - break; - case OP_IXOR: - values [ins->dreg] = LLVMBuildXor (builder, lhs, rhs, ""); - break; - case OP_FADD: - values [ins->dreg] = LLVMBuildFAdd (builder, lhs, rhs, ""); - break; - case OP_FSUB: - values [ins->dreg] = LLVMBuildFSub (builder, lhs, rhs, ""); - break; - case OP_FMUL: - values [ins->dreg] = LLVMBuildFMul (builder, lhs, rhs, ""); - break; - case OP_FDIV: - values [ins->dreg] = LLVMBuildFDiv (builder, lhs, rhs, ""); - break; - case OP_FMAX: - case OP_FMIN: { -#if defined(TARGET_X86) || defined(TARGET_AMD64) - LLVMValueRef args [] = { lhs, rhs }; - - gboolean is_r4 = ins->inst_c1 == MONO_TYPE_R4; - if (ins->inst_c0 == OP_FMAX) - values [ins->dreg] = call_intrins (ctx, is_r4 ? INTRINS_SSE_MAXPS : INTRINS_SSE_MAXPD, args, dname); - else - values [ins->dreg] = call_intrins (ctx, is_r4 ? INTRINS_SSE_MINPS : INTRINS_SSE_MINPD, args, dname); -#else - NOT_IMPLEMENTED; -#endif - break; - } - case OP_IMAX: { - gboolean is_unsigned = ins->inst_c1 == MONO_TYPE_U1 || ins->inst_c1 == MONO_TYPE_U2 || ins->inst_c1 == MONO_TYPE_U4 || ins->inst_c1 == MONO_TYPE_U8; - LLVMValueRef cmp = LLVMBuildICmp (builder, is_unsigned ? LLVMIntUGT : LLVMIntSGT, lhs, rhs, ""); - values [ins->dreg] = LLVMBuildSelect (builder, cmp, lhs, rhs, ""); - break; - } - case OP_IMIN: { - gboolean is_unsigned = ins->inst_c1 == MONO_TYPE_U1 || ins->inst_c1 == MONO_TYPE_U2 || ins->inst_c1 == MONO_TYPE_U4 || ins->inst_c1 == MONO_TYPE_U8; - LLVMValueRef cmp = LLVMBuildICmp (builder, is_unsigned ? LLVMIntULT : LLVMIntSLT, lhs, rhs, ""); - values [ins->dreg] = LLVMBuildSelect (builder, cmp, lhs, rhs, ""); - } - break; - - default: - g_assert_not_reached (); - } - break; - } case OP_XEXTRACT_I32: case OP_XEXTRACT_I64: case OP_XEXTRACT_R8: @@ -9090,6 +9152,11 @@ process_bb (EmitContext *ctx, MonoBasicBlock *bb) getElement = TRUE; element_idx = 1; bitcast_result = TRUE; + case SIMD_OP_ARM64_TBL: + if (mono_llvm_get_prim_size_bits (LLVMTypeOf (rhs)) == 128) + id = INTRINS_AARCH64_ADV_SIMD_TBL128; + else + id = INTRINS_AARCH64_ADV_SIMD_TBL64; break; default: g_assert_not_reached (); break; } @@ -9118,6 +9185,12 @@ process_bb (EmitContext *ctx, MonoBasicBlock *bb) case SIMD_OP_ARM64_SHA1C: id = INTRINS_AARCH64_SHA1C; getLowerElement = TRUE; arg_idx = 1; break; case SIMD_OP_ARM64_SHA1M: id = INTRINS_AARCH64_SHA1M; getLowerElement = TRUE; arg_idx = 1; break; case SIMD_OP_ARM64_SHA1P: id = INTRINS_AARCH64_SHA1P; getLowerElement = TRUE; arg_idx = 1; break; + case SIMD_OP_ARM64_TBX: + if (mono_llvm_get_prim_size_bits (LLVMTypeOf (lhs)) == 128) + id = INTRINS_AARCH64_ADV_SIMD_TBX128; + else + id = INTRINS_AARCH64_ADV_SIMD_TBX64; + break; default: g_assert_not_reached (); break; } LLVMValueRef args [] = { lhs, rhs, arg3 }; @@ -9174,7 +9247,7 @@ process_bb (EmitContext *ctx, MonoBasicBlock *bb) LLVMValueRef xor = LLVMBuildXor (builder, shr, lhs, ""); LLVMValueRef mul = LLVMBuildShl (builder, xor, one, ""); LLVMValueRef add = LLVMBuildOr (builder, mul, one, ""); - + LLVMValueRef args [2]; args [0] = add; args [1] = LLVMConstInt (LLVMInt1Type (), 0, FALSE); @@ -9197,6 +9270,211 @@ process_bb (EmitContext *ctx, MonoBasicBlock *bb) values [ins->dreg] = LLVMBuildTrunc (builder, hi64, LLVMInt64Type (), ""); break; } + case OP_ARM64_SQRT_SCALAR: { + int iid = ins->inst_c0 == MONO_TYPE_R8 ? INTRINS_SQRT : INTRINS_SQRTF; + LLVMTypeRef t = LLVMTypeOf (lhs); + LLVMValueRef scalar = LLVMBuildExtractElement (builder, lhs, const_int32 (0), ""); + LLVMValueRef result = call_intrins (ctx, iid, &scalar, "arm64_sqrt_scalar"); + values [ins->dreg] = LLVMBuildInsertElement (builder, LLVMGetUndef (t), result, const_int32 (0), ""); + break; + } + case OP_ARM64_STP: + case OP_ARM64_STP_SCALAR: + case OP_ARM64_STNP: + case OP_ARM64_STNP_SCALAR: { + LLVMTypeRef lhs_t = LLVMTypeOf (lhs); + unsigned int lhs_elems = LLVMGetVectorSize (lhs_t); + LLVMTypeRef lhs_elt_t = LLVMGetElementType (lhs_t); + LLVMValueRef val = NULL; + LLVMTypeRef dst_t = lhs_t; + switch (ins->opcode) { + case OP_ARM64_STP: + case OP_ARM64_STNP: { + dst_t = LLVMPointerType (LLVMVectorType (lhs_elt_t, lhs_elems * 2), 0); + val = concatenate_vectors (ctx, rhs, arg3); + break; + } + case OP_ARM64_STP_SCALAR: + case OP_ARM64_STNP_SCALAR: { + val = LLVMBuildShuffleVector (builder, rhs, arg3, create_const_vector_2_i32 (0, 2), ""); + break; + } + } + LLVMValueRef addr = convert (ctx, lhs, dst_t); + LLVMValueRef store = mono_llvm_build_store (builder, val, addr, FALSE, LLVM_BARRIER_NONE); + switch (ins->opcode) { + case OP_ARM64_STNP: + case OP_ARM64_STNP_SCALAR: + set_nontemporal_flag (store); + } + break; + } + case OP_ARM64_ST1: { + LLVMValueRef store = mono_llvm_build_store (builder, rhs, lhs, FALSE, LLVM_BARRIER_NONE); + LLVMSetValueName (store, "arm64_st1"); + break; + } + case OP_ARM64_ST1_SCALAR: { + // XXXih: TODO: unroll arg3 + LLVMValueRef val = LLVMBuildExtractElement (builder, rhs, const_int32 (0), "arm64_st1_scalar"); + mono_llvm_build_store (builder, val, lhs, FALSE, LLVM_BARRIER_NONE); + break; + } + case OP_ARM64_SUBHN: + case OP_ARM64_SUBHN2: { + LLVMValueRef l = lhs; + LLVMValueRef r = rhs; + if (ins->opcode == OP_ARM64_SUBHN2) { + l = rhs; + r = arg3; + } + LLVMValueRef result = LLVMBuildSub (builder, l, r, ""); + LLVMTypeRef l_t = LLVMTypeOf (l); + LLVMTypeRef l_elt_t = LLVMGetElementType (l_t); + unsigned int elem_bits = mono_llvm_get_prim_size_bits (l_elt_t); + unsigned int elems = LLVMGetVectorSize (l_t); + int shift_vals [MAX_VECTOR_ELEMS] = { 0 }; + for (unsigned int i = 0; i < elems; ++i) { + shift_vals [i] = elem_bits / 2; + } + result = LLVMBuildLShr (builder, result, create_const_vector (l_elt_t, shift_vals, elems), ""); + result = LLVMBuildTrunc (builder, result, LLVMVectorType (LLVMIntType (elem_bits / 2), elems), ""); + if (ins->opcode == OP_ARM64_SUBHN2) + result = concatenate_vectors (ctx, lhs, result); + values [ins->dreg] = result; + break; + } + case OP_ARM64_RSUBHN: + case OP_ARM64_RSUBHN2: { + LLVMValueRef args [2] = { lhs, rhs }; + if (ins->opcode == OP_ARM64_RSUBHN2) { + args [0] = rhs; + args [1] = arg3; + } + llvm_ovr_tag_t ovr_tag = ovr_tag_from_mono_vector_class (ins->klass); + LLVMValueRef result = call_overloaded_intrins (ctx, INTRINS_AARCH64_ADV_SIMD_RSUBHN, ovr_tag, args, ""); + if (ins->opcode == OP_ARM64_RSUBHN2) + result = concatenate_vectors (ctx, lhs, result); + values [ins->dreg] = result; + break; + } + case OP_ARM64_SSUB: + case OP_ARM64_USUB: + case OP_ARM64_SSUB2: + case OP_ARM64_USUB2: { + LLVMTypeRef ret_t = simd_class_to_llvm_type (ctx, ins->klass); + gboolean is_unsigned = FALSE; + gboolean select_high = FALSE; + switch (ins->opcode) { + case OP_ARM64_USUB: is_unsigned = TRUE; break; + case OP_ARM64_SSUB2: select_high = TRUE; break; + case OP_ARM64_USUB2: select_high = TRUE; is_unsigned = TRUE; break; + } + LLVMValueRef args [2] = { lhs, rhs }; + int mask [MAX_VECTOR_ELEMS] = { 0 }; + for (int i = 0; i < 2; ++i) { + LLVMValueRef arg = args [i]; + LLVMTypeRef arg_t = LLVMTypeOf (arg); + unsigned int elems = LLVMGetVectorSize (arg_t) / 2; + int laneix = select_high ? elems : 0; + for (int i = 0; i < elems; ++i) { + mask [i] = laneix; + ++laneix; + } + arg = LLVMBuildShuffleVector (builder, arg, LLVMGetUndef (arg_t), create_const_vector_i32 (mask, elems), ""); + if (is_unsigned) + arg = LLVMBuildZExt (builder, arg, ret_t, ""); + else + arg = LLVMBuildSExt (builder, arg, ret_t, ""); + args [i] = arg; + } + values [ins->dreg] = LLVMBuildSub (builder, args [0], args [1], "arm64_sub"); + break; + } + case OP_ARM64_SXTL: + case OP_ARM64_SXTL2: + case OP_ARM64_UXTL: + case OP_ARM64_UXTL2: { + LLVMTypeRef t = LLVMTypeOf (lhs); + unsigned int elem_bits = LLVMGetIntTypeWidth (LLVMGetElementType (t)); + unsigned int src_elems = LLVMGetVectorSize (t); + unsigned int dst_elems = src_elems; + LLVMValueRef arg = lhs; + switch (ins->opcode) { + case OP_ARM64_SXTL2: + case OP_ARM64_UXTL2: + arg = extract_high_elements (ctx, lhs); + dst_elems = LLVMGetVectorSize (LLVMTypeOf (arg)); + } + LLVMTypeRef result_t = LLVMVectorType (LLVMIntType (elem_bits * 2), dst_elems); + LLVMValueRef result = NULL; + switch (ins->opcode) { + case OP_ARM64_SXTL: + case OP_ARM64_SXTL2: + result = LLVMBuildSExt (builder, arg, result_t, "arm64_sxtl"); + break; + default: + result = LLVMBuildZExt (builder, arg, result_t, "arm64_uxtl"); + } + values [ins->dreg] = result; + break; + } + case OP_ARM64_TRN1: + case OP_ARM64_TRN2: { + LLVMTypeRef t = LLVMTypeOf (lhs); + unsigned int src_elems = LLVMGetVectorSize (t); + int mask [MAX_VECTOR_ELEMS] = { 0 }; + int laneix = ins->opcode == OP_ARM64_TRN2 ? 1 : 0; + for (unsigned int i = 0; i < src_elems; i += 2) { + mask [i] = laneix; + mask [i + 1] = laneix + src_elems; + laneix += 2; + } + values [ins->dreg] = LLVMBuildShuffleVector (builder, lhs, rhs, create_const_vector_i32 (mask, src_elems), "arm64_uzp"); + } + case OP_ARM64_UZP1: + case OP_ARM64_UZP2: { + LLVMTypeRef t = LLVMTypeOf (lhs); + unsigned int src_elems = LLVMGetVectorSize (t); + int mask [MAX_VECTOR_ELEMS] = { 0 }; + int laneix = ins->opcode == OP_ARM64_UZP2 ? 1 : 0; + for (unsigned int i = 0; i < src_elems; ++i) { + mask [i] = laneix; + laneix += 2; + } + values [ins->dreg] = LLVMBuildShuffleVector (builder, lhs, rhs, create_const_vector_i32 (mask, src_elems), "arm64_uzp"); + break; + } + case OP_ARM64_ZIP1: + case OP_ARM64_ZIP2: { + LLVMTypeRef t = LLVMTypeOf (lhs); + unsigned int src_elems = LLVMGetVectorSize (t); + int mask [MAX_VECTOR_ELEMS] = { 0 }; + int laneix = ins->opcode == OP_ARM64_ZIP2 ? src_elems / 2 : 0; + for (unsigned int i = 0; i < src_elems; i += 2) { + mask [i] = laneix; + mask [i + 1] = laneix + src_elems; + } + values [ins->dreg] = LLVMBuildShuffleVector (builder, lhs, rhs, create_const_vector_i32 (mask, src_elems), "arm64_zip"); + break; + } + case OP_XOP_OVR_X_X: { + IntrinsicId iid = (IntrinsicId) ins->inst_c0; + llvm_ovr_tag_t ovr_tag = ins->inst_c1; + values [ins->dreg] = call_overloaded_intrins (ctx, iid, ovr_tag, &lhs, ""); + break; + } + case OP_XOP_OVR_X_X_X: { + IntrinsicId iid = (IntrinsicId) ins->inst_c0; + llvm_ovr_tag_t ovr_tag = ins->inst_c1; + LLVMValueRef args [] = { lhs, rhs }; + values [ins->dreg] = call_overloaded_intrins (ctx, iid, ovr_tag, args, ""); + break; + } + case OP_ARM64_ZERO_UPPER: { + values [ins->dreg] = select_lowest_element (ctx, lhs); + break; + } #endif case OP_DUMMY_USE: @@ -10512,12 +10790,28 @@ add_intrinsic (LLVMModuleRef module, int id) return; } + if (intrin_arm64_ovr [id] != 0) { + llvm_ovr_tag_t spec = intrin_arm64_ovr [id]; + for (int vw = 0; vw < LLVM_VectorWidths; ++vw) { + for (int ew = 0; ew < LLVM_ElementWidths; ++ew) { + llvm_ovr_tag_t test = (LLVM_Vector64 << vw) | (LLVM_Int8 << ew); + if (spec & test) { + intrins = add_intrins1 (module, id, intrin_types [vw][ew]); + int ovr_id = int_from_id_and_ovr_tag (test, id); + g_hash_table_insert (intrins_id_to_intrins, GINT_TO_POINTER (ovr_id), intrins); + } + } + } + return; + } + /* Register overloaded intrinsics */ switch (id) { #define INTRINS(intrin_name, llvm_id) #define INTRINS_OVR(intrin_name, llvm_id, llvm_type) case INTRINS_ ## intrin_name: intrins = add_intrins1(module, id, llvm_type); break; #define INTRINS_OVR_2_ARG(intrin_name, llvm_id, llvm_type1, llvm_type2) case INTRINS_ ## intrin_name: intrins = add_intrins2(module, id, llvm_type1, llvm_type2); break; #define INTRINS_OVR_3_ARG(intrin_name, llvm_id, llvm_type1, llvm_type2, llvm_type3) case INTRINS_ ## intrin_name: intrins = add_intrins3(module, id, llvm_type1, llvm_type2, llvm_type3); break; + #define INTRINS_OVR_TAG(...) #include "llvm-intrinsics.h" default: @@ -10583,12 +10877,19 @@ add_types (MonoLLVMModule *module) void mono_llvm_init (gboolean enable_jit) { - sse_i1_t = type_to_sse_type (MONO_TYPE_I1); - sse_i2_t = type_to_sse_type (MONO_TYPE_I2); - sse_i4_t = type_to_sse_type (MONO_TYPE_I4); - sse_i8_t = type_to_sse_type (MONO_TYPE_I8); - sse_r4_t = type_to_sse_type (MONO_TYPE_R4); - sse_r8_t = type_to_sse_type (MONO_TYPE_R8); + intrin_types [0][0] = v64_i1_t = LLVMVectorType (LLVMInt8Type (), 8); + intrin_types [0][1] = v64_i2_t = LLVMVectorType (LLVMInt16Type (), 4); + intrin_types [0][2] = v64_i4_t = LLVMVectorType (LLVMInt32Type (), 2); + intrin_types [0][3] = v64_i8_t = LLVMVectorType (LLVMInt64Type (), 1); + intrin_types [0][4] = v64_r4_t = LLVMVectorType (LLVMFloatType (), 2); + intrin_types [0][5] = v64_r8_t = LLVMVectorType (LLVMDoubleType (), 1); + + intrin_types [1][0] = v128_i1_t = sse_i1_t = type_to_sse_type (MONO_TYPE_I1); + intrin_types [1][1] = v128_i2_t = sse_i2_t = type_to_sse_type (MONO_TYPE_I2); + intrin_types [1][2] = v128_i4_t = sse_i4_t = type_to_sse_type (MONO_TYPE_I4); + intrin_types [1][3] = v128_i8_t = sse_i8_t = type_to_sse_type (MONO_TYPE_I8); + intrin_types [1][4] = v128_r4_t = sse_r4_t = type_to_sse_type (MONO_TYPE_R4); + intrin_types [1][5] = v128_r8_t = sse_r8_t = type_to_sse_type (MONO_TYPE_R8); intrins_id_to_intrins = g_hash_table_new (NULL, NULL); diff --git a/src/mono/mono/mini/mini-ops.h b/src/mono/mono/mini/mini-ops.h index 93036193ce11bd..9964c8842c4a90 100644 --- a/src/mono/mono/mini/mini-ops.h +++ b/src/mono/mono/mini/mini-ops.h @@ -830,6 +830,8 @@ MINI_OP(OP_EXPAND_R4, "expand_r4", XREG, FREG, NONE) MINI_OP(OP_EXPAND_I8, "expand_i8", XREG, IREG, NONE) MINI_OP(OP_EXPAND_R8, "expand_r8", XREG, FREG, NONE) +MINI_OP(OP_XXOR, "xxor", XREG, XREG, XREG) + #endif #if defined(TARGET_X86) || defined(TARGET_AMD64) || defined(TARGET_WASM) @@ -1022,7 +1024,6 @@ MINI_OP(OP_SSE_UNPACKHI, "sse_unpackhi", XREG, XREG, XREG) MINI_OP(OP_SSE_SHUFFLE, "sse_shuffle", XREG, XREG, XREG) MINI_OP(OP_SSE_AND, "sse_and", XREG, XREG, XREG) MINI_OP(OP_SSE_OR, "sse_or", XREG, XREG, XREG) -MINI_OP(OP_SSE_XOR, "sse_xor", XREG, XREG, XREG) MINI_OP(OP_SSE_ANDN, "sse_andn", XREG, XREG, XREG) MINI_OP(OP_SSE_ADDSS, "sse_addss", XREG, XREG, XREG) MINI_OP(OP_SSE_SUBSS, "sse_subss", XREG, XREG, XREG) @@ -1552,6 +1553,9 @@ MINI_OP(OP_XOP_I4_I4_I4, "xop_i4_i4_i4", IREG, IREG, IREG) MINI_OP(OP_XOP_I4_I4_I8, "xop_i4_i4_i8", IREG, IREG, LREG) MINI_OP3(OP_XOP_X_X_X_X, "xop_x_x_x_x", XREG, XREG, XREG, XREG) +MINI_OP(OP_XOP_OVR_X_X, "xop_ovr_x", XREG, XREG, NONE) +MINI_OP(OP_XOP_OVR_X_X_X, "xop_ovr_x_x", XREG, XREG, XREG) + MINI_OP(OP_XCAST, "xcast", XREG, XREG, NONE) /* Extract element of vector */ /* The index is assumed to be in range */ @@ -1581,4 +1585,30 @@ MINI_OP(OP_LSCNT32, "lscnt32", IREG, IREG, NONE) MINI_OP(OP_LSCNT64, "lscnt64", LREG, LREG, NONE) MINI_OP(OP_ARM64_SMULH, "arm64_smulh", LREG, LREG, LREG) MINI_OP(OP_ARM64_UMULH, "arm64_umulh", LREG, LREG, LREG) +MINI_OP(OP_ARM64_SXTL, "arm64_sxtl", XREG, XREG, NONE) +MINI_OP(OP_ARM64_SXTL2, "arm64_sxtl2", XREG, XREG, NONE) +MINI_OP(OP_ARM64_SQRT_SCALAR, "arm64_sqrt_scalar", XREG, XREG, NONE) +MINI_OP3(OP_ARM64_STP_SCALAR, "arm64_stp_scalar", NONE, IREG, XREG, XREG) +MINI_OP3(OP_ARM64_STP, "arm64_stp", NONE, IREG, XREG, XREG) +MINI_OP3(OP_ARM64_STNP_SCALAR, "arm64_stnp_scalar", NONE, IREG, XREG, XREG) +MINI_OP3(OP_ARM64_STNP, "arm64_stnp", NONE, IREG, XREG, XREG) +MINI_OP(OP_ARM64_ST1, "arm64_st1", NONE, IREG, XREG) +MINI_OP3(OP_ARM64_ST1_SCALAR, "arm64_st1_scalar", NONE, IREG, XREG, IREG) +MINI_OP(OP_ARM64_SUBHN, "arm64_subhn", XREG, XREG, NONE) +MINI_OP3(OP_ARM64_SUBHN2, "arm64_subhn2", XREG, XREG, XREG, XREG) +MINI_OP(OP_ARM64_RSUBHN, "arm64_rsubhn", XREG, XREG, XREG) +MINI_OP3(OP_ARM64_RSUBHN2, "arm64_rsubhn2", XREG, XREG, XREG, XREG) +MINI_OP(OP_ARM64_SSUB, "arm64_ssub", XREG, XREG, XREG) +MINI_OP(OP_ARM64_USUB, "arm64_usub", XREG, XREG, XREG) +MINI_OP(OP_ARM64_SSUB2, "arm64_ssub2", XREG, XREG, XREG) +MINI_OP(OP_ARM64_USUB2, "arm64_usub2", XREG, XREG, XREG) +MINI_OP(OP_ARM64_UXTL, "arm64_uxtl", XREG, XREG, NONE) +MINI_OP(OP_ARM64_UXTL2, "arm64_uxtl2", XREG, XREG, NONE) +MINI_OP(OP_ARM64_TRN1, "arm64_trn1", XREG, XREG, XREG) +MINI_OP(OP_ARM64_TRN2, "arm64_trn2", XREG, XREG, XREG) +MINI_OP(OP_ARM64_UZP1, "arm64_uzp1", XREG, XREG, XREG) +MINI_OP(OP_ARM64_UZP2, "arm64_uzp2", XREG, XREG, XREG) +MINI_OP(OP_ARM64_ZIP1, "arm64_zip1", XREG, XREG, XREG) +MINI_OP(OP_ARM64_ZIP2, "arm64_zip2", XREG, XREG, XREG) +MINI_OP(OP_ARM64_ZERO_UPPER, "arm64_zero_upper", XREG, XREG, NONE) #endif // TARGET_ARM64 diff --git a/src/mono/mono/mini/mini.h b/src/mono/mono/mini/mini.h index e5fdbde9c441c2..34d3a14ad83fc4 100644 --- a/src/mono/mono/mini/mini.h +++ b/src/mono/mono/mini/mini.h @@ -2996,7 +2996,9 @@ typedef enum { SIMD_OP_ARM64_SHA256SU0, SIMD_OP_ARM64_SHA256SU1, SIMD_OP_ARM64_PMULL64_LOWER, - SIMD_OP_ARM64_PMULL64_UPPER + SIMD_OP_ARM64_PMULL64_UPPER, + SIMD_OP_ARM64_TBL, + SIMD_OP_ARM64_TBX, } SimdOp; const char *mono_arch_xregname (int reg); diff --git a/src/mono/mono/mini/simd-intrinsics.c b/src/mono/mono/mini/simd-intrinsics.c index 2318b9f4be936f..ec01d532830447 100644 --- a/src/mono/mono/mini/simd-intrinsics.c +++ b/src/mono/mono/mini/simd-intrinsics.c @@ -11,6 +11,7 @@ #include "ir-emit.h" #ifdef ENABLE_LLVM #include "mini-llvm.h" +#include "mini-llvm-cpp.h" #endif #include "mono/utils/bsearch.h" #include @@ -99,7 +100,7 @@ static int lookup_intrins (guint16 *intrinsics, int size, MonoMethod *cmethod) { const guint16 *result = (const guint16 *)mono_binary_search (cmethod->name, intrinsics, size / sizeof (guint16), sizeof (guint16), &simd_intrinsic_compare_by_name); - + if (result == NULL) return -1; else @@ -270,6 +271,20 @@ get_vector_t_elem_type (MonoType *vector_type) return etype; } +static gboolean +type_is_unsigned (MonoType *type) { + MonoClass *klass = mono_class_from_mono_type_internal (type); + MonoType *etype = mono_class_get_context (klass)->class_inst->type_argv [0]; + switch (etype->type) { + case MONO_TYPE_U1: + case MONO_TYPE_U2: + case MONO_TYPE_U4: + case MONO_TYPE_U8: + return TRUE; + } + return FALSE; +} + static int type_to_expand_op (MonoType *type) { @@ -955,6 +970,35 @@ static SimdIntrinsic advsimd_methods [] = { {SN_AbsoluteCompareGreaterThanOrEqual}, {SN_AbsoluteCompareLessThan}, {SN_AbsoluteCompareLessThanOrEqual}, + {SN_SignExtendWideningLower, OP_ARM64_SXTL}, + {SN_SignExtendWideningUpper, OP_ARM64_SXTL2}, + {SN_SqrtScalar, OP_ARM64_SQRT_SCALAR}, + {SN_Sqrt}, + {SN_Store, OP_ARM64_ST1}, + {SN_StorePair, OP_ARM64_STP}, + {SN_StorePairNonTemporal, OP_ARM64_STNP}, + {SN_StorePairScalar, OP_ARM64_STP_SCALAR}, + {SN_StorePairScalarNonTemporal, OP_ARM64_STNP_SCALAR}, + {SN_StoreSelectedScalar, OP_ARM64_ST1_SCALAR}, + {SN_SubtractHighNarrowingLower, OP_ARM64_SUBHN}, + {SN_SubtractHighNarrowingUpper, OP_ARM64_SUBHN2}, + {SN_SubtractRoundedHighNarrowingLower, OP_ARM64_RSUBHN}, + {SN_SubtractRoundedHighNarrowingUpper, OP_ARM64_RSUBHN2}, + {SN_SubtractSaturateScalar}, + {SN_SubtractScalar}, + {SN_SubtractWideningUpper}, + {SN_Subtract}, + {SN_TransposeEven, OP_ARM64_TRN1}, + {SN_TransposeOdd, OP_ARM64_TRN2}, + {SN_UnzipEven, OP_ARM64_UZP1}, + {SN_UnzipOdd, OP_ARM64_UZP2}, + {SN_VectorTableLookup, OP_XOP_X_X_X, SIMD_OP_ARM64_TBL}, + {SN_VectorTableLookupExtension, OP_XOP_X_X_X_X, SIMD_OP_ARM64_TBX}, + {SN_Xor, OP_XXOR}, + {SN_ZeroExtendWideningLower, OP_ARM64_UXTL}, + {SN_ZeroExtendWideningUpper, OP_ARM64_UXTL2}, + {SN_ZipHigh, OP_ARM64_ZIP2}, + {SN_ZipLow, OP_ARM64_ZIP1}, {SN_get_IsSupported}, }; @@ -973,7 +1017,7 @@ MonoInst *emit_absolute_compare (MonoCompile *cfg, MonoClass *klass, MonoMethodS default: g_assert_not_reached(); } - + return emit_simd_ins_for_sig (cfg, klass, OP_XOP_X_X_X, op, arg0_type, fsig, args); } @@ -1039,16 +1083,16 @@ emit_arm64_intrinsics ( } if (feature == MONO_CPU_ARM64_NEON) { + SimdOp op = (SimdOp) 0; + IntrinsicId iid = (IntrinsicId) -1; switch (id) { case SN_Abs: { - SimdOp op = (SimdOp)0; - // HACK: Temporary, while Vector64 support is completed MonoClass *arg0_klass = mono_class_from_mono_type_internal (fsig->params [0]); if (m_class_get_name (arg0_klass), "Vector64`1") mono_emit_jit_icall (cfg, mono_throw_platform_not_supported, NULL); - switch (get_underlying_type (fsig->params [0])) { + switch (arg0_type) { case MONO_TYPE_R8: op = SIMD_OP_ARM64_DABS; break; @@ -1097,43 +1141,73 @@ emit_arm64_intrinsics ( } case SN_AbsSaturate: { - SimdOp op = (SimdOp)0; - switch (get_underlying_type (fsig->params [0])) { - case MONO_TYPE_I1: - op = SIMD_OP_ARM64_I8ABS_SATURATE; - break; - case MONO_TYPE_I2: - op = SIMD_OP_ARM64_I16ABS_SATURATE; - break; - case MONO_TYPE_I4: - op = SIMD_OP_ARM64_I32ABS_SATURATE; - break; - case MONO_TYPE_I8: - op = SIMD_OP_ARM64_I64ABS_SATURATE; - break; + switch (arg0_type) { + case MONO_TYPE_I1: op = SIMD_OP_ARM64_I8ABS_SATURATE; break; + case MONO_TYPE_I2: op = SIMD_OP_ARM64_I16ABS_SATURATE; break; + case MONO_TYPE_I4: op = SIMD_OP_ARM64_I32ABS_SATURATE; break; + case MONO_TYPE_I8: op = SIMD_OP_ARM64_I64ABS_SATURATE; break; + default: g_assert_not_reached (); } return emit_simd_ins_for_sig (cfg, klass, OP_XOP_X_X, op, arg0_type, fsig, args); } case SN_AbsScalar: { - SimdOp op = (SimdOp)0; - switch (get_underlying_type (fsig->params [0])) { - case MONO_TYPE_I1: - op = SIMD_OP_ARM64_I8ABS_SATURATE; - break; - case MONO_TYPE_I2: - op = SIMD_OP_ARM64_I16ABS_SATURATE; - break; - case MONO_TYPE_I4: - op = SIMD_OP_ARM64_I32ABS_SATURATE; - break; - case MONO_TYPE_I8: - op = SIMD_OP_ARM64_I64ABS_SATURATE; - break; + switch (arg0_type) { + case MONO_TYPE_I1: op = SIMD_OP_ARM64_I8ABS_SATURATE; break; + case MONO_TYPE_I2: op = SIMD_OP_ARM64_I16ABS_SATURATE; break; + case MONO_TYPE_I4: op = SIMD_OP_ARM64_I32ABS_SATURATE; break; + case MONO_TYPE_I8: op = SIMD_OP_ARM64_I64ABS_SATURATE; break; + default: g_assert_not_reached (); } return emit_simd_ins_for_sig (cfg, klass, OP_XOP_X_X, op, arg0_type, fsig, args); - } + } + case SN_Sqrt: { + llvm_ovr_tag_t tag = ovr_tag_from_mono_vector_type (fsig->ret); + return emit_simd_ins_for_sig (cfg, klass, OP_XOP_OVR_X_X, INTRINS_AARCH64_ADV_SIMD_FSQRT, tag, fsig, args); + } + case SN_SubtractScalar: + case SN_Subtract: { + gboolean is_float = FALSE; + switch (arg0_type) { + case MONO_TYPE_R4: case MONO_TYPE_R8: is_float = TRUE; + } + op = is_float ? OP_FSUB : OP_ISUB; + MonoInst *ret = emit_simd_ins_for_sig (cfg, klass, OP_XBINOP, op, arg0_type, fsig, args); + if (id == SN_SubtractScalar) + ret = emit_simd_ins (cfg, klass, OP_ARM64_ZERO_UPPER, ret->dreg, -1); + return ret; + } + case SN_SubtractSaturate: + case SN_SubtractSaturateScalar: { + llvm_ovr_tag_t tag = ovr_tag_from_mono_vector_type (fsig->ret); + gboolean is_unsigned = type_is_unsigned (fsig->ret); + iid = is_unsigned ? INTRINS_AARCH64_ADV_SIMD_UQSUB : INTRINS_AARCH64_ADV_SIMD_SQSUB; + MonoInst *ret = emit_simd_ins_for_sig (cfg, klass, OP_XOP_OVR_X_X_X, iid, tag, fsig, args); + /* LLVM has intrinsic functions for only the 32 and 64-bit forms of the scalar variants + * of NEON uqsub and sqsub. The CoreCLR runtime tests for these intrinsics look like they + * assert that all lanes > 0 are zeroed out, so just use the vector variant of these + * instructions here and then set the upper "non-scalar" bits to zero. + */ + if (id == SN_SubtractSaturateScalar) + ret = emit_simd_ins (cfg, klass, OP_ARM64_ZERO_UPPER, ret->dreg, -1); + return ret; + } + case SN_SubtractWideningLower: + case SN_SubtractWideningUpper: { + gboolean is_upper = id == SN_SubtractWideningUpper; + MonoTypeEnum ret_t = get_underlying_type (fsig->params [1]); + int op = is_upper ? OP_ARM64_SSUB2 : OP_ARM64_SSUB; + switch (ret_t) { + case MONO_TYPE_U1: + case MONO_TYPE_U2: + case MONO_TYPE_U4: + case MONO_TYPE_U8: + op = is_upper ? OP_ARM64_USUB2 : OP_ARM64_USUB; + break; + } + return emit_simd_ins_for_sig (cfg, klass, op, 0, 0, fsig, args); + } default: g_assert_not_reached (); } @@ -1231,7 +1305,7 @@ static SimdIntrinsic sse_methods [] = { {SN_SubtractScalar, OP_SSE_SUBSS}, {SN_UnpackHigh, OP_SSE_UNPACKHI}, {SN_UnpackLow, OP_SSE_UNPACKLO}, - {SN_Xor, OP_SSE_XOR}, + {SN_Xor, OP_XXOR}, {SN_get_IsSupported} }; @@ -1343,7 +1417,7 @@ static SimdIntrinsic sse2_methods [] = { {SN_SumAbsoluteDifferences, OP_XOP_X_X_X, SIMD_OP_SSE_PSADBW}, {SN_UnpackHigh, OP_SSE_UNPACKHI}, {SN_UnpackLow, OP_SSE_UNPACKLO}, - {SN_Xor, OP_SSE_XOR}, + {SN_Xor, OP_XXOR}, {SN_get_IsSupported} }; diff --git a/src/mono/mono/mini/simd-methods.h b/src/mono/mono/mini/simd-methods.h index 6b529cb95480b5..1619dcb9315f15 100644 --- a/src/mono/mono/mini/simd-methods.h +++ b/src/mono/mono/mini/simd-methods.h @@ -255,3 +255,27 @@ METHOD(AbsoluteCompareGreaterThan) METHOD(AbsoluteCompareGreaterThanOrEqual) METHOD(AbsoluteCompareLessThan) METHOD(AbsoluteCompareLessThanOrEqual) +METHOD(SignExtendWideningLower) +METHOD(SignExtendWideningUpper) +METHOD(StorePair) +METHOD(StorePairNonTemporal) +METHOD(StorePairScalar) +METHOD(StorePairScalarNonTemporal) +METHOD(StoreSelectedScalar) +METHOD(SubtractHighNarrowingLower) +METHOD(SubtractHighNarrowingUpper) +METHOD(SubtractRoundedHighNarrowingLower) +METHOD(SubtractRoundedHighNarrowingUpper) +METHOD(SubtractSaturateScalar) +METHOD(SubtractWideningLower) +METHOD(SubtractWideningUpper) +METHOD(TransposeEven) +METHOD(TransposeOdd) +METHOD(UnzipEven) +METHOD(UnzipOdd) +METHOD(VectorTableLookup) +METHOD(VectorTableLookupExtension) +METHOD(ZeroExtendWideningLower) +METHOD(ZeroExtendWideningUpper) +METHOD(ZipHigh) +METHOD(ZipLow) From 706cf0efdae5ab0f2f560d1fec60de3c4f035181 Mon Sep 17 00:00:00 2001 From: Imran Hameed Date: Thu, 4 Mar 2021 20:03:24 -0800 Subject: [PATCH 02/58] Add some shifts --- src/mono/mono/mini/llvm-intrinsics.h | 21 +++ src/mono/mono/mini/mini-llvm.c | 246 ++++++++++++++++++++++++--- src/mono/mono/mini/mini-ops.h | 73 ++++++-- src/mono/mono/mini/simd-intrinsics.c | 85 ++++++++- src/mono/mono/mini/simd-methods.h | 58 +++++++ 5 files changed, 439 insertions(+), 44 deletions(-) diff --git a/src/mono/mono/mini/llvm-intrinsics.h b/src/mono/mono/mini/llvm-intrinsics.h index a6073cc6c61f87..9daa663f4809e1 100644 --- a/src/mono/mono/mini/llvm-intrinsics.h +++ b/src/mono/mono/mini/llvm-intrinsics.h @@ -314,7 +314,28 @@ INTRINS_OVR_TAG(AARCH64_ADV_SIMD_UQSUB, aarch64_neon_uqsub, V64 | V128 | I1 | I2 INTRINS_OVR_TAG(AARCH64_ADV_SIMD_SQSUB, aarch64_neon_sqsub, V64 | V128 | I1 | I2 | I4 | I8) INTRINS_OVR_TAG(AARCH64_ADV_SIMD_RSUBHN, aarch64_neon_rsubhn, V64 | I1 | I2 | I4) INTRINS_OVR_TAG(AARCH64_ADV_SIMD_FSQRT, sqrt, V64 | V128 | R4 | R8) +INTRINS_OVR_TAG(AARCH64_ADV_SIMD_UQSHRN, aarch64_neon_uqshrn, V64 | I1 | I2 | I4) // Constant shift +INTRINS_OVR_TAG(AARCH64_ADV_SIMD_RSHRN, aarch64_neon_rshrn, V64 | I1 | I2 | I4) // Constant shift +INTRINS_OVR_TAG(AARCH64_ADV_SIMD_SQRSHRN, aarch64_neon_sqrshrn, V64 | I1 | I2 | I4) // Constant shift +INTRINS_OVR_TAG(AARCH64_ADV_SIMD_SQRSHRUN, aarch64_neon_sqrshrun, V64 | I1 | I2 | I4) // Constant shift +INTRINS_OVR_TAG(AARCH64_ADV_SIMD_SQSHRN, aarch64_neon_sqshrn, V64 | I1 | I2 | I4) // Constant shift +INTRINS_OVR_TAG(AARCH64_ADV_SIMD_SQSHRUN, aarch64_neon_sqshrun, V64 | I1 | I2 | I4) // Constant shift +INTRINS_OVR_TAG(AARCH64_ADV_SIMD_UQRSHRN, aarch64_neon_uqrshrn, V64 | I1 | I2 | I4) // Constant shift + +INTRINS_OVR_TAG(AARCH64_ADV_SIMD_SQRSHL, aarch64_neon_sqrshl, V64 | V128 | I1 | I2 | I4 | I8) // Variable shift +INTRINS_OVR_TAG(AARCH64_ADV_SIMD_SQSHL, aarch64_neon_sqshl, V64 | V128 | I1 | I2 | I4 | I8) // Variable shift +INTRINS_OVR_TAG(AARCH64_ADV_SIMD_SRSHL, aarch64_neon_srshl, V64 | V128 | I1 | I2 | I4 | I8) // Variable shift +INTRINS_OVR_TAG(AARCH64_ADV_SIMD_SSHL, aarch64_neon_sshl, V64 | V128 | I1 | I2 | I4 | I8) // Variable shift +INTRINS_OVR_TAG(AARCH64_ADV_SIMD_UQRSHL, aarch64_neon_uqrshl, V64 | V128 | I1 | I2 | I4 | I8) // Variable shift +INTRINS_OVR_TAG(AARCH64_ADV_SIMD_UQSHL, aarch64_neon_uqshl, V64 | V128 | I1 | I2 | I4 | I8) // Variable shift +INTRINS_OVR_TAG(AARCH64_ADV_SIMD_URSHL, aarch64_neon_urshl, V64 | V128 | I1 | I2 | I4 | I8) // Variable shift +INTRINS_OVR_TAG(AARCH64_ADV_SIMD_USHL, aarch64_neon_ushl, V64 | V128 | I1 | I2 | I4 | I8) // Variable shift + +INTRINS_OVR_TAG(AARCH64_ADV_SIMD_SQSHLU, aarch64_neon_sqshlu, V64 | V128 | I1 | I2 | I4 | I8) // Constant shift + +INTRINS_OVR_TAG(AARCH64_ADV_SIMD_SLI, aarch64_neon_vsli, V64 | V128 | I1 | I2 | I4 | I8) // Constant shift +INTRINS_OVR_TAG(AARCH64_ADV_SIMD_SRI, aarch64_neon_vsri, V64 | V128 | I1 | I2 | I4 | I8) // Constant shift #endif #undef INTRINS diff --git a/src/mono/mono/mini/mini-llvm.c b/src/mono/mono/mini/mini-llvm.c index 9d452ee929cb70..4815d8e30e411d 100644 --- a/src/mono/mono/mini/mini-llvm.c +++ b/src/mono/mono/mini/mini-llvm.c @@ -4682,6 +4682,28 @@ emit_landing_pad (EmitContext *ctx, int group_index, int group_size) return lpad_bb; } +static LLVMValueRef +broadcast_element (EmitContext *ctx, LLVMValueRef elem, int count) +{ + LLVMTypeRef t = LLVMTypeOf (elem); + LLVMTypeRef init_vec_t = LLVMVectorType (t, 1); + LLVMValueRef undef = LLVMGetUndef (init_vec_t); + LLVMValueRef vec = LLVMBuildInsertElement (ctx->builder, undef, elem, const_int32 (0), ""); + LLVMValueRef select_zero = LLVMConstNull (LLVMVectorType (LLVMInt32Type (), count)); + return LLVMBuildShuffleVector (ctx->builder, vec, undef, select_zero, "broadcast"); +} + +static LLVMValueRef +create_shift_vector (EmitContext *ctx, LLVMValueRef shiftarg, LLVMValueRef shiftamt) +{ + LLVMTypeRef t = LLVMTypeOf (shiftarg); + unsigned int elems = LLVMGetVectorSize (t); + LLVMTypeRef elem_t = LLVMGetElementType (t); + shiftamt = convert_full (ctx, shiftamt, elem_t, TRUE); + shiftamt = broadcast_element (ctx, shiftamt, elems); + return shiftamt; +} + static LLVMValueRef create_const_vector (LLVMTypeRef t, const int *vals, int count) { @@ -9270,6 +9292,191 @@ process_bb (EmitContext *ctx, MonoBasicBlock *bb) values [ins->dreg] = LLVMBuildTrunc (builder, hi64, LLVMInt64Type (), ""); break; } + case OP_ARM64_SHL: + case OP_ARM64_SSHR: + case OP_ARM64_SSRA: + case OP_ARM64_USHR: + case OP_ARM64_USRA: { + gboolean right = FALSE; + gboolean add = FALSE; + gboolean arith = FALSE; + switch (ins->opcode) { + case OP_ARM64_USHR: right = TRUE; break; + case OP_ARM64_USRA: right = TRUE; add = TRUE; break; + case OP_ARM64_SSHR: arith = TRUE; break; + case OP_ARM64_SSRA: arith = TRUE; add = TRUE; break; + } + LLVMValueRef shiftarg = lhs; + LLVMValueRef shift = rhs; + if (add) { + shiftarg = rhs; + shift = arg3; + } + shift = create_shift_vector (ctx, shiftarg, shift); + LLVMValueRef result = NULL; + if (right) + result = LLVMBuildLShr (builder, shiftarg, shift, ""); + else if (arith) + result = LLVMBuildAShr (builder, shiftarg, shift, ""); + else + result = LLVMBuildShl (builder, shiftarg, shift, ""); + if (add) + result = LLVMBuildAdd (builder, lhs, result, "arm64_usra"); + values [ins->dreg] = result; + break; + } + case OP_ARM64_SHRN: + case OP_ARM64_SHRN2: { + LLVMValueRef shiftarg = lhs; + LLVMValueRef shift = rhs; + gboolean high = ins->opcode == OP_ARM64_SHRN2; + if (high) { + shiftarg = rhs; + shift = arg3; + } + shift = create_shift_vector (ctx, shiftarg, shift); + LLVMValueRef result = LLVMBuildLShr (builder, shiftarg, shift, "shrn"); + if (high) + result = concatenate_vectors (ctx, lhs, result); + values [ins->dreg] = result; + break; + } + case OP_ARM64_UQSHRN: + case OP_ARM64_UQSHRN2: { + // XXXih: TODO: ins->klass is wrong for *2 + // XXXih: TODO: unroll count/rhs/arg3 + LLVMValueRef shiftarg = lhs; + LLVMValueRef shift = rhs; + gboolean high = ins->opcode == OP_ARM64_UQSHRN2; + if (high) { + shiftarg = rhs; + shift = arg3; + } + llvm_ovr_tag_t ovr_tag = ovr_tag_from_mono_vector_class (ins->klass); + LLVMValueRef args [] = { shiftarg, shift }; + LLVMValueRef result = call_overloaded_intrins (ctx, INTRINS_AARCH64_ADV_SIMD_UQSHRN, ovr_tag, args, ""); + if (high) + result = concatenate_vectors (ctx, lhs, result); + break; + } + case OP_ARM64_SRSHR: + case OP_ARM64_SRSRA: + case OP_ARM64_URSHR: + case OP_ARM64_URSRA: { + llvm_ovr_tag_t ovr_tag = ovr_tag_from_mono_vector_class (ins->klass); + LLVMValueRef shiftarg = lhs; + LLVMValueRef shift = rhs; + gboolean right = FALSE; + gboolean add = FALSE; + switch (ins->opcode) { + case OP_ARM64_URSHR: right = TRUE; break; + case OP_ARM64_URSRA: right = TRUE; add = TRUE; break; + case OP_ARM64_SRSRA: add = TRUE; break; + } + int iid = 0; + switch (ins->opcode) { + case OP_ARM64_URSHR: + case OP_ARM64_URSRA: iid = INTRINS_AARCH64_ADV_SIMD_URSHL; break; + case OP_ARM64_SRSHR: + case OP_ARM64_SRSRA: iid = INTRINS_AARCH64_ADV_SIMD_SRSHL; break; + } + if (add) { + shiftarg = rhs; + shift = arg3; + } + if (right) + shift = LLVMBuildNeg (builder, shift, ""); + shift = create_shift_vector (ctx, shiftarg, shift); + LLVMValueRef args [] = { shiftarg, shift }; + LLVMValueRef result = call_overloaded_intrins (ctx, iid, ovr_tag, args, ""); + if (add) + result = LLVMBuildAdd (builder, result, lhs, ""); + values [ins->dreg] = result; + break; + } + case OP_ARM64_RSHRN: + case OP_ARM64_RSHRN2: + case OP_ARM64_SQRSHRN: + case OP_ARM64_SQRSHRN2: + case OP_ARM64_SQRSHRUN: + case OP_ARM64_SQRSHRUN2: + case OP_ARM64_SQSHRN: + case OP_ARM64_SQSHRN2: + case OP_ARM64_SQSHRUN: + case OP_ARM64_SQSHRUN2: + case OP_ARM64_UQRSHRN: + case OP_ARM64_UQRSHRN2: { + // XXXih: TODO: ins->klass is wrong for *2 + // XXXih: TODO: unroll count/rhs/arg3 + LLVMValueRef args [2] = { lhs, rhs }; + gboolean high = FALSE; + int iid = 0; + switch (ins->opcode) { + case OP_ARM64_RSHRN: iid = INTRINS_AARCH64_ADV_SIMD_RSHRN; + case OP_ARM64_RSHRN2: high = TRUE; break; + case OP_ARM64_UQRSHRN: iid = INTRINS_AARCH64_ADV_SIMD_UQRSHRN; + case OP_ARM64_UQRSHRN2: high = TRUE; break; + case OP_ARM64_SQRSHRN: iid = INTRINS_AARCH64_ADV_SIMD_SQRSHRN; + case OP_ARM64_SQRSHRN2: high = TRUE; break; + case OP_ARM64_SQRSHRUN: iid = INTRINS_AARCH64_ADV_SIMD_SQRSHRUN; + case OP_ARM64_SQRSHRUN2: high = TRUE; break; + case OP_ARM64_SQSHRN: iid = INTRINS_AARCH64_ADV_SIMD_SQSHRN; + case OP_ARM64_SQSHRN2: high = TRUE; break; + case OP_ARM64_SQSHRUN: iid = INTRINS_AARCH64_ADV_SIMD_SQSHRUN; + case OP_ARM64_SQSHRUN2: high = TRUE; break; + } + if (high) { + args [0] = rhs; + args [1] = arg3; + } + llvm_ovr_tag_t ovr_tag = ovr_tag_from_mono_vector_class (ins->klass); + LLVMValueRef result = call_overloaded_intrins (ctx, iid, ovr_tag, args, ""); + if (high) + result = concatenate_vectors (ctx, lhs, result); + values [ins->dreg] = result; + break; + } + case OP_ARM64_UQSHL_IMM: + case OP_ARM64_SQSHL_IMM: { + llvm_ovr_tag_t ovr_tag = ovr_tag_from_mono_vector_class (ins->klass); + LLVMValueRef shift = create_shift_vector (ctx, lhs, rhs); + LLVMValueRef args [] = { lhs, shift }; + int iid = 0; + switch (ins->opcode) { + case OP_ARM64_UQSHL_IMM: iid = INTRINS_AARCH64_ADV_SIMD_UQSHL; break; + case OP_ARM64_SQSHL_IMM: iid = INTRINS_AARCH64_ADV_SIMD_SQSHL; break; + } + values [ins->dreg] = call_overloaded_intrins (ctx, iid, ovr_tag, args, ""); + break; + } + case OP_ARM64_SQSHLU: { + // XXXih: TODO: unroll count/rhs + llvm_ovr_tag_t ovr_tag = ovr_tag_from_mono_vector_class (ins->klass); + LLVMValueRef shift = create_shift_vector (ctx, lhs, rhs); + LLVMValueRef args [] = { lhs, shift }; + int iid = INTRINS_AARCH64_ADV_SIMD_SQSHLU; break; + values [ins->dreg] = call_overloaded_intrins (ctx, iid, ovr_tag, args, ""); + break; + } + case OP_ARM64_USHLL: + case OP_ARM64_USHLL2: { + gboolean high = ins->opcode == OP_ARM64_USHLL2; + LLVMValueRef result = lhs; + if (high) + result = extract_high_elements (ctx, result); + result = LLVMBuildShl (builder, result, create_shift_vector (ctx, result, rhs), "arm64_ushll"); + values [ins->dreg] = result; + break; + } + case OP_ARM64_SLI: + case OP_ARM64_SRI: { + // XXXih: TODO: unroll count/arg3 + LLVMValueRef args [3] = { lhs, rhs, arg3 }; + llvm_ovr_tag_t ovr_tag = ovr_tag_from_mono_vector_class (ins->klass); + int iid = ins->opcode == OP_ARM64_SLI ? INTRINS_AARCH64_ADV_SIMD_SLI : INTRINS_AARCH64_ADV_SIMD_SRI; + values [ins->dreg] = call_overloaded_intrins (ctx, iid, ovr_tag, args, ""); + break; + } case OP_ARM64_SQRT_SCALAR: { int iid = ins->inst_c0 == MONO_TYPE_R8 ? INTRINS_SQRT : INTRINS_SQRTF; LLVMTypeRef t = LLVMTypeOf (lhs); @@ -9322,24 +9529,23 @@ process_bb (EmitContext *ctx, MonoBasicBlock *bb) } case OP_ARM64_SUBHN: case OP_ARM64_SUBHN2: { - LLVMValueRef l = lhs; - LLVMValueRef r = rhs; - if (ins->opcode == OP_ARM64_SUBHN2) { - l = rhs; - r = arg3; - } - LLVMValueRef result = LLVMBuildSub (builder, l, r, ""); - LLVMTypeRef l_t = LLVMTypeOf (l); - LLVMTypeRef l_elt_t = LLVMGetElementType (l_t); - unsigned int elem_bits = mono_llvm_get_prim_size_bits (l_elt_t); - unsigned int elems = LLVMGetVectorSize (l_t); + LLVMValueRef args [2] = { lhs, rhs }; + gboolean high = ins->opcode == OP_ARM64_SUBHN2; + if (high) { + args [0] = rhs; + args [1] = arg3; + } + LLVMValueRef result = LLVMBuildSub (builder, args [0], args [1], ""); + LLVMTypeRef t = LLVMTypeOf (args [0]); + LLVMTypeRef elt_t = LLVMGetElementType (t); + unsigned int elem_bits = mono_llvm_get_prim_size_bits (elt_t); + unsigned int elems = LLVMGetVectorSize (t); int shift_vals [MAX_VECTOR_ELEMS] = { 0 }; - for (unsigned int i = 0; i < elems; ++i) { + for (unsigned int i = 0; i < elems; ++i) shift_vals [i] = elem_bits / 2; - } - result = LLVMBuildLShr (builder, result, create_const_vector (l_elt_t, shift_vals, elems), ""); + result = LLVMBuildLShr (builder, result, create_const_vector (elt_t, shift_vals, elems), ""); result = LLVMBuildTrunc (builder, result, LLVMVectorType (LLVMIntType (elem_bits / 2), elems), ""); - if (ins->opcode == OP_ARM64_SUBHN2) + if (high) result = concatenate_vectors (ctx, lhs, result); values [ins->dreg] = result; break; @@ -9347,13 +9553,14 @@ process_bb (EmitContext *ctx, MonoBasicBlock *bb) case OP_ARM64_RSUBHN: case OP_ARM64_RSUBHN2: { LLVMValueRef args [2] = { lhs, rhs }; - if (ins->opcode == OP_ARM64_RSUBHN2) { + gboolean high = ins->opcode == OP_ARM64_RSUBHN2; + if (high) { args [0] = rhs; args [1] = arg3; } llvm_ovr_tag_t ovr_tag = ovr_tag_from_mono_vector_class (ins->klass); LLVMValueRef result = call_overloaded_intrins (ctx, INTRINS_AARCH64_ADV_SIMD_RSUBHN, ovr_tag, args, ""); - if (ins->opcode == OP_ARM64_RSUBHN2) + if (high) result = concatenate_vectors (ctx, lhs, result); values [ins->dreg] = result; break; @@ -9431,6 +9638,7 @@ process_bb (EmitContext *ctx, MonoBasicBlock *bb) laneix += 2; } values [ins->dreg] = LLVMBuildShuffleVector (builder, lhs, rhs, create_const_vector_i32 (mask, src_elems), "arm64_uzp"); + break; } case OP_ARM64_UZP1: case OP_ARM64_UZP2: { @@ -9460,13 +9668,13 @@ process_bb (EmitContext *ctx, MonoBasicBlock *bb) } case OP_XOP_OVR_X_X: { IntrinsicId iid = (IntrinsicId) ins->inst_c0; - llvm_ovr_tag_t ovr_tag = ins->inst_c1; + llvm_ovr_tag_t ovr_tag = ovr_tag_from_mono_vector_class (ins->klass); values [ins->dreg] = call_overloaded_intrins (ctx, iid, ovr_tag, &lhs, ""); break; } case OP_XOP_OVR_X_X_X: { IntrinsicId iid = (IntrinsicId) ins->inst_c0; - llvm_ovr_tag_t ovr_tag = ins->inst_c1; + llvm_ovr_tag_t ovr_tag = ovr_tag_from_mono_vector_class (ins->klass); LLVMValueRef args [] = { lhs, rhs }; values [ins->dreg] = call_overloaded_intrins (ctx, iid, ovr_tag, args, ""); break; diff --git a/src/mono/mono/mini/mini-ops.h b/src/mono/mono/mini/mini-ops.h index 9964c8842c4a90..154e077753736d 100644 --- a/src/mono/mono/mini/mini-ops.h +++ b/src/mono/mono/mini/mini-ops.h @@ -1583,32 +1583,73 @@ MINI_OP(OP_POPCNT64, "popcnt64", LREG, LREG, NONE) #ifdef TARGET_ARM64 MINI_OP(OP_LSCNT32, "lscnt32", IREG, IREG, NONE) MINI_OP(OP_LSCNT64, "lscnt64", LREG, LREG, NONE) + +MINI_OP(OP_ARM64_RSUBHN, "arm64_rsubhn", XREG, XREG, XREG) MINI_OP(OP_ARM64_SMULH, "arm64_smulh", LREG, LREG, LREG) -MINI_OP(OP_ARM64_UMULH, "arm64_umulh", LREG, LREG, LREG) -MINI_OP(OP_ARM64_SXTL, "arm64_sxtl", XREG, XREG, NONE) -MINI_OP(OP_ARM64_SXTL2, "arm64_sxtl2", XREG, XREG, NONE) MINI_OP(OP_ARM64_SQRT_SCALAR, "arm64_sqrt_scalar", XREG, XREG, NONE) -MINI_OP3(OP_ARM64_STP_SCALAR, "arm64_stp_scalar", NONE, IREG, XREG, XREG) -MINI_OP3(OP_ARM64_STP, "arm64_stp", NONE, IREG, XREG, XREG) -MINI_OP3(OP_ARM64_STNP_SCALAR, "arm64_stnp_scalar", NONE, IREG, XREG, XREG) -MINI_OP3(OP_ARM64_STNP, "arm64_stnp", NONE, IREG, XREG, XREG) +MINI_OP(OP_ARM64_SSUB, "arm64_ssub", XREG, XREG, XREG) +MINI_OP(OP_ARM64_SSUB2, "arm64_ssub2", XREG, XREG, XREG) MINI_OP(OP_ARM64_ST1, "arm64_st1", NONE, IREG, XREG) -MINI_OP3(OP_ARM64_ST1_SCALAR, "arm64_st1_scalar", NONE, IREG, XREG, IREG) MINI_OP(OP_ARM64_SUBHN, "arm64_subhn", XREG, XREG, NONE) -MINI_OP3(OP_ARM64_SUBHN2, "arm64_subhn2", XREG, XREG, XREG, XREG) -MINI_OP(OP_ARM64_RSUBHN, "arm64_rsubhn", XREG, XREG, XREG) -MINI_OP3(OP_ARM64_RSUBHN2, "arm64_rsubhn2", XREG, XREG, XREG, XREG) -MINI_OP(OP_ARM64_SSUB, "arm64_ssub", XREG, XREG, XREG) +MINI_OP(OP_ARM64_SXTL, "arm64_sxtl", XREG, XREG, NONE) +MINI_OP(OP_ARM64_SXTL2, "arm64_sxtl2", XREG, XREG, NONE) +MINI_OP(OP_ARM64_TRN1, "arm64_trn1", XREG, XREG, XREG) +MINI_OP(OP_ARM64_TRN2, "arm64_trn2", XREG, XREG, XREG) +MINI_OP(OP_ARM64_UMULH, "arm64_umulh", LREG, LREG, LREG) MINI_OP(OP_ARM64_USUB, "arm64_usub", XREG, XREG, XREG) -MINI_OP(OP_ARM64_SSUB2, "arm64_ssub2", XREG, XREG, XREG) MINI_OP(OP_ARM64_USUB2, "arm64_usub2", XREG, XREG, XREG) MINI_OP(OP_ARM64_UXTL, "arm64_uxtl", XREG, XREG, NONE) MINI_OP(OP_ARM64_UXTL2, "arm64_uxtl2", XREG, XREG, NONE) -MINI_OP(OP_ARM64_TRN1, "arm64_trn1", XREG, XREG, XREG) -MINI_OP(OP_ARM64_TRN2, "arm64_trn2", XREG, XREG, XREG) MINI_OP(OP_ARM64_UZP1, "arm64_uzp1", XREG, XREG, XREG) MINI_OP(OP_ARM64_UZP2, "arm64_uzp2", XREG, XREG, XREG) +MINI_OP(OP_ARM64_ZERO_UPPER, "arm64_zero_upper", XREG, XREG, NONE) MINI_OP(OP_ARM64_ZIP1, "arm64_zip1", XREG, XREG, XREG) MINI_OP(OP_ARM64_ZIP2, "arm64_zip2", XREG, XREG, XREG) -MINI_OP(OP_ARM64_ZERO_UPPER, "arm64_zero_upper", XREG, XREG, NONE) +MINI_OP3(OP_ARM64_RSUBHN2, "arm64_rsubhn2", XREG, XREG, XREG, XREG) +MINI_OP3(OP_ARM64_ST1_SCALAR, "arm64_st1_scalar", NONE, IREG, XREG, IREG) +MINI_OP3(OP_ARM64_STNP, "arm64_stnp", NONE, IREG, XREG, XREG) +MINI_OP3(OP_ARM64_STNP_SCALAR, "arm64_stnp_scalar", NONE, IREG, XREG, XREG) +MINI_OP3(OP_ARM64_STP, "arm64_stp", NONE, IREG, XREG, XREG) +MINI_OP3(OP_ARM64_STP_SCALAR, "arm64_stp_scalar", NONE, IREG, XREG, XREG) +MINI_OP3(OP_ARM64_SUBHN2, "arm64_subhn2", XREG, XREG, XREG, XREG) + +MINI_OP(OP_ARM64_SHRN, "arm64_shrn", XREG, XREG, IREG) +MINI_OP(OP_ARM64_UQSHRN, "arm64_uqshrn", XREG, XREG, IREG) +MINI_OP3(OP_ARM64_SHRN2, "arm64_shrn2", XREG, XREG, XREG, IREG) +MINI_OP3(OP_ARM64_UQSHRN2, "arm64_uqshrn2", XREG, XREG, XREG, IREG) + +MINI_OP3(OP_ARM64_SLI, "arm64_sli", XREG, XREG, XREG, IREG) +MINI_OP3(OP_ARM64_SRI, "arm64_sri", XREG, XREG, XREG, IREG) + +MINI_OP(OP_ARM64_SRSHR, "arm64_srshr", XREG, XREG, IREG) +MINI_OP(OP_ARM64_URSHR, "arm64_urshr", XREG, XREG, IREG) +MINI_OP3(OP_ARM64_SRSRA, "arm64_srsra", XREG, XREG, XREG, IREG) +MINI_OP3(OP_ARM64_URSRA, "arm64_ursra", XREG, XREG, XREG, IREG) + +MINI_OP(OP_ARM64_SHL, "arm64_shl", XREG, XREG, IREG) +MINI_OP(OP_ARM64_SSHR, "arm64_sshr", XREG, XREG, IREG) +MINI_OP(OP_ARM64_USHR, "arm64_ushr", XREG, XREG, IREG) +MINI_OP3(OP_ARM64_USRA, "arm64_usra", XREG, XREG, XREG, IREG) +MINI_OP3(OP_ARM64_SSRA, "arm64_ssra", XREG, XREG, XREG, IREG) + +MINI_OP(OP_ARM64_USHLL, "arm64_ushll", XREG, XREG, IREG) +MINI_OP(OP_ARM64_USHLL2, "arm64_ushll2", XREG, XREG, IREG) + +MINI_OP(OP_ARM64_RSHRN, "arm64_rshrn", XREG, XREG, IREG) +MINI_OP(OP_ARM64_SQRSHRN, "arm64_sqrshrn", XREG, XREG, IREG) +MINI_OP(OP_ARM64_SQRSHRUN, "arm64_sqrshrun", XREG, XREG, IREG) +MINI_OP(OP_ARM64_SQSHRN, "arm64_sqshrn", XREG, XREG, IREG) +MINI_OP(OP_ARM64_SQSHRUN, "arm64_sqshrun", XREG, XREG, IREG) +MINI_OP(OP_ARM64_UQRSHRN, "arm64_uqrshrn", XREG, XREG, IREG) +MINI_OP3(OP_ARM64_RSHRN2, "arm64_rshrn2", XREG, XREG, XREG, IREG) +MINI_OP3(OP_ARM64_SQRSHRN2, "arm64_sqrshrn2", XREG, XREG, XREG, IREG) +MINI_OP3(OP_ARM64_SQRSHRUN2, "arm64_sqrshrun2", XREG, XREG, XREG, IREG) +MINI_OP3(OP_ARM64_SQSHRN2, "arm64_sqshrn2", XREG, XREG, XREG, IREG) +MINI_OP3(OP_ARM64_SQSHRUN2, "arm64_sqshrun2", XREG, XREG, XREG, IREG) +MINI_OP3(OP_ARM64_UQRSHRN2, "arm64_uqrshrn2", XREG, XREG, XREG, IREG) + +MINI_OP(OP_ARM64_UQSHL_IMM, "arm64_uqshl_imm", XREG, XREG, IREG) +MINI_OP(OP_ARM64_SQSHL_IMM, "arm64_uqshl_imm", XREG, XREG, IREG) + +MINI_OP(OP_ARM64_SQSHLU, "arm64_sqshlu", XREG, XREG, IREG) #endif // TARGET_ARM64 diff --git a/src/mono/mono/mini/simd-intrinsics.c b/src/mono/mono/mini/simd-intrinsics.c index ec01d532830447..1f6f6dea9875dc 100644 --- a/src/mono/mono/mini/simd-intrinsics.c +++ b/src/mono/mono/mini/simd-intrinsics.c @@ -963,17 +963,78 @@ static SimdIntrinsic sha256_methods [] = { }; static SimdIntrinsic advsimd_methods [] = { - {SN_Abs}, {SN_AbsSaturate}, {SN_AbsScalar}, - {SN_AbsoluteCompareGreaterThan}, {SN_AbsoluteCompareGreaterThanOrEqual}, - {SN_AbsoluteCompareLessThan}, + {SN_AbsoluteCompareGreaterThan}, {SN_AbsoluteCompareLessThanOrEqual}, + {SN_AbsoluteCompareLessThan}, + {SN_Abs}, + {SN_ShiftArithmetic, OP_XOP_OVR_X_X_X, INTRINS_AARCH64_ADV_SIMD_SSHL}, + {SN_ShiftArithmeticRounded, OP_XOP_OVR_X_X_X, INTRINS_AARCH64_ADV_SIMD_SRSHL}, + {SN_ShiftArithmeticRoundedSaturate, OP_XOP_OVR_X_X_X, INTRINS_AARCH64_ADV_SIMD_SQRSHL}, + {SN_ShiftArithmeticRoundedSaturateScalar, OP_XOP_OVR_X_X_X, INTRINS_AARCH64_ADV_SIMD_SQRSHL}, + {SN_ShiftArithmeticRoundedScalar, OP_XOP_OVR_X_X_X, INTRINS_AARCH64_ADV_SIMD_SRSHL}, + {SN_ShiftArithmeticSaturate, OP_XOP_OVR_X_X_X, INTRINS_AARCH64_ADV_SIMD_SQSHL}, + {SN_ShiftArithmeticSaturateScalar, OP_XOP_OVR_X_X_X, INTRINS_AARCH64_ADV_SIMD_SQSHL}, + {SN_ShiftArithmeticScalar, OP_XOP_OVR_X_X_X, INTRINS_AARCH64_ADV_SIMD_SSHL}, + {SN_ShiftLeftAndInsert, OP_ARM64_SRI}, + {SN_ShiftLeftAndInsertScalar, OP_ARM64_SRI}, + {SN_ShiftLeftLogical, OP_ARM64_SHL}, + {SN_ShiftLeftLogicalSaturateScalar}, + {SN_ShiftLeftLogicalSaturateUnsigned, OP_ARM64_SQSHLU}, + {SN_ShiftLeftLogicalSaturateUnsignedScalar, OP_ARM64_SQSHLU}, + {SN_ShiftLeftLogicalSaturate}, + {SN_ShiftLeftLogicalScalar, OP_ARM64_SHL}, + {SN_ShiftLeftLogicalWideningLower, OP_ARM64_USHLL}, + {SN_ShiftLeftLogicalWideningUpper, OP_ARM64_USHLL2}, + {SN_ShiftLogical, OP_XOP_OVR_X_X_X, INTRINS_AARCH64_ADV_SIMD_USHL}, + {SN_ShiftLogicalRounded, OP_XOP_OVR_X_X_X, INTRINS_AARCH64_ADV_SIMD_URSHL}, + {SN_ShiftLogicalRoundedSaturate, OP_XOP_OVR_X_X_X, INTRINS_AARCH64_ADV_SIMD_UQRSHL}, + {SN_ShiftLogicalRoundedSaturateScalar, OP_XOP_OVR_X_X_X, INTRINS_AARCH64_ADV_SIMD_UQRSHL}, + {SN_ShiftLogicalRoundedScalar, OP_XOP_OVR_X_X_X, INTRINS_AARCH64_ADV_SIMD_URSHL}, + {SN_ShiftLogicalSaturate, OP_XOP_OVR_X_X_X, INTRINS_AARCH64_ADV_SIMD_UQSHL}, + {SN_ShiftLogicalSaturateScalar, OP_XOP_OVR_X_X_X, INTRINS_AARCH64_ADV_SIMD_UQSHL}, + {SN_ShiftLogicalScalar, OP_XOP_OVR_X_X_X, INTRINS_AARCH64_ADV_SIMD_USHL}, + {SN_ShiftRightAndInsert, OP_ARM64_SRI}, + {SN_ShiftRightAndInsertScalar, OP_ARM64_SRI}, + {SN_ShiftRightArithmetic, OP_ARM64_SSHR}, + {SN_ShiftRightArithmeticAdd, OP_ARM64_SSRA}, + {SN_ShiftRightArithmeticAddScalar, OP_ARM64_SSRA}, + {SN_ShiftRightArithmeticNarrowingSaturateLower, OP_ARM64_SQSHRN}, + {SN_ShiftRightArithmeticNarrowingSaturateUnsignedLower, OP_ARM64_SQSHRUN}, + {SN_ShiftRightArithmeticNarrowingSaturateUnsignedUpper, OP_ARM64_SQSHRUN2}, + {SN_ShiftRightArithmeticNarrowingSaturateUpper, OP_ARM64_SQSHRN2}, + {SN_ShiftRightArithmeticRounded, OP_ARM64_SRSHR}, + {SN_ShiftRightArithmeticRoundedAdd, OP_ARM64_SRSRA}, + {SN_ShiftRightArithmeticRoundedAddScalar, OP_ARM64_SRSRA}, + {SN_ShiftRightArithmeticRoundedNarrowingSaturateLower, OP_ARM64_SQRSHRN}, + {SN_ShiftRightArithmeticRoundedNarrowingSaturateUnsignedLower, OP_ARM64_SQRSHRUN}, + {SN_ShiftRightArithmeticRoundedNarrowingSaturateUnsignedUpper, OP_ARM64_SQRSHRUN2}, + {SN_ShiftRightArithmeticRoundedNarrowingSaturateUpper, OP_ARM64_SQRSHRN2}, + {SN_ShiftRightArithmeticRoundedScalar, OP_ARM64_SRSHR}, + {SN_ShiftRightArithmeticScalar, OP_ARM64_SSHR}, + {SN_ShiftRightLogical, OP_ARM64_USHR}, + {SN_ShiftRightLogicalAdd, OP_ARM64_USRA}, + {SN_ShiftRightLogicalAddScalar, OP_ARM64_USRA}, + {SN_ShiftRightLogicalNarrowingLower, OP_ARM64_SHRN}, + {SN_ShiftRightLogicalNarrowingSaturateLower, OP_ARM64_UQSHRN}, + {SN_ShiftRightLogicalNarrowingSaturateUpper, OP_ARM64_UQSHRN2}, + {SN_ShiftRightLogicalNarrowingUpper, OP_ARM64_SHRN2}, + {SN_ShiftRightLogicalRounded, OP_ARM64_URSHR}, + {SN_ShiftRightLogicalRoundedAdd, OP_ARM64_URSRA}, + {SN_ShiftRightLogicalRoundedAddScalar, OP_ARM64_URSRA}, + {SN_ShiftRightLogicalRoundedNarrowingLower, OP_ARM64_RSHRN}, + {SN_ShiftRightLogicalRoundedNarrowingSaturateLower, OP_ARM64_UQRSHRN}, + {SN_ShiftRightLogicalRoundedNarrowingSaturateScalar}, + {SN_ShiftRightLogicalRoundedNarrowingSaturateUpper, OP_ARM64_UQRSHRN2}, + {SN_ShiftRightLogicalRoundedNarrowingUpper, OP_ARM64_RSHRN2}, + {SN_ShiftRightLogicalRoundedScalar, OP_ARM64_URSHR}, + {SN_ShiftRightLogicalScalar, OP_ARM64_USHR}, {SN_SignExtendWideningLower, OP_ARM64_SXTL}, {SN_SignExtendWideningUpper, OP_ARM64_SXTL2}, + {SN_Sqrt, OP_XOP_OVR_X_X, INTRINS_AARCH64_ADV_SIMD_FSQRT}, {SN_SqrtScalar, OP_ARM64_SQRT_SCALAR}, - {SN_Sqrt}, {SN_Store, OP_ARM64_ST1}, {SN_StorePair, OP_ARM64_STP}, {SN_StorePairNonTemporal, OP_ARM64_STNP}, @@ -1162,9 +1223,16 @@ emit_arm64_intrinsics ( } return emit_simd_ins_for_sig (cfg, klass, OP_XOP_X_X, op, arg0_type, fsig, args); } - case SN_Sqrt: { - llvm_ovr_tag_t tag = ovr_tag_from_mono_vector_type (fsig->ret); - return emit_simd_ins_for_sig (cfg, klass, OP_XOP_OVR_X_X, INTRINS_AARCH64_ADV_SIMD_FSQRT, tag, fsig, args); + case SN_ShiftLeftLogicalSaturate: + case SN_ShiftLeftLogicalSaturateScalar: { + gboolean is_unsigned = type_is_unsigned (fsig->ret); + op = is_unsigned ? OP_ARM64_UQSHL_IMM : OP_ARM64_SQSHL_IMM; + return emit_simd_ins_for_sig (cfg, klass, op, 0, arg0_type, fsig, args); + } + case SN_ShiftRightLogicalRoundedNarrowingSaturateScalar: { + MonoInst *ret = emit_simd_ins_for_sig (cfg, klass, OP_ARM64_UQRSHRN, 0, arg0_type, fsig, args); + ret = emit_simd_ins (cfg, klass, OP_ARM64_ZERO_UPPER, ret->dreg, -1); + return ret; } case SN_SubtractScalar: case SN_Subtract: { @@ -1180,10 +1248,9 @@ emit_arm64_intrinsics ( } case SN_SubtractSaturate: case SN_SubtractSaturateScalar: { - llvm_ovr_tag_t tag = ovr_tag_from_mono_vector_type (fsig->ret); gboolean is_unsigned = type_is_unsigned (fsig->ret); iid = is_unsigned ? INTRINS_AARCH64_ADV_SIMD_UQSUB : INTRINS_AARCH64_ADV_SIMD_SQSUB; - MonoInst *ret = emit_simd_ins_for_sig (cfg, klass, OP_XOP_OVR_X_X_X, iid, tag, fsig, args); + MonoInst *ret = emit_simd_ins_for_sig (cfg, klass, OP_XOP_OVR_X_X_X, iid, 0, fsig, args); /* LLVM has intrinsic functions for only the 32 and 64-bit forms of the scalar variants * of NEON uqsub and sqsub. The CoreCLR runtime tests for these intrinsics look like they * assert that all lanes > 0 are zeroed out, so just use the vector variant of these diff --git a/src/mono/mono/mini/simd-methods.h b/src/mono/mono/mini/simd-methods.h index 1619dcb9315f15..f68a277a0439fb 100644 --- a/src/mono/mono/mini/simd-methods.h +++ b/src/mono/mono/mini/simd-methods.h @@ -255,6 +255,64 @@ METHOD(AbsoluteCompareGreaterThan) METHOD(AbsoluteCompareGreaterThanOrEqual) METHOD(AbsoluteCompareLessThan) METHOD(AbsoluteCompareLessThanOrEqual) +METHOD(ShiftArithmetic) +METHOD(ShiftArithmeticRounded) +METHOD(ShiftArithmeticRoundedSaturate) +METHOD(ShiftArithmeticRoundedSaturateScalar) +METHOD(ShiftArithmeticRoundedScalar) +METHOD(ShiftArithmeticSaturate) +METHOD(ShiftArithmeticSaturateScalar) +METHOD(ShiftArithmeticScalar) +METHOD(ShiftLeftAndInsert) +METHOD(ShiftLeftAndInsertScalar) +METHOD(ShiftLeftLogicalSaturate) +METHOD(ShiftLeftLogicalSaturateScalar) +METHOD(ShiftLeftLogicalSaturateUnsigned) +METHOD(ShiftLeftLogicalSaturateUnsignedScalar) +METHOD(ShiftLeftLogicalScalar) +METHOD(ShiftLeftLogicalWideningLower) +METHOD(ShiftLeftLogicalWideningUpper) +METHOD(ShiftLogical) +METHOD(ShiftLogicalRounded) +METHOD(ShiftLogicalRoundedSaturate) +METHOD(ShiftLogicalRoundedSaturateScalar) +METHOD(ShiftLogicalRoundedScalar) +METHOD(ShiftLogicalSaturate) +METHOD(ShiftLogicalSaturateScalar) +METHOD(ShiftLogicalScalar) +METHOD(ShiftRightAndInsert) +METHOD(ShiftRightAndInsertScalar) +METHOD(ShiftRightArithmeticAdd) +METHOD(ShiftRightArithmeticAddScalar) +METHOD(ShiftRightArithmeticNarrowingSaturateLower) +METHOD(ShiftRightArithmeticNarrowingSaturateUnsignedLower) +METHOD(ShiftRightArithmeticNarrowingSaturateUnsignedUpper) +METHOD(ShiftRightArithmeticNarrowingSaturateUpper) +METHOD(ShiftRightArithmeticRounded) +METHOD(ShiftRightArithmeticRoundedAdd) +METHOD(ShiftRightArithmeticRoundedAddScalar) +METHOD(ShiftRightArithmeticRoundedNarrowingSaturateLower) +METHOD(ShiftRightArithmeticRoundedNarrowingSaturateUnsignedLower) +METHOD(ShiftRightArithmeticRoundedNarrowingSaturateUnsignedUpper) +METHOD(ShiftRightArithmeticRoundedNarrowingSaturateUpper) +METHOD(ShiftRightArithmeticRoundedScalar) +METHOD(ShiftRightArithmeticScalar) +METHOD(ShiftRightLogicalAdd) +METHOD(ShiftRightLogicalAddScalar) +METHOD(ShiftRightLogicalNarrowingLower) +METHOD(ShiftRightLogicalNarrowingSaturateLower) +METHOD(ShiftRightLogicalNarrowingSaturateUpper) +METHOD(ShiftRightLogicalNarrowingUpper) +METHOD(ShiftRightLogicalRounded) +METHOD(ShiftRightLogicalRoundedAdd) +METHOD(ShiftRightLogicalRoundedAddScalar) +METHOD(ShiftRightLogicalRoundedNarrowingLower) +METHOD(ShiftRightLogicalRoundedNarrowingSaturateLower) +METHOD(ShiftRightLogicalRoundedNarrowingSaturateScalar) +METHOD(ShiftRightLogicalRoundedNarrowingSaturateUpper) +METHOD(ShiftRightLogicalRoundedNarrowingUpper) +METHOD(ShiftRightLogicalRoundedScalar) +METHOD(ShiftRightLogicalScalar) METHOD(SignExtendWideningLower) METHOD(SignExtendWideningUpper) METHOD(StorePair) From 79514da35ea3f2a53149b8d42052f6f15d8b3069 Mon Sep 17 00:00:00 2001 From: Imran Hameed Date: Fri, 5 Mar 2021 10:35:41 -0800 Subject: [PATCH 03/58] Implement rounding --- src/mono/mono/mini/llvm-intrinsics.h | 8 +++++ src/mono/mono/mini/mini-llvm-cpp.h | 28 ++++++++++------- src/mono/mono/mini/mini-llvm.c | 47 +++++++++++++++++++--------- src/mono/mono/mini/mini-ops.h | 2 ++ src/mono/mono/mini/simd-intrinsics.c | 10 ++++++ src/mono/mono/mini/simd-methods.h | 4 +++ 6 files changed, 74 insertions(+), 25 deletions(-) diff --git a/src/mono/mono/mini/llvm-intrinsics.h b/src/mono/mono/mini/llvm-intrinsics.h index 9daa663f4809e1..a4fcc098138b6e 100644 --- a/src/mono/mono/mini/llvm-intrinsics.h +++ b/src/mono/mono/mini/llvm-intrinsics.h @@ -12,6 +12,7 @@ * To define an overloaded intrinsic with three arguments */ +#define Scalar 0 #define V64 LLVM_Vector64 #define V128 LLVM_Vector128 #define I1 LLVM_Int8 @@ -310,6 +311,12 @@ INTRINS_OVR(AARCH64_ADV_SIMD_TBX128, aarch64_neon_tbx1, sse_i1_t) INTRINS_OVR(AARCH64_ADV_SIMD_TBL64, aarch64_neon_tbl1, LLVMVectorType (LLVMInt8Type (), 8)) INTRINS_OVR(AARCH64_ADV_SIMD_TBL128, aarch64_neon_tbl1, sse_i1_t) +INTRINS_OVR_TAG(AARCH64_ADV_SIMD_FRINTA, round, Scalar | V64 | V128 | R4 | R8) +INTRINS_OVR_TAG(AARCH64_ADV_SIMD_FRINTN, aarch64_neon_frintn, Scalar | V64 | V128 | R4 | R8) +INTRINS_OVR_TAG(AARCH64_ADV_SIMD_FRINTM, floor, Scalar | V64 | V128 | R4 | R8) +INTRINS_OVR_TAG(AARCH64_ADV_SIMD_FRINTP, ceil, Scalar | V64 | V128 | R4 | R8) +INTRINS_OVR_TAG(AARCH64_ADV_SIMD_FRINTZ, trunc, Scalar | V64 | V128 | R4 | R8) + INTRINS_OVR_TAG(AARCH64_ADV_SIMD_UQSUB, aarch64_neon_uqsub, V64 | V128 | I1 | I2 | I4 | I8) INTRINS_OVR_TAG(AARCH64_ADV_SIMD_SQSUB, aarch64_neon_sqsub, V64 | V128 | I1 | I2 | I4 | I8) INTRINS_OVR_TAG(AARCH64_ADV_SIMD_RSUBHN, aarch64_neon_rsubhn, V64 | I1 | I2 | I4) @@ -343,6 +350,7 @@ INTRINS_OVR_TAG(AARCH64_ADV_SIMD_SRI, aarch64_neon_vsri, V64 | V128 | I1 | I2 | #undef INTRINS_OVR_2_ARG #undef INTRINS_OVR_3_ARG #undef INTRINS_OVR_TAG +#undef Scalar #undef V64 #undef V128 #undef I1 diff --git a/src/mono/mono/mini/mini-llvm-cpp.h b/src/mono/mono/mini/mini-llvm-cpp.h index b50e5aa8a0aa31..b85f3a2b825628 100644 --- a/src/mono/mono/mini/mini-llvm-cpp.h +++ b/src/mono/mono/mini/mini-llvm-cpp.h @@ -40,21 +40,27 @@ typedef enum { } IntrinsicId; enum { - LLVM_Vector64 = 1 << 0, - LLVM_Vector128 = 1 << 1, - LLVM_VectorWidths = 2, - - LLVM_Int8 = 1 << 2, - LLVM_Int16 = 1 << 3, - LLVM_Int32 = 1 << 4, - LLVM_Int64 = 1 << 5, - LLVM_Float32 = 1 << 6, - LLVM_Float64 = 1 << 7, - LLVM_ElementWidths = 6, + LLVM_Vector64 = 1 << 0, + LLVM_Vector128 = 1 << 1, + LLVM_VectorWidths = 3, // Scalar, 64, 128 + + LLVM_Int8 = 1 << 2, + LLVM_Int16 = 1 << 3, + LLVM_Int32 = 1 << 4, + LLVM_Int64 = 1 << 5, + LLVM_Float32 = 1 << 6, + LLVM_Float64 = 1 << 7, + LLVM_ElementWidths = 6, }; typedef uint8_t llvm_ovr_tag_t; +static inline llvm_ovr_tag_t +ovr_tag_to_scalar (llvm_ovr_tag_t tag) +{ + return tag & 0xfc; +} + llvm_ovr_tag_t ovr_tag_from_mono_vector_class (MonoClass *klass); diff --git a/src/mono/mono/mini/mini-llvm.c b/src/mono/mono/mini/mini-llvm.c index 4815d8e30e411d..d3ee5476470023 100644 --- a/src/mono/mono/mini/mini-llvm.c +++ b/src/mono/mono/mini/mini-llvm.c @@ -9679,6 +9679,17 @@ process_bb (EmitContext *ctx, MonoBasicBlock *bb) values [ins->dreg] = call_overloaded_intrins (ctx, iid, ovr_tag, args, ""); break; } + case OP_XOP_OVR_SCALAR_X_X: { + IntrinsicId iid = (IntrinsicId) ins->inst_c0; + LLVMTypeRef t = LLVMTypeOf (lhs); + llvm_ovr_tag_t ovr_tag = ovr_tag_from_mono_vector_class (ins->klass); + ovr_tag = ovr_tag_to_scalar (ovr_tag); + LLVMValueRef result = LLVMBuildExtractElement (builder, lhs, const_int32 (0), "xop_ovr_scalar_x_x"); + result = call_overloaded_intrins (ctx, iid, ovr_tag, &lhs, ""); + result = LLVMBuildInsertElement (builder, LLVMGetUndef (t), result, const_int32 (0), ""); + values [ins->dreg] = result; + break; + } case OP_ARM64_ZERO_UPPER: { values [ins->dreg] = select_lowest_element (ctx, lhs); break; @@ -11002,7 +11013,8 @@ add_intrinsic (LLVMModuleRef module, int id) llvm_ovr_tag_t spec = intrin_arm64_ovr [id]; for (int vw = 0; vw < LLVM_VectorWidths; ++vw) { for (int ew = 0; ew < LLVM_ElementWidths; ++ew) { - llvm_ovr_tag_t test = (LLVM_Vector64 << vw) | (LLVM_Int8 << ew); + llvm_ovr_tag_t vec_bit = vw == 0 ? 0 : LLVM_Vector64 << vw; + llvm_ovr_tag_t test = vec_bit | (LLVM_Int8 << ew); if (spec & test) { intrins = add_intrins1 (module, id, intrin_types [vw][ew]); int ovr_id = int_from_id_and_ovr_tag (test, id); @@ -11085,19 +11097,26 @@ add_types (MonoLLVMModule *module) void mono_llvm_init (gboolean enable_jit) { - intrin_types [0][0] = v64_i1_t = LLVMVectorType (LLVMInt8Type (), 8); - intrin_types [0][1] = v64_i2_t = LLVMVectorType (LLVMInt16Type (), 4); - intrin_types [0][2] = v64_i4_t = LLVMVectorType (LLVMInt32Type (), 2); - intrin_types [0][3] = v64_i8_t = LLVMVectorType (LLVMInt64Type (), 1); - intrin_types [0][4] = v64_r4_t = LLVMVectorType (LLVMFloatType (), 2); - intrin_types [0][5] = v64_r8_t = LLVMVectorType (LLVMDoubleType (), 1); - - intrin_types [1][0] = v128_i1_t = sse_i1_t = type_to_sse_type (MONO_TYPE_I1); - intrin_types [1][1] = v128_i2_t = sse_i2_t = type_to_sse_type (MONO_TYPE_I2); - intrin_types [1][2] = v128_i4_t = sse_i4_t = type_to_sse_type (MONO_TYPE_I4); - intrin_types [1][3] = v128_i8_t = sse_i8_t = type_to_sse_type (MONO_TYPE_I8); - intrin_types [1][4] = v128_r4_t = sse_r4_t = type_to_sse_type (MONO_TYPE_R4); - intrin_types [1][5] = v128_r8_t = sse_r8_t = type_to_sse_type (MONO_TYPE_R8); + intrin_types [0][0] = LLVMInt8Type (); + intrin_types [0][1] = LLVMInt16Type (); + intrin_types [0][2] = LLVMInt32Type (); + intrin_types [0][3] = LLVMInt64Type (); + intrin_types [0][4] = LLVMFloatType (); + intrin_types [0][5] = LLVMDoubleType (); + + intrin_types [1][0] = v64_i1_t = LLVMVectorType (LLVMInt8Type (), 8); + intrin_types [1][1] = v64_i2_t = LLVMVectorType (LLVMInt16Type (), 4); + intrin_types [1][2] = v64_i4_t = LLVMVectorType (LLVMInt32Type (), 2); + intrin_types [1][3] = v64_i8_t = LLVMVectorType (LLVMInt64Type (), 1); + intrin_types [1][4] = v64_r4_t = LLVMVectorType (LLVMFloatType (), 2); + intrin_types [1][5] = v64_r8_t = LLVMVectorType (LLVMDoubleType (), 1); + + intrin_types [2][0] = v128_i1_t = sse_i1_t = type_to_sse_type (MONO_TYPE_I1); + intrin_types [2][1] = v128_i2_t = sse_i2_t = type_to_sse_type (MONO_TYPE_I2); + intrin_types [2][2] = v128_i4_t = sse_i4_t = type_to_sse_type (MONO_TYPE_I4); + intrin_types [2][3] = v128_i8_t = sse_i8_t = type_to_sse_type (MONO_TYPE_I8); + intrin_types [2][4] = v128_r4_t = sse_r4_t = type_to_sse_type (MONO_TYPE_R4); + intrin_types [2][5] = v128_r8_t = sse_r8_t = type_to_sse_type (MONO_TYPE_R8); intrins_id_to_intrins = g_hash_table_new (NULL, NULL); diff --git a/src/mono/mono/mini/mini-ops.h b/src/mono/mono/mini/mini-ops.h index 154e077753736d..d701447d391dd3 100644 --- a/src/mono/mono/mini/mini-ops.h +++ b/src/mono/mono/mini/mini-ops.h @@ -1555,6 +1555,7 @@ MINI_OP3(OP_XOP_X_X_X_X, "xop_x_x_x_x", XREG, XREG, XREG, XREG) MINI_OP(OP_XOP_OVR_X_X, "xop_ovr_x", XREG, XREG, NONE) MINI_OP(OP_XOP_OVR_X_X_X, "xop_ovr_x_x", XREG, XREG, XREG) +MINI_OP(OP_XOP_OVR_SCALAR_X_X, "xop_ovr_scalar_x_x", XREG, XREG, NONE) MINI_OP(OP_XCAST, "xcast", XREG, XREG, NONE) /* Extract element of vector */ @@ -1652,4 +1653,5 @@ MINI_OP(OP_ARM64_UQSHL_IMM, "arm64_uqshl_imm", XREG, XREG, IREG) MINI_OP(OP_ARM64_SQSHL_IMM, "arm64_uqshl_imm", XREG, XREG, IREG) MINI_OP(OP_ARM64_SQSHLU, "arm64_sqshlu", XREG, XREG, IREG) + #endif // TARGET_ARM64 diff --git a/src/mono/mono/mini/simd-intrinsics.c b/src/mono/mono/mini/simd-intrinsics.c index 1f6f6dea9875dc..d976158b906d1c 100644 --- a/src/mono/mono/mini/simd-intrinsics.c +++ b/src/mono/mono/mini/simd-intrinsics.c @@ -970,6 +970,16 @@ static SimdIntrinsic advsimd_methods [] = { {SN_AbsoluteCompareLessThanOrEqual}, {SN_AbsoluteCompareLessThan}, {SN_Abs}, + {SN_RoundAwayFromZero, OP_XOP_OVR_X_X, INTRINS_AARCH64_ADV_SIMD_FRINTA}, + {SN_RoundAwayFromZeroScalar, OP_XOP_OVR_SCALAR_X_X, INTRINS_AARCH64_ADV_SIMD_FRINTA}, + {SN_RoundToNearest, OP_XOP_OVR_X_X, INTRINS_AARCH64_ADV_SIMD_FRINTN}, + {SN_RoundToNearestScalar, OP_XOP_OVR_SCALAR_X_X, INTRINS_AARCH64_ADV_SIMD_FRINTN}, + {SN_RoundToNegativeInfinity, OP_XOP_OVR_X_X, INTRINS_AARCH64_ADV_SIMD_FRINTM}, + {SN_RoundToNegativeInfinityScalar, OP_XOP_OVR_SCALAR_X_X, INTRINS_AARCH64_ADV_SIMD_FRINTM}, + {SN_RoundToPositiveInfinity, OP_XOP_OVR_X_X, INTRINS_AARCH64_ADV_SIMD_FRINTP}, + {SN_RoundToPositiveInfinityScalar, OP_XOP_OVR_SCALAR_X_X, INTRINS_AARCH64_ADV_SIMD_FRINTP}, + {SN_RoundToZero, OP_XOP_OVR_X_X, INTRINS_AARCH64_ADV_SIMD_FRINTZ}, + {SN_RoundToZeroScalar, OP_XOP_OVR_SCALAR_X_X, INTRINS_AARCH64_ADV_SIMD_FRINTZ}, {SN_ShiftArithmetic, OP_XOP_OVR_X_X_X, INTRINS_AARCH64_ADV_SIMD_SSHL}, {SN_ShiftArithmeticRounded, OP_XOP_OVR_X_X_X, INTRINS_AARCH64_ADV_SIMD_SRSHL}, {SN_ShiftArithmeticRoundedSaturate, OP_XOP_OVR_X_X_X, INTRINS_AARCH64_ADV_SIMD_SQRSHL}, diff --git a/src/mono/mono/mini/simd-methods.h b/src/mono/mono/mini/simd-methods.h index f68a277a0439fb..36afbdaca7821c 100644 --- a/src/mono/mono/mini/simd-methods.h +++ b/src/mono/mono/mini/simd-methods.h @@ -255,6 +255,10 @@ METHOD(AbsoluteCompareGreaterThan) METHOD(AbsoluteCompareGreaterThanOrEqual) METHOD(AbsoluteCompareLessThan) METHOD(AbsoluteCompareLessThanOrEqual) +METHOD(RoundAwayFromZero) +METHOD(RoundAwayFromZeroScalar) +METHOD(RoundToNearest) +METHOD(RoundToNearestScalar) METHOD(ShiftArithmetic) METHOD(ShiftArithmeticRounded) METHOD(ShiftArithmeticRoundedSaturate) From 6cc259688bc3c7cddb7b4d29aa728c273efee2fd Mon Sep 17 00:00:00 2001 From: Imran Hameed Date: Fri, 5 Mar 2021 11:09:27 -0800 Subject: [PATCH 04/58] Implement ReverseElement{Bits,8,16,32} --- src/mono/mono/mini/llvm-intrinsics.h | 11 +++++++---- src/mono/mono/mini/mini-llvm-cpp.h | 3 ++- src/mono/mono/mini/mini-llvm.c | 26 +++++++++++++++++++++++++- src/mono/mono/mini/mini-ops.h | 4 ++++ src/mono/mono/mini/simd-intrinsics.c | 4 ++++ src/mono/mono/mini/simd-methods.h | 3 +++ 6 files changed, 45 insertions(+), 6 deletions(-) diff --git a/src/mono/mono/mini/llvm-intrinsics.h b/src/mono/mono/mini/llvm-intrinsics.h index a4fcc098138b6e..65d60c5458f8b4 100644 --- a/src/mono/mono/mini/llvm-intrinsics.h +++ b/src/mono/mono/mini/llvm-intrinsics.h @@ -306,10 +306,8 @@ INTRINS_OVR_2_ARG(AARCH64_ADV_SIMD_ABS_COMPARE_LT_FLOAT, aarch64_neon_facgt, sse INTRINS_OVR_2_ARG(AARCH64_ADV_SIMD_ABS_COMPARE_LT_DOUBLE, aarch64_neon_facgt, sse_i4_t, sse_r8_t) INTRINS_OVR_2_ARG(AARCH64_ADV_SIMD_ABS_COMPARE_LTE_FLOAT, aarch64_neon_facge, sse_i4_t, sse_r4_t) INTRINS_OVR_2_ARG(AARCH64_ADV_SIMD_ABS_COMPARE_LTE_DOUBLE, aarch64_neon_facge, sse_i4_t, sse_r8_t) -INTRINS_OVR(AARCH64_ADV_SIMD_TBX64, aarch64_neon_tbx1, LLVMVectorType (LLVMInt8Type (), 8)) -INTRINS_OVR(AARCH64_ADV_SIMD_TBX128, aarch64_neon_tbx1, sse_i1_t) -INTRINS_OVR(AARCH64_ADV_SIMD_TBL64, aarch64_neon_tbl1, LLVMVectorType (LLVMInt8Type (), 8)) -INTRINS_OVR(AARCH64_ADV_SIMD_TBL128, aarch64_neon_tbl1, sse_i1_t) + +INTRINS_OVR_TAG(AARCH64_ADV_SIMD_RBIT, aarch64_neon_rbit, V64 | V128 | I1) INTRINS_OVR_TAG(AARCH64_ADV_SIMD_FRINTA, round, Scalar | V64 | V128 | R4 | R8) INTRINS_OVR_TAG(AARCH64_ADV_SIMD_FRINTN, aarch64_neon_frintn, Scalar | V64 | V128 | R4 | R8) @@ -343,6 +341,11 @@ INTRINS_OVR_TAG(AARCH64_ADV_SIMD_SQSHLU, aarch64_neon_sqshlu, V64 | V128 | I1 | INTRINS_OVR_TAG(AARCH64_ADV_SIMD_SLI, aarch64_neon_vsli, V64 | V128 | I1 | I2 | I4 | I8) // Constant shift INTRINS_OVR_TAG(AARCH64_ADV_SIMD_SRI, aarch64_neon_vsri, V64 | V128 | I1 | I2 | I4 | I8) // Constant shift + +INTRINS_OVR(AARCH64_ADV_SIMD_TBX64, aarch64_neon_tbx1, LLVMVectorType (LLVMInt8Type (), 8)) +INTRINS_OVR(AARCH64_ADV_SIMD_TBX128, aarch64_neon_tbx1, sse_i1_t) +INTRINS_OVR(AARCH64_ADV_SIMD_TBL64, aarch64_neon_tbl1, LLVMVectorType (LLVMInt8Type (), 8)) +INTRINS_OVR(AARCH64_ADV_SIMD_TBL128, aarch64_neon_tbl1, sse_i1_t) #endif #undef INTRINS diff --git a/src/mono/mono/mini/mini-llvm-cpp.h b/src/mono/mono/mini/mini-llvm-cpp.h index b85f3a2b825628..45402e43ed53df 100644 --- a/src/mono/mono/mini/mini-llvm-cpp.h +++ b/src/mono/mono/mini/mini-llvm-cpp.h @@ -27,7 +27,8 @@ G_BEGIN_DECLS /* An intrinsic id. The lower 24 bits are used to store a mono-specific ID. The * next 8 bits store arm64 overload tag bits. In the configuration of LLVM 9 we - * use, there are 7017 total intrinsics defined in IntrinsicEnums.inc. + * use, there are 7017 total intrinsics defined in IntrinsicEnums.inc, so only + * 13 bits are needed to label each intrinsic overload group. */ typedef enum { #define INTRINS(id, llvm_id) INTRINS_ ## id, diff --git a/src/mono/mono/mini/mini-llvm.c b/src/mono/mono/mini/mini-llvm.c index d3ee5476470023..48a80d5a398009 100644 --- a/src/mono/mono/mini/mini-llvm.c +++ b/src/mono/mono/mini/mini-llvm.c @@ -9292,6 +9292,30 @@ process_bb (EmitContext *ctx, MonoBasicBlock *bb) values [ins->dreg] = LLVMBuildTrunc (builder, hi64, LLVMInt64Type (), ""); break; } + case OP_ARM64_REV16: + case OP_ARM64_REV32: + case OP_ARM64_REV64: { + unsigned int tmp_bits = 0; + switch (ins->opcode) { + case OP_ARM64_REV16: tmp_bits = 8; break; + case OP_ARM64_REV32: tmp_bits = 16; break; + case OP_ARM64_REV64: tmp_bits = 32; break; + } + LLVMTypeRef t = LLVMTypeOf (lhs); + unsigned int t_bits = mono_llvm_get_prim_size_bits (t); + unsigned int tmp_elements = t_bits / tmp_bits; + LLVMTypeRef tmp_t = LLVMVectorType (LLVMIntType (tmp_bits), tmp_elements); + LLVMValueRef tmp = LLVMBuildBitCast (builder, lhs, tmp_t, "arm64_rev"); + int mask [MAX_VECTOR_ELEMS] = { 0 }; + for (unsigned int i = 0; i < tmp_elements; i += 2) { + mask [i] = i + 1; + mask [i] = i; + } + LLVMValueRef result = LLVMBuildShuffleVector(builder, tmp, LLVMGetUndef (tmp_t), create_const_vector_i32 (mask, tmp_elements), ""); + result = LLVMBuildBitCast (builder, result, t, ""); + values [ins->dreg] = result; + break; + } case OP_ARM64_SHL: case OP_ARM64_SSHR: case OP_ARM64_SSRA: @@ -11013,7 +11037,7 @@ add_intrinsic (LLVMModuleRef module, int id) llvm_ovr_tag_t spec = intrin_arm64_ovr [id]; for (int vw = 0; vw < LLVM_VectorWidths; ++vw) { for (int ew = 0; ew < LLVM_ElementWidths; ++ew) { - llvm_ovr_tag_t vec_bit = vw == 0 ? 0 : LLVM_Vector64 << vw; + llvm_ovr_tag_t vec_bit = LLVM_Vector128 >> ((LLVM_VectorWidths - 1) - vw); llvm_ovr_tag_t test = vec_bit | (LLVM_Int8 << ew); if (spec & test) { intrins = add_intrins1 (module, id, intrin_types [vw][ew]); diff --git a/src/mono/mono/mini/mini-ops.h b/src/mono/mono/mini/mini-ops.h index d701447d391dd3..e7df2ea2dffd61 100644 --- a/src/mono/mono/mini/mini-ops.h +++ b/src/mono/mono/mini/mini-ops.h @@ -1654,4 +1654,8 @@ MINI_OP(OP_ARM64_SQSHL_IMM, "arm64_uqshl_imm", XREG, XREG, IREG) MINI_OP(OP_ARM64_SQSHLU, "arm64_sqshlu", XREG, XREG, IREG) +MINI_OP(OP_ARM64_REV16, "arm64_rev16", XREG, XREG, NONE) +MINI_OP(OP_ARM64_REV32, "arm64_rev32", XREG, XREG, NONE) +MINI_OP(OP_ARM64_REV64, "arm64_rev64", XREG, XREG, NONE) + #endif // TARGET_ARM64 diff --git a/src/mono/mono/mini/simd-intrinsics.c b/src/mono/mono/mini/simd-intrinsics.c index d976158b906d1c..3a1ea137200025 100644 --- a/src/mono/mono/mini/simd-intrinsics.c +++ b/src/mono/mono/mini/simd-intrinsics.c @@ -970,6 +970,10 @@ static SimdIntrinsic advsimd_methods [] = { {SN_AbsoluteCompareLessThanOrEqual}, {SN_AbsoluteCompareLessThan}, {SN_Abs}, + {SN_ReverseElementBits, OP_XOP_OVR_X_X, INTRINS_AARCH64_ADV_SIMD_RBIT}, + {SN_ReverseElement16, OP_ARM64_REV32}, + {SN_ReverseElement32, OP_ARM64_REV64}, + {SN_ReverseElement8, OP_ARM64_REV16}, {SN_RoundAwayFromZero, OP_XOP_OVR_X_X, INTRINS_AARCH64_ADV_SIMD_FRINTA}, {SN_RoundAwayFromZeroScalar, OP_XOP_OVR_SCALAR_X_X, INTRINS_AARCH64_ADV_SIMD_FRINTA}, {SN_RoundToNearest, OP_XOP_OVR_X_X, INTRINS_AARCH64_ADV_SIMD_FRINTN}, diff --git a/src/mono/mono/mini/simd-methods.h b/src/mono/mono/mini/simd-methods.h index 36afbdaca7821c..667537a229c029 100644 --- a/src/mono/mono/mini/simd-methods.h +++ b/src/mono/mono/mini/simd-methods.h @@ -255,6 +255,9 @@ METHOD(AbsoluteCompareGreaterThan) METHOD(AbsoluteCompareGreaterThanOrEqual) METHOD(AbsoluteCompareLessThan) METHOD(AbsoluteCompareLessThanOrEqual) +METHOD(ReverseElement16) +METHOD(ReverseElement32) +METHOD(ReverseElement8) METHOD(RoundAwayFromZero) METHOD(RoundAwayFromZeroScalar) METHOD(RoundToNearest) From f0e2e4152346fd8916fbbd7092c473da46299982 Mon Sep 17 00:00:00 2001 From: Imran Hameed Date: Fri, 5 Mar 2021 12:47:34 -0800 Subject: [PATCH 05/58] Add reciprocal fp/u32 operations --- src/mono/mono/mini/llvm-intrinsics.h | 8 ++++++++ src/mono/mono/mini/mini-llvm-cpp.cpp | 7 ------- src/mono/mono/mini/mini-llvm-cpp.h | 3 --- src/mono/mono/mini/simd-intrinsics.c | 22 ++++++++++++++++++++++ src/mono/mono/mini/simd-methods.h | 9 +++++++++ 5 files changed, 39 insertions(+), 10 deletions(-) diff --git a/src/mono/mono/mini/llvm-intrinsics.h b/src/mono/mono/mini/llvm-intrinsics.h index 65d60c5458f8b4..6e1d4917854b8a 100644 --- a/src/mono/mono/mini/llvm-intrinsics.h +++ b/src/mono/mono/mini/llvm-intrinsics.h @@ -307,6 +307,14 @@ INTRINS_OVR_2_ARG(AARCH64_ADV_SIMD_ABS_COMPARE_LT_DOUBLE, aarch64_neon_facgt, ss INTRINS_OVR_2_ARG(AARCH64_ADV_SIMD_ABS_COMPARE_LTE_FLOAT, aarch64_neon_facge, sse_i4_t, sse_r4_t) INTRINS_OVR_2_ARG(AARCH64_ADV_SIMD_ABS_COMPARE_LTE_DOUBLE, aarch64_neon_facge, sse_i4_t, sse_r8_t) +INTRINS_OVR_TAG(AARCH64_ADV_SIMD_URECPE, aarch64_neon_urecpe, V64 | V128 | I4) +INTRINS_OVR_TAG(AARCH64_ADV_SIMD_FRECPE, aarch64_neon_frecpe, Scalar | V64 | V128 | R4 | R8) +INTRINS_OVR_TAG(AARCH64_ADV_SIMD_FRECPX, aarch64_neon_frecpx, Scalar | R4 | R8) +INTRINS_OVR_TAG(AARCH64_ADV_SIMD_URSQRTE, aarch64_neon_ursqrte, V64 | V128 | I4) +INTRINS_OVR_TAG(AARCH64_ADV_SIMD_FRSQRTE, aarch64_neon_frsqrte, Scalar | V64 | V128) +INTRINS_OVR_TAG(AARCH64_ADV_SIMD_FRSQRTS, aarch64_neon_frsqrts, Scalar | V64 | V128 | R4 | R8) +INTRINS_OVR_TAG(AARCH64_ADV_SIMD_FRECPS, aarch64_neon_frecps, Scalar | V64 | V128 | R4 | R8) + INTRINS_OVR_TAG(AARCH64_ADV_SIMD_RBIT, aarch64_neon_rbit, V64 | V128 | I1) INTRINS_OVR_TAG(AARCH64_ADV_SIMD_FRINTA, round, Scalar | V64 | V128 | R4 | R8) diff --git a/src/mono/mono/mini/mini-llvm-cpp.cpp b/src/mono/mono/mini/mini-llvm-cpp.cpp index f37703def760d1..ba7c6fb8623f6f 100644 --- a/src/mono/mono/mini/mini-llvm-cpp.cpp +++ b/src/mono/mono/mini/mini-llvm-cpp.cpp @@ -680,13 +680,6 @@ ovr_tag_from_mono_vector_class (MonoClass *klass) { return ret; } -llvm_ovr_tag_t -ovr_tag_from_mono_vector_type (MonoType *type) -{ - MonoClass *klass = mono_class_from_mono_type_internal (type); - return ovr_tag_from_mono_vector_class (klass); -} - /* * mono_llvm_register_intrinsic: * diff --git a/src/mono/mono/mini/mini-llvm-cpp.h b/src/mono/mono/mini/mini-llvm-cpp.h index 45402e43ed53df..31d04161282732 100644 --- a/src/mono/mono/mini/mini-llvm-cpp.h +++ b/src/mono/mono/mini/mini-llvm-cpp.h @@ -65,9 +65,6 @@ ovr_tag_to_scalar (llvm_ovr_tag_t tag) llvm_ovr_tag_t ovr_tag_from_mono_vector_class (MonoClass *klass); -llvm_ovr_tag_t -ovr_tag_from_mono_vector_type (MonoType *type); - /* * Keep in sync with the enum in utils/mono-memory-model.h. */ diff --git a/src/mono/mono/mini/simd-intrinsics.c b/src/mono/mono/mini/simd-intrinsics.c index 3a1ea137200025..86b5508dde47ad 100644 --- a/src/mono/mono/mini/simd-intrinsics.c +++ b/src/mono/mono/mini/simd-intrinsics.c @@ -970,6 +970,13 @@ static SimdIntrinsic advsimd_methods [] = { {SN_AbsoluteCompareLessThanOrEqual}, {SN_AbsoluteCompareLessThan}, {SN_Abs}, + {SN_ReciprocalEstimate}, + {SN_ReciprocalEstimateScalar, OP_XOP_OVR_SCALAR_X_X, INTRINS_AARCH64_ADV_SIMD_FRECPE}, + {SN_ReciprocalExponentScalar, OP_XOP_OVR_SCALAR_X_X, INTRINS_AARCH64_ADV_SIMD_FRECPX}, + {SN_ReciprocalSquareRootEstimate}, + {SN_ReciprocalSquareRootEstimateScalar, OP_XOP_OVR_SCALAR_X_X, INTRINS_AARCH64_ADV_SIMD_FRSQRTE}, + {SN_ReciprocalStep, OP_XOP_OVR_X_X, INTRINS_AARCH64_ADV_SIMD_FRECPS}, + {SN_ReciprocalStepScalar, OP_XOP_OVR_SCALAR_X_X, INTRINS_AARCH64_ADV_SIMD_FRECPS}, {SN_ReverseElementBits, OP_XOP_OVR_X_X, INTRINS_AARCH64_ADV_SIMD_RBIT}, {SN_ReverseElement16, OP_ARM64_REV32}, {SN_ReverseElement32, OP_ARM64_REV64}, @@ -1237,6 +1244,21 @@ emit_arm64_intrinsics ( } return emit_simd_ins_for_sig (cfg, klass, OP_XOP_X_X, op, arg0_type, fsig, args); } + case SN_ReciprocalEstimate: + case SN_ReciprocalSquareRootEstimate: { + gboolean is_float = FALSE; + switch (arg0_type) { + case MONO_TYPE_R4: case MONO_TYPE_R8: is_float = TRUE; + } + switch (id) { + case SN_ReciprocalEstimate: + iid = is_float ? INTRINS_AARCH64_ADV_SIMD_FRECPE: INTRINS_AARCH64_ADV_SIMD_URECPE; + break; + case SN_ReciprocalSquareRootEstimate: + iid = is_float ? INTRINS_AARCH64_ADV_SIMD_FRSQRTE : INTRINS_AARCH64_ADV_SIMD_URSQRTE; + } + return emit_simd_ins_for_sig (cfg, klass, OP_XOP_OVR_X_X, iid, arg0_type, fsig, args); + } case SN_ShiftLeftLogicalSaturate: case SN_ShiftLeftLogicalSaturateScalar: { gboolean is_unsigned = type_is_unsigned (fsig->ret); diff --git a/src/mono/mono/mini/simd-methods.h b/src/mono/mono/mini/simd-methods.h index 667537a229c029..76f775b56f8774 100644 --- a/src/mono/mono/mini/simd-methods.h +++ b/src/mono/mono/mini/simd-methods.h @@ -255,6 +255,15 @@ METHOD(AbsoluteCompareGreaterThan) METHOD(AbsoluteCompareGreaterThanOrEqual) METHOD(AbsoluteCompareLessThan) METHOD(AbsoluteCompareLessThanOrEqual) +METHOD(ReciprocalEstimate) +METHOD(ReciprocalEstimateScalar) +METHOD(ReciprocalExponentScalar) +METHOD(ReciprocalSquareRootEstimate) +METHOD(ReciprocalSquareRootEstimateScalar) +METHOD(ReciprocalSquareRootStep) +METHOD(ReciprocalSquareRootStepScalar) +METHOD(ReciprocalStep) +METHOD(ReciprocalStepScalar) METHOD(ReverseElement16) METHOD(ReverseElement32) METHOD(ReverseElement8) From cca921bc2a15ac8b4bc0f93e293ec9d0622e3eda Mon Sep 17 00:00:00 2001 From: Imran Hameed Date: Fri, 5 Mar 2021 13:47:37 -0800 Subject: [PATCH 06/58] Add some bitwise operations --- src/mono/mono/mini/llvm-intrinsics.h | 5 +++ src/mono/mono/mini/mini-llvm-cpp.h | 7 ++++ src/mono/mono/mini/mini-llvm.c | 49 +++++++++++++++++++++------- src/mono/mono/mini/mini-ops.h | 7 ++-- src/mono/mono/mini/simd-intrinsics.c | 19 +++++++---- src/mono/mono/mini/simd-methods.h | 3 ++ 6 files changed, 71 insertions(+), 19 deletions(-) diff --git a/src/mono/mono/mini/llvm-intrinsics.h b/src/mono/mono/mini/llvm-intrinsics.h index 6e1d4917854b8a..9a4b4c33a4f453 100644 --- a/src/mono/mono/mini/llvm-intrinsics.h +++ b/src/mono/mono/mini/llvm-intrinsics.h @@ -307,6 +307,11 @@ INTRINS_OVR_2_ARG(AARCH64_ADV_SIMD_ABS_COMPARE_LT_DOUBLE, aarch64_neon_facgt, ss INTRINS_OVR_2_ARG(AARCH64_ADV_SIMD_ABS_COMPARE_LTE_FLOAT, aarch64_neon_facge, sse_i4_t, sse_r4_t) INTRINS_OVR_2_ARG(AARCH64_ADV_SIMD_ABS_COMPARE_LTE_DOUBLE, aarch64_neon_facge, sse_i4_t, sse_r8_t) +INTRINS_OVR_TAG(AARCH64_ADV_SIMD_PMUL, aarch64_neon_pmul, V64 | V128 | I1) +INTRINS_OVR_TAG(AARCH64_ADV_SIMD_PMULL, aarch64_neon_pmull, V128 | I2) + +INTRINS_OVR_TAG(AARCH64_ADV_SIMD_CNT, ctpop, V64 | V128 | I8) + INTRINS_OVR_TAG(AARCH64_ADV_SIMD_URECPE, aarch64_neon_urecpe, V64 | V128 | I4) INTRINS_OVR_TAG(AARCH64_ADV_SIMD_FRECPE, aarch64_neon_frecpe, Scalar | V64 | V128 | R4 | R8) INTRINS_OVR_TAG(AARCH64_ADV_SIMD_FRECPX, aarch64_neon_frecpx, Scalar | R4 | R8) diff --git a/src/mono/mono/mini/mini-llvm-cpp.h b/src/mono/mono/mini/mini-llvm-cpp.h index 31d04161282732..6169a214d1e617 100644 --- a/src/mono/mono/mini/mini-llvm-cpp.h +++ b/src/mono/mono/mini/mini-llvm-cpp.h @@ -40,6 +40,13 @@ typedef enum { INTRINS_NUM } IntrinsicId; +typedef enum { + XBINOP_FORCEINT_Not, + XBINOP_FORCEINT_Or, + XBINOP_FORCEINT_OrNot, + XBINOP_FORCEINT_Xor, +} XBinOpId; + enum { LLVM_Vector64 = 1 << 0, LLVM_Vector128 = 1 << 1, diff --git a/src/mono/mono/mini/mini-llvm.c b/src/mono/mono/mini/mini-llvm.c index 48a80d5a398009..ed510654ecfc76 100644 --- a/src/mono/mono/mini/mini-llvm.c +++ b/src/mono/mono/mini/mini-llvm.c @@ -7093,16 +7093,6 @@ process_bb (EmitContext *ctx, MonoBasicBlock *bb) mono_llvm_build_aligned_store (builder, values [ins->sreg1], dest, FALSE, 1); break; } - case OP_XXOR: { - LLVMTypeRef t = LLVMTypeOf (lhs); - unsigned int bit_width = mono_llvm_get_prim_size_bits (t); - LLVMTypeRef intermediate_t = LLVMVectorType (LLVMInt8Type (), bit_width / 8); - LLVMValueRef lhs_i8 = convert (ctx, lhs, intermediate_t); - LLVMValueRef rhs_i8 = convert (ctx, rhs, intermediate_t); - LLVMValueRef result = LLVMBuildXor (builder, lhs_i8, rhs_i8, ""); - values [ins->dreg] = LLVMBuildBitCast (builder, result, t, ""); - break; - } case OP_XBINOP: { switch (ins->inst_c0) { case OP_IADD: @@ -7165,6 +7155,34 @@ process_bb (EmitContext *ctx, MonoBasicBlock *bb) } break; } + case OP_XBINOP_FORCEINT: { + LLVMTypeRef t = LLVMTypeOf (lhs); + LLVMTypeRef elem_t = LLVMGetElementType (t); + unsigned int elems = LLVMGetVectorSize (t); + unsigned int elem_bits = mono_llvm_get_prim_size_bits (elem_t); + LLVMTypeRef intermediate_elem_t = LLVMIntType (elem_bits); + LLVMTypeRef intermediate_t = LLVMVectorType (intermediate_elem_t, elems); + LLVMValueRef lhs_int = convert (ctx, lhs, intermediate_t); + LLVMValueRef rhs_int = convert (ctx, rhs, intermediate_t); + LLVMValueRef result = NULL; + switch (ins->inst_c0) { + case XBINOP_FORCEINT_Or: + result = LLVMBuildOr (builder, lhs_int, rhs_int, ""); + break; + case XBINOP_FORCEINT_OrNot: + result = LLVMBuildXor (builder, lhs_int, rhs_int, ""); + break; + case XBINOP_FORCEINT_Xor: { + int vals [MAX_VECTOR_ELEMS] = { 0 }; + for (unsigned int i = 0; i < elems; ++i) + vals [i] = -1; + result = LLVMBuildXor (builder, rhs_int, create_const_vector (intermediate_elem_t, vals, elems), ""); + result = LLVMBuildOr (builder, result, lhs_int, ""); + break; + } + } + values [ins->dreg] = LLVMBuildBitCast (builder, result, t, ""); + } #endif // defined(TARGET_X86) || defined(TARGET_AMD64) || defined(TARGET_ARM64) || defined(TARGET_WASM) #if defined(TARGET_X86) || defined(TARGET_AMD64) || defined(TARGET_WASM) @@ -9250,7 +9268,6 @@ process_bb (EmitContext *ctx, MonoBasicBlock *bb) result = LLVMBuildInsertElement (ctx->builder, LLVMConstNull (t), result, const_int32 (0), ""); } values [ins->dreg] = result; - break; } case OP_LSCNT32: @@ -9292,6 +9309,16 @@ process_bb (EmitContext *ctx, MonoBasicBlock *bb) values [ins->dreg] = LLVMBuildTrunc (builder, hi64, LLVMInt64Type (), ""); break; } + case OP_ARM64_PMULL: + case OP_ARM64_PMULL2: { + gboolean high = ins->opcode == OP_ARM64_PMULL2; + LLVMValueRef val = lhs; + if (high) + val = extract_high_elements (ctx, val); + LLVMValueRef result = call_overloaded_intrins (ctx, INTRINS_AARCH64_ADV_SIMD_PMULL, LLVM_Vector128 | LLVM_Int16, &val, "arm64_pmull"); + values [ins->dreg] = result; + break; + } case OP_ARM64_REV16: case OP_ARM64_REV32: case OP_ARM64_REV64: { diff --git a/src/mono/mono/mini/mini-ops.h b/src/mono/mono/mini/mini-ops.h index e7df2ea2dffd61..0f47eba6e90a59 100644 --- a/src/mono/mono/mini/mini-ops.h +++ b/src/mono/mono/mini/mini-ops.h @@ -830,8 +830,6 @@ MINI_OP(OP_EXPAND_R4, "expand_r4", XREG, FREG, NONE) MINI_OP(OP_EXPAND_I8, "expand_i8", XREG, IREG, NONE) MINI_OP(OP_EXPAND_R8, "expand_r8", XREG, FREG, NONE) -MINI_OP(OP_XXOR, "xxor", XREG, XREG, XREG) - #endif #if defined(TARGET_X86) || defined(TARGET_AMD64) || defined(TARGET_WASM) @@ -1536,6 +1534,7 @@ MINI_OP(OP_XCOMPARE_FP, "xcompare_fp", XREG, XREG, XREG) * Generic SIMD operations, the rest of the JIT doesn't care about the exact operation. */ MINI_OP(OP_XBINOP, "xbinop", XREG, XREG, XREG) +MINI_OP(OP_XBINOP_FORCEINT, "xbinop_coerce", XREG, XREG, XREG) /* inst_c0 contains a SimdOp, inst_c1 might contain additional data */ MINI_OP(OP_XOP, "xop", NONE, NONE, NONE) MINI_OP(OP_XOP_X_I, "xop_x_i", XREG, IREG, NONE) @@ -1658,4 +1657,8 @@ MINI_OP(OP_ARM64_REV16, "arm64_rev16", XREG, XREG, NONE) MINI_OP(OP_ARM64_REV32, "arm64_rev32", XREG, XREG, NONE) MINI_OP(OP_ARM64_REV64, "arm64_rev64", XREG, XREG, NONE) +MINI_OP(OP_ARM64_PMULL, "arm64_pmull", XREG, XREG, NONE) +MINI_OP(OP_ARM64_PMULL2, "arm64_pmull2", XREG, XREG, NONE) + + #endif // TARGET_ARM64 diff --git a/src/mono/mono/mini/simd-intrinsics.c b/src/mono/mono/mini/simd-intrinsics.c index 86b5508dde47ad..5f51be879bcf8d 100644 --- a/src/mono/mono/mini/simd-intrinsics.c +++ b/src/mono/mono/mini/simd-intrinsics.c @@ -970,17 +970,24 @@ static SimdIntrinsic advsimd_methods [] = { {SN_AbsoluteCompareLessThanOrEqual}, {SN_AbsoluteCompareLessThan}, {SN_Abs}, - {SN_ReciprocalEstimate}, + {SN_Not, OP_XBINOP_FORCEINT, XBINOP_FORCEINT_Not}, + {SN_Or, OP_XBINOP_FORCEINT, XBINOP_FORCEINT_Or}, + {SN_OrNot, OP_XBINOP_FORCEINT, XBINOP_FORCEINT_OrNot}, + {SN_PolynomialMultiply, OP_XOP_OVR_X_X_X, INTRINS_AARCH64_ADV_SIMD_PMUL}, + {SN_PolynomialMultiplyWideningLower, OP_ARM64_PMULL}, + {SN_PolynomialMultiplyWideningUpper, OP_ARM64_PMULL2}, + {SN_PopCount, OP_XOP_OVR_SCALAR_X_X, INTRINS_AARCH64_ADV_SIMD_CNT}, {SN_ReciprocalEstimateScalar, OP_XOP_OVR_SCALAR_X_X, INTRINS_AARCH64_ADV_SIMD_FRECPE}, + {SN_ReciprocalEstimate}, {SN_ReciprocalExponentScalar, OP_XOP_OVR_SCALAR_X_X, INTRINS_AARCH64_ADV_SIMD_FRECPX}, - {SN_ReciprocalSquareRootEstimate}, {SN_ReciprocalSquareRootEstimateScalar, OP_XOP_OVR_SCALAR_X_X, INTRINS_AARCH64_ADV_SIMD_FRSQRTE}, + {SN_ReciprocalSquareRootEstimate}, {SN_ReciprocalStep, OP_XOP_OVR_X_X, INTRINS_AARCH64_ADV_SIMD_FRECPS}, {SN_ReciprocalStepScalar, OP_XOP_OVR_SCALAR_X_X, INTRINS_AARCH64_ADV_SIMD_FRECPS}, - {SN_ReverseElementBits, OP_XOP_OVR_X_X, INTRINS_AARCH64_ADV_SIMD_RBIT}, {SN_ReverseElement16, OP_ARM64_REV32}, {SN_ReverseElement32, OP_ARM64_REV64}, {SN_ReverseElement8, OP_ARM64_REV16}, + {SN_ReverseElementBits, OP_XOP_OVR_X_X, INTRINS_AARCH64_ADV_SIMD_RBIT}, {SN_RoundAwayFromZero, OP_XOP_OVR_X_X, INTRINS_AARCH64_ADV_SIMD_FRINTA}, {SN_RoundAwayFromZeroScalar, OP_XOP_OVR_SCALAR_X_X, INTRINS_AARCH64_ADV_SIMD_FRINTA}, {SN_RoundToNearest, OP_XOP_OVR_X_X, INTRINS_AARCH64_ADV_SIMD_FRINTN}, @@ -1076,7 +1083,7 @@ static SimdIntrinsic advsimd_methods [] = { {SN_UnzipOdd, OP_ARM64_UZP2}, {SN_VectorTableLookup, OP_XOP_X_X_X, SIMD_OP_ARM64_TBL}, {SN_VectorTableLookupExtension, OP_XOP_X_X_X_X, SIMD_OP_ARM64_TBX}, - {SN_Xor, OP_XXOR}, + {SN_Xor, OP_XBINOP_FORCEINT, XBINOP_FORCEINT_Xor}, {SN_ZeroExtendWideningLower, OP_ARM64_UXTL}, {SN_ZeroExtendWideningUpper, OP_ARM64_UXTL2}, {SN_ZipHigh, OP_ARM64_ZIP2}, @@ -1408,7 +1415,7 @@ static SimdIntrinsic sse_methods [] = { {SN_SubtractScalar, OP_SSE_SUBSS}, {SN_UnpackHigh, OP_SSE_UNPACKHI}, {SN_UnpackLow, OP_SSE_UNPACKLO}, - {SN_Xor, OP_XXOR}, + {SN_Xor, OP_XBINOP_COERCE, XBINOP_Xor}, {SN_get_IsSupported} }; @@ -1520,7 +1527,7 @@ static SimdIntrinsic sse2_methods [] = { {SN_SumAbsoluteDifferences, OP_XOP_X_X_X, SIMD_OP_SSE_PSADBW}, {SN_UnpackHigh, OP_SSE_UNPACKHI}, {SN_UnpackLow, OP_SSE_UNPACKLO}, - {SN_Xor, OP_XXOR}, + {SN_Xor, OP_XBINOP_COERCE, XBINOP_Xor}, {SN_get_IsSupported} }; diff --git a/src/mono/mono/mini/simd-methods.h b/src/mono/mono/mini/simd-methods.h index 76f775b56f8774..ece715068835f5 100644 --- a/src/mono/mono/mini/simd-methods.h +++ b/src/mono/mono/mini/simd-methods.h @@ -255,6 +255,9 @@ METHOD(AbsoluteCompareGreaterThan) METHOD(AbsoluteCompareGreaterThanOrEqual) METHOD(AbsoluteCompareLessThan) METHOD(AbsoluteCompareLessThanOrEqual) +METHOD(Not) +METHOD(OrNot) +METHOD(PolynomialMultiply) METHOD(ReciprocalEstimate) METHOD(ReciprocalEstimateScalar) METHOD(ReciprocalExponentScalar) From dad5fed3583b97188f0c2011d4c2062613f40093 Mon Sep 17 00:00:00 2001 From: Imran Hameed Date: Fri, 5 Mar 2021 15:33:34 -0800 Subject: [PATCH 07/58] Add negation --- src/mono/mono/mini/llvm-intrinsics.h | 4 ++- src/mono/mono/mini/mini-llvm-cpp.h | 2 +- src/mono/mono/mini/mini-llvm.c | 49 +++++++++++++++++++++------- src/mono/mono/mini/mini-ops.h | 3 ++ src/mono/mono/mini/simd-intrinsics.c | 10 ++++++ src/mono/mono/mini/simd-methods.h | 4 +++ 6 files changed, 58 insertions(+), 14 deletions(-) diff --git a/src/mono/mono/mini/llvm-intrinsics.h b/src/mono/mono/mini/llvm-intrinsics.h index 9a4b4c33a4f453..758ec0dc8ee2e7 100644 --- a/src/mono/mono/mini/llvm-intrinsics.h +++ b/src/mono/mono/mini/llvm-intrinsics.h @@ -307,6 +307,8 @@ INTRINS_OVR_2_ARG(AARCH64_ADV_SIMD_ABS_COMPARE_LT_DOUBLE, aarch64_neon_facgt, ss INTRINS_OVR_2_ARG(AARCH64_ADV_SIMD_ABS_COMPARE_LTE_FLOAT, aarch64_neon_facge, sse_i4_t, sse_r4_t) INTRINS_OVR_2_ARG(AARCH64_ADV_SIMD_ABS_COMPARE_LTE_DOUBLE, aarch64_neon_facge, sse_i4_t, sse_r8_t) +INTRINS_OVR_TAG(AARCH64_ADV_SIMD_SQNEG, aarch64_neon_sqneg, V64 | V128 | I1 | I2 | I4 | I8) + INTRINS_OVR_TAG(AARCH64_ADV_SIMD_PMUL, aarch64_neon_pmul, V64 | V128 | I1) INTRINS_OVR_TAG(AARCH64_ADV_SIMD_PMULL, aarch64_neon_pmull, V128 | I2) @@ -316,7 +318,7 @@ INTRINS_OVR_TAG(AARCH64_ADV_SIMD_URECPE, aarch64_neon_urecpe, V64 | V128 | I4) INTRINS_OVR_TAG(AARCH64_ADV_SIMD_FRECPE, aarch64_neon_frecpe, Scalar | V64 | V128 | R4 | R8) INTRINS_OVR_TAG(AARCH64_ADV_SIMD_FRECPX, aarch64_neon_frecpx, Scalar | R4 | R8) INTRINS_OVR_TAG(AARCH64_ADV_SIMD_URSQRTE, aarch64_neon_ursqrte, V64 | V128 | I4) -INTRINS_OVR_TAG(AARCH64_ADV_SIMD_FRSQRTE, aarch64_neon_frsqrte, Scalar | V64 | V128) +INTRINS_OVR_TAG(AARCH64_ADV_SIMD_FRSQRTE, aarch64_neon_frsqrte, Scalar | V64 | V128| R4 | R8) INTRINS_OVR_TAG(AARCH64_ADV_SIMD_FRSQRTS, aarch64_neon_frsqrts, Scalar | V64 | V128 | R4 | R8) INTRINS_OVR_TAG(AARCH64_ADV_SIMD_FRECPS, aarch64_neon_frecps, Scalar | V64 | V128 | R4 | R8) diff --git a/src/mono/mono/mini/mini-llvm-cpp.h b/src/mono/mono/mini/mini-llvm-cpp.h index 6169a214d1e617..e8175f580715e8 100644 --- a/src/mono/mono/mini/mini-llvm-cpp.h +++ b/src/mono/mono/mini/mini-llvm-cpp.h @@ -64,7 +64,7 @@ enum { typedef uint8_t llvm_ovr_tag_t; static inline llvm_ovr_tag_t -ovr_tag_to_scalar (llvm_ovr_tag_t tag) +ovr_tag_force_scalar (llvm_ovr_tag_t tag) { return tag & 0xfc; } diff --git a/src/mono/mono/mini/mini-llvm.c b/src/mono/mono/mini/mini-llvm.c index ed510654ecfc76..3e8506de42474c 100644 --- a/src/mono/mono/mini/mini-llvm.c +++ b/src/mono/mono/mini/mini-llvm.c @@ -4775,6 +4775,18 @@ concatenate_vectors (EmitContext *ctx, LLVMValueRef xs, LLVMValueRef ys) return LLVMBuildShuffleVector (ctx->builder, xs, ys, create_const_vector_i32 (mask, elems), ""); } +static LLVMValueRef +scalar_from_vector (EmitContext *ctx, LLVMValueRef xs) +{ + return LLVMBuildExtractElement (ctx->builder, xs, const_int32 (0), "v2s"); +} + +static LLVMValueRef +vector_from_scalar (EmitContext *ctx, LLVMValueRef type_donor, LLVMValueRef x) +{ + return LLVMBuildInsertElement (ctx->builder, LLVMGetUndef (LLVMTypeOf (type_donor)), x, const_int32 (0), "s2v"); +} + static void emit_llvmonly_handler_start (EmitContext *ctx, MonoBasicBlock *bb, LLVMBasicBlockRef cbb) { @@ -7170,16 +7182,12 @@ process_bb (EmitContext *ctx, MonoBasicBlock *bb) result = LLVMBuildOr (builder, lhs_int, rhs_int, ""); break; case XBINOP_FORCEINT_OrNot: - result = LLVMBuildXor (builder, lhs_int, rhs_int, ""); - break; - case XBINOP_FORCEINT_Xor: { - int vals [MAX_VECTOR_ELEMS] = { 0 }; - for (unsigned int i = 0; i < elems; ++i) - vals [i] = -1; - result = LLVMBuildXor (builder, rhs_int, create_const_vector (intermediate_elem_t, vals, elems), ""); + result = LLVMBuildNot (builder, rhs_int, ""); result = LLVMBuildOr (builder, result, lhs_int, ""); break; - } + case XBINOP_FORCEINT_Xor: + result = LLVMBuildXor (builder, lhs_int, rhs_int, ""); + break; } values [ins->dreg] = LLVMBuildBitCast (builder, result, t, ""); } @@ -9309,6 +9317,24 @@ process_bb (EmitContext *ctx, MonoBasicBlock *bb) values [ins->dreg] = LLVMBuildTrunc (builder, hi64, LLVMInt64Type (), ""); break; } + case OP_ARM64_XNEG: + case OP_ARM64_XNEG_SCALAR: { + gboolean scalar = ins->opcode == OP_ARM64_XNEG_SCALAR; + gboolean is_float = FALSE; + switch (ins->inst_c1) { + case MONO_TYPE_R4: case MONO_TYPE_R8: is_float = TRUE; + } + LLVMValueRef result = lhs; + if (scalar) + result = scalar_from_vector (ctx, result); + if (is_float) + result = LLVMBuildFNeg (builder, result, "arm64_xneg"); + else + result = LLVMBuildNeg (builder, result, "arm64_xneg"); + if (scalar) + result = vector_from_scalar (ctx, lhs, result); + values [ins->dreg] = result; + } case OP_ARM64_PMULL: case OP_ARM64_PMULL2: { gboolean high = ins->opcode == OP_ARM64_PMULL2; @@ -9732,12 +9758,11 @@ process_bb (EmitContext *ctx, MonoBasicBlock *bb) } case OP_XOP_OVR_SCALAR_X_X: { IntrinsicId iid = (IntrinsicId) ins->inst_c0; - LLVMTypeRef t = LLVMTypeOf (lhs); llvm_ovr_tag_t ovr_tag = ovr_tag_from_mono_vector_class (ins->klass); - ovr_tag = ovr_tag_to_scalar (ovr_tag); - LLVMValueRef result = LLVMBuildExtractElement (builder, lhs, const_int32 (0), "xop_ovr_scalar_x_x"); + ovr_tag = ovr_tag_force_scalar (ovr_tag); + LLVMValueRef result = scalar_from_vector (ctx, lhs); result = call_overloaded_intrins (ctx, iid, ovr_tag, &lhs, ""); - result = LLVMBuildInsertElement (builder, LLVMGetUndef (t), result, const_int32 (0), ""); + result = vector_from_scalar (ctx, lhs, result); values [ins->dreg] = result; break; } diff --git a/src/mono/mono/mini/mini-ops.h b/src/mono/mono/mini/mini-ops.h index 0f47eba6e90a59..f6a1a3a2997f45 100644 --- a/src/mono/mono/mini/mini-ops.h +++ b/src/mono/mono/mini/mini-ops.h @@ -1660,5 +1660,8 @@ MINI_OP(OP_ARM64_REV64, "arm64_rev64", XREG, XREG, NONE) MINI_OP(OP_ARM64_PMULL, "arm64_pmull", XREG, XREG, NONE) MINI_OP(OP_ARM64_PMULL2, "arm64_pmull2", XREG, XREG, NONE) +MINI_OP(OP_ARM64_XNEG, "arm64_xneg", XREG, XREG, NONE) +MINI_OP(OP_ARM64_XNEG_SCALAR, "arm64_xneg_scalar", XREG, XREG, NONE) + #endif // TARGET_ARM64 diff --git a/src/mono/mono/mini/simd-intrinsics.c b/src/mono/mono/mini/simd-intrinsics.c index 5f51be879bcf8d..5c747a4f577acd 100644 --- a/src/mono/mono/mini/simd-intrinsics.c +++ b/src/mono/mono/mini/simd-intrinsics.c @@ -970,6 +970,10 @@ static SimdIntrinsic advsimd_methods [] = { {SN_AbsoluteCompareLessThanOrEqual}, {SN_AbsoluteCompareLessThan}, {SN_Abs}, + {SN_Negate, OP_ARM64_XNEG}, + {SN_NegateSaturate, OP_XOP_OVR_X_X, INTRINS_AARCH64_ADV_SIMD_SQNEG}, + {SN_NegateSaturateScalar}, + {SN_NegateScalar, OP_ARM64_XNEG_SCALAR}, {SN_Not, OP_XBINOP_FORCEINT, XBINOP_FORCEINT_Not}, {SN_Or, OP_XBINOP_FORCEINT, XBINOP_FORCEINT_Or}, {SN_OrNot, OP_XBINOP_FORCEINT, XBINOP_FORCEINT_OrNot}, @@ -1251,6 +1255,11 @@ emit_arm64_intrinsics ( } return emit_simd_ins_for_sig (cfg, klass, OP_XOP_X_X, op, arg0_type, fsig, args); } + case SN_NegateSaturateScalar: { + MonoInst *ret = emit_simd_ins_for_sig (cfg, klass, OP_XOP_OVR_X_X, INTRINS_AARCH64_ADV_SIMD_SQNEG, arg0_type, fsig, args); + ret = emit_simd_ins (cfg, klass, OP_ARM64_ZERO_UPPER, ret->dreg, -1); + return ret; + } case SN_ReciprocalEstimate: case SN_ReciprocalSquareRootEstimate: { gboolean is_float = FALSE; @@ -1263,6 +1272,7 @@ emit_arm64_intrinsics ( break; case SN_ReciprocalSquareRootEstimate: iid = is_float ? INTRINS_AARCH64_ADV_SIMD_FRSQRTE : INTRINS_AARCH64_ADV_SIMD_URSQRTE; + break; } return emit_simd_ins_for_sig (cfg, klass, OP_XOP_OVR_X_X, iid, arg0_type, fsig, args); } diff --git a/src/mono/mono/mini/simd-methods.h b/src/mono/mono/mini/simd-methods.h index ece715068835f5..60f3370aa6576b 100644 --- a/src/mono/mono/mini/simd-methods.h +++ b/src/mono/mono/mini/simd-methods.h @@ -255,6 +255,10 @@ METHOD(AbsoluteCompareGreaterThan) METHOD(AbsoluteCompareGreaterThanOrEqual) METHOD(AbsoluteCompareLessThan) METHOD(AbsoluteCompareLessThanOrEqual) +METHOD(Negate) +METHOD(NegateSaturate) +METHOD(NegateSaturateScalar) +METHOD(NegateScalar) METHOD(Not) METHOD(OrNot) METHOD(PolynomialMultiply) From 63414496af52326bf34e7dd81234d4e44dd553fd Mon Sep 17 00:00:00 2001 From: Imran Hameed Date: Fri, 5 Mar 2021 15:47:10 -0800 Subject: [PATCH 08/58] Add every AdvSimd symbol name --- src/mono/mono/mini/simd-methods.h | 199 ++++++++++++++++++++++++++++++ 1 file changed, 199 insertions(+) diff --git a/src/mono/mono/mini/simd-methods.h b/src/mono/mono/mini/simd-methods.h index 60f3370aa6576b..673543d4389554 100644 --- a/src/mono/mono/mini/simd-methods.h +++ b/src/mono/mono/mini/simd-methods.h @@ -250,11 +250,205 @@ METHOD(ScheduleUpdate1) METHOD(MixColumns) //AdvSimd METHOD(AbsSaturate) +METHOD(AbsSaturateScalar) METHOD(AbsScalar) METHOD(AbsoluteCompareGreaterThan) METHOD(AbsoluteCompareGreaterThanOrEqual) +METHOD(AbsoluteCompareGreaterThanOrEqualScalar) +METHOD(AbsoluteCompareGreaterThanScalar) METHOD(AbsoluteCompareLessThan) METHOD(AbsoluteCompareLessThanOrEqual) +METHOD(AbsoluteCompareLessThanOrEqualScalar) +METHOD(AbsoluteCompareLessThanScalar) +METHOD(AbsoluteDifference) +METHOD(AbsoluteDifferenceAdd) +METHOD(AbsoluteDifferenceScalar) +METHOD(AbsoluteDifferenceWideningLower) +METHOD(AbsoluteDifferenceWideningLowerAndAdd) +METHOD(AbsoluteDifferenceWideningUpper) +METHOD(AbsoluteDifferenceWideningUpperAndAdd) +METHOD(AddAcross) +METHOD(AddAcrossWidening) +METHOD(AddHighNarrowingLower) +METHOD(AddHighNarrowingUpper) +METHOD(AddPairwise) +METHOD(AddPairwiseScalar) +METHOD(AddPairwiseWidening) +METHOD(AddPairwiseWideningAndAdd) +METHOD(AddPairwiseWideningAndAddScalar) +METHOD(AddPairwiseWideningScalar) +METHOD(AddRoundedHighNarrowingLower) +METHOD(AddRoundedHighNarrowingUpper) +METHOD(AddSaturateScalar) +METHOD(AddWideningLower) +METHOD(AddWideningUpper) +METHOD(BitwiseClear) +METHOD(BitwiseSelect) +METHOD(CompareEqualScalar) +METHOD(CompareGreaterThanOrEqualScalar) +METHOD(CompareGreaterThanScalar) +METHOD(CompareLessThanOrEqualScalar) +METHOD(CompareLessThanScalar) +METHOD(CompareTest) +METHOD(CompareTestScalar) +METHOD(ConvertToDoubleScalar) +METHOD(ConvertToDoubleUpper) +METHOD(ConvertToInt32RoundAwayFromZero) +METHOD(ConvertToInt32RoundAwayFromZeroScalar) +METHOD(ConvertToInt32RoundToEven) +METHOD(ConvertToInt32RoundToEvenScalar) +METHOD(ConvertToInt32RoundToNegativeInfinity) +METHOD(ConvertToInt32RoundToNegativeInfinityScalar) +METHOD(ConvertToInt32RoundToPositiveInfinity) +METHOD(ConvertToInt32RoundToPositiveInfinityScalar) +METHOD(ConvertToInt32RoundToZero) +METHOD(ConvertToInt32RoundToZeroScalar) +METHOD(ConvertToInt64RoundAwayFromZero) +METHOD(ConvertToInt64RoundAwayFromZeroScalar) +METHOD(ConvertToInt64RoundToEven) +METHOD(ConvertToInt64RoundToEvenScalar) +METHOD(ConvertToInt64RoundToNegativeInfinity) +METHOD(ConvertToInt64RoundToNegativeInfinityScalar) +METHOD(ConvertToInt64RoundToPositiveInfinity) +METHOD(ConvertToInt64RoundToPositiveInfinityScalar) +METHOD(ConvertToInt64RoundToZero) +METHOD(ConvertToInt64RoundToZeroScalar) +METHOD(ConvertToSingleLower) +METHOD(ConvertToSingleRoundToOddLower) +METHOD(ConvertToSingleRoundToOddUpper) +METHOD(ConvertToSingleScalar) +METHOD(ConvertToSingleUpper) +METHOD(ConvertToUInt32RoundAwayFromZero) +METHOD(ConvertToUInt32RoundAwayFromZeroScalar) +METHOD(ConvertToUInt32RoundToEven) +METHOD(ConvertToUInt32RoundToEvenScalar) +METHOD(ConvertToUInt32RoundToNegativeInfinity) +METHOD(ConvertToUInt32RoundToNegativeInfinityScalar) +METHOD(ConvertToUInt32RoundToPositiveInfinity) +METHOD(ConvertToUInt32RoundToPositiveInfinityScalar) +METHOD(ConvertToUInt32RoundToZero) +METHOD(ConvertToUInt32RoundToZeroScalar) +METHOD(ConvertToUInt64RoundAwayFromZero) +METHOD(ConvertToUInt64RoundAwayFromZeroScalar) +METHOD(ConvertToUInt64RoundToEven) +METHOD(ConvertToUInt64RoundToEvenScalar) +METHOD(ConvertToUInt64RoundToNegativeInfinity) +METHOD(ConvertToUInt64RoundToNegativeInfinityScalar) +METHOD(ConvertToUInt64RoundToPositiveInfinity) +METHOD(ConvertToUInt64RoundToPositiveInfinityScalar) +METHOD(ConvertToUInt64RoundToZero) +METHOD(ConvertToUInt64RoundToZeroScalar) +METHOD(DuplicateSelectedScalarToVector128) +METHOD(DuplicateSelectedScalarToVector64) +METHOD(DuplicateToVector128) +METHOD(DuplicateToVector64) +METHOD(ExtractNarrowingLower) +METHOD(ExtractNarrowingSaturateLower) +METHOD(ExtractNarrowingSaturateScalar) +METHOD(ExtractNarrowingSaturateUnsignedLower) +METHOD(ExtractNarrowingSaturateUnsignedScalar) +METHOD(ExtractNarrowingSaturateUnsignedUpper) +METHOD(ExtractNarrowingSaturateUpper) +METHOD(ExtractNarrowingUpper) +METHOD(ExtractVector128) +METHOD(ExtractVector64) +METHOD(FusedAddHalving) +METHOD(FusedAddRoundedHalving) +METHOD(FusedMultiplyAdd) +METHOD(FusedMultiplyAddByScalar) +METHOD(FusedMultiplyAddBySelectedScalar) +METHOD(FusedMultiplyAddNegatedScalar) +METHOD(FusedMultiplyAddScalar) +METHOD(FusedMultiplyAddScalarBySelectedScalar) +METHOD(FusedMultiplySubtract) +METHOD(FusedMultiplySubtractByScalar) +METHOD(FusedMultiplySubtractBySelectedScalar) +METHOD(FusedMultiplySubtractNegatedScalar) +METHOD(FusedMultiplySubtractScalar) +METHOD(FusedMultiplySubtractScalarBySelectedScalar) +METHOD(FusedSubtractHalving) +METHOD(InsertScalar) +METHOD(InsertSelectedScalar) +METHOD(LoadAndInsertScalar) +METHOD(LoadAndReplicateToVector128) +METHOD(LoadAndReplicateToVector64) +METHOD(LoadVector64) +METHOD(MaxAcross) +METHOD(MaxNumber) +METHOD(MaxNumberAcross) +METHOD(MaxNumberPairwise) +METHOD(MaxNumberPairwiseScalar) +METHOD(MaxNumberScalar) +METHOD(MaxPairwise) +METHOD(MaxPairwiseScalar) +METHOD(MinAcross) +METHOD(MinNumber) +METHOD(MinNumberAcross) +METHOD(MinNumberPairwise) +METHOD(MinNumberPairwiseScalar) +METHOD(MinNumberScalar) +METHOD(MinPairwise) +METHOD(MinPairwiseScalar) +METHOD(MultiplyAdd) +METHOD(MultiplyAddByScalar) +METHOD(MultiplyAddBySelectedScalar) +METHOD(MultiplyByScalar) +METHOD(MultiplyBySelectedScalar) +METHOD(MultiplyBySelectedScalarWideningLower) +METHOD(MultiplyBySelectedScalarWideningLowerAndAdd) +METHOD(MultiplyBySelectedScalarWideningLowerAndSubtract) +METHOD(MultiplyBySelectedScalarWideningUpper) +METHOD(MultiplyBySelectedScalarWideningUpperAndAdd) +METHOD(MultiplyBySelectedScalarWideningUpperAndSubtract) +METHOD(MultiplyDoublingByScalarSaturateHigh) +METHOD(MultiplyDoublingBySelectedScalarSaturateHigh) +METHOD(MultiplyDoublingSaturateHigh) +METHOD(MultiplyDoublingSaturateHighScalar) +METHOD(MultiplyDoublingScalarBySelectedScalarSaturateHigh) +METHOD(MultiplyDoublingWideningAndAddSaturateScalar) +METHOD(MultiplyDoublingWideningAndSubtractSaturateScalar) +METHOD(MultiplyDoublingWideningLowerAndAddSaturate) +METHOD(MultiplyDoublingWideningLowerAndSubtractSaturate) +METHOD(MultiplyDoublingWideningLowerByScalarAndAddSaturate) +METHOD(MultiplyDoublingWideningLowerByScalarAndSubtractSaturate) +METHOD(MultiplyDoublingWideningLowerBySelectedScalarAndAddSaturate) +METHOD(MultiplyDoublingWideningLowerBySelectedScalarAndSubtractSaturate) +METHOD(MultiplyDoublingWideningSaturateLower) +METHOD(MultiplyDoublingWideningSaturateLowerByScalar) +METHOD(MultiplyDoublingWideningSaturateLowerBySelectedScalar) +METHOD(MultiplyDoublingWideningSaturateScalar) +METHOD(MultiplyDoublingWideningSaturateScalarBySelectedScalar) +METHOD(MultiplyDoublingWideningSaturateUpper) +METHOD(MultiplyDoublingWideningSaturateUpperByScalar) +METHOD(MultiplyDoublingWideningSaturateUpperBySelectedScalar) +METHOD(MultiplyDoublingWideningScalarBySelectedScalarAndAddSaturate) +METHOD(MultiplyDoublingWideningScalarBySelectedScalarAndSubtractSaturate) +METHOD(MultiplyDoublingWideningUpperAndAddSaturate) +METHOD(MultiplyDoublingWideningUpperAndSubtractSaturate) +METHOD(MultiplyDoublingWideningUpperByScalarAndAddSaturate) +METHOD(MultiplyDoublingWideningUpperByScalarAndSubtractSaturate) +METHOD(MultiplyDoublingWideningUpperBySelectedScalarAndAddSaturate) +METHOD(MultiplyDoublingWideningUpperBySelectedScalarAndSubtractSaturate) +METHOD(MultiplyExtended) +METHOD(MultiplyExtendedByScalar) +METHOD(MultiplyExtendedBySelectedScalar) +METHOD(MultiplyExtendedScalar) +METHOD(MultiplyExtendedScalarBySelectedScalar) +METHOD(MultiplyRoundedDoublingByScalarSaturateHigh) +METHOD(MultiplyRoundedDoublingBySelectedScalarSaturateHigh) +METHOD(MultiplyRoundedDoublingSaturateHigh) +METHOD(MultiplyRoundedDoublingSaturateHighScalar) +METHOD(MultiplyRoundedDoublingScalarBySelectedScalarSaturateHigh) +METHOD(MultiplyScalarBySelectedScalar) +METHOD(MultiplySubtract) +METHOD(MultiplySubtractByScalar) +METHOD(MultiplySubtractBySelectedScalar) +METHOD(MultiplyWideningLower) +METHOD(MultiplyWideningLowerAndAdd) +METHOD(MultiplyWideningLowerAndSubtract) +METHOD(MultiplyWideningUpper) +METHOD(MultiplyWideningUpperAndAdd) +METHOD(MultiplyWideningUpperAndSubtract) METHOD(Negate) METHOD(NegateSaturate) METHOD(NegateSaturateScalar) @@ -308,14 +502,18 @@ METHOD(ShiftRightAndInsertScalar) METHOD(ShiftRightArithmeticAdd) METHOD(ShiftRightArithmeticAddScalar) METHOD(ShiftRightArithmeticNarrowingSaturateLower) +METHOD(ShiftRightArithmeticNarrowingSaturateScalar) METHOD(ShiftRightArithmeticNarrowingSaturateUnsignedLower) +METHOD(ShiftRightArithmeticNarrowingSaturateUnsignedScalar) METHOD(ShiftRightArithmeticNarrowingSaturateUnsignedUpper) METHOD(ShiftRightArithmeticNarrowingSaturateUpper) METHOD(ShiftRightArithmeticRounded) METHOD(ShiftRightArithmeticRoundedAdd) METHOD(ShiftRightArithmeticRoundedAddScalar) METHOD(ShiftRightArithmeticRoundedNarrowingSaturateLower) +METHOD(ShiftRightArithmeticRoundedNarrowingSaturateScalar) METHOD(ShiftRightArithmeticRoundedNarrowingSaturateUnsignedLower) +METHOD(ShiftRightArithmeticRoundedNarrowingSaturateUnsignedScalar) METHOD(ShiftRightArithmeticRoundedNarrowingSaturateUnsignedUpper) METHOD(ShiftRightArithmeticRoundedNarrowingSaturateUpper) METHOD(ShiftRightArithmeticRoundedScalar) @@ -324,6 +522,7 @@ METHOD(ShiftRightLogicalAdd) METHOD(ShiftRightLogicalAddScalar) METHOD(ShiftRightLogicalNarrowingLower) METHOD(ShiftRightLogicalNarrowingSaturateLower) +METHOD(ShiftRightLogicalNarrowingSaturateScalar) METHOD(ShiftRightLogicalNarrowingSaturateUpper) METHOD(ShiftRightLogicalNarrowingUpper) METHOD(ShiftRightLogicalRounded) From 8580569ea09793e05748c7f261b39f433e431195 Mon Sep 17 00:00:00 2001 From: Imran Hameed Date: Fri, 5 Mar 2021 17:17:11 -0800 Subject: [PATCH 09/58] Minor cleanup --- src/mono/mono/mini/llvm-intrinsics.h | 4 +- src/mono/mono/mini/mini-llvm.c | 102 ++++++++++++++++----------- src/mono/mono/mini/mini-ops.h | 1 + src/mono/mono/mini/simd-intrinsics.c | 80 ++++++++++----------- 4 files changed, 102 insertions(+), 85 deletions(-) diff --git a/src/mono/mono/mini/llvm-intrinsics.h b/src/mono/mono/mini/llvm-intrinsics.h index 758ec0dc8ee2e7..92e22fa4fbd3a6 100644 --- a/src/mono/mono/mini/llvm-intrinsics.h +++ b/src/mono/mono/mini/llvm-intrinsics.h @@ -330,8 +330,8 @@ INTRINS_OVR_TAG(AARCH64_ADV_SIMD_FRINTM, floor, Scalar | V64 | V128 | R4 | R8) INTRINS_OVR_TAG(AARCH64_ADV_SIMD_FRINTP, ceil, Scalar | V64 | V128 | R4 | R8) INTRINS_OVR_TAG(AARCH64_ADV_SIMD_FRINTZ, trunc, Scalar | V64 | V128 | R4 | R8) -INTRINS_OVR_TAG(AARCH64_ADV_SIMD_UQSUB, aarch64_neon_uqsub, V64 | V128 | I1 | I2 | I4 | I8) -INTRINS_OVR_TAG(AARCH64_ADV_SIMD_SQSUB, aarch64_neon_sqsub, V64 | V128 | I1 | I2 | I4 | I8) +INTRINS_OVR_TAG(AARCH64_ADV_SIMD_UQSUB, aarch64_neon_uqsub, Scalar | V64 | V128 | I1 | I2 | I4 | I8) +INTRINS_OVR_TAG(AARCH64_ADV_SIMD_SQSUB, aarch64_neon_sqsub, Scalar | V64 | V128 | I1 | I2 | I4 | I8) INTRINS_OVR_TAG(AARCH64_ADV_SIMD_RSUBHN, aarch64_neon_rsubhn, V64 | I1 | I2 | I4) INTRINS_OVR_TAG(AARCH64_ADV_SIMD_FSQRT, sqrt, V64 | V128 | R4 | R8) INTRINS_OVR_TAG(AARCH64_ADV_SIMD_UQSHRN, aarch64_neon_uqshrn, V64 | I1 | I2 | I4) // Constant shift diff --git a/src/mono/mono/mini/mini-llvm.c b/src/mono/mono/mini/mini-llvm.c index 3e8506de42474c..da80c65b09a1af 100644 --- a/src/mono/mono/mini/mini-llvm.c +++ b/src/mono/mono/mini/mini-llvm.c @@ -9321,7 +9321,7 @@ process_bb (EmitContext *ctx, MonoBasicBlock *bb) case OP_ARM64_XNEG_SCALAR: { gboolean scalar = ins->opcode == OP_ARM64_XNEG_SCALAR; gboolean is_float = FALSE; - switch (ins->inst_c1) { + switch (inst_c1_type (ins)) { case MONO_TYPE_R4: case MONO_TYPE_R8: is_float = TRUE; } LLVMValueRef result = lhs; @@ -9566,31 +9566,28 @@ process_bb (EmitContext *ctx, MonoBasicBlock *bb) case OP_ARM64_STP_SCALAR: case OP_ARM64_STNP: case OP_ARM64_STNP_SCALAR: { + gboolean nontemporal = FALSE; + gboolean scalar = FALSE; + switch (ins->opcode) { + case OP_ARM64_STNP: nontemporal = TRUE; break; + case OP_ARM64_STNP_SCALAR: nontemporal = TRUE; scalar = TRUE; break; + case OP_ARM64_STP_SCALAR: scalar = TRUE; break; + } LLVMTypeRef lhs_t = LLVMTypeOf (lhs); unsigned int lhs_elems = LLVMGetVectorSize (lhs_t); LLVMTypeRef lhs_elt_t = LLVMGetElementType (lhs_t); LLVMValueRef val = NULL; LLVMTypeRef dst_t = lhs_t; - switch (ins->opcode) { - case OP_ARM64_STP: - case OP_ARM64_STNP: { + if (scalar) + val = LLVMBuildShuffleVector (builder, rhs, arg3, create_const_vector_2_i32 (0, 2), ""); + else { dst_t = LLVMPointerType (LLVMVectorType (lhs_elt_t, lhs_elems * 2), 0); val = concatenate_vectors (ctx, rhs, arg3); - break; - } - case OP_ARM64_STP_SCALAR: - case OP_ARM64_STNP_SCALAR: { - val = LLVMBuildShuffleVector (builder, rhs, arg3, create_const_vector_2_i32 (0, 2), ""); - break; - } } LLVMValueRef addr = convert (ctx, lhs, dst_t); LLVMValueRef store = mono_llvm_build_store (builder, val, addr, FALSE, LLVM_BARRIER_NONE); - switch (ins->opcode) { - case OP_ARM64_STNP: - case OP_ARM64_STNP_SCALAR: + if (nontemporal) set_nontemporal_flag (store); - } break; } case OP_ARM64_ST1: { @@ -9648,24 +9645,17 @@ process_bb (EmitContext *ctx, MonoBasicBlock *bb) case OP_ARM64_USUB2: { LLVMTypeRef ret_t = simd_class_to_llvm_type (ctx, ins->klass); gboolean is_unsigned = FALSE; - gboolean select_high = FALSE; + gboolean high = FALSE; switch (ins->opcode) { case OP_ARM64_USUB: is_unsigned = TRUE; break; - case OP_ARM64_SSUB2: select_high = TRUE; break; - case OP_ARM64_USUB2: select_high = TRUE; is_unsigned = TRUE; break; + case OP_ARM64_SSUB2: high = TRUE; break; + case OP_ARM64_USUB2: high = TRUE; is_unsigned = TRUE; break; } LLVMValueRef args [2] = { lhs, rhs }; - int mask [MAX_VECTOR_ELEMS] = { 0 }; for (int i = 0; i < 2; ++i) { LLVMValueRef arg = args [i]; - LLVMTypeRef arg_t = LLVMTypeOf (arg); - unsigned int elems = LLVMGetVectorSize (arg_t) / 2; - int laneix = select_high ? elems : 0; - for (int i = 0; i < elems; ++i) { - mask [i] = laneix; - ++laneix; - } - arg = LLVMBuildShuffleVector (builder, arg, LLVMGetUndef (arg_t), create_const_vector_i32 (mask, elems), ""); + if (high) + arg = extract_high_elements (ctx, arg); if (is_unsigned) arg = LLVMBuildZExt (builder, arg, ret_t, ""); else @@ -9679,36 +9669,38 @@ process_bb (EmitContext *ctx, MonoBasicBlock *bb) case OP_ARM64_SXTL2: case OP_ARM64_UXTL: case OP_ARM64_UXTL2: { + gboolean high = FALSE; + gboolean is_unsigned = FALSE; + switch (ins->opcode) { + case OP_ARM64_SXTL2: high = TRUE; break; + case OP_ARM64_UXTL: is_unsigned = TRUE; break; + case OP_ARM64_UXTL2: high = TRUE; is_unsigned = TRUE; break; + } LLVMTypeRef t = LLVMTypeOf (lhs); unsigned int elem_bits = LLVMGetIntTypeWidth (LLVMGetElementType (t)); unsigned int src_elems = LLVMGetVectorSize (t); unsigned int dst_elems = src_elems; LLVMValueRef arg = lhs; - switch (ins->opcode) { - case OP_ARM64_SXTL2: - case OP_ARM64_UXTL2: + if (high) { arg = extract_high_elements (ctx, lhs); dst_elems = LLVMGetVectorSize (LLVMTypeOf (arg)); } LLVMTypeRef result_t = LLVMVectorType (LLVMIntType (elem_bits * 2), dst_elems); LLVMValueRef result = NULL; - switch (ins->opcode) { - case OP_ARM64_SXTL: - case OP_ARM64_SXTL2: - result = LLVMBuildSExt (builder, arg, result_t, "arm64_sxtl"); - break; - default: + if (is_unsigned) result = LLVMBuildZExt (builder, arg, result_t, "arm64_uxtl"); - } + else + result = LLVMBuildSExt (builder, arg, result_t, "arm64_sxtl"); values [ins->dreg] = result; break; } case OP_ARM64_TRN1: case OP_ARM64_TRN2: { + gboolean high = ins->opcode == OP_ARM64_TRN2; LLVMTypeRef t = LLVMTypeOf (lhs); unsigned int src_elems = LLVMGetVectorSize (t); int mask [MAX_VECTOR_ELEMS] = { 0 }; - int laneix = ins->opcode == OP_ARM64_TRN2 ? 1 : 0; + int laneix = high ? 1 : 0; for (unsigned int i = 0; i < src_elems; i += 2) { mask [i] = laneix; mask [i + 1] = laneix + src_elems; @@ -9719,10 +9711,11 @@ process_bb (EmitContext *ctx, MonoBasicBlock *bb) } case OP_ARM64_UZP1: case OP_ARM64_UZP2: { + gboolean high = ins->opcode == OP_ARM64_UZP2; LLVMTypeRef t = LLVMTypeOf (lhs); unsigned int src_elems = LLVMGetVectorSize (t); int mask [MAX_VECTOR_ELEMS] = { 0 }; - int laneix = ins->opcode == OP_ARM64_UZP2 ? 1 : 0; + int laneix = high ? 1 : 0; for (unsigned int i = 0; i < src_elems; ++i) { mask [i] = laneix; laneix += 2; @@ -9732,10 +9725,11 @@ process_bb (EmitContext *ctx, MonoBasicBlock *bb) } case OP_ARM64_ZIP1: case OP_ARM64_ZIP2: { + gboolean high = ins->opcode == OP_ARM64_ZIP2; LLVMTypeRef t = LLVMTypeOf (lhs); unsigned int src_elems = LLVMGetVectorSize (t); int mask [MAX_VECTOR_ELEMS] = { 0 }; - int laneix = ins->opcode == OP_ARM64_ZIP2 ? src_elems / 2 : 0; + int laneix = high ? src_elems / 2 : 0; for (unsigned int i = 0; i < src_elems; i += 2) { mask [i] = laneix; mask [i + 1] = laneix + src_elems; @@ -9759,10 +9753,32 @@ process_bb (EmitContext *ctx, MonoBasicBlock *bb) case OP_XOP_OVR_SCALAR_X_X: { IntrinsicId iid = (IntrinsicId) ins->inst_c0; llvm_ovr_tag_t ovr_tag = ovr_tag_from_mono_vector_class (ins->klass); - ovr_tag = ovr_tag_force_scalar (ovr_tag); - LLVMValueRef result = scalar_from_vector (ctx, lhs); + /* LLVM 9 NEON intrinsic functions have scalar overloads. Unfortunately + * only overloads for 32 and 64-bit integers and floating point types are + * supported. 8 and 16-bit integers are unsupported, and will fail during + * instruction selection. This is worked around by using a vector + * operation and then explicitly clearing the upper bits of the register. + */ + gboolean fake_scalar_op = FALSE; +#if defined(TARGET_ARM64) + switch (inst_c1_type (ins)) { + case MONO_TYPE_U1: + case MONO_TYPE_I1: + case MONO_TYPE_U2: + case MONO_TYPE_I2: + fake_scalar_op = TRUE; + } +#endif + LLVMValueRef result = lhs; + if (!fake_scalar_op) { + ovr_tag = ovr_tag_force_scalar (ovr_tag); + result = scalar_from_vector (ctx, result); + } result = call_overloaded_intrins (ctx, iid, ovr_tag, &lhs, ""); - result = vector_from_scalar (ctx, lhs, result); + if (!fake_scalar_op) + result = vector_from_scalar (ctx, lhs, result); + else + result = select_lowest_element (ctx, result); values [ins->dreg] = result; break; } diff --git a/src/mono/mono/mini/mini-ops.h b/src/mono/mono/mini/mini-ops.h index f6a1a3a2997f45..ee225cc2128ff4 100644 --- a/src/mono/mono/mini/mini-ops.h +++ b/src/mono/mono/mini/mini-ops.h @@ -1663,5 +1663,6 @@ MINI_OP(OP_ARM64_PMULL2, "arm64_pmull2", XREG, XREG, NONE) MINI_OP(OP_ARM64_XNEG, "arm64_xneg", XREG, XREG, NONE) MINI_OP(OP_ARM64_XNEG_SCALAR, "arm64_xneg_scalar", XREG, XREG, NONE) +MINI_OP3(OP_ARM64_SMLSL2, "arm64_smlsl2", XREG, XREG, XREG, XREG) #endif // TARGET_ARM64 diff --git a/src/mono/mono/mini/simd-intrinsics.c b/src/mono/mono/mini/simd-intrinsics.c index 5c747a4f577acd..fe01bcccdb8761 100644 --- a/src/mono/mono/mini/simd-intrinsics.c +++ b/src/mono/mono/mini/simd-intrinsics.c @@ -54,13 +54,16 @@ enum { static int register_size; +#define None 0 + typedef struct { - // One of the SN_ constants - guint16 id; - // ins->opcode - int op; - // ins->inst_c0 - int instc0; + uint16_t id; // One of the SN_ constants + uint16_t default_op; // ins->opcode + uint16_t default_instc0; // ins->inst_c0 + uint16_t unsigned_op; + uint16_t unsigned_instc0; + uint16_t floating_op; + uint16_t floating_instc0; } SimdIntrinsic; static const SimdIntrinsic unsupported [] = { {SN_get_IsSupported} }; @@ -397,8 +400,31 @@ emit_hardware_intrinsics ( id = info->id; - if (info->op != 0) - return emit_simd_ins_for_sig (cfg, klass, info->op, info->instc0, arg0_type, fsig, args); + uint16_t op = info->default_op; + uint16_t c0 = info->default_instc0; + gboolean is_unsigned = FALSE; + gboolean is_float = FALSE; + switch (arg0_type) { + case MONO_TYPE_U1: + case MONO_TYPE_U2: + case MONO_TYPE_U4: + case MONO_TYPE_U8: + is_unsigned = TRUE; + break; + case MONO_TYPE_R4: + case MONO_TYPE_R8: + is_float = TRUE; + break; + } + if (is_unsigned && info->unsigned_op != 0) { + op = info->unsigned_op; + c0 = info->unsigned_instc0; + } else if (is_float && info->floating_op != 0) { + op = info->floating_op; + c0 = info->floating_instc0; + } + if (op != 0) + return emit_simd_ins_for_sig (cfg, klass, op, c0, arg0_type, fsig, args); } support_probe_complete: if (id == SN_get_IsSupported) { @@ -925,7 +951,7 @@ static SimdIntrinsic armbase_methods [] = { {SN_LeadingZeroCount}, {SN_MultiplyHigh}, {SN_ReverseElementBits}, - {SN_get_IsSupported} + {SN_get_IsSupported}, }; static SimdIntrinsic crc32_methods [] = { @@ -970,6 +996,7 @@ static SimdIntrinsic advsimd_methods [] = { {SN_AbsoluteCompareLessThanOrEqual}, {SN_AbsoluteCompareLessThan}, {SN_Abs}, + {SN_MultiplyWideningUpperAndSubtract, OP_ARM64_SMLSL2}, {SN_Negate, OP_ARM64_XNEG}, {SN_NegateSaturate, OP_XOP_OVR_X_X, INTRINS_AARCH64_ADV_SIMD_SQNEG}, {SN_NegateSaturateScalar}, @@ -1077,9 +1104,11 @@ static SimdIntrinsic advsimd_methods [] = { {SN_SubtractHighNarrowingUpper, OP_ARM64_SUBHN2}, {SN_SubtractRoundedHighNarrowingLower, OP_ARM64_RSUBHN}, {SN_SubtractRoundedHighNarrowingUpper, OP_ARM64_RSUBHN2}, - {SN_SubtractSaturateScalar}, + {SN_SubtractSaturate, OP_XOP_OVR_X_X_X, INTRINS_AARCH64_ADV_SIMD_SQSUB, OP_XOP_OVR_X_X_X, INTRINS_AARCH64_ADV_SIMD_UQSUB}, + {SN_SubtractSaturateScalar, OP_XOP_OVR_SCALAR_X_X, INTRINS_AARCH64_ADV_SIMD_SQSUB, OP_XOP_OVR_SCALAR_X_X, INTRINS_AARCH64_ADV_SIMD_UQSUB}, {SN_SubtractScalar}, - {SN_SubtractWideningUpper}, + {SN_SubtractWideningLower, OP_ARM64_SSUB, None, OP_ARM64_USUB}, + {SN_SubtractWideningUpper, OP_ARM64_SSUB2, None, OP_ARM64_USUB2}, {SN_Subtract}, {SN_TransposeEven, OP_ARM64_TRN1}, {SN_TransposeOdd, OP_ARM64_TRN2}, @@ -1299,35 +1328,6 @@ emit_arm64_intrinsics ( ret = emit_simd_ins (cfg, klass, OP_ARM64_ZERO_UPPER, ret->dreg, -1); return ret; } - case SN_SubtractSaturate: - case SN_SubtractSaturateScalar: { - gboolean is_unsigned = type_is_unsigned (fsig->ret); - iid = is_unsigned ? INTRINS_AARCH64_ADV_SIMD_UQSUB : INTRINS_AARCH64_ADV_SIMD_SQSUB; - MonoInst *ret = emit_simd_ins_for_sig (cfg, klass, OP_XOP_OVR_X_X_X, iid, 0, fsig, args); - /* LLVM has intrinsic functions for only the 32 and 64-bit forms of the scalar variants - * of NEON uqsub and sqsub. The CoreCLR runtime tests for these intrinsics look like they - * assert that all lanes > 0 are zeroed out, so just use the vector variant of these - * instructions here and then set the upper "non-scalar" bits to zero. - */ - if (id == SN_SubtractSaturateScalar) - ret = emit_simd_ins (cfg, klass, OP_ARM64_ZERO_UPPER, ret->dreg, -1); - return ret; - } - case SN_SubtractWideningLower: - case SN_SubtractWideningUpper: { - gboolean is_upper = id == SN_SubtractWideningUpper; - MonoTypeEnum ret_t = get_underlying_type (fsig->params [1]); - int op = is_upper ? OP_ARM64_SSUB2 : OP_ARM64_SSUB; - switch (ret_t) { - case MONO_TYPE_U1: - case MONO_TYPE_U2: - case MONO_TYPE_U4: - case MONO_TYPE_U8: - op = is_upper ? OP_ARM64_USUB2 : OP_ARM64_USUB; - break; - } - return emit_simd_ins_for_sig (cfg, klass, op, 0, 0, fsig, args); - } default: g_assert_not_reached (); } From a9bc75ec40d43759b4300f6f17530b6ec14ee912 Mon Sep 17 00:00:00 2001 From: Imran Hameed Date: Fri, 5 Mar 2021 18:17:15 -0800 Subject: [PATCH 10/58] Checkpoint Remove `MonoLLVMModule::intrins_by_id`, which doesn't do anything other than serve as a lookup table for data contained in `intrins_id_to_intrins` Don't emit table-driven intrinsics when the corresponding intrinsic group isn't fully supported. --- src/mono/mono/mini/llvm-intrinsics.h | 8 +- src/mono/mono/mini/mini-llvm-cpp.h | 38 +++++-- src/mono/mono/mini/mini-llvm.c | 157 +++++++++++++-------------- src/mono/mono/mini/mini-ops.h | 14 ++- src/mono/mono/mini/mini.h | 2 - src/mono/mono/mini/simd-intrinsics.c | 55 ++++------ 6 files changed, 139 insertions(+), 135 deletions(-) diff --git a/src/mono/mono/mini/llvm-intrinsics.h b/src/mono/mono/mini/llvm-intrinsics.h index 92e22fa4fbd3a6..dca90b8adece78 100644 --- a/src/mono/mono/mini/llvm-intrinsics.h +++ b/src/mono/mono/mini/llvm-intrinsics.h @@ -12,7 +12,7 @@ * To define an overloaded intrinsic with three arguments */ -#define Scalar 0 +#define Scalar LLVM_Scalar #define V64 LLVM_Vector64 #define V128 LLVM_Vector128 #define I1 LLVM_Int8 @@ -357,10 +357,8 @@ INTRINS_OVR_TAG(AARCH64_ADV_SIMD_SQSHLU, aarch64_neon_sqshlu, V64 | V128 | I1 | INTRINS_OVR_TAG(AARCH64_ADV_SIMD_SLI, aarch64_neon_vsli, V64 | V128 | I1 | I2 | I4 | I8) // Constant shift INTRINS_OVR_TAG(AARCH64_ADV_SIMD_SRI, aarch64_neon_vsri, V64 | V128 | I1 | I2 | I4 | I8) // Constant shift -INTRINS_OVR(AARCH64_ADV_SIMD_TBX64, aarch64_neon_tbx1, LLVMVectorType (LLVMInt8Type (), 8)) -INTRINS_OVR(AARCH64_ADV_SIMD_TBX128, aarch64_neon_tbx1, sse_i1_t) -INTRINS_OVR(AARCH64_ADV_SIMD_TBL64, aarch64_neon_tbl1, LLVMVectorType (LLVMInt8Type (), 8)) -INTRINS_OVR(AARCH64_ADV_SIMD_TBL128, aarch64_neon_tbl1, sse_i1_t) +INTRINS_OVR_TAG(AARCH64_ADV_SIMD_TBX1, aarch64_neon_tbx1, V64 | V128 | I1) +INTRINS_OVR_TAG(AARCH64_ADV_SIMD_TBL1, aarch64_neon_tbl1, V64 | V128 | I1) #endif #undef INTRINS diff --git a/src/mono/mono/mini/mini-llvm-cpp.h b/src/mono/mono/mini/mini-llvm-cpp.h index e8175f580715e8..e397eea5d3135a 100644 --- a/src/mono/mono/mini/mini-llvm-cpp.h +++ b/src/mono/mono/mini/mini-llvm-cpp.h @@ -48,30 +48,44 @@ typedef enum { } XBinOpId; enum { - LLVM_Vector64 = 1 << 0, - LLVM_Vector128 = 1 << 1, - LLVM_VectorWidths = 3, // Scalar, 64, 128 - - LLVM_Int8 = 1 << 2, - LLVM_Int16 = 1 << 3, - LLVM_Int32 = 1 << 4, - LLVM_Int64 = 1 << 5, - LLVM_Float32 = 1 << 6, - LLVM_Float64 = 1 << 7, + LLVM_Scalar = 1 << 0, + LLVM_Vector64 = 1 << 1, + LLVM_Vector128 = 1 << 2, + LLVM_VectorWidths = 3, + LLVM_VectorMask = 0x7, + + LLVM_Int8 = 1 << 3, + LLVM_Int16 = 1 << 4, + LLVM_Int32 = 1 << 5, + LLVM_Int64 = 1 << 6, + LLVM_Float32 = 1 << 7, + LLVM_Float64 = 1 << 8, LLVM_ElementWidths = 6, }; -typedef uint8_t llvm_ovr_tag_t; +typedef uint16_t llvm_ovr_tag_t; static inline llvm_ovr_tag_t ovr_tag_force_scalar (llvm_ovr_tag_t tag) { - return tag & 0xfc; + return (tag & ~LLVM_VectorMask) | LLVM_Scalar; +} + +static inline llvm_ovr_tag_t +ovr_tag_smaller_vector (llvm_ovr_tag_t tag) +{ + return (tag & ~LLVM_VectorMask) | ((tag & LLVM_VectorMask) >> 1); } llvm_ovr_tag_t ovr_tag_from_mono_vector_class (MonoClass *klass); +static int +int_from_id_and_ovr_tag (int id, llvm_ovr_tag_t ovr_tag) +{ + return (((int) ovr_tag) << 23) | id; +} + /* * Keep in sync with the enum in utils/mono-memory-model.h. */ diff --git a/src/mono/mono/mini/mini-llvm.c b/src/mono/mono/mini/mini-llvm.c index da80c65b09a1af..304fa146eb201e 100644 --- a/src/mono/mono/mini/mini-llvm.c +++ b/src/mono/mono/mini/mini-llvm.c @@ -92,7 +92,6 @@ typedef struct { gboolean emit_dwarf; int max_got_offset; LLVMValueRef personality; - LLVMValueRef *intrins_by_id; gpointer gc_poll_cold_wrapper_compiled; /* For AOT */ @@ -4753,7 +4752,7 @@ extract_high_elements (EmitContext *ctx, LLVMValueRef src_vec) } static LLVMValueRef -select_lowest_element (EmitContext *ctx, LLVMValueRef vec) +keep_lowest_element (EmitContext *ctx, LLVMValueRef vec) { int mask [MAX_VECTOR_ELEMS] = { 0 }; LLVMTypeRef t = LLVMTypeOf (vec); @@ -4987,12 +4986,6 @@ get_float_const (MonoCompile *cfg, float val) return LLVMConstFPExt (LLVMConstReal (LLVMFloatType (), val), LLVMDoubleType ()); } -static int -int_from_id_and_ovr_tag (int id, llvm_ovr_tag_t ovr_tag) -{ - return (((int) ovr_tag) << 24) | id; -} - static LLVMValueRef call_overloaded_intrins (EmitContext *ctx, int id, llvm_ovr_tag_t ovr_tag, LLVMValueRef *args, const char *name) { @@ -7105,45 +7098,54 @@ process_bb (EmitContext *ctx, MonoBasicBlock *bb) mono_llvm_build_aligned_store (builder, values [ins->sreg1], dest, FALSE, 1); break; } - case OP_XBINOP: { + case OP_XBINOP: + case OP_XBINOP_SCALAR: { + gboolean scalar = ins->opcode == OP_XBINOP_SCALAR; + LLVMValueRef result = NULL; + LLVMValueRef args [] = { lhs, rhs }; + if (scalar) + for (int i = 0; i < 2; ++i) + args [i] = scalar_from_vector (ctx, args [i]); + LLVMValueRef l = args [0]; + LLVMValueRef r = args [1]; switch (ins->inst_c0) { case OP_IADD: - values [ins->dreg] = LLVMBuildAdd (builder, lhs, rhs, ""); + result = LLVMBuildAdd (builder, l, r, ""); break; case OP_ISUB: - values [ins->dreg] = LLVMBuildSub (builder, lhs, rhs, ""); + result = LLVMBuildSub (builder, l, r, ""); break; case OP_IAND: - values [ins->dreg] = LLVMBuildAnd (builder, lhs, rhs, ""); + result = LLVMBuildAnd (builder, l, r, ""); break; case OP_IOR: - values [ins->dreg] = LLVMBuildOr (builder, lhs, rhs, ""); + result = LLVMBuildOr (builder, l, r, ""); break; case OP_IXOR: - values [ins->dreg] = LLVMBuildXor (builder, lhs, rhs, ""); + result = LLVMBuildXor (builder, l, r, ""); break; case OP_FADD: - values [ins->dreg] = LLVMBuildFAdd (builder, lhs, rhs, ""); + result = LLVMBuildFAdd (builder, l, r, ""); break; case OP_FSUB: - values [ins->dreg] = LLVMBuildFSub (builder, lhs, rhs, ""); + result = LLVMBuildFSub (builder, l, r, ""); break; case OP_FMUL: - values [ins->dreg] = LLVMBuildFMul (builder, lhs, rhs, ""); + result = LLVMBuildFMul (builder, l, r, ""); break; case OP_FDIV: - values [ins->dreg] = LLVMBuildFDiv (builder, lhs, rhs, ""); + result = LLVMBuildFDiv (builder, l, r, ""); break; case OP_FMAX: case OP_FMIN: { #if defined(TARGET_X86) || defined(TARGET_AMD64) - LLVMValueRef args [] = { lhs, rhs }; + LLVMValueRef args [] = { l, r }; gboolean is_r4 = ins->inst_c1 == MONO_TYPE_R4; if (ins->inst_c0 == OP_FMAX) - values [ins->dreg] = call_intrins (ctx, is_r4 ? INTRINS_SSE_MAXPS : INTRINS_SSE_MAXPD, args, dname); + result = call_intrins (ctx, is_r4 ? INTRINS_SSE_MAXPS : INTRINS_SSE_MAXPD, args, dname); else - values [ins->dreg] = call_intrins (ctx, is_r4 ? INTRINS_SSE_MINPS : INTRINS_SSE_MINPD, args, dname); + result = call_intrins (ctx, is_r4 ? INTRINS_SSE_MINPS : INTRINS_SSE_MINPD, args, dname); #else NOT_IMPLEMENTED; #endif @@ -7151,20 +7153,23 @@ process_bb (EmitContext *ctx, MonoBasicBlock *bb) } case OP_IMAX: { gboolean is_unsigned = ins->inst_c1 == MONO_TYPE_U1 || ins->inst_c1 == MONO_TYPE_U2 || ins->inst_c1 == MONO_TYPE_U4 || ins->inst_c1 == MONO_TYPE_U8; - LLVMValueRef cmp = LLVMBuildICmp (builder, is_unsigned ? LLVMIntUGT : LLVMIntSGT, lhs, rhs, ""); - values [ins->dreg] = LLVMBuildSelect (builder, cmp, lhs, rhs, ""); + LLVMValueRef cmp = LLVMBuildICmp (builder, is_unsigned ? LLVMIntUGT : LLVMIntSGT, l, r, ""); + result = LLVMBuildSelect (builder, cmp, l, r, ""); break; } case OP_IMIN: { gboolean is_unsigned = ins->inst_c1 == MONO_TYPE_U1 || ins->inst_c1 == MONO_TYPE_U2 || ins->inst_c1 == MONO_TYPE_U4 || ins->inst_c1 == MONO_TYPE_U8; - LLVMValueRef cmp = LLVMBuildICmp (builder, is_unsigned ? LLVMIntULT : LLVMIntSLT, lhs, rhs, ""); - values [ins->dreg] = LLVMBuildSelect (builder, cmp, lhs, rhs, ""); + LLVMValueRef cmp = LLVMBuildICmp (builder, is_unsigned ? LLVMIntULT : LLVMIntSLT, l, r, ""); + result = LLVMBuildSelect (builder, cmp, l, r, ""); + break; } - break; default: g_assert_not_reached (); } + if (scalar) + result = vector_from_scalar (ctx, lhs, result); + values [ins->dreg] = result; break; } case OP_XBINOP_FORCEINT: { @@ -7190,6 +7195,7 @@ process_bb (EmitContext *ctx, MonoBasicBlock *bb) break; } values [ins->dreg] = LLVMBuildBitCast (builder, result, t, ""); + break; } #endif // defined(TARGET_X86) || defined(TARGET_AMD64) || defined(TARGET_ARM64) || defined(TARGET_WASM) @@ -9200,12 +9206,6 @@ process_bb (EmitContext *ctx, MonoBasicBlock *bb) getElement = TRUE; element_idx = 1; bitcast_result = TRUE; - case SIMD_OP_ARM64_TBL: - if (mono_llvm_get_prim_size_bits (LLVMTypeOf (rhs)) == 128) - id = INTRINS_AARCH64_ADV_SIMD_TBL128; - else - id = INTRINS_AARCH64_ADV_SIMD_TBL64; - break; default: g_assert_not_reached (); break; } LLVMValueRef arg1 = rhs; @@ -9233,12 +9233,6 @@ process_bb (EmitContext *ctx, MonoBasicBlock *bb) case SIMD_OP_ARM64_SHA1C: id = INTRINS_AARCH64_SHA1C; getLowerElement = TRUE; arg_idx = 1; break; case SIMD_OP_ARM64_SHA1M: id = INTRINS_AARCH64_SHA1M; getLowerElement = TRUE; arg_idx = 1; break; case SIMD_OP_ARM64_SHA1P: id = INTRINS_AARCH64_SHA1P; getLowerElement = TRUE; arg_idx = 1; break; - case SIMD_OP_ARM64_TBX: - if (mono_llvm_get_prim_size_bits (LLVMTypeOf (lhs)) == 128) - id = INTRINS_AARCH64_ADV_SIMD_TBX128; - else - id = INTRINS_AARCH64_ADV_SIMD_TBX64; - break; default: g_assert_not_reached (); break; } LLVMValueRef args [] = { lhs, rhs, arg3 }; @@ -9573,15 +9567,15 @@ process_bb (EmitContext *ctx, MonoBasicBlock *bb) case OP_ARM64_STNP_SCALAR: nontemporal = TRUE; scalar = TRUE; break; case OP_ARM64_STP_SCALAR: scalar = TRUE; break; } - LLVMTypeRef lhs_t = LLVMTypeOf (lhs); - unsigned int lhs_elems = LLVMGetVectorSize (lhs_t); - LLVMTypeRef lhs_elt_t = LLVMGetElementType (lhs_t); + LLVMTypeRef rhs_t = LLVMTypeOf (rhs); LLVMValueRef val = NULL; - LLVMTypeRef dst_t = lhs_t; + LLVMTypeRef dst_t = LLVMPointerType (rhs_t, 0); if (scalar) val = LLVMBuildShuffleVector (builder, rhs, arg3, create_const_vector_2_i32 (0, 2), ""); else { - dst_t = LLVMPointerType (LLVMVectorType (lhs_elt_t, lhs_elems * 2), 0); + unsigned int rhs_elems = LLVMGetVectorSize (rhs_t); + LLVMTypeRef rhs_elt_t = LLVMGetElementType (rhs_t); + dst_t = LLVMPointerType (LLVMVectorType (rhs_elt_t, rhs_elems * 2), 0); val = concatenate_vectors (ctx, rhs, arg3); } LLVMValueRef addr = convert (ctx, lhs, dst_t); @@ -9591,13 +9585,15 @@ process_bb (EmitContext *ctx, MonoBasicBlock *bb) break; } case OP_ARM64_ST1: { - LLVMValueRef store = mono_llvm_build_store (builder, rhs, lhs, FALSE, LLVM_BARRIER_NONE); - LLVMSetValueName (store, "arm64_st1"); + LLVMTypeRef t = LLVMTypeOf (rhs); + LLVMValueRef address = convert (ctx, lhs, LLVMPointerType (t, 0)); + unsigned int bytes = mono_llvm_get_prim_size_bits (t) / 8; + mono_llvm_build_aligned_store (builder, rhs, address, FALSE, bytes); break; } case OP_ARM64_ST1_SCALAR: { // XXXih: TODO: unroll arg3 - LLVMValueRef val = LLVMBuildExtractElement (builder, rhs, const_int32 (0), "arm64_st1_scalar"); + LLVMValueRef val = LLVMBuildExtractElement (builder, rhs, arg3, "arm64_st1_scalar"); mono_llvm_build_store (builder, val, lhs, FALSE, LLVM_BARRIER_NONE); break; } @@ -9628,11 +9624,12 @@ process_bb (EmitContext *ctx, MonoBasicBlock *bb) case OP_ARM64_RSUBHN2: { LLVMValueRef args [2] = { lhs, rhs }; gboolean high = ins->opcode == OP_ARM64_RSUBHN2; + llvm_ovr_tag_t ovr_tag = ovr_tag_from_mono_vector_class (ins->klass); if (high) { args [0] = rhs; args [1] = arg3; + ovr_tag = ovr_tag_smaller_vector (ovr_tag); } - llvm_ovr_tag_t ovr_tag = ovr_tag_from_mono_vector_class (ins->klass); LLVMValueRef result = call_overloaded_intrins (ctx, INTRINS_AARCH64_ADV_SIMD_RSUBHN, ovr_tag, args, ""); if (high) result = concatenate_vectors (ctx, lhs, result); @@ -9654,7 +9651,8 @@ process_bb (EmitContext *ctx, MonoBasicBlock *bb) LLVMValueRef args [2] = { lhs, rhs }; for (int i = 0; i < 2; ++i) { LLVMValueRef arg = args [i]; - if (high) + LLVMTypeRef arg_t = LLVMTypeOf (arg); + if (high && arg_t != ret_t) arg = extract_high_elements (ctx, arg); if (is_unsigned) arg = LLVMBuildZExt (builder, arg, ret_t, ""); @@ -9733,6 +9731,7 @@ process_bb (EmitContext *ctx, MonoBasicBlock *bb) for (unsigned int i = 0; i < src_elems; i += 2) { mask [i] = laneix; mask [i + 1] = laneix + src_elems; + ++laneix; } values [ins->dreg] = LLVMBuildShuffleVector (builder, lhs, rhs, create_const_vector_i32 (mask, src_elems), "arm64_zip"); break; @@ -9750,7 +9749,20 @@ process_bb (EmitContext *ctx, MonoBasicBlock *bb) values [ins->dreg] = call_overloaded_intrins (ctx, iid, ovr_tag, args, ""); break; } - case OP_XOP_OVR_SCALAR_X_X: { + case OP_XOP_OVR_X_X_X_X: { + IntrinsicId iid = (IntrinsicId) ins->inst_c0; + llvm_ovr_tag_t ovr_tag = ovr_tag_from_mono_vector_class (ins->klass); + LLVMValueRef args [] = { lhs, rhs, arg3 }; + values [ins->dreg] = call_overloaded_intrins (ctx, iid, ovr_tag, args, ""); + break; + } + case OP_XOP_OVR_SCALAR_X_X: + case OP_XOP_OVR_SCALAR_X_X_X: { + int num_args = 0; + switch (ins->opcode) { + case OP_XOP_OVR_SCALAR_X_X: num_args = 1; break; + case OP_XOP_OVR_SCALAR_X_X_X: num_args = 2; break; + } IntrinsicId iid = (IntrinsicId) ins->inst_c0; llvm_ovr_tag_t ovr_tag = ovr_tag_from_mono_vector_class (ins->klass); /* LLVM 9 NEON intrinsic functions have scalar overloads. Unfortunately @@ -9759,31 +9771,33 @@ process_bb (EmitContext *ctx, MonoBasicBlock *bb) * instruction selection. This is worked around by using a vector * operation and then explicitly clearing the upper bits of the register. */ - gboolean fake_scalar_op = FALSE; -#if defined(TARGET_ARM64) + gboolean arm64_fake_scalar_op = FALSE; switch (inst_c1_type (ins)) { case MONO_TYPE_U1: case MONO_TYPE_I1: case MONO_TYPE_U2: case MONO_TYPE_I2: - fake_scalar_op = TRUE; + arm64_fake_scalar_op = TRUE; } +#if !defined(TARGET_ARM64) + arm64_fake_scalar_op = FALSE; #endif - LLVMValueRef result = lhs; - if (!fake_scalar_op) { + LLVMValueRef args [2] = { lhs, rhs }; + if (!arm64_fake_scalar_op) { ovr_tag = ovr_tag_force_scalar (ovr_tag); - result = scalar_from_vector (ctx, result); + for (int i = 0; i < num_args; ++i) + args [i] = scalar_from_vector (ctx, args [i]); } - result = call_overloaded_intrins (ctx, iid, ovr_tag, &lhs, ""); - if (!fake_scalar_op) + LLVMValueRef result = call_overloaded_intrins (ctx, iid, ovr_tag, args, ""); + if (!arm64_fake_scalar_op) result = vector_from_scalar (ctx, lhs, result); else - result = select_lowest_element (ctx, result); + result = keep_lowest_element (ctx, result); values [ins->dreg] = result; break; } case OP_ARM64_ZERO_UPPER: { - values [ins->dreg] = select_lowest_element (ctx, lhs); + values [ins->dreg] = keep_lowest_element (ctx, lhs); break; } #endif @@ -10192,8 +10206,6 @@ mono_llvm_emit_method (MonoCompile *cfg) ctx->lmodule = ctx->module->lmodule; } else { ctx->lmodule = LLVMModuleCreateWithName (g_strdup_printf ("jit-module-%s", cfg->method->name)); - /* Reset this as it contains values from lmodule */ - memset (ctx->module->intrins_by_id, 0, sizeof (LLVMValueRef) * INTRINS_NUM); } ctx->llvm_only = ctx->module->llvm_only; #ifdef TARGET_WASM @@ -11106,10 +11118,11 @@ add_intrinsic (LLVMModuleRef module, int id) for (int vw = 0; vw < LLVM_VectorWidths; ++vw) { for (int ew = 0; ew < LLVM_ElementWidths; ++ew) { llvm_ovr_tag_t vec_bit = LLVM_Vector128 >> ((LLVM_VectorWidths - 1) - vw); - llvm_ovr_tag_t test = vec_bit | (LLVM_Int8 << ew); - if (spec & test) { + llvm_ovr_tag_t elem_bit = LLVM_Int8 << ew; + llvm_ovr_tag_t test = vec_bit | elem_bit; + if ((spec & test) == test) { intrins = add_intrins1 (module, id, intrin_types [vw][ew]); - int ovr_id = int_from_id_and_ovr_tag (test, id); + int ovr_id = int_from_id_and_ovr_tag (id, test); g_hash_table_insert (intrins_id_to_intrins, GINT_TO_POINTER (ovr_id), intrins); } } @@ -11147,19 +11160,7 @@ get_intrins_from_module (LLVMModuleRef lmodule, int id) static LLVMValueRef get_intrins (EmitContext *ctx, int id) { - MonoLLVMModule *module = ctx->module; - LLVMValueRef res; - - /* - * Every method is emitted into its own module so - * we can add intrinsics on demand. - */ - res = module->intrins_by_id [id]; - if (!res) { - res = get_intrins_from_module (ctx->lmodule, id); - module->intrins_by_id [id] = res; - } - return res; + return get_intrins_from_module (ctx->lmodule, id); } static void @@ -11288,7 +11289,6 @@ mono_llvm_create_aot_module (MonoAssembly *assembly, const char *global_prefix, module->max_got_offset = initial_got_size; module->context = LLVMGetGlobalContext (); module->cfgs = g_ptr_array_new (); - module->intrins_by_id = g_new0 (LLVMValueRef, INTRINS_NUM); module->aotconst_vars = g_hash_table_new (NULL, NULL); module->llvm_types = g_hash_table_new (NULL, NULL); module->plt_entries = g_hash_table_new (g_str_hash, g_str_equal); @@ -12407,7 +12407,6 @@ init_jit_module (void) module = g_new0 (MonoLLVMModule, 1); module->context = LLVMGetGlobalContext (); - module->intrins_by_id = g_new0 (LLVMValueRef, INTRINS_NUM); module->mono_ee = (MonoEERef*)mono_llvm_create_ee (&module->ee); diff --git a/src/mono/mono/mini/mini-ops.h b/src/mono/mono/mini/mini-ops.h index ee225cc2128ff4..74fe0ddbffb2e6 100644 --- a/src/mono/mono/mini/mini-ops.h +++ b/src/mono/mono/mini/mini-ops.h @@ -1535,6 +1535,7 @@ MINI_OP(OP_XCOMPARE_FP, "xcompare_fp", XREG, XREG, XREG) */ MINI_OP(OP_XBINOP, "xbinop", XREG, XREG, XREG) MINI_OP(OP_XBINOP_FORCEINT, "xbinop_coerce", XREG, XREG, XREG) +MINI_OP(OP_XBINOP_SCALAR, "xbinop", XREG, XREG, XREG) /* inst_c0 contains a SimdOp, inst_c1 might contain additional data */ MINI_OP(OP_XOP, "xop", NONE, NONE, NONE) MINI_OP(OP_XOP_X_I, "xop_x_i", XREG, IREG, NONE) @@ -1552,9 +1553,11 @@ MINI_OP(OP_XOP_I4_I4_I4, "xop_i4_i4_i4", IREG, IREG, IREG) MINI_OP(OP_XOP_I4_I4_I8, "xop_i4_i4_i8", IREG, IREG, LREG) MINI_OP3(OP_XOP_X_X_X_X, "xop_x_x_x_x", XREG, XREG, XREG, XREG) -MINI_OP(OP_XOP_OVR_X_X, "xop_ovr_x", XREG, XREG, NONE) -MINI_OP(OP_XOP_OVR_X_X_X, "xop_ovr_x_x", XREG, XREG, XREG) +MINI_OP(OP_XOP_OVR_X_X, "xop_ovr_x_x", XREG, XREG, NONE) +MINI_OP(OP_XOP_OVR_X_X_X, "xop_ovr_x_x_x", XREG, XREG, XREG) +MINI_OP3(OP_XOP_OVR_X_X_X_X, "xop_ovr_x_x_x_x", XREG, XREG, XREG, XREG) MINI_OP(OP_XOP_OVR_SCALAR_X_X, "xop_ovr_scalar_x_x", XREG, XREG, NONE) +MINI_OP(OP_XOP_OVR_SCALAR_X_X_X, "xop_ovr_scalar_x_x_x", XREG, XREG, XREG) MINI_OP(OP_XCAST, "xcast", XREG, XREG, NONE) /* Extract element of vector */ @@ -1584,13 +1587,11 @@ MINI_OP(OP_POPCNT64, "popcnt64", LREG, LREG, NONE) MINI_OP(OP_LSCNT32, "lscnt32", IREG, IREG, NONE) MINI_OP(OP_LSCNT64, "lscnt64", LREG, LREG, NONE) -MINI_OP(OP_ARM64_RSUBHN, "arm64_rsubhn", XREG, XREG, XREG) MINI_OP(OP_ARM64_SMULH, "arm64_smulh", LREG, LREG, LREG) MINI_OP(OP_ARM64_SQRT_SCALAR, "arm64_sqrt_scalar", XREG, XREG, NONE) MINI_OP(OP_ARM64_SSUB, "arm64_ssub", XREG, XREG, XREG) MINI_OP(OP_ARM64_SSUB2, "arm64_ssub2", XREG, XREG, XREG) MINI_OP(OP_ARM64_ST1, "arm64_st1", NONE, IREG, XREG) -MINI_OP(OP_ARM64_SUBHN, "arm64_subhn", XREG, XREG, NONE) MINI_OP(OP_ARM64_SXTL, "arm64_sxtl", XREG, XREG, NONE) MINI_OP(OP_ARM64_SXTL2, "arm64_sxtl2", XREG, XREG, NONE) MINI_OP(OP_ARM64_TRN1, "arm64_trn1", XREG, XREG, XREG) @@ -1605,12 +1606,15 @@ MINI_OP(OP_ARM64_UZP2, "arm64_uzp2", XREG, XREG, XREG) MINI_OP(OP_ARM64_ZERO_UPPER, "arm64_zero_upper", XREG, XREG, NONE) MINI_OP(OP_ARM64_ZIP1, "arm64_zip1", XREG, XREG, XREG) MINI_OP(OP_ARM64_ZIP2, "arm64_zip2", XREG, XREG, XREG) -MINI_OP3(OP_ARM64_RSUBHN2, "arm64_rsubhn2", XREG, XREG, XREG, XREG) MINI_OP3(OP_ARM64_ST1_SCALAR, "arm64_st1_scalar", NONE, IREG, XREG, IREG) MINI_OP3(OP_ARM64_STNP, "arm64_stnp", NONE, IREG, XREG, XREG) MINI_OP3(OP_ARM64_STNP_SCALAR, "arm64_stnp_scalar", NONE, IREG, XREG, XREG) MINI_OP3(OP_ARM64_STP, "arm64_stp", NONE, IREG, XREG, XREG) MINI_OP3(OP_ARM64_STP_SCALAR, "arm64_stp_scalar", NONE, IREG, XREG, XREG) + +MINI_OP(OP_ARM64_RSUBHN, "arm64_rsubhn", XREG, XREG, XREG) +MINI_OP(OP_ARM64_SUBHN, "arm64_subhn", XREG, XREG, XREG) +MINI_OP3(OP_ARM64_RSUBHN2, "arm64_rsubhn2", XREG, XREG, XREG, XREG) MINI_OP3(OP_ARM64_SUBHN2, "arm64_subhn2", XREG, XREG, XREG, XREG) MINI_OP(OP_ARM64_SHRN, "arm64_shrn", XREG, XREG, IREG) diff --git a/src/mono/mono/mini/mini.h b/src/mono/mono/mini/mini.h index 34d3a14ad83fc4..39e4673cb761d9 100644 --- a/src/mono/mono/mini/mini.h +++ b/src/mono/mono/mini/mini.h @@ -2997,8 +2997,6 @@ typedef enum { SIMD_OP_ARM64_SHA256SU1, SIMD_OP_ARM64_PMULL64_LOWER, SIMD_OP_ARM64_PMULL64_UPPER, - SIMD_OP_ARM64_TBL, - SIMD_OP_ARM64_TBX, } SimdOp; const char *mono_arch_xregname (int reg); diff --git a/src/mono/mono/mini/simd-intrinsics.c b/src/mono/mono/mini/simd-intrinsics.c index fe01bcccdb8761..61a4c2fb87ce80 100644 --- a/src/mono/mono/mini/simd-intrinsics.c +++ b/src/mono/mono/mini/simd-intrinsics.c @@ -375,6 +375,8 @@ emit_hardware_intrinsics ( MonoInst *ins = NULL; gboolean supported = FALSE; MonoTypeEnum arg0_type = fsig->param_count > 0 ? get_underlying_type (fsig->params [0]) : MONO_TYPE_VOID; + uint16_t op = 0; + uint16_t c0 = 0; if (intrin_group) { const SimdIntrinsic *intrinsics = intrin_group->intrinsics; int intrinsics_size = intrin_group->intrinsics_size; @@ -393,15 +395,10 @@ emit_hardware_intrinsics ( else supported = TRUE; -#if defined(TARGET_ARM64) - // HACK: Mark AdvSimd as unsupported until it's completely implemented - if (feature == MONO_CPU_ARM64_NEON) supported = FALSE; -#endif - id = info->id; - uint16_t op = info->default_op; - uint16_t c0 = info->default_instc0; + op = info->default_op; + c0 = info->default_instc0; gboolean is_unsigned = FALSE; gboolean is_float = FALSE; switch (arg0_type) { @@ -423,8 +420,6 @@ emit_hardware_intrinsics ( op = info->floating_op; c0 = info->floating_instc0; } - if (op != 0) - return emit_simd_ins_for_sig (cfg, klass, op, c0, arg0_type, fsig, args); } support_probe_complete: if (id == SN_get_IsSupported) { @@ -442,6 +437,8 @@ emit_hardware_intrinsics ( return NULL; } } + if (op != 0) + return emit_simd_ins_for_sig (cfg, klass, op, c0, arg0_type, fsig, args); return custom_emit (cfg, fsig, args, klass, intrin_group, info, id, arg0_type, is_64bit); } @@ -988,14 +985,20 @@ static SimdIntrinsic sha256_methods [] = { {SN_get_IsSupported} }; +// This table must be kept in sorted order. ASCII } is sorted after alphanumeric +// characters, so blind use of your editor's "sort lines" facility will +// mis-order the lines. +// +// In Vim you can use `sort /.*{[0-9A-z]*/ r` to sort this table. + static SimdIntrinsic advsimd_methods [] = { + {SN_Abs}, {SN_AbsSaturate}, {SN_AbsScalar}, - {SN_AbsoluteCompareGreaterThanOrEqual}, {SN_AbsoluteCompareGreaterThan}, - {SN_AbsoluteCompareLessThanOrEqual}, + {SN_AbsoluteCompareGreaterThanOrEqual}, {SN_AbsoluteCompareLessThan}, - {SN_Abs}, + {SN_AbsoluteCompareLessThanOrEqual}, {SN_MultiplyWideningUpperAndSubtract, OP_ARM64_SMLSL2}, {SN_Negate, OP_ARM64_XNEG}, {SN_NegateSaturate, OP_XOP_OVR_X_X, INTRINS_AARCH64_ADV_SIMD_SQNEG}, @@ -1008,11 +1011,11 @@ static SimdIntrinsic advsimd_methods [] = { {SN_PolynomialMultiplyWideningLower, OP_ARM64_PMULL}, {SN_PolynomialMultiplyWideningUpper, OP_ARM64_PMULL2}, {SN_PopCount, OP_XOP_OVR_SCALAR_X_X, INTRINS_AARCH64_ADV_SIMD_CNT}, - {SN_ReciprocalEstimateScalar, OP_XOP_OVR_SCALAR_X_X, INTRINS_AARCH64_ADV_SIMD_FRECPE}, {SN_ReciprocalEstimate}, + {SN_ReciprocalEstimateScalar, OP_XOP_OVR_SCALAR_X_X, INTRINS_AARCH64_ADV_SIMD_FRECPE}, {SN_ReciprocalExponentScalar, OP_XOP_OVR_SCALAR_X_X, INTRINS_AARCH64_ADV_SIMD_FRECPX}, - {SN_ReciprocalSquareRootEstimateScalar, OP_XOP_OVR_SCALAR_X_X, INTRINS_AARCH64_ADV_SIMD_FRSQRTE}, {SN_ReciprocalSquareRootEstimate}, + {SN_ReciprocalSquareRootEstimateScalar, OP_XOP_OVR_SCALAR_X_X, INTRINS_AARCH64_ADV_SIMD_FRSQRTE}, {SN_ReciprocalStep, OP_XOP_OVR_X_X, INTRINS_AARCH64_ADV_SIMD_FRECPS}, {SN_ReciprocalStepScalar, OP_XOP_OVR_SCALAR_X_X, INTRINS_AARCH64_ADV_SIMD_FRECPS}, {SN_ReverseElement16, OP_ARM64_REV32}, @@ -1040,10 +1043,10 @@ static SimdIntrinsic advsimd_methods [] = { {SN_ShiftLeftAndInsert, OP_ARM64_SRI}, {SN_ShiftLeftAndInsertScalar, OP_ARM64_SRI}, {SN_ShiftLeftLogical, OP_ARM64_SHL}, + {SN_ShiftLeftLogicalSaturate}, {SN_ShiftLeftLogicalSaturateScalar}, {SN_ShiftLeftLogicalSaturateUnsigned, OP_ARM64_SQSHLU}, {SN_ShiftLeftLogicalSaturateUnsignedScalar, OP_ARM64_SQSHLU}, - {SN_ShiftLeftLogicalSaturate}, {SN_ShiftLeftLogicalScalar, OP_ARM64_SHL}, {SN_ShiftLeftLogicalWideningLower, OP_ARM64_USHLL}, {SN_ShiftLeftLogicalWideningUpper, OP_ARM64_USHLL2}, @@ -1100,22 +1103,22 @@ static SimdIntrinsic advsimd_methods [] = { {SN_StorePairScalar, OP_ARM64_STP_SCALAR}, {SN_StorePairScalarNonTemporal, OP_ARM64_STNP_SCALAR}, {SN_StoreSelectedScalar, OP_ARM64_ST1_SCALAR}, + {SN_Subtract, OP_XBINOP, OP_ISUB, None, None, OP_XBINOP_SCALAR, OP_FSUB}, {SN_SubtractHighNarrowingLower, OP_ARM64_SUBHN}, {SN_SubtractHighNarrowingUpper, OP_ARM64_SUBHN2}, {SN_SubtractRoundedHighNarrowingLower, OP_ARM64_RSUBHN}, {SN_SubtractRoundedHighNarrowingUpper, OP_ARM64_RSUBHN2}, {SN_SubtractSaturate, OP_XOP_OVR_X_X_X, INTRINS_AARCH64_ADV_SIMD_SQSUB, OP_XOP_OVR_X_X_X, INTRINS_AARCH64_ADV_SIMD_UQSUB}, - {SN_SubtractSaturateScalar, OP_XOP_OVR_SCALAR_X_X, INTRINS_AARCH64_ADV_SIMD_SQSUB, OP_XOP_OVR_SCALAR_X_X, INTRINS_AARCH64_ADV_SIMD_UQSUB}, - {SN_SubtractScalar}, + {SN_SubtractSaturateScalar, OP_XOP_OVR_SCALAR_X_X_X, INTRINS_AARCH64_ADV_SIMD_SQSUB, OP_XOP_OVR_SCALAR_X_X_X, INTRINS_AARCH64_ADV_SIMD_UQSUB}, + {SN_SubtractScalar, OP_XBINOP_SCALAR, OP_ISUB, None, None, OP_XBINOP_SCALAR, OP_FSUB}, {SN_SubtractWideningLower, OP_ARM64_SSUB, None, OP_ARM64_USUB}, {SN_SubtractWideningUpper, OP_ARM64_SSUB2, None, OP_ARM64_USUB2}, - {SN_Subtract}, {SN_TransposeEven, OP_ARM64_TRN1}, {SN_TransposeOdd, OP_ARM64_TRN2}, {SN_UnzipEven, OP_ARM64_UZP1}, {SN_UnzipOdd, OP_ARM64_UZP2}, - {SN_VectorTableLookup, OP_XOP_X_X_X, SIMD_OP_ARM64_TBL}, - {SN_VectorTableLookupExtension, OP_XOP_X_X_X_X, SIMD_OP_ARM64_TBX}, + {SN_VectorTableLookup, OP_XOP_OVR_X_X_X, INTRINS_AARCH64_ADV_SIMD_TBL1}, + {SN_VectorTableLookupExtension, OP_XOP_OVR_X_X_X_X, INTRINS_AARCH64_ADV_SIMD_TBX1}, {SN_Xor, OP_XBINOP_FORCEINT, XBINOP_FORCEINT_Xor}, {SN_ZeroExtendWideningLower, OP_ARM64_UXTL}, {SN_ZeroExtendWideningUpper, OP_ARM64_UXTL2}, @@ -1316,18 +1319,6 @@ emit_arm64_intrinsics ( ret = emit_simd_ins (cfg, klass, OP_ARM64_ZERO_UPPER, ret->dreg, -1); return ret; } - case SN_SubtractScalar: - case SN_Subtract: { - gboolean is_float = FALSE; - switch (arg0_type) { - case MONO_TYPE_R4: case MONO_TYPE_R8: is_float = TRUE; - } - op = is_float ? OP_FSUB : OP_ISUB; - MonoInst *ret = emit_simd_ins_for_sig (cfg, klass, OP_XBINOP, op, arg0_type, fsig, args); - if (id == SN_SubtractScalar) - ret = emit_simd_ins (cfg, klass, OP_ARM64_ZERO_UPPER, ret->dreg, -1); - return ret; - } default: g_assert_not_reached (); } From 8d1690777090a35a37669145be46f383c1f3dfd5 Mon Sep 17 00:00:00 2001 From: Imran Hameed Date: Fri, 5 Mar 2021 22:06:01 -0800 Subject: [PATCH 11/58] Implement some multiplication ops --- src/mono/mono/mini/llvm-intrinsics.h | 6 ++ src/mono/mono/mini/mini-llvm-cpp.cpp | 20 ----- src/mono/mono/mini/mini-llvm-cpp.h | 62 -------------- src/mono/mono/mini/mini-llvm.c | 120 +++++++++++++++++++++++---- src/mono/mono/mini/mini-ops.h | 21 +++++ src/mono/mono/mini/mini.c | 20 +++++ src/mono/mono/mini/mini.h | 61 ++++++++++++++ src/mono/mono/mini/simd-intrinsics.c | 14 +++- 8 files changed, 224 insertions(+), 100 deletions(-) diff --git a/src/mono/mono/mini/llvm-intrinsics.h b/src/mono/mono/mini/llvm-intrinsics.h index dca90b8adece78..43c596e69b4f18 100644 --- a/src/mono/mono/mini/llvm-intrinsics.h +++ b/src/mono/mono/mini/llvm-intrinsics.h @@ -307,6 +307,12 @@ INTRINS_OVR_2_ARG(AARCH64_ADV_SIMD_ABS_COMPARE_LT_DOUBLE, aarch64_neon_facgt, ss INTRINS_OVR_2_ARG(AARCH64_ADV_SIMD_ABS_COMPARE_LTE_FLOAT, aarch64_neon_facge, sse_i4_t, sse_r4_t) INTRINS_OVR_2_ARG(AARCH64_ADV_SIMD_ABS_COMPARE_LTE_DOUBLE, aarch64_neon_facge, sse_i4_t, sse_r8_t) +INTRINS_OVR_TAG(AARCH64_ADV_SIMD_SQDMULL, aarch64_neon_sqdmull, V128 | I4 | I8) +INTRINS_OVR_TAG(AARCH64_ADV_SIMD_SQRDMULH, aarch64_neon_sqrdmulh, V64 | V128 | I2 | I4) + +INTRINS_OVR_TAG(AARCH64_ADV_SIMD_SMULL, aarch64_neon_smull, V128 | I2 | I4 | I8) +INTRINS_OVR_TAG(AARCH64_ADV_SIMD_UMULL, aarch64_neon_umull, V128 | I2 | I4 | I8) + INTRINS_OVR_TAG(AARCH64_ADV_SIMD_SQNEG, aarch64_neon_sqneg, V64 | V128 | I1 | I2 | I4 | I8) INTRINS_OVR_TAG(AARCH64_ADV_SIMD_PMUL, aarch64_neon_pmul, V64 | V128 | I1) diff --git a/src/mono/mono/mini/mini-llvm-cpp.cpp b/src/mono/mono/mini/mini-llvm-cpp.cpp index ba7c6fb8623f6f..5d8c2abf0b0a83 100644 --- a/src/mono/mono/mini/mini-llvm-cpp.cpp +++ b/src/mono/mono/mini/mini-llvm-cpp.cpp @@ -660,26 +660,6 @@ is_overloaded_intrins (IntrinsicId id) return false; } -llvm_ovr_tag_t -ovr_tag_from_mono_vector_class (MonoClass *klass) { - int size = mono_class_value_size (klass, NULL); - llvm_ovr_tag_t ret = 0; - switch (size) { - case 8: ret |= LLVM_Vector64; break; - case 16: ret |= LLVM_Vector128; break; - } - MonoType *etype = mono_class_get_context (klass)->class_inst->type_argv [0]; - switch (etype->type) { - case MONO_TYPE_I1: case MONO_TYPE_U1: ret |= LLVM_Int8; break; - case MONO_TYPE_I2: case MONO_TYPE_U2: ret |= LLVM_Int16; break; - case MONO_TYPE_I4: case MONO_TYPE_U4: ret |= LLVM_Int32; break; - case MONO_TYPE_I8: case MONO_TYPE_U8: ret |= LLVM_Int64; break; - case MONO_TYPE_R4: ret |= LLVM_Float32; break; - case MONO_TYPE_R8: ret |= LLVM_Float64; break; - } - return ret; -} - /* * mono_llvm_register_intrinsic: * diff --git a/src/mono/mono/mini/mini-llvm-cpp.h b/src/mono/mono/mini/mini-llvm-cpp.h index e397eea5d3135a..82badd50d01bdd 100644 --- a/src/mono/mono/mini/mini-llvm-cpp.h +++ b/src/mono/mono/mini/mini-llvm-cpp.h @@ -24,68 +24,6 @@ G_BEGIN_DECLS - -/* An intrinsic id. The lower 24 bits are used to store a mono-specific ID. The - * next 8 bits store arm64 overload tag bits. In the configuration of LLVM 9 we - * use, there are 7017 total intrinsics defined in IntrinsicEnums.inc, so only - * 13 bits are needed to label each intrinsic overload group. - */ -typedef enum { -#define INTRINS(id, llvm_id) INTRINS_ ## id, -#define INTRINS_OVR(id, llvm_id, ty) INTRINS_ ## id, -#define INTRINS_OVR_2_ARG(id, llvm_id, ty1, ty2) INTRINS_ ## id, -#define INTRINS_OVR_3_ARG(id, llvm_id, ty1, ty2, ty3) INTRINS_ ## id, -#define INTRINS_OVR_TAG(id, ...) INTRINS_ ## id, -#include "llvm-intrinsics.h" - INTRINS_NUM -} IntrinsicId; - -typedef enum { - XBINOP_FORCEINT_Not, - XBINOP_FORCEINT_Or, - XBINOP_FORCEINT_OrNot, - XBINOP_FORCEINT_Xor, -} XBinOpId; - -enum { - LLVM_Scalar = 1 << 0, - LLVM_Vector64 = 1 << 1, - LLVM_Vector128 = 1 << 2, - LLVM_VectorWidths = 3, - LLVM_VectorMask = 0x7, - - LLVM_Int8 = 1 << 3, - LLVM_Int16 = 1 << 4, - LLVM_Int32 = 1 << 5, - LLVM_Int64 = 1 << 6, - LLVM_Float32 = 1 << 7, - LLVM_Float64 = 1 << 8, - LLVM_ElementWidths = 6, -}; - -typedef uint16_t llvm_ovr_tag_t; - -static inline llvm_ovr_tag_t -ovr_tag_force_scalar (llvm_ovr_tag_t tag) -{ - return (tag & ~LLVM_VectorMask) | LLVM_Scalar; -} - -static inline llvm_ovr_tag_t -ovr_tag_smaller_vector (llvm_ovr_tag_t tag) -{ - return (tag & ~LLVM_VectorMask) | ((tag & LLVM_VectorMask) >> 1); -} - -llvm_ovr_tag_t -ovr_tag_from_mono_vector_class (MonoClass *klass); - -static int -int_from_id_and_ovr_tag (int id, llvm_ovr_tag_t ovr_tag) -{ - return (((int) ovr_tag) << 23) | id; -} - /* * Keep in sync with the enum in utils/mono-memory-model.h. */ diff --git a/src/mono/mono/mini/mini-llvm.c b/src/mono/mono/mini/mini-llvm.c index 304fa146eb201e..8b4149e0dea690 100644 --- a/src/mono/mono/mini/mini-llvm.c +++ b/src/mono/mono/mini/mini-llvm.c @@ -9311,6 +9311,97 @@ process_bb (EmitContext *ctx, MonoBasicBlock *bb) values [ins->dreg] = LLVMBuildTrunc (builder, hi64, LLVMInt64Type (), ""); break; } + case OP_ARM64_SQDMLSL2: { + llvm_ovr_tag_t ovr_tag = ovr_tag_from_mono_vector_class (ins->klass); + LLVMValueRef args [] = { rhs, arg3 }; + for (int i = 0; i < 2; ++i) + args [i] = extract_high_elements (ctx, args [i]); + LLVMValueRef result = call_overloaded_intrins (ctx, INTRINS_AARCH64_ADV_SIMD_SQDMULL, ovr_tag, args, ""); + LLVMValueRef args2 [] = { lhs, result }; + result = call_overloaded_intrins (ctx, INTRINS_AARCH64_ADV_SIMD_SQSUB, ovr_tag, args2, ""); + values [ins->dreg] = result; + break; + } + case OP_ARM64_SQRDMULH_SCALAR: + case OP_ARM64_SQRDMULH_SEL: { + gboolean sel = ins->opcode == OP_ARM64_SQRDMULH_SEL; + // XXXih: TODO: unroll arg3, bounds checks for arg3 + llvm_ovr_tag_t ovr_tag = ovr_tag_from_mono_vector_class (ins->klass); + LLVMValueRef lane = arg3; + if (!sel) + lane = const_int32 (0); + LLVMTypeRef t = LLVMTypeOf (lhs); + unsigned int elems = LLVMGetVectorSize (t); + LLVMValueRef arg = LLVMBuildExtractElement (builder, rhs, lane, ""); + arg = broadcast_element (ctx, arg, elems); + LLVMValueRef args [] = { lhs, arg }; + LLVMValueRef result = call_overloaded_intrins (ctx, INTRINS_AARCH64_ADV_SIMD_SQRDMULH, ovr_tag, args, ""); + values [ins->dreg] = result; + break; + } + case OP_ARM64_FMUL_SEL: { + LLVMValueRef mul2 = LLVMBuildExtractElement (builder, rhs, arg3, ""); + LLVMValueRef mul1 = scalar_from_vector (ctx, lhs); + LLVMValueRef result = LLVMBuildFMul (builder, mul1, mul2, "arm64_fmul_sel"); + result = vector_from_scalar (ctx, lhs, result); + values [ins->dreg] = result; + break; + } + case OP_ARM64_MLS: + case OP_ARM64_MLS_SCALAR: { + gboolean scalar = ins->opcode == OP_ARM64_MLS_SCALAR; + LLVMTypeRef mul_t = LLVMTypeOf (rhs); + unsigned int elems = LLVMGetVectorSize (mul_t); + LLVMValueRef mul2 = arg3; + if (scalar) + mul2 = broadcast_element (ctx, scalar_from_vector (ctx, mul2), elems); + LLVMValueRef result = LLVMBuildMul (builder, rhs, mul2, ""); + result = LLVMBuildSub (builder, lhs, result, ""); + values [ins->dreg] = result; + break; + } + case OP_ARM64_SMULL: + case OP_ARM64_SMULL2: + case OP_ARM64_UMULL: + case OP_ARM64_UMULL2: + case OP_ARM64_SMLAL: + case OP_ARM64_SMLAL2: + case OP_ARM64_UMLAL: + case OP_ARM64_UMLAL2: + case OP_ARM64_SMLSL: + case OP_ARM64_SMLSL2: + case OP_ARM64_UMLSL: + case OP_ARM64_UMLSL2: { + llvm_ovr_tag_t ovr_tag = ovr_tag_from_mono_vector_class (ins->klass); + gboolean is_unsigned = FALSE; + gboolean high = FALSE; + gboolean add = FALSE; + gboolean subtract = FALSE; + switch (ins->opcode) { + case OP_ARM64_SMULL2: high = TRUE; case OP_ARM64_SMULL: break; + case OP_ARM64_UMULL2: high = TRUE; case OP_ARM64_UMULL: is_unsigned = TRUE; break; + case OP_ARM64_SMLAL2: high = TRUE; case OP_ARM64_SMLAL: add = TRUE; break; + case OP_ARM64_UMLAL2: high = TRUE; case OP_ARM64_UMLAL: add = TRUE; is_unsigned = TRUE; break; + case OP_ARM64_SMLSL2: high = TRUE; case OP_ARM64_SMLSL: subtract = TRUE; break; + case OP_ARM64_UMLSL2: high = TRUE; case OP_ARM64_UMLSL: subtract = TRUE; is_unsigned = TRUE; break; + } + int iid = is_unsigned ? INTRINS_AARCH64_ADV_SIMD_UMULL : INTRINS_AARCH64_ADV_SIMD_SMULL; + LLVMValueRef intrin_args [] = { lhs, rhs }; + if (add || subtract) { + intrin_args [0] = rhs; + intrin_args [1] = arg3; + } + if (high) + for (int i = 0; i < 2; ++i) + intrin_args [i] = extract_high_elements (ctx, intrin_args [i]); + LLVMValueRef result = call_overloaded_intrins (ctx, iid, ovr_tag, intrin_args, ""); + if (add) + result = LLVMBuildAdd (builder, lhs, result, ""); + if (subtract) + result = LLVMBuildSub (builder, lhs, result, ""); + values [ins->dreg] = result; + break; + } case OP_ARM64_XNEG: case OP_ARM64_XNEG_SCALAR: { gboolean scalar = ins->opcode == OP_ARM64_XNEG_SCALAR; @@ -9328,6 +9419,7 @@ process_bb (EmitContext *ctx, MonoBasicBlock *bb) if (scalar) result = vector_from_scalar (ctx, lhs, result); values [ins->dreg] = result; + break; } case OP_ARM64_PMULL: case OP_ARM64_PMULL2: { @@ -9356,7 +9448,7 @@ process_bb (EmitContext *ctx, MonoBasicBlock *bb) int mask [MAX_VECTOR_ELEMS] = { 0 }; for (unsigned int i = 0; i < tmp_elements; i += 2) { mask [i] = i + 1; - mask [i] = i; + mask [i + 1] = i; } LLVMValueRef result = LLVMBuildShuffleVector(builder, tmp, LLVMGetUndef (tmp_t), create_const_vector_i32 (mask, tmp_elements), ""); result = LLVMBuildBitCast (builder, result, t, ""); @@ -9414,16 +9506,16 @@ process_bb (EmitContext *ctx, MonoBasicBlock *bb) } case OP_ARM64_UQSHRN: case OP_ARM64_UQSHRN2: { - // XXXih: TODO: ins->klass is wrong for *2 // XXXih: TODO: unroll count/rhs/arg3 + llvm_ovr_tag_t ovr_tag = ovr_tag_from_mono_vector_class (ins->klass); LLVMValueRef shiftarg = lhs; LLVMValueRef shift = rhs; gboolean high = ins->opcode == OP_ARM64_UQSHRN2; if (high) { shiftarg = rhs; shift = arg3; + ovr_tag = ovr_tag_smaller_vector (ovr_tag); } - llvm_ovr_tag_t ovr_tag = ovr_tag_from_mono_vector_class (ins->klass); LLVMValueRef args [] = { shiftarg, shift }; LLVMValueRef result = call_overloaded_intrins (ctx, INTRINS_AARCH64_ADV_SIMD_UQSHRN, ovr_tag, args, ""); if (high) @@ -9477,30 +9569,24 @@ process_bb (EmitContext *ctx, MonoBasicBlock *bb) case OP_ARM64_SQSHRUN2: case OP_ARM64_UQRSHRN: case OP_ARM64_UQRSHRN2: { - // XXXih: TODO: ins->klass is wrong for *2 // XXXih: TODO: unroll count/rhs/arg3 + llvm_ovr_tag_t ovr_tag = ovr_tag_from_mono_vector_class (ins->klass); LLVMValueRef args [2] = { lhs, rhs }; gboolean high = FALSE; int iid = 0; switch (ins->opcode) { - case OP_ARM64_RSHRN: iid = INTRINS_AARCH64_ADV_SIMD_RSHRN; - case OP_ARM64_RSHRN2: high = TRUE; break; - case OP_ARM64_UQRSHRN: iid = INTRINS_AARCH64_ADV_SIMD_UQRSHRN; - case OP_ARM64_UQRSHRN2: high = TRUE; break; - case OP_ARM64_SQRSHRN: iid = INTRINS_AARCH64_ADV_SIMD_SQRSHRN; - case OP_ARM64_SQRSHRN2: high = TRUE; break; - case OP_ARM64_SQRSHRUN: iid = INTRINS_AARCH64_ADV_SIMD_SQRSHRUN; - case OP_ARM64_SQRSHRUN2: high = TRUE; break; - case OP_ARM64_SQSHRN: iid = INTRINS_AARCH64_ADV_SIMD_SQSHRN; - case OP_ARM64_SQSHRN2: high = TRUE; break; - case OP_ARM64_SQSHRUN: iid = INTRINS_AARCH64_ADV_SIMD_SQSHRUN; - case OP_ARM64_SQSHRUN2: high = TRUE; break; + case OP_ARM64_RSHRN: iid = INTRINS_AARCH64_ADV_SIMD_RSHRN; case OP_ARM64_RSHRN2: high = TRUE; break; + case OP_ARM64_UQRSHRN: iid = INTRINS_AARCH64_ADV_SIMD_UQRSHRN; case OP_ARM64_UQRSHRN2: high = TRUE; break; + case OP_ARM64_SQRSHRN: iid = INTRINS_AARCH64_ADV_SIMD_SQRSHRN; case OP_ARM64_SQRSHRN2: high = TRUE; break; + case OP_ARM64_SQRSHRUN: iid = INTRINS_AARCH64_ADV_SIMD_SQRSHRUN; case OP_ARM64_SQRSHRUN2: high = TRUE; break; + case OP_ARM64_SQSHRN: iid = INTRINS_AARCH64_ADV_SIMD_SQSHRN; case OP_ARM64_SQSHRN2: high = TRUE; break; + case OP_ARM64_SQSHRUN: iid = INTRINS_AARCH64_ADV_SIMD_SQSHRUN; case OP_ARM64_SQSHRUN2: high = TRUE; break; } if (high) { args [0] = rhs; args [1] = arg3; + ovr_tag = ovr_tag_smaller_vector (ovr_tag); } - llvm_ovr_tag_t ovr_tag = ovr_tag_from_mono_vector_class (ins->klass); LLVMValueRef result = call_overloaded_intrins (ctx, iid, ovr_tag, args, ""); if (high) result = concatenate_vectors (ctx, lhs, result); diff --git a/src/mono/mono/mini/mini-ops.h b/src/mono/mono/mini/mini-ops.h index 74fe0ddbffb2e6..e6083334507dae 100644 --- a/src/mono/mono/mini/mini-ops.h +++ b/src/mono/mono/mini/mini-ops.h @@ -1667,6 +1667,27 @@ MINI_OP(OP_ARM64_PMULL2, "arm64_pmull2", XREG, XREG, NONE) MINI_OP(OP_ARM64_XNEG, "arm64_xneg", XREG, XREG, NONE) MINI_OP(OP_ARM64_XNEG_SCALAR, "arm64_xneg_scalar", XREG, XREG, NONE) +MINI_OP(OP_ARM64_SMULL, "arm64_smull", XREG, XREG, XREG) +MINI_OP(OP_ARM64_SMULL2, "arm64_smull2", XREG, XREG, XREG) +MINI_OP(OP_ARM64_UMULL, "arm64_umull", XREG, XREG, XREG) +MINI_OP(OP_ARM64_UMULL2, "arm64_umull2", XREG, XREG, XREG) +MINI_OP3(OP_ARM64_SMLAL, "arm64_smlal", XREG, XREG, XREG, XREG) +MINI_OP3(OP_ARM64_SMLAL2, "arm64_smlal2", XREG, XREG, XREG, XREG) +MINI_OP3(OP_ARM64_UMLAL, "arm64_umlal", XREG, XREG, XREG, XREG) +MINI_OP3(OP_ARM64_UMLAL2, "arm64_umlal2", XREG, XREG, XREG, XREG) +MINI_OP3(OP_ARM64_SMLSL, "arm64_smlsl", XREG, XREG, XREG, XREG) MINI_OP3(OP_ARM64_SMLSL2, "arm64_smlsl2", XREG, XREG, XREG, XREG) +MINI_OP3(OP_ARM64_UMLSL, "arm64_umlsl", XREG, XREG, XREG, XREG) +MINI_OP3(OP_ARM64_UMLSL2, "arm64_umlsl2", XREG, XREG, XREG, XREG) + +MINI_OP3(OP_ARM64_MLS_SCALAR, "arm64_mls_scalar", XREG, XREG, XREG, XREG) +MINI_OP3(OP_ARM64_MLS, "arm64_mls", XREG, XREG, XREG, XREG) + +MINI_OP3(OP_ARM64_FMUL_SEL, "arm64_fmul_sel", XREG, XREG, XREG, IREG) + +MINI_OP3(OP_ARM64_SQRDMULH_SEL, "arm64_sqrdmulh_sel", XREG, XREG, XREG, IREG) +MINI_OP(OP_ARM64_SQRDMULH_SCALAR, "arm64_sqrdmulh_scalar", XREG, XREG, XREG) + +MINI_OP3(OP_ARM64_SQDMLSL2, "arm64_sqdmlsl2", XREG, XREG, XREG, XREG) #endif // TARGET_ARM64 diff --git a/src/mono/mono/mini/mini.c b/src/mono/mono/mini/mini.c index e219d5d616de53..4c3a0ab09fb0ae 100644 --- a/src/mono/mono/mini/mini.c +++ b/src/mono/mono/mini/mini.c @@ -4323,3 +4323,23 @@ mini_get_cpu_features (MonoCompile* cfg) // apply parameters passed via -mattr return (features | mono_cpu_features_enabled) & ~mono_cpu_features_disabled; } + +llvm_ovr_tag_t +ovr_tag_from_mono_vector_class (MonoClass *klass) { + int size = mono_class_value_size (klass, NULL); + llvm_ovr_tag_t ret = 0; + switch (size) { + case 8: ret |= LLVM_Vector64; break; + case 16: ret |= LLVM_Vector128; break; + } + MonoType *etype = mono_class_get_context (klass)->class_inst->type_argv [0]; + switch (etype->type) { + case MONO_TYPE_I1: case MONO_TYPE_U1: ret |= LLVM_Int8; break; + case MONO_TYPE_I2: case MONO_TYPE_U2: ret |= LLVM_Int16; break; + case MONO_TYPE_I4: case MONO_TYPE_U4: ret |= LLVM_Int32; break; + case MONO_TYPE_I8: case MONO_TYPE_U8: ret |= LLVM_Int64; break; + case MONO_TYPE_R4: ret |= LLVM_Float32; break; + case MONO_TYPE_R8: ret |= LLVM_Float64; break; + } + return ret; +} diff --git a/src/mono/mono/mini/mini.h b/src/mono/mono/mini/mini.h index 39e4673cb761d9..91c3b4b3b49e89 100644 --- a/src/mono/mono/mini/mini.h +++ b/src/mono/mono/mini/mini.h @@ -2999,6 +2999,67 @@ typedef enum { SIMD_OP_ARM64_PMULL64_UPPER, } SimdOp; +/* An intrinsic id. The lower 23 bits are used to store a mono-specific ID. The + * next 9 bits store overload tag bits. In the configuration of LLVM 9 we use, + * there are 7017 total intrinsics defined in IntrinsicEnums.inc, so only 13 + * bits are needed to label each intrinsic overload group. + */ +typedef enum { +#define INTRINS(id, llvm_id) INTRINS_ ## id, +#define INTRINS_OVR(id, llvm_id, ty) INTRINS_ ## id, +#define INTRINS_OVR_2_ARG(id, llvm_id, ty1, ty2) INTRINS_ ## id, +#define INTRINS_OVR_3_ARG(id, llvm_id, ty1, ty2, ty3) INTRINS_ ## id, +#define INTRINS_OVR_TAG(id, ...) INTRINS_ ## id, +#include "llvm-intrinsics.h" + INTRINS_NUM +} IntrinsicId; + +typedef enum { + XBINOP_FORCEINT_Not, + XBINOP_FORCEINT_Or, + XBINOP_FORCEINT_OrNot, + XBINOP_FORCEINT_Xor, +} XBinOpId; + +enum { + LLVM_Scalar = 1 << 0, + LLVM_Vector64 = 1 << 1, + LLVM_Vector128 = 1 << 2, + LLVM_VectorWidths = 3, + LLVM_VectorMask = 0x7, + + LLVM_Int8 = 1 << 3, + LLVM_Int16 = 1 << 4, + LLVM_Int32 = 1 << 5, + LLVM_Int64 = 1 << 6, + LLVM_Float32 = 1 << 7, + LLVM_Float64 = 1 << 8, + LLVM_ElementWidths = 6, +}; + +typedef uint16_t llvm_ovr_tag_t; + +static inline llvm_ovr_tag_t +ovr_tag_force_scalar (llvm_ovr_tag_t tag) +{ + return (tag & ~LLVM_VectorMask) | LLVM_Scalar; +} + +static inline llvm_ovr_tag_t +ovr_tag_smaller_vector (llvm_ovr_tag_t tag) +{ + return (tag & ~LLVM_VectorMask) | ((tag & LLVM_VectorMask) >> 1); +} + +llvm_ovr_tag_t +ovr_tag_from_mono_vector_class (MonoClass *klass); + +static int +int_from_id_and_ovr_tag (int id, llvm_ovr_tag_t ovr_tag) +{ + return (((int) ovr_tag) << 23) | id; +} + const char *mono_arch_xregname (int reg); MonoCPUFeatures mono_arch_get_cpu_features (void); diff --git a/src/mono/mono/mini/simd-intrinsics.c b/src/mono/mono/mini/simd-intrinsics.c index 61a4c2fb87ce80..0cc958cc7b64c7 100644 --- a/src/mono/mono/mini/simd-intrinsics.c +++ b/src/mono/mono/mini/simd-intrinsics.c @@ -999,7 +999,19 @@ static SimdIntrinsic advsimd_methods [] = { {SN_AbsoluteCompareGreaterThanOrEqual}, {SN_AbsoluteCompareLessThan}, {SN_AbsoluteCompareLessThanOrEqual}, - {SN_MultiplyWideningUpperAndSubtract, OP_ARM64_SMLSL2}, + {SN_MultiplyRoundedDoublingByScalarSaturateHigh, OP_ARM64_SQRDMULH_SCALAR}, + {SN_MultiplyRoundedDoublingBySelectedScalarSaturateHigh, OP_ARM64_SQRDMULH_SEL}, + {SN_MultiplyRoundedDoublingSaturateHigh, OP_XOP_OVR_X_X_X, INTRINS_AARCH64_ADV_SIMD_SQRDMULH}, + {SN_MultiplyScalar, OP_XBINOP_SCALAR, OP_FMUL}, + {SN_MultiplyScalarBySelectedScalar, OP_ARM64_FMUL_SEL}, + {SN_MultiplySubtract, OP_ARM64_MLS}, + {SN_MultiplySubtractByScalar, OP_ARM64_MLS_SCALAR}, + {SN_MultiplyWideningLower, OP_ARM64_SMULL, None, OP_ARM64_UMULL}, + {SN_MultiplyWideningLowerAndAdd, OP_ARM64_SMLAL, None, OP_ARM64_UMLAL}, + {SN_MultiplyWideningLowerAndSubtract, OP_ARM64_SMLSL, None, OP_ARM64_UMLSL}, + {SN_MultiplyWideningUpper, OP_ARM64_SMULL2, None, OP_ARM64_UMULL2}, + {SN_MultiplyWideningUpperAndAdd, OP_ARM64_SMLAL2, None, OP_ARM64_UMLAL2}, + {SN_MultiplyWideningUpperAndSubtract, OP_ARM64_SMLSL2, None, OP_ARM64_UMLSL2}, {SN_Negate, OP_ARM64_XNEG}, {SN_NegateSaturate, OP_XOP_OVR_X_X, INTRINS_AARCH64_ADV_SIMD_SQNEG}, {SN_NegateSaturateScalar}, From e5a6b06ad54ebb7ca5a27b6573e3f4a1c1e7b7a9 Mon Sep 17 00:00:00 2001 From: Imran Hameed Date: Sat, 6 Mar 2021 02:10:53 -0800 Subject: [PATCH 12/58] Fix amd64 --- src/mono/mono/mini/simd-intrinsics.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/mono/mono/mini/simd-intrinsics.c b/src/mono/mono/mini/simd-intrinsics.c index 0cc958cc7b64c7..e2c470f80df26c 100644 --- a/src/mono/mono/mini/simd-intrinsics.c +++ b/src/mono/mono/mini/simd-intrinsics.c @@ -1428,7 +1428,7 @@ static SimdIntrinsic sse_methods [] = { {SN_SubtractScalar, OP_SSE_SUBSS}, {SN_UnpackHigh, OP_SSE_UNPACKHI}, {SN_UnpackLow, OP_SSE_UNPACKLO}, - {SN_Xor, OP_XBINOP_COERCE, XBINOP_Xor}, + {SN_Xor, OP_XBINOP_FORCEINT, XBINOP_FORCEINT_Xor}, {SN_get_IsSupported} }; @@ -1540,7 +1540,7 @@ static SimdIntrinsic sse2_methods [] = { {SN_SumAbsoluteDifferences, OP_XOP_X_X_X, SIMD_OP_SSE_PSADBW}, {SN_UnpackHigh, OP_SSE_UNPACKHI}, {SN_UnpackLow, OP_SSE_UNPACKLO}, - {SN_Xor, OP_XBINOP_COERCE, XBINOP_Xor}, + {SN_Xor, OP_XBINOP_FORCEINT, XBINOP_FORCEINT_Xor}, {SN_get_IsSupported} }; @@ -2082,10 +2082,10 @@ emit_x86_intrinsics ( case SN_RoundToPositiveInfinityScalar: case SN_RoundToZeroScalar: if (fsig->param_count == 2) { - return emit_simd_ins_for_sig (cfg, klass, OP_SSE41_ROUNDS, info->instc0, arg0_type, fsig, args); + return emit_simd_ins_for_sig (cfg, klass, OP_SSE41_ROUNDS, info->default_instc0, arg0_type, fsig, args); } else { MonoInst* ins = emit_simd_ins (cfg, klass, OP_SSE41_ROUNDS, args [0]->dreg, args [0]->dreg); - ins->inst_c0 = info->instc0; + ins->inst_c0 = info->default_instc0; ins->inst_c1 = arg0_type; return ins; } From 19cbca8c8975de6e07d55d9d17b4ecd5a0e237ee Mon Sep 17 00:00:00 2001 From: Imran Hameed Date: Sat, 6 Mar 2021 03:26:14 -0800 Subject: [PATCH 13/58] More multiplication --- src/mono/mono/mini/llvm-intrinsics.h | 3 + src/mono/mono/mini/mini-llvm.c | 149 ++++++++++++++++++++++++--- src/mono/mono/mini/mini-ops.h | 35 ++++++- src/mono/mono/mini/simd-intrinsics.c | 116 +++++++++++++++++++++ 4 files changed, 286 insertions(+), 17 deletions(-) diff --git a/src/mono/mono/mini/llvm-intrinsics.h b/src/mono/mono/mini/llvm-intrinsics.h index 43c596e69b4f18..29d8cc62d02bfe 100644 --- a/src/mono/mono/mini/llvm-intrinsics.h +++ b/src/mono/mono/mini/llvm-intrinsics.h @@ -307,6 +307,9 @@ INTRINS_OVR_2_ARG(AARCH64_ADV_SIMD_ABS_COMPARE_LT_DOUBLE, aarch64_neon_facgt, ss INTRINS_OVR_2_ARG(AARCH64_ADV_SIMD_ABS_COMPARE_LTE_FLOAT, aarch64_neon_facge, sse_i4_t, sse_r4_t) INTRINS_OVR_2_ARG(AARCH64_ADV_SIMD_ABS_COMPARE_LTE_DOUBLE, aarch64_neon_facge, sse_i4_t, sse_r8_t) +INTRINS_OVR_TAG(AARCH64_ADV_SIMD_SQDMULH, aarch64_neon_sqdmulh, V64 | V128 | I2 | I4) + +INTRINS_OVR_TAG(AARCH64_ADV_SIMD_SQADD, aarch64_neon_sqadd, V128 | I4 | I8) INTRINS_OVR_TAG(AARCH64_ADV_SIMD_SQDMULL, aarch64_neon_sqdmull, V128 | I4 | I8) INTRINS_OVR_TAG(AARCH64_ADV_SIMD_SQRDMULH, aarch64_neon_sqrdmulh, V64 | V128 | I2 | I4) diff --git a/src/mono/mono/mini/mini-llvm.c b/src/mono/mono/mini/mini-llvm.c index 8b4149e0dea690..10bc819f4297aa 100644 --- a/src/mono/mono/mini/mini-llvm.c +++ b/src/mono/mono/mini/mini-llvm.c @@ -4748,7 +4748,7 @@ extract_high_elements (EmitContext *ctx, LLVMValueRef src_vec) int mask [MAX_VECTOR_ELEMS] = { 0 }; for (int i = 0; i < dst_elems; ++i) mask [i] = dst_elems + i; - return LLVMBuildShuffleVector (ctx->builder, src_vec, LLVMGetUndef (src_t), create_const_vector_i32 (mask, dst_elems), ""); + return LLVMBuildShuffleVector (ctx->builder, src_vec, LLVMGetUndef (src_t), create_const_vector_i32 (mask, dst_elems), "extract_high"); } static LLVMValueRef @@ -4760,7 +4760,7 @@ keep_lowest_element (EmitContext *ctx, LLVMValueRef vec) mask [0] = 0; for (unsigned int i = 1; i < elems; ++i) mask [i] = elems + i; - return LLVMBuildShuffleVector (ctx->builder, vec, LLVMConstNull (t), create_const_vector_i32 (mask, elems), ""); + return LLVMBuildShuffleVector (ctx->builder, vec, LLVMConstNull (t), create_const_vector_i32 (mask, elems), "keep_lowest"); } static LLVMValueRef @@ -4771,7 +4771,7 @@ concatenate_vectors (EmitContext *ctx, LLVMValueRef xs, LLVMValueRef ys) int mask [MAX_VECTOR_ELEMS] = { 0 }; for (int i = 0; i < elems; ++i) mask [i] = i; - return LLVMBuildShuffleVector (ctx->builder, xs, ys, create_const_vector_i32 (mask, elems), ""); + return LLVMBuildShuffleVector (ctx->builder, xs, ys, create_const_vector_i32 (mask, elems), "concat_vecs"); } static LLVMValueRef @@ -4780,10 +4780,16 @@ scalar_from_vector (EmitContext *ctx, LLVMValueRef xs) return LLVMBuildExtractElement (ctx->builder, xs, const_int32 (0), "v2s"); } +static LLVMValueRef +vector_from_scalar_ty (EmitContext *ctx, LLVMTypeRef type, LLVMValueRef x) +{ + return LLVMBuildInsertElement (ctx->builder, LLVMGetUndef (type), x, const_int32 (0), "s2v"); +} + static LLVMValueRef vector_from_scalar (EmitContext *ctx, LLVMValueRef type_donor, LLVMValueRef x) { - return LLVMBuildInsertElement (ctx->builder, LLVMGetUndef (LLVMTypeOf (type_donor)), x, const_int32 (0), "s2v"); + return vector_from_scalar_ty (ctx, LLVMTypeOf (type_donor), x); } static void @@ -7099,13 +7105,20 @@ process_bb (EmitContext *ctx, MonoBasicBlock *bb) break; } case OP_XBINOP: - case OP_XBINOP_SCALAR: { + case OP_XBINOP_SCALAR: + case OP_XBINOP_BYSCALAR: { gboolean scalar = ins->opcode == OP_XBINOP_SCALAR; + gboolean byscalar = ins->opcode == OP_XBINOP_BYSCALAR; LLVMValueRef result = NULL; LLVMValueRef args [] = { lhs, rhs }; if (scalar) for (int i = 0; i < 2; ++i) args [i] = scalar_from_vector (ctx, args [i]); + if (byscalar) { + LLVMTypeRef t = LLVMTypeOf (args [0]); + unsigned int elems = LLVMGetVectorSize (t); + args [1] = broadcast_element (ctx, scalar_from_vector (ctx, args [1]), elems); + } LLVMValueRef l = args [0]; LLVMValueRef r = args [1]; switch (ins->inst_c0) { @@ -7115,6 +7128,9 @@ process_bb (EmitContext *ctx, MonoBasicBlock *bb) case OP_ISUB: result = LLVMBuildSub (builder, l, r, ""); break; + case OP_IMUL: + result = LLVMBuildMul (builder, l, r, ""); + break; case OP_IAND: result = LLVMBuildAnd (builder, l, r, ""); break; @@ -9272,6 +9288,15 @@ process_bb (EmitContext *ctx, MonoBasicBlock *bb) values [ins->dreg] = result; break; } + case OP_ARM64_SELECT_SCALAR: { + LLVMValueRef result = LLVMBuildExtractElement (builder, lhs, rhs, ""); + LLVMTypeRef elem_t = LLVMTypeOf (result); + unsigned int elem_bits = mono_llvm_get_prim_size_bits (elem_t); + LLVMTypeRef t = LLVMVectorType (elem_t, 64 / elem_bits); + result = vector_from_scalar_ty (ctx, t, result); + values [ins->dreg] = result; + break; + } case OP_LSCNT32: case OP_LSCNT64: { // %shr = ashr i32 %x, 31 @@ -9311,14 +9336,55 @@ process_bb (EmitContext *ctx, MonoBasicBlock *bb) values [ins->dreg] = LLVMBuildTrunc (builder, hi64, LLVMInt64Type (), ""); break; } - case OP_ARM64_SQDMLSL2: { + case OP_ARM64_SQDMULL: + case OP_ARM64_SQDMULL_SCALAR: + case OP_ARM64_SQDMULL2: + case OP_ARM64_SQDMULL2_SCALAR: + case OP_ARM64_SQDMLAL: + case OP_ARM64_SQDMLAL_SCALAR: + case OP_ARM64_SQDMLSL: + case OP_ARM64_SQDMLSL_SCALAR: + case OP_ARM64_SQDMLAL2: + case OP_ARM64_SQDMLAL2_SCALAR: + case OP_ARM64_SQDMLSL2: + case OP_ARM64_SQDMLSL2_SCALAR: { llvm_ovr_tag_t ovr_tag = ovr_tag_from_mono_vector_class (ins->klass); - LLVMValueRef args [] = { rhs, arg3 }; - for (int i = 0; i < 2; ++i) - args [i] = extract_high_elements (ctx, args [i]); + gboolean scalar = FALSE; + gboolean add = FALSE; + gboolean subtract = FALSE; + gboolean high = FALSE; + switch (ins->opcode) { + case OP_ARM64_SQDMULL_SCALAR: scalar = TRUE; case OP_ARM64_SQDMULL: break; + case OP_ARM64_SQDMULL2_SCALAR: scalar = TRUE; case OP_ARM64_SQDMULL2: high = TRUE; break; + case OP_ARM64_SQDMLAL_SCALAR: scalar = TRUE; case OP_ARM64_SQDMLAL: add = TRUE; break; + case OP_ARM64_SQDMLSL_SCALAR: scalar = TRUE; case OP_ARM64_SQDMLSL: subtract = TRUE; break; + case OP_ARM64_SQDMLAL2_SCALAR: scalar = TRUE; case OP_ARM64_SQDMLAL2: high = TRUE; add = TRUE; break; + case OP_ARM64_SQDMLSL2_SCALAR: scalar = TRUE; case OP_ARM64_SQDMLSL2: high = TRUE; subtract = TRUE; break; + } + int iid = 0; + if (add) + iid = INTRINS_AARCH64_ADV_SIMD_SQADD; + else if (subtract) + iid = INTRINS_AARCH64_ADV_SIMD_SQSUB; + LLVMValueRef mul1 = lhs; + LLVMValueRef mul2 = rhs; + if (iid != 0) { + mul1 = rhs; + mul2 = arg3; + } + if (scalar) { + LLVMTypeRef t = LLVMTypeOf (mul1); + unsigned int elems = LLVMGetVectorSize (t); + mul2 = broadcast_element (ctx, scalar_from_vector (ctx, mul2), elems); + } + LLVMValueRef args [] = { mul1, mul2 }; + if (high) + for (int i = 0; i < 2; ++i) + args [i] = extract_high_elements (ctx, args [i]); LLVMValueRef result = call_overloaded_intrins (ctx, INTRINS_AARCH64_ADV_SIMD_SQDMULL, ovr_tag, args, ""); LLVMValueRef args2 [] = { lhs, result }; - result = call_overloaded_intrins (ctx, INTRINS_AARCH64_ADV_SIMD_SQSUB, ovr_tag, args2, ""); + if (iid != 0) + result = call_overloaded_intrins (ctx, iid, ovr_tag, args2, ""); values [ins->dreg] = result; break; } @@ -9347,37 +9413,75 @@ process_bb (EmitContext *ctx, MonoBasicBlock *bb) values [ins->dreg] = result; break; } + case OP_ARM64_MLA: + case OP_ARM64_MLA_SCALAR: case OP_ARM64_MLS: case OP_ARM64_MLS_SCALAR: { - gboolean scalar = ins->opcode == OP_ARM64_MLS_SCALAR; + gboolean scalar = FALSE; + gboolean add = FALSE; + switch (ins->opcode) { + case OP_ARM64_MLA_SCALAR: scalar = TRUE; case OP_ARM64_MLA: add = TRUE; break; + case OP_ARM64_MLS_SCALAR: scalar = TRUE; case OP_ARM64_MLS: break; + } LLVMTypeRef mul_t = LLVMTypeOf (rhs); unsigned int elems = LLVMGetVectorSize (mul_t); LLVMValueRef mul2 = arg3; if (scalar) mul2 = broadcast_element (ctx, scalar_from_vector (ctx, mul2), elems); LLVMValueRef result = LLVMBuildMul (builder, rhs, mul2, ""); - result = LLVMBuildSub (builder, lhs, result, ""); + if (add) + result = LLVMBuildAdd (builder, lhs, result, ""); + else + result = LLVMBuildSub (builder, lhs, result, ""); values [ins->dreg] = result; break; } case OP_ARM64_SMULL: + case OP_ARM64_SMULL_SCALAR: case OP_ARM64_SMULL2: + case OP_ARM64_SMULL2_SCALAR: case OP_ARM64_UMULL: + case OP_ARM64_UMULL_SCALAR: case OP_ARM64_UMULL2: + case OP_ARM64_UMULL2_SCALAR: case OP_ARM64_SMLAL: + case OP_ARM64_SMLAL_SCALAR: case OP_ARM64_SMLAL2: + case OP_ARM64_SMLAL2_SCALAR: case OP_ARM64_UMLAL: + case OP_ARM64_UMLAL_SCALAR: case OP_ARM64_UMLAL2: + case OP_ARM64_UMLAL2_SCALAR: case OP_ARM64_SMLSL: + case OP_ARM64_SMLSL_SCALAR: case OP_ARM64_SMLSL2: + case OP_ARM64_SMLSL2_SCALAR: case OP_ARM64_UMLSL: - case OP_ARM64_UMLSL2: { + case OP_ARM64_UMLSL_SCALAR: + case OP_ARM64_UMLSL2: + case OP_ARM64_UMLSL2_SCALAR: { llvm_ovr_tag_t ovr_tag = ovr_tag_from_mono_vector_class (ins->klass); gboolean is_unsigned = FALSE; gboolean high = FALSE; gboolean add = FALSE; gboolean subtract = FALSE; - switch (ins->opcode) { + gboolean scalar = FALSE; + int opcode = ins->opcode; + switch (opcode) { + case OP_ARM64_SMULL_SCALAR: scalar = TRUE; opcode = OP_ARM64_SMULL; break; + case OP_ARM64_UMULL_SCALAR: scalar = TRUE; opcode = OP_ARM64_UMULL; break; + case OP_ARM64_SMLAL_SCALAR: scalar = TRUE; opcode = OP_ARM64_SMLAL; break; + case OP_ARM64_UMLAL_SCALAR: scalar = TRUE; opcode = OP_ARM64_UMLAL; break; + case OP_ARM64_SMLSL_SCALAR: scalar = TRUE; opcode = OP_ARM64_SMLSL; break; + case OP_ARM64_UMLSL_SCALAR: scalar = TRUE; opcode = OP_ARM64_UMLSL; break; + case OP_ARM64_SMULL2_SCALAR: scalar = TRUE; opcode = OP_ARM64_SMULL2; break; + case OP_ARM64_UMULL2_SCALAR: scalar = TRUE; opcode = OP_ARM64_UMULL2; break; + case OP_ARM64_SMLAL2_SCALAR: scalar = TRUE; opcode = OP_ARM64_SMLAL2; break; + case OP_ARM64_UMLAL2_SCALAR: scalar = TRUE; opcode = OP_ARM64_UMLAL2; break; + case OP_ARM64_SMLSL2_SCALAR: scalar = TRUE; opcode = OP_ARM64_SMLSL2; break; + case OP_ARM64_UMLSL2_SCALAR: scalar = TRUE; opcode = OP_ARM64_UMLSL2; break; + } + switch (opcode) { case OP_ARM64_SMULL2: high = TRUE; case OP_ARM64_SMULL: break; case OP_ARM64_UMULL2: high = TRUE; case OP_ARM64_UMULL: is_unsigned = TRUE; break; case OP_ARM64_SMLAL2: high = TRUE; case OP_ARM64_SMLAL: add = TRUE; break; @@ -9391,6 +9495,13 @@ process_bb (EmitContext *ctx, MonoBasicBlock *bb) intrin_args [0] = rhs; intrin_args [1] = arg3; } + if (scalar) { + LLVMValueRef sarg = intrin_args [1]; + LLVMTypeRef t = LLVMTypeOf (intrin_args [0]); + unsigned int elems = LLVMGetVectorSize (t); + sarg = broadcast_element (ctx, scalar_from_vector (ctx, sarg), elems); + intrin_args [1] = sarg; + } if (high) for (int i = 0; i < 2; ++i) intrin_args [i] = extract_high_elements (ctx, intrin_args [i]); @@ -9842,6 +9953,16 @@ process_bb (EmitContext *ctx, MonoBasicBlock *bb) values [ins->dreg] = call_overloaded_intrins (ctx, iid, ovr_tag, args, ""); break; } + case OP_XOP_OVR_BYSCALAR_X_X_X: { + IntrinsicId iid = (IntrinsicId) ins->inst_c0; + llvm_ovr_tag_t ovr_tag = ovr_tag_from_mono_vector_class (ins->klass); + LLVMTypeRef t = LLVMTypeOf (lhs); + unsigned int elems = LLVMGetVectorSize (t); + LLVMValueRef arg2 = broadcast_element (ctx, scalar_from_vector (ctx, rhs), elems); + LLVMValueRef args [] = { lhs, arg2 }; + values [ins->dreg] = call_overloaded_intrins (ctx, iid, ovr_tag, args, ""); + break; + } case OP_XOP_OVR_SCALAR_X_X: case OP_XOP_OVR_SCALAR_X_X_X: { int num_args = 0; diff --git a/src/mono/mono/mini/mini-ops.h b/src/mono/mono/mini/mini-ops.h index e6083334507dae..eea066268601a1 100644 --- a/src/mono/mono/mini/mini-ops.h +++ b/src/mono/mono/mini/mini-ops.h @@ -1534,8 +1534,9 @@ MINI_OP(OP_XCOMPARE_FP, "xcompare_fp", XREG, XREG, XREG) * Generic SIMD operations, the rest of the JIT doesn't care about the exact operation. */ MINI_OP(OP_XBINOP, "xbinop", XREG, XREG, XREG) -MINI_OP(OP_XBINOP_FORCEINT, "xbinop_coerce", XREG, XREG, XREG) -MINI_OP(OP_XBINOP_SCALAR, "xbinop", XREG, XREG, XREG) +MINI_OP(OP_XBINOP_FORCEINT, "xbinop_forceint", XREG, XREG, XREG) +MINI_OP(OP_XBINOP_SCALAR, "xbinop_scalar", XREG, XREG, XREG) +MINI_OP(OP_XBINOP_BYSCALAR, "xbinop_byscalar", XREG, XREG, XREG) /* inst_c0 contains a SimdOp, inst_c1 might contain additional data */ MINI_OP(OP_XOP, "xop", NONE, NONE, NONE) MINI_OP(OP_XOP_X_I, "xop_x_i", XREG, IREG, NONE) @@ -1558,6 +1559,7 @@ MINI_OP(OP_XOP_OVR_X_X_X, "xop_ovr_x_x_x", XREG, XREG, XREG) MINI_OP3(OP_XOP_OVR_X_X_X_X, "xop_ovr_x_x_x_x", XREG, XREG, XREG, XREG) MINI_OP(OP_XOP_OVR_SCALAR_X_X, "xop_ovr_scalar_x_x", XREG, XREG, NONE) MINI_OP(OP_XOP_OVR_SCALAR_X_X_X, "xop_ovr_scalar_x_x_x", XREG, XREG, XREG) +MINI_OP(OP_XOP_OVR_BYSCALAR_X_X_X, "xop_ovr_byscalar_x_x_x", XREG, XREG, XREG) MINI_OP(OP_XCAST, "xcast", XREG, XREG, NONE) /* Extract element of vector */ @@ -1668,26 +1670,53 @@ MINI_OP(OP_ARM64_XNEG, "arm64_xneg", XREG, XREG, NONE) MINI_OP(OP_ARM64_XNEG_SCALAR, "arm64_xneg_scalar", XREG, XREG, NONE) MINI_OP(OP_ARM64_SMULL, "arm64_smull", XREG, XREG, XREG) +MINI_OP(OP_ARM64_SMULL_SCALAR, "arm64_smull_scalar", XREG, XREG, XREG) MINI_OP(OP_ARM64_SMULL2, "arm64_smull2", XREG, XREG, XREG) +MINI_OP(OP_ARM64_SMULL2_SCALAR, "arm64_smull2_scalar", XREG, XREG, XREG) MINI_OP(OP_ARM64_UMULL, "arm64_umull", XREG, XREG, XREG) +MINI_OP(OP_ARM64_UMULL_SCALAR, "arm64_umull_scalar", XREG, XREG, XREG) MINI_OP(OP_ARM64_UMULL2, "arm64_umull2", XREG, XREG, XREG) +MINI_OP(OP_ARM64_UMULL2_SCALAR, "arm64_umull2_scalar", XREG, XREG, XREG) MINI_OP3(OP_ARM64_SMLAL, "arm64_smlal", XREG, XREG, XREG, XREG) +MINI_OP3(OP_ARM64_SMLAL_SCALAR, "arm64_smlal_scalar", XREG, XREG, XREG, XREG) MINI_OP3(OP_ARM64_SMLAL2, "arm64_smlal2", XREG, XREG, XREG, XREG) +MINI_OP3(OP_ARM64_SMLAL2_SCALAR, "arm64_smlal2_scalar", XREG, XREG, XREG, XREG) MINI_OP3(OP_ARM64_UMLAL, "arm64_umlal", XREG, XREG, XREG, XREG) +MINI_OP3(OP_ARM64_UMLAL_SCALAR, "arm64_umlal_scalar", XREG, XREG, XREG, XREG) MINI_OP3(OP_ARM64_UMLAL2, "arm64_umlal2", XREG, XREG, XREG, XREG) +MINI_OP3(OP_ARM64_UMLAL2_SCALAR, "arm64_umlal2_scalar", XREG, XREG, XREG, XREG) MINI_OP3(OP_ARM64_SMLSL, "arm64_smlsl", XREG, XREG, XREG, XREG) +MINI_OP3(OP_ARM64_SMLSL_SCALAR, "arm64_smlsl_scalar", XREG, XREG, XREG, XREG) MINI_OP3(OP_ARM64_SMLSL2, "arm64_smlsl2", XREG, XREG, XREG, XREG) +MINI_OP3(OP_ARM64_SMLSL2_SCALAR, "arm64_smlsl2_scalar", XREG, XREG, XREG, XREG) MINI_OP3(OP_ARM64_UMLSL, "arm64_umlsl", XREG, XREG, XREG, XREG) +MINI_OP3(OP_ARM64_UMLSL_SCALAR, "arm64_umlsl_scalar", XREG, XREG, XREG, XREG) MINI_OP3(OP_ARM64_UMLSL2, "arm64_umlsl2", XREG, XREG, XREG, XREG) +MINI_OP3(OP_ARM64_UMLSL2_SCALAR, "arm64_umlsl2_scalar", XREG, XREG, XREG, XREG) -MINI_OP3(OP_ARM64_MLS_SCALAR, "arm64_mls_scalar", XREG, XREG, XREG, XREG) MINI_OP3(OP_ARM64_MLS, "arm64_mls", XREG, XREG, XREG, XREG) +MINI_OP3(OP_ARM64_MLS_SCALAR, "arm64_mls_scalar", XREG, XREG, XREG, XREG) +MINI_OP3(OP_ARM64_MLA, "arm64_mla", XREG, XREG, XREG, XREG) +MINI_OP3(OP_ARM64_MLA_SCALAR, "arm64_mla_scalar", XREG, XREG, XREG, XREG) MINI_OP3(OP_ARM64_FMUL_SEL, "arm64_fmul_sel", XREG, XREG, XREG, IREG) MINI_OP3(OP_ARM64_SQRDMULH_SEL, "arm64_sqrdmulh_sel", XREG, XREG, XREG, IREG) MINI_OP(OP_ARM64_SQRDMULH_SCALAR, "arm64_sqrdmulh_scalar", XREG, XREG, XREG) +MINI_OP(OP_ARM64_SQDMULL, "arm64_sqdmull", XREG, XREG, XREG) +MINI_OP(OP_ARM64_SQDMULL_SCALAR, "arm64_sqdmull", XREG, XREG, XREG) +MINI_OP(OP_ARM64_SQDMULL2, "arm64_sqdmull2", XREG, XREG, XREG) +MINI_OP(OP_ARM64_SQDMULL2_SCALAR, "arm64_sqdmull2", XREG, XREG, XREG) +MINI_OP3(OP_ARM64_SQDMLSL, "arm64_sqdmlsl", XREG, XREG, XREG, XREG) +MINI_OP3(OP_ARM64_SQDMLSL_SCALAR, "arm64_sqdmlsl", XREG, XREG, XREG, XREG) +MINI_OP3(OP_ARM64_SQDMLAL, "arm64_sqdmlal", XREG, XREG, XREG, XREG) +MINI_OP3(OP_ARM64_SQDMLAL_SCALAR, "arm64_sqdmlal_scalar", XREG, XREG, XREG, XREG) MINI_OP3(OP_ARM64_SQDMLSL2, "arm64_sqdmlsl2", XREG, XREG, XREG, XREG) +MINI_OP3(OP_ARM64_SQDMLSL2_SCALAR, "arm64_sqdmlsl2_scalar", XREG, XREG, XREG, XREG) +MINI_OP3(OP_ARM64_SQDMLAL2, "arm64_sqdmlal2", XREG, XREG, XREG, XREG) +MINI_OP3(OP_ARM64_SQDMLAL2_SCALAR, "arm64_sqdmlal2_scalar", XREG, XREG, XREG, XREG) + +MINI_OP(OP_ARM64_SELECT_SCALAR, "arm64_select_scalar", XREG, XREG, IREG) #endif // TARGET_ARM64 diff --git a/src/mono/mono/mini/simd-intrinsics.c b/src/mono/mono/mini/simd-intrinsics.c index e2c470f80df26c..8f0f3a2592af00 100644 --- a/src/mono/mono/mini/simd-intrinsics.c +++ b/src/mono/mono/mini/simd-intrinsics.c @@ -288,6 +288,18 @@ type_is_unsigned (MonoType *type) { return FALSE; } +static gboolean +type_is_float (MonoType *type) { + MonoClass *klass = mono_class_from_mono_type_internal (type); + MonoType *etype = mono_class_get_context (klass)->class_inst->type_argv [0]; + switch (etype->type) { + case MONO_TYPE_R4: + case MONO_TYPE_R8: + return TRUE; + } + return FALSE; +} + static int type_to_expand_op (MonoType *type) { @@ -999,6 +1011,39 @@ static SimdIntrinsic advsimd_methods [] = { {SN_AbsoluteCompareGreaterThanOrEqual}, {SN_AbsoluteCompareLessThan}, {SN_AbsoluteCompareLessThanOrEqual}, + {SN_Multiply, OP_XBINOP, OP_IMUL, None, None, OP_XBINOP, OP_FMUL}, + {SN_MultiplyAdd, OP_ARM64_MLA}, + {SN_MultiplyAddByScalar, OP_ARM64_MLA_SCALAR}, + {SN_MultiplyAddBySelectedScalar}, + {SN_MultiplyByScalar, OP_XBINOP_BYSCALAR, OP_IMUL, None, None, OP_XBINOP_BYSCALAR, OP_FMUL}, + {SN_MultiplyBySelectedScalar}, + {SN_MultiplyBySelectedScalarWideningLower}, + {SN_MultiplyBySelectedScalarWideningLowerAndAdd}, + {SN_MultiplyBySelectedScalarWideningLowerAndSubtract}, + {SN_MultiplyBySelectedScalarWideningUpper}, + {SN_MultiplyBySelectedScalarWideningUpperAndAdd}, + {SN_MultiplyBySelectedScalarWideningUpperAndSubtract}, + {SN_MultiplyDoublingByScalarSaturateHigh, OP_XOP_OVR_BYSCALAR_X_X_X, INTRINS_AARCH64_ADV_SIMD_SQDMULH}, + {SN_MultiplyDoublingBySelectedScalarSaturateHigh}, + {SN_MultiplyDoublingSaturateHigh, OP_XOP_OVR_X_X_X, INTRINS_AARCH64_ADV_SIMD_SQDMULH}, + {SN_MultiplyDoublingWideningLowerAndAddSaturate, OP_ARM64_SQDMLAL}, + {SN_MultiplyDoublingWideningLowerAndSubtractSaturate, OP_ARM64_SQDMLSL}, + {SN_MultiplyDoublingWideningLowerByScalarAndAddSaturate, OP_ARM64_SQDMLAL_SCALAR}, + {SN_MultiplyDoublingWideningLowerByScalarAndSubtractSaturate, OP_ARM64_SQDMLSL_SCALAR}, + {SN_MultiplyDoublingWideningLowerBySelectedScalarAndAddSaturate}, + {SN_MultiplyDoublingWideningLowerBySelectedScalarAndSubtractSaturate}, + {SN_MultiplyDoublingWideningSaturateLower, OP_ARM64_SQDMULL}, + {SN_MultiplyDoublingWideningSaturateLowerByScalar, OP_ARM64_SQDMULL_SCALAR}, + {SN_MultiplyDoublingWideningSaturateLowerBySelectedScalar}, + {SN_MultiplyDoublingWideningSaturateUpper, OP_ARM64_SQDMULL2}, + {SN_MultiplyDoublingWideningSaturateUpperByScalar, OP_ARM64_SQDMULL2_SCALAR}, + {SN_MultiplyDoublingWideningSaturateUpperBySelectedScalar}, + {SN_MultiplyDoublingWideningUpperAndAddSaturate, OP_ARM64_SQDMLAL2}, + {SN_MultiplyDoublingWideningUpperAndSubtractSaturate, OP_ARM64_SQDMLSL2}, + {SN_MultiplyDoublingWideningUpperByScalarAndAddSaturate, OP_ARM64_SQDMLAL2_SCALAR}, + {SN_MultiplyDoublingWideningUpperByScalarAndSubtractSaturate, OP_ARM64_SQDMLSL2_SCALAR}, + {SN_MultiplyDoublingWideningUpperBySelectedScalarAndAddSaturate}, + {SN_MultiplyDoublingWideningUpperBySelectedScalarAndSubtractSaturate}, {SN_MultiplyRoundedDoublingByScalarSaturateHigh, OP_ARM64_SQRDMULH_SCALAR}, {SN_MultiplyRoundedDoublingBySelectedScalarSaturateHigh, OP_ARM64_SQRDMULH_SEL}, {SN_MultiplyRoundedDoublingSaturateHigh, OP_XOP_OVR_X_X_X, INTRINS_AARCH64_ADV_SIMD_SQRDMULH}, @@ -1006,6 +1051,7 @@ static SimdIntrinsic advsimd_methods [] = { {SN_MultiplyScalarBySelectedScalar, OP_ARM64_FMUL_SEL}, {SN_MultiplySubtract, OP_ARM64_MLS}, {SN_MultiplySubtractByScalar, OP_ARM64_MLS_SCALAR}, + {SN_MultiplySubtractBySelectedScalar}, {SN_MultiplyWideningLower, OP_ARM64_SMULL, None, OP_ARM64_UMULL}, {SN_MultiplyWideningLowerAndAdd, OP_ARM64_SMLAL, None, OP_ARM64_UMLAL}, {SN_MultiplyWideningLowerAndSubtract, OP_ARM64_SMLSL, None, OP_ARM64_UMLSL}, @@ -1299,6 +1345,76 @@ emit_arm64_intrinsics ( } return emit_simd_ins_for_sig (cfg, klass, OP_XOP_X_X, op, arg0_type, fsig, args); } + case SN_MultiplyBySelectedScalar: + case SN_MultiplyBySelectedScalarWideningLower: + case SN_MultiplyBySelectedScalarWideningUpper: + case SN_MultiplyDoublingBySelectedScalarSaturateHigh: + case SN_MultiplyDoublingWideningSaturateLowerBySelectedScalar: + case SN_MultiplyDoublingWideningSaturateUpperBySelectedScalar: { + gboolean is_unsigned = type_is_unsigned (fsig->ret); + gboolean is_float = type_is_float (fsig->ret); + int opcode = 0; + int c0 = 0; + switch (id) { + case SN_MultiplyBySelectedScalar: opcode = OP_XBINOP_BYSCALAR; c0 = OP_IMUL; break; + case SN_MultiplyBySelectedScalarWideningLower: opcode = OP_ARM64_SMULL_SCALAR; break; + case SN_MultiplyBySelectedScalarWideningUpper: opcode = OP_ARM64_SMULL2_SCALAR; break; + case SN_MultiplyDoublingBySelectedScalarSaturateHigh: opcode = OP_XOP_OVR_BYSCALAR_X_X_X; c0 = INTRINS_AARCH64_ADV_SIMD_SQDMULH; break; + case SN_MultiplyDoublingWideningSaturateLowerBySelectedScalar: opcode = OP_ARM64_SQDMULL_SCALAR; break; + case SN_MultiplyDoublingWideningSaturateUpperBySelectedScalar: opcode = OP_ARM64_SQDMULL2_SCALAR; break; + default: g_assert_not_reached(); + } + if (is_unsigned) + switch (opcode) { + case OP_ARM64_SMULL_SCALAR: opcode = OP_ARM64_UMULL_SCALAR; break; + case OP_ARM64_SMULL2_SCALAR: opcode = OP_ARM64_UMULL2_SCALAR; break; + } + if (is_float) + switch (opcode) { + case OP_XBINOP_BYSCALAR: c0 = OP_FMUL; + } + MonoInst *scalar = emit_simd_ins (cfg, klass, OP_ARM64_SELECT_SCALAR, args [1]->dreg, args [2]->dreg); + MonoInst *ret = emit_simd_ins (cfg, klass, opcode, args [0]->dreg, scalar->dreg); + ret->inst_c0 = c0; + return ret; + } + case SN_MultiplyAddBySelectedScalar: + case SN_MultiplySubtractBySelectedScalar: + case SN_MultiplyBySelectedScalarWideningLowerAndAdd: + case SN_MultiplyBySelectedScalarWideningLowerAndSubtract: + case SN_MultiplyBySelectedScalarWideningUpperAndAdd: + case SN_MultiplyBySelectedScalarWideningUpperAndSubtract: + case SN_MultiplyDoublingWideningLowerBySelectedScalarAndAddSaturate: + case SN_MultiplyDoublingWideningLowerBySelectedScalarAndSubtractSaturate: + case SN_MultiplyDoublingWideningUpperBySelectedScalarAndAddSaturate: + case SN_MultiplyDoublingWideningUpperBySelectedScalarAndSubtractSaturate: { + gboolean is_unsigned = type_is_unsigned (fsig->ret); + int opcode = 0; + switch (id) { + case SN_MultiplyAddBySelectedScalar: opcode = OP_ARM64_MLA_SCALAR; break; + case SN_MultiplySubtractBySelectedScalar: opcode = OP_ARM64_MLS_SCALAR; break; + case SN_MultiplyBySelectedScalarWideningLowerAndAdd: opcode = OP_ARM64_SMLAL_SCALAR; break; + case SN_MultiplyBySelectedScalarWideningLowerAndSubtract: opcode = OP_ARM64_SMLSL_SCALAR; break; + case SN_MultiplyBySelectedScalarWideningUpperAndAdd: opcode = OP_ARM64_SMLAL2_SCALAR; break; + case SN_MultiplyBySelectedScalarWideningUpperAndSubtract: opcode = OP_ARM64_SMLSL2_SCALAR; break; + case SN_MultiplyDoublingWideningLowerBySelectedScalarAndAddSaturate: opcode = OP_ARM64_SQDMLAL_SCALAR; break; + case SN_MultiplyDoublingWideningLowerBySelectedScalarAndSubtractSaturate: opcode = OP_ARM64_SQDMLSL_SCALAR; break; + case SN_MultiplyDoublingWideningUpperBySelectedScalarAndAddSaturate: opcode = OP_ARM64_SQDMLAL2_SCALAR; break; + case SN_MultiplyDoublingWideningUpperBySelectedScalarAndSubtractSaturate: opcode = OP_ARM64_SQDMLSL2_SCALAR; break; + default: g_assert_not_reached(); + } + if (is_unsigned) + switch (opcode) { + case OP_ARM64_SMLAL_SCALAR: opcode = OP_ARM64_UMLAL_SCALAR; break; + case OP_ARM64_SMLSL_SCALAR: opcode = OP_ARM64_UMLSL_SCALAR; break; + case OP_ARM64_SMLAL2_SCALAR: opcode = OP_ARM64_UMLAL2_SCALAR; break; + case OP_ARM64_SMLSL2_SCALAR: opcode = OP_ARM64_UMLSL2_SCALAR; break; + } + MonoInst *scalar = emit_simd_ins (cfg, klass, OP_ARM64_SELECT_SCALAR, args [2]->dreg, args [3]->dreg); + MonoInst *ret = emit_simd_ins (cfg, klass, opcode, args [0]->dreg, args [1]->dreg); + ret->sreg3 = scalar->dreg; + return ret; + } case SN_NegateSaturateScalar: { MonoInst *ret = emit_simd_ins_for_sig (cfg, klass, OP_XOP_OVR_X_X, INTRINS_AARCH64_ADV_SIMD_SQNEG, arg0_type, fsig, args); ret = emit_simd_ins (cfg, klass, OP_ARM64_ZERO_UPPER, ret->dreg, -1); From 6a88dc73a026a8c06b7f16babd29a24d00ff545a Mon Sep 17 00:00:00 2001 From: Imran Hameed Date: Sat, 6 Mar 2021 13:24:54 -0800 Subject: [PATCH 14/58] Implement min/max --- src/mono/mono/mini/llvm-intrinsics.h | 22 +++++++++ src/mono/mono/mini/mini-llvm.c | 67 +++++++++++++++++----------- src/mono/mono/mini/mini-ops.h | 3 ++ src/mono/mono/mini/simd-intrinsics.c | 24 ++++++++++ 4 files changed, 91 insertions(+), 25 deletions(-) diff --git a/src/mono/mono/mini/llvm-intrinsics.h b/src/mono/mono/mini/llvm-intrinsics.h index 29d8cc62d02bfe..ec80871ac77a28 100644 --- a/src/mono/mono/mini/llvm-intrinsics.h +++ b/src/mono/mono/mini/llvm-intrinsics.h @@ -307,6 +307,28 @@ INTRINS_OVR_2_ARG(AARCH64_ADV_SIMD_ABS_COMPARE_LT_DOUBLE, aarch64_neon_facgt, ss INTRINS_OVR_2_ARG(AARCH64_ADV_SIMD_ABS_COMPARE_LTE_FLOAT, aarch64_neon_facge, sse_i4_t, sse_r4_t) INTRINS_OVR_2_ARG(AARCH64_ADV_SIMD_ABS_COMPARE_LTE_DOUBLE, aarch64_neon_facge, sse_i4_t, sse_r8_t) +INTRINS_OVR_TAG(AARCH64_ADV_SIMD_SMAX, aarch64_neon_smax, V64 | V128 | I1 | I2 | I4 | I8) +INTRINS_OVR_TAG(AARCH64_ADV_SIMD_UMAX, aarch64_neon_umax, V64 | V128 | I1 | I2 | I4 | I8) +INTRINS_OVR_TAG(AARCH64_ADV_SIMD_FMAX, aarch64_neon_fmax, Scalar | V64 | V128 | R4 | R8) +INTRINS_OVR_TAG(AARCH64_ADV_SIMD_SMIN, aarch64_neon_smin, V64 | V128 | I1 | I2 | I4 | I8) +INTRINS_OVR_TAG(AARCH64_ADV_SIMD_UMIN, aarch64_neon_umin, V64 | V128 | I1 | I2 | I4 | I8) +INTRINS_OVR_TAG(AARCH64_ADV_SIMD_FMIN, aarch64_neon_fmin, Scalar | V64 | V128 | R4 | R8) + +INTRINS_OVR_TAG(AARCH64_ADV_SIMD_FMAXP, aarch64_neon_fmaxp, V64 | V128 | R4 | R8) +INTRINS_OVR_TAG(AARCH64_ADV_SIMD_SMAXP, aarch64_neon_smaxp, V64 | V128 | I1 | I2 | I4 | I8) +INTRINS_OVR_TAG(AARCH64_ADV_SIMD_UMAXP, aarch64_neon_umaxp, V64 | V128 | I1 | I2 | I4 | I8) +INTRINS_OVR_TAG(AARCH64_ADV_SIMD_FMINP, aarch64_neon_fminp, V64 | V128 | R4 | R8) +INTRINS_OVR_TAG(AARCH64_ADV_SIMD_SMINP, aarch64_neon_sminp, V64 | V128 | I1 | I2 | I4 | I8) +INTRINS_OVR_TAG(AARCH64_ADV_SIMD_UMINP, aarch64_neon_uminp, V64 | V128 | I1 | I2 | I4 | I8) + +INTRINS_OVR_TAG(AARCH64_ADV_SIMD_FMAXNM, aarch64_neon_fmaxnm, Scalar | V64 | V128 | R4 | R8) +INTRINS_OVR_TAG(AARCH64_ADV_SIMD_FMINNM, aarch64_neon_fminnm, Scalar | V64 | V128 | R4 | R8) + +INTRINS_OVR_2_ARG(AARCH64_ADV_SIMD_FMAXV_F32, aarch64_neon_fminv, r4_t, v64_r4_t) +INTRINS_OVR_2_ARG(AARCH64_ADV_SIMD_FMAXV_F64, aarch64_neon_fminv, r8_t, v128_r8_t) +INTRINS_OVR_2_ARG(AARCH64_ADV_SIMD_FMINV_F32, aarch64_neon_fminv, r4_t, v64_r4_t) +INTRINS_OVR_2_ARG(AARCH64_ADV_SIMD_FMINV_F64, aarch64_neon_fminv, r8_t, v128_r8_t) + INTRINS_OVR_TAG(AARCH64_ADV_SIMD_SQDMULH, aarch64_neon_sqdmulh, V64 | V128 | I2 | I4) INTRINS_OVR_TAG(AARCH64_ADV_SIMD_SQADD, aarch64_neon_sqadd, V128 | I4 | I8) diff --git a/src/mono/mono/mini/mini-llvm.c b/src/mono/mono/mini/mini-llvm.c index 10bc819f4297aa..db725ccf6bbaf7 100644 --- a/src/mono/mono/mini/mini-llvm.c +++ b/src/mono/mono/mini/mini-llvm.c @@ -299,6 +299,7 @@ static LLVMRealPredicate fpcond_to_llvm_cond [] = { static MonoLLVMModule aot_module; static GHashTable *intrins_id_to_intrins; +static LLVMTypeRef i1_t, i2_t, i4_t, i8_t, r4_t, r8_t; static LLVMTypeRef sse_i1_t, sse_i2_t, sse_i4_t, sse_i8_t, sse_r4_t, sse_r8_t; static LLVMTypeRef v64_i1_t, v64_i2_t, v64_i4_t, v64_i8_t, v64_r4_t, v64_r8_t; static LLVMTypeRef v128_i1_t, v128_i2_t, v128_i4_t, v128_i8_t, v128_r4_t, v128_r8_t; @@ -4780,6 +4781,12 @@ scalar_from_vector (EmitContext *ctx, LLVMValueRef xs) return LLVMBuildExtractElement (ctx->builder, xs, const_int32 (0), "v2s"); } +static LLVMValueRef +vector_zero_from_scalar (EmitContext *ctx, LLVMTypeRef type, LLVMValueRef x) +{ + return LLVMBuildInsertElement (ctx->builder, LLVMConstNull (type), x, const_int32 (0), "s2vz"); +} + static LLVMValueRef vector_from_scalar_ty (EmitContext *ctx, LLVMTypeRef type, LLVMValueRef x) { @@ -9257,33 +9264,43 @@ process_bb (EmitContext *ctx, MonoBasicBlock *bb) values [ins->dreg] = call_intrins (ctx, id, args, ""); break; } + case OP_XOP_SX_X: case OP_XOP_X_X: { IntrinsicId id = (IntrinsicId)0; + gboolean pack_result = FALSE; gboolean getLowerElement = FALSE; - switch (ins->inst_c0) { - case SIMD_OP_AES_IMC: id = INTRINS_AARCH64_AESIMC; break; - case SIMD_OP_ARM64_AES_AESMC: id = INTRINS_AARCH64_AESMC; break; - case SIMD_OP_ARM64_FABS: id = INTRINS_AARCH64_ADV_SIMD_ABS_FLOAT; break; - case SIMD_OP_ARM64_DABS: id = INTRINS_AARCH64_ADV_SIMD_ABS_DOUBLE; break; - case SIMD_OP_ARM64_I8ABS: id = INTRINS_AARCH64_ADV_SIMD_ABS_INT8; break; - case SIMD_OP_ARM64_I16ABS: id = INTRINS_AARCH64_ADV_SIMD_ABS_INT16; break; - case SIMD_OP_ARM64_I32ABS: id = INTRINS_AARCH64_ADV_SIMD_ABS_INT32; break; - case SIMD_OP_ARM64_I64ABS: id = INTRINS_AARCH64_ADV_SIMD_ABS_INT64; break; - case SIMD_OP_ARM64_I8ABS_SATURATE: id = INTRINS_AARCH64_ADV_SIMD_ABS_SATURATE_INT8; break; - case SIMD_OP_ARM64_I16ABS_SATURATE: id = INTRINS_AARCH64_ADV_SIMD_ABS_SATURATE_INT16; break; - case SIMD_OP_ARM64_I32ABS_SATURATE: id = INTRINS_AARCH64_ADV_SIMD_ABS_SATURATE_INT32; break; - case SIMD_OP_ARM64_I64ABS_SATURATE: id = INTRINS_AARCH64_ADV_SIMD_ABS_SATURATE_INT64; break; - case SIMD_OP_ARM64_SHA1H: id = INTRINS_AARCH64_SHA1H; getLowerElement = TRUE; break; - default: g_assert_not_reached (); break; + switch (ins->opcode) { + case OP_XOP_SX_X: + pack_result = TRUE; + id = ins->inst_c0; + break; + default: + switch (ins->inst_c0) { + case SIMD_OP_AES_IMC: id = INTRINS_AARCH64_AESIMC; break; + case SIMD_OP_ARM64_AES_AESMC: id = INTRINS_AARCH64_AESMC; break; + case SIMD_OP_ARM64_FABS: id = INTRINS_AARCH64_ADV_SIMD_ABS_FLOAT; break; + case SIMD_OP_ARM64_DABS: id = INTRINS_AARCH64_ADV_SIMD_ABS_DOUBLE; break; + case SIMD_OP_ARM64_I8ABS: id = INTRINS_AARCH64_ADV_SIMD_ABS_INT8; break; + case SIMD_OP_ARM64_I16ABS: id = INTRINS_AARCH64_ADV_SIMD_ABS_INT16; break; + case SIMD_OP_ARM64_I32ABS: id = INTRINS_AARCH64_ADV_SIMD_ABS_INT32; break; + case SIMD_OP_ARM64_I64ABS: id = INTRINS_AARCH64_ADV_SIMD_ABS_INT64; break; + case SIMD_OP_ARM64_I8ABS_SATURATE: id = INTRINS_AARCH64_ADV_SIMD_ABS_SATURATE_INT8; break; + case SIMD_OP_ARM64_I16ABS_SATURATE: id = INTRINS_AARCH64_ADV_SIMD_ABS_SATURATE_INT16; break; + case SIMD_OP_ARM64_I32ABS_SATURATE: id = INTRINS_AARCH64_ADV_SIMD_ABS_SATURATE_INT32; break; + case SIMD_OP_ARM64_I64ABS_SATURATE: id = INTRINS_AARCH64_ADV_SIMD_ABS_SATURATE_INT64; break; + case SIMD_OP_ARM64_SHA1H: id = INTRINS_AARCH64_SHA1H; getLowerElement = TRUE; break; + default: g_assert_not_reached (); break; + } } + if (getLowerElement) + pack_result = TRUE; LLVMValueRef arg0 = lhs; - LLVMValueRef result; if (getLowerElement) arg0 = LLVMBuildExtractElement (ctx->builder, arg0, const_int32 (0), ""); - result = call_intrins (ctx, id, &arg0, ""); - if (getLowerElement) { + LLVMValueRef result = call_intrins (ctx, id, &arg0, ""); + if (pack_result) { LLVMTypeRef t = simd_class_to_llvm_type (ctx, ins->klass); - result = LLVMBuildInsertElement (ctx->builder, LLVMConstNull (t), result, const_int32 (0), ""); + result = vector_zero_from_scalar (ctx, t, result); } values [ins->dreg] = result; break; @@ -11397,12 +11414,12 @@ add_types (MonoLLVMModule *module) void mono_llvm_init (gboolean enable_jit) { - intrin_types [0][0] = LLVMInt8Type (); - intrin_types [0][1] = LLVMInt16Type (); - intrin_types [0][2] = LLVMInt32Type (); - intrin_types [0][3] = LLVMInt64Type (); - intrin_types [0][4] = LLVMFloatType (); - intrin_types [0][5] = LLVMDoubleType (); + intrin_types [0][0] = i1_t = LLVMInt8Type (); + intrin_types [0][1] = i2_t = LLVMInt16Type (); + intrin_types [0][2] = i4_t = LLVMInt32Type (); + intrin_types [0][3] = i8_t = LLVMInt64Type (); + intrin_types [0][4] = r4_t = LLVMFloatType (); + intrin_types [0][5] = r8_t = LLVMDoubleType (); intrin_types [1][0] = v64_i1_t = LLVMVectorType (LLVMInt8Type (), 8); intrin_types [1][1] = v64_i2_t = LLVMVectorType (LLVMInt16Type (), 4); diff --git a/src/mono/mono/mini/mini-ops.h b/src/mono/mono/mini/mini-ops.h index eea066268601a1..a6c25d259a4bc5 100644 --- a/src/mono/mono/mini/mini-ops.h +++ b/src/mono/mono/mini/mini-ops.h @@ -1554,6 +1554,9 @@ MINI_OP(OP_XOP_I4_I4_I4, "xop_i4_i4_i4", IREG, IREG, IREG) MINI_OP(OP_XOP_I4_I4_I8, "xop_i4_i4_i8", IREG, IREG, LREG) MINI_OP3(OP_XOP_X_X_X_X, "xop_x_x_x_x", XREG, XREG, XREG, XREG) +/* SX: an LLVM scalar that will be wrapped in a vector */ +MINI_OP(OP_XOP_SX_X, "xop_sx_x", XREG, XREG, NONE) + MINI_OP(OP_XOP_OVR_X_X, "xop_ovr_x_x", XREG, XREG, NONE) MINI_OP(OP_XOP_OVR_X_X_X, "xop_ovr_x_x_x", XREG, XREG, XREG) MINI_OP3(OP_XOP_OVR_X_X_X_X, "xop_ovr_x_x_x_x", XREG, XREG, XREG, XREG) diff --git a/src/mono/mono/mini/simd-intrinsics.c b/src/mono/mono/mini/simd-intrinsics.c index 8f0f3a2592af00..488a3b21307457 100644 --- a/src/mono/mono/mini/simd-intrinsics.c +++ b/src/mono/mono/mini/simd-intrinsics.c @@ -1011,6 +1011,17 @@ static SimdIntrinsic advsimd_methods [] = { {SN_AbsoluteCompareGreaterThanOrEqual}, {SN_AbsoluteCompareLessThan}, {SN_AbsoluteCompareLessThanOrEqual}, + {SN_Max, OP_XOP_OVR_X_X_X, INTRINS_AARCH64_ADV_SIMD_SMAX, OP_XOP_OVR_X_X_X, INTRINS_AARCH64_ADV_SIMD_UMAX, OP_XOP_OVR_X_X_X, INTRINS_AARCH64_ADV_SIMD_FMAX}, + {SN_MaxNumber, OP_XOP_OVR_X_X_X, INTRINS_AARCH64_ADV_SIMD_FMAXNM}, + {SN_MaxNumberScalar, OP_XOP_OVR_SCALAR_X_X_X, INTRINS_AARCH64_ADV_SIMD_FMAXNM}, + {SN_MaxPairwise, OP_XOP_OVR_X_X_X, INTRINS_AARCH64_ADV_SIMD_SMAXP, OP_XOP_OVR_X_X_X, INTRINS_AARCH64_ADV_SIMD_UMAXP, OP_XOP_OVR_X_X_X, INTRINS_AARCH64_ADV_SIMD_FMAXP}, + {SN_Min, OP_XOP_OVR_X_X_X, INTRINS_AARCH64_ADV_SIMD_SMIN, OP_XOP_OVR_X_X_X, INTRINS_AARCH64_ADV_SIMD_UMIN, OP_XOP_OVR_X_X_X, INTRINS_AARCH64_ADV_SIMD_FMIN}, + {SN_MinNumber, OP_XOP_OVR_X_X_X, INTRINS_AARCH64_ADV_SIMD_FMINNM}, + {SN_MinNumberScalar, OP_XOP_OVR_SCALAR_X_X_X, INTRINS_AARCH64_ADV_SIMD_FMINNM}, + {SN_MinPairwise, OP_XOP_OVR_X_X_X, INTRINS_AARCH64_ADV_SIMD_SMINP, OP_XOP_OVR_X_X_X, INTRINS_AARCH64_ADV_SIMD_UMINP, OP_XOP_OVR_X_X_X, INTRINS_AARCH64_ADV_SIMD_FMINP}, + {SN_MinPairwiseScalar}, + {SN_MinPairwiseScalar}, + {SN_MinScalar, OP_XOP_OVR_SCALAR_X_X_X, INTRINS_AARCH64_ADV_SIMD_FMIN}, {SN_Multiply, OP_XBINOP, OP_IMUL, None, None, OP_XBINOP, OP_FMUL}, {SN_MultiplyAdd, OP_ARM64_MLA}, {SN_MultiplyAddByScalar, OP_ARM64_MLA_SCALAR}, @@ -1345,6 +1356,19 @@ emit_arm64_intrinsics ( } return emit_simd_ins_for_sig (cfg, klass, OP_XOP_X_X, op, arg0_type, fsig, args); } + case SN_MaxPairwiseScalar: + case SN_MinPairwiseScalar: { + int iid = 0; + switch (id) { + case SN_MaxPairwiseScalar: + iid = arg0_type == MONO_TYPE_R4 ? INTRINS_AARCH64_ADV_SIMD_FMAXV_F32 : INTRINS_AARCH64_ADV_SIMD_FMAXV_F64; + break; + case SN_MinPairwiseScalar: + iid = arg0_type == MONO_TYPE_R4 ? INTRINS_AARCH64_ADV_SIMD_FMINV_F32 : INTRINS_AARCH64_ADV_SIMD_FMINV_F64; + break; + } + return emit_simd_ins_for_sig (cfg, klass, OP_XOP_SX_X, iid, arg0_type, fsig, args); + } case SN_MultiplyBySelectedScalar: case SN_MultiplyBySelectedScalarWideningLower: case SN_MultiplyBySelectedScalarWideningUpper: From a82ad4da52cafcd57cc0077d1a045a0d91d2d337 Mon Sep 17 00:00:00 2001 From: Imran Hameed Date: Sat, 6 Mar 2021 13:46:00 -0800 Subject: [PATCH 15/58] Add load ops --- src/mono/mono/mini/mini-llvm.c | 25 +++++++++++++++++++++++++ src/mono/mono/mini/mini-ops.h | 11 ++++++++--- src/mono/mono/mini/simd-intrinsics.c | 5 +++++ 3 files changed, 38 insertions(+), 3 deletions(-) diff --git a/src/mono/mono/mini/mini-llvm.c b/src/mono/mono/mini/mini-llvm.c index db725ccf6bbaf7..058a347e3e7b08 100644 --- a/src/mono/mono/mini/mini-llvm.c +++ b/src/mono/mono/mini/mini-llvm.c @@ -9798,6 +9798,31 @@ process_bb (EmitContext *ctx, MonoBasicBlock *bb) set_nontemporal_flag (store); break; } + case OP_ARM64_LD1_INSERT: { + LLVMTypeRef ret_t = simd_class_to_llvm_type (ctx, ins->klass); + unsigned int bytes = mono_llvm_get_prim_size_bits (ret_t) / 8; + LLVMValueRef address = arg3; + LLVMValueRef result = mono_llvm_build_aligned_load (builder, address, "arm64_ld1_insert", FALSE, bytes); + result = LLVMBuildInsertElement (builder, lhs, result, rhs, "arm64_ld1_insert"); + values [ins->dreg] = result; + break; + } + case OP_ARM64_LD1R: + case OP_ARM64_LD1: { + gboolean replicate = ins->opcode == OP_ARM64_LD1R; + LLVMTypeRef ret_t = simd_class_to_llvm_type (ctx, ins->klass); + unsigned int bytes = mono_llvm_get_prim_size_bits (ret_t) / 8; + LLVMValueRef address = lhs; + if (!replicate) + address = convert (ctx, address, LLVMPointerType (ret_t, 0)); + LLVMValueRef result = mono_llvm_build_aligned_load (builder, address, "arm64_ld1", FALSE, bytes); + if (replicate) { + unsigned int elems = LLVMGetVectorSize (ret_t); + result = broadcast_element (ctx, result, elems); + } + values [ins->dreg] = result; + break; + } case OP_ARM64_ST1: { LLVMTypeRef t = LLVMTypeOf (rhs); LLVMValueRef address = convert (ctx, lhs, LLVMPointerType (t, 0)); diff --git a/src/mono/mono/mini/mini-ops.h b/src/mono/mono/mini/mini-ops.h index a6c25d259a4bc5..df5791eaf3d292 100644 --- a/src/mono/mono/mini/mini-ops.h +++ b/src/mono/mono/mini/mini-ops.h @@ -1592,13 +1592,18 @@ MINI_OP(OP_POPCNT64, "popcnt64", LREG, LREG, NONE) MINI_OP(OP_LSCNT32, "lscnt32", IREG, IREG, NONE) MINI_OP(OP_LSCNT64, "lscnt64", LREG, LREG, NONE) +MINI_OP3(OP_ARM64_LD1_INSERT, "arm64_ld1_insert", XREG, IREG, XREG, IREG) +MINI_OP(OP_ARM64_LD1, "arm64_ld1", XREG, IREG, NONE) +MINI_OP(OP_ARM64_LD1R, "arm64_ld1r", XREG, IREG, NONE) + +MINI_OP(OP_ARM64_ST1, "arm64_st1", NONE, IREG, XREG) +MINI_OP(OP_ARM64_SXTL, "arm64_sxtl", XREG, XREG, NONE) +MINI_OP(OP_ARM64_SXTL2, "arm64_sxtl2", XREG, XREG, NONE) + MINI_OP(OP_ARM64_SMULH, "arm64_smulh", LREG, LREG, LREG) MINI_OP(OP_ARM64_SQRT_SCALAR, "arm64_sqrt_scalar", XREG, XREG, NONE) MINI_OP(OP_ARM64_SSUB, "arm64_ssub", XREG, XREG, XREG) MINI_OP(OP_ARM64_SSUB2, "arm64_ssub2", XREG, XREG, XREG) -MINI_OP(OP_ARM64_ST1, "arm64_st1", NONE, IREG, XREG) -MINI_OP(OP_ARM64_SXTL, "arm64_sxtl", XREG, XREG, NONE) -MINI_OP(OP_ARM64_SXTL2, "arm64_sxtl2", XREG, XREG, NONE) MINI_OP(OP_ARM64_TRN1, "arm64_trn1", XREG, XREG, XREG) MINI_OP(OP_ARM64_TRN2, "arm64_trn2", XREG, XREG, XREG) MINI_OP(OP_ARM64_UMULH, "arm64_umulh", LREG, LREG, LREG) diff --git a/src/mono/mono/mini/simd-intrinsics.c b/src/mono/mono/mini/simd-intrinsics.c index 488a3b21307457..4f98aad14357f4 100644 --- a/src/mono/mono/mini/simd-intrinsics.c +++ b/src/mono/mono/mini/simd-intrinsics.c @@ -1011,6 +1011,11 @@ static SimdIntrinsic advsimd_methods [] = { {SN_AbsoluteCompareGreaterThanOrEqual}, {SN_AbsoluteCompareLessThan}, {SN_AbsoluteCompareLessThanOrEqual}, + {SN_LoadAndInsertScalar, OP_ARM64_LD1_INSERT}, + {SN_LoadAndReplicateToVector128, OP_ARM64_LD1R}, + {SN_LoadAndReplicateToVector64, OP_ARM64_LD1R}, + {SN_LoadVector128, OP_ARM64_LD1}, + {SN_LoadVector64, OP_ARM64_LD1}, {SN_Max, OP_XOP_OVR_X_X_X, INTRINS_AARCH64_ADV_SIMD_SMAX, OP_XOP_OVR_X_X_X, INTRINS_AARCH64_ADV_SIMD_UMAX, OP_XOP_OVR_X_X_X, INTRINS_AARCH64_ADV_SIMD_FMAX}, {SN_MaxNumber, OP_XOP_OVR_X_X_X, INTRINS_AARCH64_ADV_SIMD_FMAXNM}, {SN_MaxNumberScalar, OP_XOP_OVR_SCALAR_X_X_X, INTRINS_AARCH64_ADV_SIMD_FMAXNM}, From 52035668e169a186b22dfc1990ea729199f9967c Mon Sep 17 00:00:00 2001 From: Imran Hameed Date: Sat, 6 Mar 2021 13:54:59 -0800 Subject: [PATCH 16/58] Add clz/cls --- src/mono/mono/mini/llvm-intrinsics.h | 3 +++ src/mono/mono/mini/mini-llvm.c | 13 +++++++++++++ src/mono/mono/mini/mini-ops.h | 2 ++ src/mono/mono/mini/simd-intrinsics.c | 2 ++ 4 files changed, 20 insertions(+) diff --git a/src/mono/mono/mini/llvm-intrinsics.h b/src/mono/mono/mini/llvm-intrinsics.h index ec80871ac77a28..02bf6e56319ba2 100644 --- a/src/mono/mono/mini/llvm-intrinsics.h +++ b/src/mono/mono/mini/llvm-intrinsics.h @@ -307,6 +307,9 @@ INTRINS_OVR_2_ARG(AARCH64_ADV_SIMD_ABS_COMPARE_LT_DOUBLE, aarch64_neon_facgt, ss INTRINS_OVR_2_ARG(AARCH64_ADV_SIMD_ABS_COMPARE_LTE_FLOAT, aarch64_neon_facge, sse_i4_t, sse_r4_t) INTRINS_OVR_2_ARG(AARCH64_ADV_SIMD_ABS_COMPARE_LTE_DOUBLE, aarch64_neon_facge, sse_i4_t, sse_r8_t) +INTRINS_OVR_TAG(AARCH64_ADV_SIMD_CLS, aarch64_neon_cls, V64 | V128 | I1 | I2 | I4 | I8) +INTRINS_OVR_TAG(AARCH64_ADV_SIMD_CLZ, ctlz, V64 | V128 | I1 | I2 | I4 | I8) + INTRINS_OVR_TAG(AARCH64_ADV_SIMD_SMAX, aarch64_neon_smax, V64 | V128 | I1 | I2 | I4 | I8) INTRINS_OVR_TAG(AARCH64_ADV_SIMD_UMAX, aarch64_neon_umax, V64 | V128 | I1 | I2 | I4 | I8) INTRINS_OVR_TAG(AARCH64_ADV_SIMD_FMAX, aarch64_neon_fmax, Scalar | V64 | V128 | R4 | R8) diff --git a/src/mono/mono/mini/mini-llvm.c b/src/mono/mono/mini/mini-llvm.c index 058a347e3e7b08..0655bd07f0038e 100644 --- a/src/mono/mono/mini/mini-llvm.c +++ b/src/mono/mono/mini/mini-llvm.c @@ -328,6 +328,12 @@ set_failure (EmitContext *ctx, const char *message) ctx->cfg->disable_llvm = TRUE; } +static LLVMValueRef +const_int1 (int v) +{ + return LLVMConstInt (LLVMInt1Type (), v ? 1 : 0, FALSE); +} + static LLVMValueRef const_int32 (int v) { @@ -9353,6 +9359,13 @@ process_bb (EmitContext *ctx, MonoBasicBlock *bb) values [ins->dreg] = LLVMBuildTrunc (builder, hi64, LLVMInt64Type (), ""); break; } + case OP_ARM64_CLZ: { + llvm_ovr_tag_t ovr_tag = ovr_tag_from_mono_vector_class (ins->klass); + LLVMValueRef args [] = { lhs, const_int1 (0) }; + LLVMValueRef result = call_overloaded_intrins (ctx, INTRINS_AARCH64_ADV_SIMD_CLZ, ovr_tag, args, ""); + values [ins->dreg] = result; + break; + } case OP_ARM64_SQDMULL: case OP_ARM64_SQDMULL_SCALAR: case OP_ARM64_SQDMULL2: diff --git a/src/mono/mono/mini/mini-ops.h b/src/mono/mono/mini/mini-ops.h index df5791eaf3d292..5425977517de59 100644 --- a/src/mono/mono/mini/mini-ops.h +++ b/src/mono/mono/mini/mini-ops.h @@ -1592,6 +1592,8 @@ MINI_OP(OP_POPCNT64, "popcnt64", LREG, LREG, NONE) MINI_OP(OP_LSCNT32, "lscnt32", IREG, IREG, NONE) MINI_OP(OP_LSCNT64, "lscnt64", LREG, LREG, NONE) +MINI_OP(OP_ARM64_CLZ, "arm64_clz", XREG, XREG, NONE) + MINI_OP3(OP_ARM64_LD1_INSERT, "arm64_ld1_insert", XREG, IREG, XREG, IREG) MINI_OP(OP_ARM64_LD1, "arm64_ld1", XREG, IREG, NONE) MINI_OP(OP_ARM64_LD1R, "arm64_ld1r", XREG, IREG, NONE) diff --git a/src/mono/mono/mini/simd-intrinsics.c b/src/mono/mono/mini/simd-intrinsics.c index 4f98aad14357f4..82af50e48f886b 100644 --- a/src/mono/mono/mini/simd-intrinsics.c +++ b/src/mono/mono/mini/simd-intrinsics.c @@ -1011,6 +1011,8 @@ static SimdIntrinsic advsimd_methods [] = { {SN_AbsoluteCompareGreaterThanOrEqual}, {SN_AbsoluteCompareLessThan}, {SN_AbsoluteCompareLessThanOrEqual}, + {SN_LeadingSignCount, OP_XOP_OVR_X_X, INTRINS_AARCH64_ADV_SIMD_CLS}, + {SN_LeadingZeroCount, OP_ARM64_CLZ}, {SN_LoadAndInsertScalar, OP_ARM64_LD1_INSERT}, {SN_LoadAndReplicateToVector128, OP_ARM64_LD1R}, {SN_LoadAndReplicateToVector64, OP_ARM64_LD1R}, From da21e6fd96f05b56cd428424974d20bc4ac9cf49 Mon Sep 17 00:00:00 2001 From: Imran Hameed Date: Sat, 6 Mar 2021 14:36:30 -0800 Subject: [PATCH 17/58] Implement insert/insertscalar --- src/mono/mono/mini/mini-llvm.c | 25 ++++++++++++++++++++++++- src/mono/mono/mini/mini-ops.h | 6 +++--- src/mono/mono/mini/simd-intrinsics.c | 27 +++++++++++++++++++++++++++ 3 files changed, 54 insertions(+), 4 deletions(-) diff --git a/src/mono/mono/mini/mini-llvm.c b/src/mono/mono/mini/mini-llvm.c index 0655bd07f0038e..4f67c1679c533b 100644 --- a/src/mono/mono/mini/mini-llvm.c +++ b/src/mono/mono/mini/mini-llvm.c @@ -7229,6 +7229,7 @@ process_bb (EmitContext *ctx, MonoBasicBlock *bb) #endif // defined(TARGET_X86) || defined(TARGET_AMD64) || defined(TARGET_ARM64) || defined(TARGET_WASM) #if defined(TARGET_X86) || defined(TARGET_AMD64) || defined(TARGET_WASM) + case OP_PADDB: case OP_PADDW: case OP_PADDD: @@ -7462,7 +7463,6 @@ process_bb (EmitContext *ctx, MonoBasicBlock *bb) values [ins->dreg] = LLVMBuildZExt (builder, values [ins->dreg], LLVMInt32Type (), ""); break; } - case OP_INSERT_I1: values [ins->dreg] = LLVMBuildInsertElement (builder, values [ins->sreg1], convert (ctx, values [ins->sreg2], LLVMInt8Type ()), LLVMConstInt (LLVMInt32Type (), ins->inst_c0, FALSE), dname); break; @@ -9186,6 +9186,15 @@ process_bb (EmitContext *ctx, MonoBasicBlock *bb) #endif #if defined(TARGET_ARM64) + case OP_EXTRACT_I1: + case OP_EXTRACT_I2: + case OP_EXTRACT_I4: + case OP_EXTRACT_I8: + case OP_EXTRACT_R4: + case OP_EXTRACT_R8: { + values [ins->dreg] = LLVMBuildExtractElement (builder, lhs, const_int32 (ins->inst_c0), "extract"); + break; + } case OP_XOP_I4_I4: case OP_XOP_I8_I8: { IntrinsicId id = (IntrinsicId)0; @@ -9311,6 +9320,20 @@ process_bb (EmitContext *ctx, MonoBasicBlock *bb) values [ins->dreg] = result; break; } + case OP_XINSERT_I1: + case OP_XINSERT_I2: + case OP_XINSERT_I4: + case OP_XINSERT_I8: + case OP_XINSERT_R4: + case OP_XINSERT_R8: { + LLVMTypeRef t = LLVMTypeOf (lhs); + LLVMTypeRef elem_t = LLVMGetElementType (t); + MonoTypeEnum primty = inst_c1_type (ins); + LLVMValueRef val = convert_full(ctx, rhs, elem_t, primitive_type_is_unsigned (primty)); + LLVMValueRef result = LLVMBuildInsertElement (builder, lhs, val, arg3, "xinsert"); + values [ins->dreg] = result; + break; + } case OP_ARM64_SELECT_SCALAR: { LLVMValueRef result = LLVMBuildExtractElement (builder, lhs, rhs, ""); LLVMTypeRef elem_t = LLVMTypeOf (result); diff --git a/src/mono/mono/mini/mini-ops.h b/src/mono/mono/mini/mini-ops.h index 5425977517de59..27099298966218 100644 --- a/src/mono/mono/mini/mini-ops.h +++ b/src/mono/mono/mini/mini-ops.h @@ -787,13 +787,13 @@ MINI_OP(OP_NOT_NULL, "not_null", NONE, IREG, NONE) #if defined(TARGET_X86) || defined(TARGET_AMD64) || defined(TARGET_WASM) || defined(TARGET_ARM64) -MINI_OP(OP_EXTRACT_I4, "extract_i4", IREG, XREG, NONE) MINI_OP(OP_ICONV_TO_R4_RAW, "iconv_to_r4_raw", FREG, IREG, NONE) -MINI_OP(OP_EXTRACT_I2, "extract_i2", IREG, XREG, NONE) -MINI_OP(OP_EXTRACT_U2, "extract_u2", IREG, XREG, NONE) MINI_OP(OP_EXTRACT_I1, "extract_i1", IREG, XREG, NONE) MINI_OP(OP_EXTRACT_U1, "extract_u1", IREG, XREG, NONE) +MINI_OP(OP_EXTRACT_I2, "extract_i2", IREG, XREG, NONE) +MINI_OP(OP_EXTRACT_U2, "extract_u2", IREG, XREG, NONE) +MINI_OP(OP_EXTRACT_I4, "extract_i4", IREG, XREG, NONE) MINI_OP(OP_EXTRACT_R4, "extract_r4", FREG, XREG, NONE) MINI_OP(OP_EXTRACT_R8, "extract_r8", FREG, XREG, NONE) MINI_OP(OP_EXTRACT_I8, "extract_i8", LREG, XREG, NONE) diff --git a/src/mono/mono/mini/simd-intrinsics.c b/src/mono/mono/mini/simd-intrinsics.c index 82af50e48f886b..cd23364d320c26 100644 --- a/src/mono/mono/mini/simd-intrinsics.c +++ b/src/mono/mono/mini/simd-intrinsics.c @@ -1011,6 +1011,8 @@ static SimdIntrinsic advsimd_methods [] = { {SN_AbsoluteCompareGreaterThanOrEqual}, {SN_AbsoluteCompareLessThan}, {SN_AbsoluteCompareLessThanOrEqual}, + {SN_Insert}, + {SN_InsertScalar}, {SN_LeadingSignCount, OP_XOP_OVR_X_X, INTRINS_AARCH64_ADV_SIMD_CLS}, {SN_LeadingZeroCount, OP_ARM64_CLZ}, {SN_LoadAndInsertScalar, OP_ARM64_LD1_INSERT}, @@ -1363,6 +1365,31 @@ emit_arm64_intrinsics ( } return emit_simd_ins_for_sig (cfg, klass, OP_XOP_X_X, op, arg0_type, fsig, args); } + case SN_InsertScalar: + case SN_Insert: { + int insert_op = 0; + int extract_op = 0; + printf ("XXXih: arg0_type = %d\n", arg0_type); + switch (arg0_type) { + case MONO_TYPE_I1: case MONO_TYPE_U1: insert_op = OP_XINSERT_I1; extract_op = OP_EXTRACT_I1; break; + case MONO_TYPE_I2: case MONO_TYPE_U2: insert_op = OP_XINSERT_I2; extract_op = OP_EXTRACT_I2; break; + case MONO_TYPE_I4: case MONO_TYPE_U4: insert_op = OP_XINSERT_I4; extract_op = OP_EXTRACT_I4; break; + case MONO_TYPE_I8: case MONO_TYPE_U8: insert_op = OP_XINSERT_I8; extract_op = OP_EXTRACT_I8; break; + case MONO_TYPE_R4: insert_op = OP_XINSERT_R4; extract_op = OP_EXTRACT_R4; break; + case MONO_TYPE_R8: insert_op = OP_XINSERT_R8; extract_op = OP_EXTRACT_R8; break; + default: g_assert_not_reached (); + } + int val_src_reg = args [2]->dreg; + if (id == SN_InsertScalar) { + MonoInst *ins = emit_simd_ins (cfg, klass, extract_op, val_src_reg, -1); + ins->inst_c0 = 0; + val_src_reg = ins->dreg; + } + MonoInst *ins = emit_simd_ins (cfg, klass, insert_op, args [0]->dreg, val_src_reg); + ins->sreg3 = args [1]->dreg; + ins->inst_c1 = arg0_type; + return ins; + } case SN_MaxPairwiseScalar: case SN_MinPairwiseScalar: { int iid = 0; From 94b739689666311ba1ae6ef02cfd98028ec05c58 Mon Sep 17 00:00:00 2001 From: Imran Hameed Date: Sat, 6 Mar 2021 15:38:30 -0800 Subject: [PATCH 18/58] Add fused ops --- src/mono/mono/mini/llvm-intrinsics.h | 10 +++++++ src/mono/mono/mini/mini-llvm.c | 43 ++++++++++++++++++++++++++-- src/mono/mono/mini/mini-ops.h | 9 ++++++ src/mono/mono/mini/simd-intrinsics.c | 10 ++++++- 4 files changed, 69 insertions(+), 3 deletions(-) diff --git a/src/mono/mono/mini/llvm-intrinsics.h b/src/mono/mono/mini/llvm-intrinsics.h index 02bf6e56319ba2..88e32683a1625e 100644 --- a/src/mono/mono/mini/llvm-intrinsics.h +++ b/src/mono/mono/mini/llvm-intrinsics.h @@ -307,6 +307,16 @@ INTRINS_OVR_2_ARG(AARCH64_ADV_SIMD_ABS_COMPARE_LT_DOUBLE, aarch64_neon_facgt, ss INTRINS_OVR_2_ARG(AARCH64_ADV_SIMD_ABS_COMPARE_LTE_FLOAT, aarch64_neon_facge, sse_i4_t, sse_r4_t) INTRINS_OVR_2_ARG(AARCH64_ADV_SIMD_ABS_COMPARE_LTE_DOUBLE, aarch64_neon_facge, sse_i4_t, sse_r8_t) +INTRINS_OVR_TAG(AARCH64_ADV_SIMD_SRHADD, aarch64_neon_srhadd, V64 | V128 | I1 | I2 | I4) +INTRINS_OVR_TAG(AARCH64_ADV_SIMD_URHADD, aarch64_neon_urhadd, V64 | V128 | I1 | I2 | I4) + +INTRINS_OVR_TAG(AARCH64_ADV_SIMD_FMA, fma, Scalar | V64 | V128 | R4 | R8) + +INTRINS_OVR_TAG(AARCH64_ADV_SIMD_SHADD, aarch64_neon_shadd, V64 | V128 | I1 | I2 | I4) +INTRINS_OVR_TAG(AARCH64_ADV_SIMD_UHADD, aarch64_neon_uhadd, V64 | V128 | I1 | I2 | I4) +INTRINS_OVR_TAG(AARCH64_ADV_SIMD_SHSUB, aarch64_neon_shsub, V64 | V128 | I1 | I2 | I4) +INTRINS_OVR_TAG(AARCH64_ADV_SIMD_UHSUB, aarch64_neon_uhsub, V64 | V128 | I1 | I2 | I4) + INTRINS_OVR_TAG(AARCH64_ADV_SIMD_CLS, aarch64_neon_cls, V64 | V128 | I1 | I2 | I4 | I8) INTRINS_OVR_TAG(AARCH64_ADV_SIMD_CLZ, ctlz, V64 | V128 | I1 | I2 | I4 | I8) diff --git a/src/mono/mono/mini/mini-llvm.c b/src/mono/mono/mini/mini-llvm.c index 4f67c1679c533b..943deba63d9d3a 100644 --- a/src/mono/mono/mini/mini-llvm.c +++ b/src/mono/mono/mini/mini-llvm.c @@ -9389,6 +9389,43 @@ process_bb (EmitContext *ctx, MonoBasicBlock *bb) values [ins->dreg] = result; break; } + case OP_ARM64_FMSUB: + case OP_ARM64_FMSUB_SCALAR: + case OP_ARM64_FNMSUB_SCALAR: + case OP_ARM64_FMADD: + case OP_ARM64_FMADD_SCALAR: + case OP_ARM64_FNMADD_SCALAR: { + llvm_ovr_tag_t ovr_tag = ovr_tag_from_mono_vector_class (ins->klass); + gboolean scalar = FALSE; + gboolean negate = FALSE; + gboolean subtract = FALSE; + switch (ins->opcode) { + case OP_ARM64_FMSUB: subtract = TRUE; break; + case OP_ARM64_FMSUB_SCALAR: subtract = TRUE; scalar = TRUE; break; + case OP_ARM64_FNMSUB_SCALAR: subtract = TRUE; scalar = TRUE; negate = TRUE; break; + case OP_ARM64_FMADD: break; + case OP_ARM64_FMADD_SCALAR: scalar = TRUE; break; + case OP_ARM64_FNMADD_SCALAR: scalar = TRUE; negate = TRUE; break; + } + // llvm.fma argument order: mulop1, mulop2, addend + LLVMValueRef args [] = { rhs, arg3, lhs }; + if (scalar) { + ovr_tag = ovr_tag_force_scalar (ovr_tag); + for (int i = 0; i < 3; ++i) + args [i] = scalar_from_vector (ctx, args [i]); + } + if (subtract) + args [0] = LLVMBuildFNeg (builder, args [0], "arm64_fma_sub"); + if (negate) { + args [0] = LLVMBuildFNeg (builder, args [0], "arm64_fma_negate"); + args [2] = LLVMBuildFNeg (builder, args [2], "arm64_fma_negate"); + } + LLVMValueRef result = call_overloaded_intrins (ctx, INTRINS_AARCH64_ADV_SIMD_FMA, ovr_tag, args, "arm64_fma"); + if (scalar) + result = vector_from_scalar (ctx, lhs, result); + values [ins->dreg] = result; + break; + } case OP_ARM64_SQDMULL: case OP_ARM64_SQDMULL_SCALAR: case OP_ARM64_SQDMULL2: @@ -10042,11 +10079,13 @@ process_bb (EmitContext *ctx, MonoBasicBlock *bb) break; } case OP_XOP_OVR_SCALAR_X_X: - case OP_XOP_OVR_SCALAR_X_X_X: { + case OP_XOP_OVR_SCALAR_X_X_X: + case OP_XOP_OVR_SCALAR_X_X_X_X: { int num_args = 0; switch (ins->opcode) { case OP_XOP_OVR_SCALAR_X_X: num_args = 1; break; case OP_XOP_OVR_SCALAR_X_X_X: num_args = 2; break; + case OP_XOP_OVR_SCALAR_X_X_X_X: num_args = 3; break; } IntrinsicId iid = (IntrinsicId) ins->inst_c0; llvm_ovr_tag_t ovr_tag = ovr_tag_from_mono_vector_class (ins->klass); @@ -10067,7 +10106,7 @@ process_bb (EmitContext *ctx, MonoBasicBlock *bb) #if !defined(TARGET_ARM64) arm64_fake_scalar_op = FALSE; #endif - LLVMValueRef args [2] = { lhs, rhs }; + LLVMValueRef args [3] = { lhs, rhs, arg3 }; if (!arm64_fake_scalar_op) { ovr_tag = ovr_tag_force_scalar (ovr_tag); for (int i = 0; i < num_args; ++i) diff --git a/src/mono/mono/mini/mini-ops.h b/src/mono/mono/mini/mini-ops.h index 27099298966218..2b57d4bb25e281 100644 --- a/src/mono/mono/mini/mini-ops.h +++ b/src/mono/mono/mini/mini-ops.h @@ -1562,6 +1562,7 @@ MINI_OP(OP_XOP_OVR_X_X_X, "xop_ovr_x_x_x", XREG, XREG, XREG) MINI_OP3(OP_XOP_OVR_X_X_X_X, "xop_ovr_x_x_x_x", XREG, XREG, XREG, XREG) MINI_OP(OP_XOP_OVR_SCALAR_X_X, "xop_ovr_scalar_x_x", XREG, XREG, NONE) MINI_OP(OP_XOP_OVR_SCALAR_X_X_X, "xop_ovr_scalar_x_x_x", XREG, XREG, XREG) +MINI_OP3(OP_XOP_OVR_SCALAR_X_X_X_X, "xop_ovr_scalar_x_x_x_x", XREG, XREG, XREG, XREG) MINI_OP(OP_XOP_OVR_BYSCALAR_X_X_X, "xop_ovr_byscalar_x_x_x", XREG, XREG, XREG) MINI_OP(OP_XCAST, "xcast", XREG, XREG, NONE) @@ -1727,6 +1728,14 @@ MINI_OP3(OP_ARM64_SQDMLSL2_SCALAR, "arm64_sqdmlsl2_scalar", XREG, XREG, XREG, XR MINI_OP3(OP_ARM64_SQDMLAL2, "arm64_sqdmlal2", XREG, XREG, XREG, XREG) MINI_OP3(OP_ARM64_SQDMLAL2_SCALAR, "arm64_sqdmlal2_scalar", XREG, XREG, XREG, XREG) +MINI_OP3(OP_ARM64_FMSUB, "arm64_fmsub_scalar", XREG, XREG, XREG, XREG) +MINI_OP3(OP_ARM64_FMSUB_SCALAR, "arm64_fmsub_scalar", XREG, XREG, XREG, XREG) +MINI_OP3(OP_ARM64_FNMSUB_SCALAR, "arm64_fnmsub_scalar", XREG, XREG, XREG, XREG) + +MINI_OP3(OP_ARM64_FMADD, "arm64_fmadd_scalar", XREG, XREG, XREG, XREG) +MINI_OP3(OP_ARM64_FMADD_SCALAR, "arm64_fmadd_scalar", XREG, XREG, XREG, XREG) +MINI_OP3(OP_ARM64_FNMADD_SCALAR, "arm64_fnmadd_scalar", XREG, XREG, XREG, XREG) + MINI_OP(OP_ARM64_SELECT_SCALAR, "arm64_select_scalar", XREG, XREG, IREG) #endif // TARGET_ARM64 diff --git a/src/mono/mono/mini/simd-intrinsics.c b/src/mono/mono/mini/simd-intrinsics.c index cd23364d320c26..0748493a88e266 100644 --- a/src/mono/mono/mini/simd-intrinsics.c +++ b/src/mono/mono/mini/simd-intrinsics.c @@ -1011,6 +1011,15 @@ static SimdIntrinsic advsimd_methods [] = { {SN_AbsoluteCompareGreaterThanOrEqual}, {SN_AbsoluteCompareLessThan}, {SN_AbsoluteCompareLessThanOrEqual}, + {SN_FusedAddHalving, OP_XOP_OVR_X_X_X, INTRINS_AARCH64_ADV_SIMD_SHADD, OP_XOP_OVR_X_X_X, INTRINS_AARCH64_ADV_SIMD_UHADD}, + {SN_FusedAddRoundedHalving, OP_XOP_OVR_X_X_X, INTRINS_AARCH64_ADV_SIMD_SRHADD, OP_XOP_OVR_X_X_X, INTRINS_AARCH64_ADV_SIMD_URHADD}, + {SN_FusedMultiplyAdd, OP_ARM64_FMADD}, + {SN_FusedMultiplyAddNegatedScalar, OP_ARM64_FNMADD_SCALAR}, + {SN_FusedMultiplyAddScalar, OP_ARM64_FMADD_SCALAR}, + {SN_FusedMultiplySubtract, OP_ARM64_FMSUB}, + {SN_FusedMultiplySubtractNegatedScalar, OP_ARM64_FNMSUB_SCALAR}, + {SN_FusedMultiplySubtractScalar, OP_ARM64_FMSUB_SCALAR}, + {SN_FusedSubtractHalving, OP_XOP_OVR_X_X_X, INTRINS_AARCH64_ADV_SIMD_SHSUB, OP_XOP_OVR_X_X_X, INTRINS_AARCH64_ADV_SIMD_UHSUB}, {SN_Insert}, {SN_InsertScalar}, {SN_LeadingSignCount, OP_XOP_OVR_X_X, INTRINS_AARCH64_ADV_SIMD_CLS}, @@ -1369,7 +1378,6 @@ emit_arm64_intrinsics ( case SN_Insert: { int insert_op = 0; int extract_op = 0; - printf ("XXXih: arg0_type = %d\n", arg0_type); switch (arg0_type) { case MONO_TYPE_I1: case MONO_TYPE_U1: insert_op = OP_XINSERT_I1; extract_op = OP_EXTRACT_I1; break; case MONO_TYPE_I2: case MONO_TYPE_U2: insert_op = OP_XINSERT_I2; extract_op = OP_EXTRACT_I2; break; From 0c5047c50ae383aef21df5a24577f6907df4f2d4 Mon Sep 17 00:00:00 2001 From: Imran Hameed Date: Sat, 6 Mar 2021 16:38:12 -0800 Subject: [PATCH 19/58] Add extract --- src/mono/mono/mini/llvm-intrinsics.h | 4 ++ src/mono/mono/mini/mini-llvm.c | 83 +++++++++++++++++++++++++++- src/mono/mono/mini/mini-ops.h | 16 ++++++ src/mono/mono/mini/simd-intrinsics.c | 32 ++++++++++- 4 files changed, 131 insertions(+), 4 deletions(-) diff --git a/src/mono/mono/mini/llvm-intrinsics.h b/src/mono/mono/mini/llvm-intrinsics.h index 88e32683a1625e..e76b292276cb86 100644 --- a/src/mono/mono/mini/llvm-intrinsics.h +++ b/src/mono/mono/mini/llvm-intrinsics.h @@ -307,6 +307,10 @@ INTRINS_OVR_2_ARG(AARCH64_ADV_SIMD_ABS_COMPARE_LT_DOUBLE, aarch64_neon_facgt, ss INTRINS_OVR_2_ARG(AARCH64_ADV_SIMD_ABS_COMPARE_LTE_FLOAT, aarch64_neon_facge, sse_i4_t, sse_r4_t) INTRINS_OVR_2_ARG(AARCH64_ADV_SIMD_ABS_COMPARE_LTE_DOUBLE, aarch64_neon_facge, sse_i4_t, sse_r8_t) +INTRINS_OVR_TAG(AARCH64_ADV_SIMD_SQXTUN, aarch64_neon_sqxtun, V64 | I1 | I2 | I4) +INTRINS_OVR_TAG(AARCH64_ADV_SIMD_SQXTN, aarch64_neon_sqxtn, V64 | I1 | I2 | I4) +INTRINS_OVR_TAG(AARCH64_ADV_SIMD_UQXTN, aarch64_neon_uqxtn, V64 | I1 | I2 | I4) + INTRINS_OVR_TAG(AARCH64_ADV_SIMD_SRHADD, aarch64_neon_srhadd, V64 | V128 | I1 | I2 | I4) INTRINS_OVR_TAG(AARCH64_ADV_SIMD_URHADD, aarch64_neon_urhadd, V64 | V128 | I1 | I2 | I4) diff --git a/src/mono/mono/mini/mini-llvm.c b/src/mono/mono/mini/mini-llvm.c index 943deba63d9d3a..30f53d428d3964 100644 --- a/src/mono/mono/mini/mini-llvm.c +++ b/src/mono/mono/mini/mini-llvm.c @@ -9186,13 +9186,55 @@ process_bb (EmitContext *ctx, MonoBasicBlock *bb) #endif #if defined(TARGET_ARM64) + case OP_EXTRACT_VAR_I1: + case OP_EXTRACT_VAR_U1: + case OP_EXTRACT_VAR_I2: + case OP_EXTRACT_VAR_U2: + case OP_EXTRACT_VAR_I4: + case OP_EXTRACT_VAR_R4: + case OP_EXTRACT_VAR_R8: + case OP_EXTRACT_VAR_I8: + case OP_EXTRACT_U1: case OP_EXTRACT_I1: + case OP_EXTRACT_U2: case OP_EXTRACT_I2: case OP_EXTRACT_I4: case OP_EXTRACT_I8: case OP_EXTRACT_R4: case OP_EXTRACT_R8: { - values [ins->dreg] = LLVMBuildExtractElement (builder, lhs, const_int32 (ins->inst_c0), "extract"); + gboolean sext = FALSE; + gboolean zext = FALSE; + switch (ins->opcode) { + case OP_EXTRACT_U1: case OP_EXTRACT_U2: zext = TRUE; break; + case OP_EXTRACT_I1: case OP_EXTRACT_I2: sext = TRUE; break; + case OP_EXTRACT_VAR_U1: case OP_EXTRACT_VAR_U2: zext = TRUE; break; + case OP_EXTRACT_VAR_I1: case OP_EXTRACT_VAR_I2: sext = TRUE; break; + } + LLVMValueRef element_ix = NULL; + switch (ins->opcode) { + case OP_EXTRACT_VAR_I1: + case OP_EXTRACT_VAR_U1: + case OP_EXTRACT_VAR_I2: + case OP_EXTRACT_VAR_U2: + case OP_EXTRACT_VAR_I4: + case OP_EXTRACT_VAR_R4: + case OP_EXTRACT_VAR_R8: + case OP_EXTRACT_VAR_I8: + element_ix = rhs; + break; + default: + element_ix = const_int32 (ins->inst_c0); + } + LLVMValueRef result = LLVMBuildExtractElement (builder, lhs, element_ix, "extract"); + /* TODO: Scalar types smaller than i32 seem to be + * normalized to i32 via zero or sign extension. + * Is this still necessary? + */ + if (zext) + result = LLVMBuildZExt (builder, result, i4_t, "extract_zext"); + else if (sext) + result = LLVMBuildSExt (builder, result, i4_t, "extract_sext"); + values [ins->dreg] = result; break; } case OP_XOP_I4_I4: @@ -9326,10 +9368,14 @@ process_bb (EmitContext *ctx, MonoBasicBlock *bb) case OP_XINSERT_I8: case OP_XINSERT_R4: case OP_XINSERT_R8: { + /* TODO: Scalar types smaller than i32 seem to be + * normalized to i32 via zero or sign extension. + * Is this still necessary? + */ LLVMTypeRef t = LLVMTypeOf (lhs); LLVMTypeRef elem_t = LLVMGetElementType (t); MonoTypeEnum primty = inst_c1_type (ins); - LLVMValueRef val = convert_full(ctx, rhs, elem_t, primitive_type_is_unsigned (primty)); + LLVMValueRef val = convert_full (ctx, rhs, elem_t, primitive_type_is_unsigned (primty)); LLVMValueRef result = LLVMBuildInsertElement (builder, lhs, val, arg3, "xinsert"); values [ins->dreg] = result; break; @@ -9382,6 +9428,39 @@ process_bb (EmitContext *ctx, MonoBasicBlock *bb) values [ins->dreg] = LLVMBuildTrunc (builder, hi64, LLVMInt64Type (), ""); break; } + case OP_ARM64_SQXTUN2: + case OP_ARM64_UQXTN2: + case OP_ARM64_SQXTN2: + case OP_ARM64_XTN: + case OP_ARM64_XTN2: { + llvm_ovr_tag_t ovr_tag = ovr_tag_from_mono_vector_class (ins->klass); + gboolean high = FALSE; + int iid = 0; + switch (ins->opcode) { + case OP_ARM64_SQXTUN2: high = TRUE; iid = INTRINS_AARCH64_ADV_SIMD_SQXTUN; break; + case OP_ARM64_UQXTN2: high = TRUE; iid = INTRINS_AARCH64_ADV_SIMD_UQXTN; break; + case OP_ARM64_SQXTN2: high = TRUE; iid = INTRINS_AARCH64_ADV_SIMD_SQXTN; break; + case OP_ARM64_XTN2: high = TRUE; break; + } + LLVMValueRef result = lhs; + if (high) { + result = rhs; + ovr_tag = ovr_tag_smaller_vector (ovr_tag); + } + LLVMTypeRef t = LLVMTypeOf (result); + LLVMTypeRef elem_t = LLVMGetElementType (t); + unsigned int elems = LLVMGetVectorSize (t); + unsigned int elem_bits = mono_llvm_get_prim_size_bits (elem_t); + LLVMTypeRef result_t = LLVMVectorType (LLVMIntType (elem_bits / 2), elems); + if (iid != 0) + result = call_overloaded_intrins (ctx, iid, ovr_tag, &result, ""); + else + result = LLVMBuildTrunc (builder, result, result_t, "arm64_xtn"); + if (high) + result = concatenate_vectors (ctx, lhs, result); + values [ins->dreg] = result; + break; + } case OP_ARM64_CLZ: { llvm_ovr_tag_t ovr_tag = ovr_tag_from_mono_vector_class (ins->klass); LLVMValueRef args [] = { lhs, const_int1 (0) }; diff --git a/src/mono/mono/mini/mini-ops.h b/src/mono/mono/mini/mini-ops.h index 2b57d4bb25e281..8f8a44c678e322 100644 --- a/src/mono/mono/mini/mini-ops.h +++ b/src/mono/mono/mini/mini-ops.h @@ -1736,6 +1736,22 @@ MINI_OP3(OP_ARM64_FMADD, "arm64_fmadd_scalar", XREG, XREG, XREG, XREG) MINI_OP3(OP_ARM64_FMADD_SCALAR, "arm64_fmadd_scalar", XREG, XREG, XREG, XREG) MINI_OP3(OP_ARM64_FNMADD_SCALAR, "arm64_fnmadd_scalar", XREG, XREG, XREG, XREG) +MINI_OP(OP_ARM64_XTN, "arm64_xtn", XREG, XREG, NONE) +MINI_OP(OP_ARM64_XTN2, "arm64_xtn2", XREG, XREG, XREG) + +MINI_OP(OP_ARM64_SQXTN2, "arm64_sqxtn2", XREG, XREG, XREG) +MINI_OP(OP_ARM64_UQXTN2, "arm64_uqxtn2", XREG, XREG, XREG) +MINI_OP(OP_ARM64_SQXTUN2, "arm64_sqxtun2", XREG, XREG, XREG) + MINI_OP(OP_ARM64_SELECT_SCALAR, "arm64_select_scalar", XREG, XREG, IREG) +MINI_OP(OP_EXTRACT_VAR_I1, "extract_var_i1", IREG, XREG, IREG) +MINI_OP(OP_EXTRACT_VAR_U1, "extract_var_u1", IREG, XREG, IREG) +MINI_OP(OP_EXTRACT_VAR_I2, "extract_var_i2", IREG, XREG, IREG) +MINI_OP(OP_EXTRACT_VAR_U2, "extract_var_u2", IREG, XREG, IREG) +MINI_OP(OP_EXTRACT_VAR_I4, "extract_var_i4", IREG, XREG, IREG) +MINI_OP(OP_EXTRACT_VAR_R4, "extract_var_r4", FREG, XREG, IREG) +MINI_OP(OP_EXTRACT_VAR_R8, "extract_var_r8", FREG, XREG, IREG) +MINI_OP(OP_EXTRACT_VAR_I8, "extract_var_i8", LREG, XREG, IREG) + #endif // TARGET_ARM64 diff --git a/src/mono/mono/mini/simd-intrinsics.c b/src/mono/mono/mini/simd-intrinsics.c index 0748493a88e266..88ccfdf7b55a51 100644 --- a/src/mono/mono/mini/simd-intrinsics.c +++ b/src/mono/mono/mini/simd-intrinsics.c @@ -1011,6 +1011,15 @@ static SimdIntrinsic advsimd_methods [] = { {SN_AbsoluteCompareGreaterThanOrEqual}, {SN_AbsoluteCompareLessThan}, {SN_AbsoluteCompareLessThanOrEqual}, + {SN_Extract}, + {SN_ExtractNarrowingLower, OP_ARM64_XTN}, + {SN_ExtractNarrowingSaturateLower, OP_XOP_OVR_X_X, INTRINS_AARCH64_ADV_SIMD_SQXTN, OP_XOP_OVR_X_X, INTRINS_AARCH64_ADV_SIMD_UQXTN}, + {SN_ExtractNarrowingSaturateUnsignedLower, OP_XOP_OVR_X_X, INTRINS_AARCH64_ADV_SIMD_SQXTUN}, + {SN_ExtractNarrowingSaturateUnsignedUpper, OP_ARM64_SQXTUN2}, + {SN_ExtractNarrowingSaturateUpper, OP_ARM64_SQXTN2, None, OP_ARM64_UQXTN2}, + {SN_ExtractNarrowingUpper, OP_ARM64_XTN2}, + {SN_Floor, OP_XOP_OVR_X_X, INTRINS_AARCH64_ADV_SIMD_FRINTM}, + {SN_FloorScalar, OP_XOP_OVR_SCALAR_X_X, INTRINS_AARCH64_ADV_SIMD_FRINTM}, {SN_FusedAddHalving, OP_XOP_OVR_X_X_X, INTRINS_AARCH64_ADV_SIMD_SHADD, OP_XOP_OVR_X_X_X, INTRINS_AARCH64_ADV_SIMD_UHADD}, {SN_FusedAddRoundedHalving, OP_XOP_OVR_X_X_X, INTRINS_AARCH64_ADV_SIMD_SRHADD, OP_XOP_OVR_X_X_X, INTRINS_AARCH64_ADV_SIMD_URHADD}, {SN_FusedMultiplyAdd, OP_ARM64_FMADD}, @@ -1374,13 +1383,32 @@ emit_arm64_intrinsics ( } return emit_simd_ins_for_sig (cfg, klass, OP_XOP_X_X, op, arg0_type, fsig, args); } + case SN_Extract: { + int extract_op = 0; + switch (arg0_type) { + case MONO_TYPE_I1: extract_op = OP_EXTRACT_VAR_U1; break; + case MONO_TYPE_U1: extract_op = OP_EXTRACT_VAR_I1; break; + case MONO_TYPE_I2: extract_op = OP_EXTRACT_VAR_U2; break; + case MONO_TYPE_U2: extract_op = OP_EXTRACT_VAR_I2; break; + case MONO_TYPE_I4: case MONO_TYPE_U4: extract_op = OP_EXTRACT_VAR_I4; break; + case MONO_TYPE_I8: case MONO_TYPE_U8: extract_op = OP_EXTRACT_VAR_I8; break; + case MONO_TYPE_R4: extract_op = OP_EXTRACT_VAR_R4; break; + case MONO_TYPE_R8: extract_op = OP_EXTRACT_VAR_R8; break; + default: g_assert_not_reached (); + } + MonoInst *ins = emit_simd_ins (cfg, klass, extract_op, args [0]->dreg, args [1]->dreg); + ins->inst_c1 = arg0_type; + return ins; + } case SN_InsertScalar: case SN_Insert: { int insert_op = 0; int extract_op = 0; switch (arg0_type) { - case MONO_TYPE_I1: case MONO_TYPE_U1: insert_op = OP_XINSERT_I1; extract_op = OP_EXTRACT_I1; break; - case MONO_TYPE_I2: case MONO_TYPE_U2: insert_op = OP_XINSERT_I2; extract_op = OP_EXTRACT_I2; break; + case MONO_TYPE_I1: insert_op = OP_XINSERT_I1; extract_op = OP_EXTRACT_U1; break; + case MONO_TYPE_U1: insert_op = OP_XINSERT_I1; extract_op = OP_EXTRACT_I1; break; + case MONO_TYPE_I2: insert_op = OP_XINSERT_I2; extract_op = OP_EXTRACT_U2; break; + case MONO_TYPE_U2: insert_op = OP_XINSERT_I2; extract_op = OP_EXTRACT_I2; break; case MONO_TYPE_I4: case MONO_TYPE_U4: insert_op = OP_XINSERT_I4; extract_op = OP_EXTRACT_I4; break; case MONO_TYPE_I8: case MONO_TYPE_U8: insert_op = OP_XINSERT_I8; extract_op = OP_EXTRACT_I8; break; case MONO_TYPE_R4: insert_op = OP_XINSERT_R4; extract_op = OP_EXTRACT_R4; break; From fabeff7dae7b81c51a1c5b4f7b26e57811af8c86 Mon Sep 17 00:00:00 2001 From: Imran Hameed Date: Sat, 6 Mar 2021 16:58:33 -0800 Subject: [PATCH 20/58] Add duplicate --- src/mono/mono/mini/simd-intrinsics.c | 51 +++++++++++++++++++++------- 1 file changed, 39 insertions(+), 12 deletions(-) diff --git a/src/mono/mono/mini/simd-intrinsics.c b/src/mono/mono/mini/simd-intrinsics.c index 88ccfdf7b55a51..e703dffc4c50e3 100644 --- a/src/mono/mono/mini/simd-intrinsics.c +++ b/src/mono/mono/mini/simd-intrinsics.c @@ -300,6 +300,22 @@ type_is_float (MonoType *type) { return FALSE; } +static int +type_to_extract_var_op (MonoTypeEnum type) +{ + switch (type) { + case MONO_TYPE_I1: return OP_EXTRACT_VAR_U1; + case MONO_TYPE_U1: return OP_EXTRACT_VAR_I1; + case MONO_TYPE_I2: return OP_EXTRACT_VAR_U2; + case MONO_TYPE_U2: return OP_EXTRACT_VAR_I2; + case MONO_TYPE_I4: case MONO_TYPE_U4: return OP_EXTRACT_VAR_I4; + case MONO_TYPE_I8: case MONO_TYPE_U8: return OP_EXTRACT_VAR_I8; + case MONO_TYPE_R4: return OP_EXTRACT_VAR_R4; + case MONO_TYPE_R8: return OP_EXTRACT_VAR_R8; + default: g_assert_not_reached (); + } +} + static int type_to_expand_op (MonoType *type) { @@ -1011,6 +1027,10 @@ static SimdIntrinsic advsimd_methods [] = { {SN_AbsoluteCompareGreaterThanOrEqual}, {SN_AbsoluteCompareLessThan}, {SN_AbsoluteCompareLessThanOrEqual}, + {SN_DuplicateSelectedScalarToVector128}, + {SN_DuplicateSelectedScalarToVector64}, + {SN_DuplicateToVector128}, + {SN_DuplicateToVector64}, {SN_Extract}, {SN_ExtractNarrowingLower, OP_ARM64_XTN}, {SN_ExtractNarrowingSaturateLower, OP_XOP_OVR_X_X, INTRINS_AARCH64_ADV_SIMD_SQXTN, OP_XOP_OVR_X_X, INTRINS_AARCH64_ADV_SIMD_UQXTN}, @@ -1383,19 +1403,26 @@ emit_arm64_intrinsics ( } return emit_simd_ins_for_sig (cfg, klass, OP_XOP_X_X, op, arg0_type, fsig, args); } - case SN_Extract: { - int extract_op = 0; - switch (arg0_type) { - case MONO_TYPE_I1: extract_op = OP_EXTRACT_VAR_U1; break; - case MONO_TYPE_U1: extract_op = OP_EXTRACT_VAR_I1; break; - case MONO_TYPE_I2: extract_op = OP_EXTRACT_VAR_U2; break; - case MONO_TYPE_U2: extract_op = OP_EXTRACT_VAR_I2; break; - case MONO_TYPE_I4: case MONO_TYPE_U4: extract_op = OP_EXTRACT_VAR_I4; break; - case MONO_TYPE_I8: case MONO_TYPE_U8: extract_op = OP_EXTRACT_VAR_I8; break; - case MONO_TYPE_R4: extract_op = OP_EXTRACT_VAR_R4; break; - case MONO_TYPE_R8: extract_op = OP_EXTRACT_VAR_R8; break; - default: g_assert_not_reached (); + case SN_DuplicateSelectedScalarToVector128: + case SN_DuplicateSelectedScalarToVector64: + case SN_DuplicateToVector64: + case SN_DuplicateToVector128: { + MonoClass *ret_klass = mono_class_from_mono_type_internal (fsig->ret); + MonoType *rtype = get_vector_t_elem_type (fsig->ret); + int scalar_src_reg = args [0]->dreg; + switch (id) { + case SN_DuplicateSelectedScalarToVector128: + case SN_DuplicateSelectedScalarToVector64: { + int extract_op = type_to_extract_var_op (arg0_type); + MonoInst *ins = emit_simd_ins (cfg, ret_klass, type_to_extract_var_op (rtype->type), args [0]->dreg, args [1]->dreg); + scalar_src_reg = ins->dreg; + break; } + } + return emit_simd_ins (cfg, ret_klass, type_to_expand_op (rtype), scalar_src_reg, -1); + } + case SN_Extract: { + int extract_op = type_to_extract_var_op (arg0_type); MonoInst *ins = emit_simd_ins (cfg, klass, extract_op, args [0]->dreg, args [1]->dreg); ins->inst_c1 = arg0_type; return ins; From 8d5e824b1c56d5a78c0550a3c01e641d45c034db Mon Sep 17 00:00:00 2001 From: Imran Hameed Date: Sat, 6 Mar 2021 17:05:44 -0800 Subject: [PATCH 21/58] Add divide --- src/mono/mono/mini/simd-intrinsics.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/mono/mono/mini/simd-intrinsics.c b/src/mono/mono/mini/simd-intrinsics.c index e703dffc4c50e3..10a1a19a0d0891 100644 --- a/src/mono/mono/mini/simd-intrinsics.c +++ b/src/mono/mono/mini/simd-intrinsics.c @@ -1027,6 +1027,8 @@ static SimdIntrinsic advsimd_methods [] = { {SN_AbsoluteCompareGreaterThanOrEqual}, {SN_AbsoluteCompareLessThan}, {SN_AbsoluteCompareLessThanOrEqual}, + {SN_Divide, OP_XBINOP, OP_FDIV}, + {SN_DivideScalar, OP_XBINOP_SCALAR, OP_FDIV}, {SN_DuplicateSelectedScalarToVector128}, {SN_DuplicateSelectedScalarToVector64}, {SN_DuplicateToVector128}, @@ -1413,7 +1415,6 @@ emit_arm64_intrinsics ( switch (id) { case SN_DuplicateSelectedScalarToVector128: case SN_DuplicateSelectedScalarToVector64: { - int extract_op = type_to_extract_var_op (arg0_type); MonoInst *ins = emit_simd_ins (cfg, ret_klass, type_to_extract_var_op (rtype->type), args [0]->dreg, args [1]->dreg); scalar_src_reg = ins->dreg; break; From 030032a89432cbb717e00251f0e15b914c479282 Mon Sep 17 00:00:00 2001 From: Imran Hameed Date: Sat, 6 Mar 2021 21:08:29 -0800 Subject: [PATCH 22/58] Add f32->{u32,i32} --- src/mono/mono/mini/llvm-intrinsics.h | 31 ++++--- src/mono/mono/mini/mini-llvm-cpp.cpp | 2 + src/mono/mono/mini/mini-llvm.c | 120 ++++++++++++++++++++++----- src/mono/mono/mini/mini-ops.h | 10 +++ src/mono/mono/mini/mini.c | 16 ++-- src/mono/mono/mini/mini.h | 73 ++++++++-------- src/mono/mono/mini/simd-intrinsics.c | 65 ++++++++++----- 7 files changed, 218 insertions(+), 99 deletions(-) diff --git a/src/mono/mono/mini/llvm-intrinsics.h b/src/mono/mono/mini/llvm-intrinsics.h index e76b292276cb86..c0bc985e31d390 100644 --- a/src/mono/mono/mini/llvm-intrinsics.h +++ b/src/mono/mono/mini/llvm-intrinsics.h @@ -12,15 +12,15 @@ * To define an overloaded intrinsic with three arguments */ -#define Scalar LLVM_Scalar -#define V64 LLVM_Vector64 -#define V128 LLVM_Vector128 -#define I1 LLVM_Int8 -#define I2 LLVM_Int16 -#define I4 LLVM_Int32 -#define I8 LLVM_Int64 -#define R4 LLVM_Float32 -#define R8 LLVM_Float64 +#define Scalar INTRIN_scalar +#define V64 INTRIN_vector64 +#define V128 INTRIN_vector128 +#define I1 INTRIN_int8 +#define I2 INTRIN_int16 +#define I4 INTRIN_int32 +#define I8 INTRIN_int64 +#define R4 INTRIN_float32 +#define R8 INTRIN_float64 INTRINS_OVR_2_ARG(MEMSET, memset, LLVMPointerType (LLVMInt8Type (), 0), LLVMInt32Type ()) INTRINS_OVR_3_ARG(MEMCPY, memcpy, LLVMPointerType (LLVMInt8Type (), 0), LLVMPointerType (LLVMInt8Type (), 0), LLVMInt32Type () ) @@ -307,6 +307,16 @@ INTRINS_OVR_2_ARG(AARCH64_ADV_SIMD_ABS_COMPARE_LT_DOUBLE, aarch64_neon_facgt, ss INTRINS_OVR_2_ARG(AARCH64_ADV_SIMD_ABS_COMPARE_LTE_FLOAT, aarch64_neon_facge, sse_i4_t, sse_r4_t) INTRINS_OVR_2_ARG(AARCH64_ADV_SIMD_ABS_COMPARE_LTE_DOUBLE, aarch64_neon_facge, sse_i4_t, sse_r8_t) +INTRINS_OVR_TAG_FTOI(AARCH64_ADV_SIMD_FCVTAS, aarch64_neon_fcvtas, Scalar | V64 | V128 | I4 | I8) +INTRINS_OVR_TAG_FTOI(AARCH64_ADV_SIMD_FCVTNS, aarch64_neon_fcvtns, Scalar | V64 | V128 | I4 | I8) +INTRINS_OVR_TAG_FTOI(AARCH64_ADV_SIMD_FCVTMS, aarch64_neon_fcvtms, Scalar | V64 | V128 | I4 | I8) +INTRINS_OVR_TAG_FTOI(AARCH64_ADV_SIMD_FCVTPS, aarch64_neon_fcvtps, Scalar | V64 | V128 | I4 | I8) + +INTRINS_OVR_TAG_FTOI(AARCH64_ADV_SIMD_FCVTAU, aarch64_neon_fcvtau, Scalar | V64 | V128 | I4 | I8) +INTRINS_OVR_TAG_FTOI(AARCH64_ADV_SIMD_FCVTNU, aarch64_neon_fcvtnu, Scalar | V64 | V128 | I4 | I8) +INTRINS_OVR_TAG_FTOI(AARCH64_ADV_SIMD_FCVTMU, aarch64_neon_fcvtmu, Scalar | V64 | V128 | I4 | I8) +INTRINS_OVR_TAG_FTOI(AARCH64_ADV_SIMD_FCVTPU, aarch64_neon_fcvtpu, Scalar | V64 | V128 | I4 | I8) + INTRINS_OVR_TAG(AARCH64_ADV_SIMD_SQXTUN, aarch64_neon_sqxtun, V64 | I1 | I2 | I4) INTRINS_OVR_TAG(AARCH64_ADV_SIMD_SQXTN, aarch64_neon_sqxtn, V64 | I1 | I2 | I4) INTRINS_OVR_TAG(AARCH64_ADV_SIMD_UQXTN, aarch64_neon_uqxtn, V64 | I1 | I2 | I4) @@ -358,7 +368,7 @@ INTRINS_OVR_TAG(AARCH64_ADV_SIMD_UMULL, aarch64_neon_umull, V128 | I2 | I4 | I8) INTRINS_OVR_TAG(AARCH64_ADV_SIMD_SQNEG, aarch64_neon_sqneg, V64 | V128 | I1 | I2 | I4 | I8) INTRINS_OVR_TAG(AARCH64_ADV_SIMD_PMUL, aarch64_neon_pmul, V64 | V128 | I1) -INTRINS_OVR_TAG(AARCH64_ADV_SIMD_PMULL, aarch64_neon_pmull, V128 | I2) +INTRINS_OVR(AARCH64_ADV_SIMD_PMULL, aarch64_neon_pmull, v128_i2_t) INTRINS_OVR_TAG(AARCH64_ADV_SIMD_CNT, ctpop, V64 | V128 | I8) @@ -414,6 +424,7 @@ INTRINS_OVR_TAG(AARCH64_ADV_SIMD_TBL1, aarch64_neon_tbl1, V64 | V128 | I1) #undef INTRINS_OVR_2_ARG #undef INTRINS_OVR_3_ARG #undef INTRINS_OVR_TAG +#undef INTRINS_OVR_TAG_FTOI #undef Scalar #undef V64 #undef V128 diff --git a/src/mono/mono/mini/mini-llvm-cpp.cpp b/src/mono/mono/mini/mini-llvm-cpp.cpp index 5d8c2abf0b0a83..dc8b81f57568e6 100644 --- a/src/mono/mono/mini/mini-llvm-cpp.cpp +++ b/src/mono/mono/mini/mini-llvm-cpp.cpp @@ -637,6 +637,7 @@ get_intrins_id (IntrinsicId id) #define INTRINS_OVR_2_ARG(id, llvm_id, ty1, ty2) case INTRINS_ ## id: intrins_id = Intrinsic::ID::llvm_id; break; #define INTRINS_OVR_3_ARG(id, llvm_id, ty1, ty2, ty3) case INTRINS_ ## id: intrins_id = Intrinsic::ID::llvm_id; break; #define INTRINS_OVR_TAG(id, llvm_id, ...) case INTRINS_ ## id: intrins_id = Intrinsic::ID::llvm_id; break; +#define INTRINS_OVR_TAG_FTOI(id, llvm_id, ...) case INTRINS_ ## id: intrins_id = Intrinsic::ID::llvm_id; break; #include "llvm-intrinsics.h" default: break; @@ -653,6 +654,7 @@ is_overloaded_intrins (IntrinsicId id) #define INTRINS_OVR_2_ARG(id, llvm_id, ty1, ty2) case INTRINS_ ## id: return true; #define INTRINS_OVR_3_ARG(id, llvm_id, ty1, ty2, ty3) case INTRINS_ ## id: return true; #define INTRINS_OVR_TAG(id, llvm_id, ...) case INTRINS_ ## id: return true; +#define INTRINS_OVR_TAG_FTOI(id, llvm_id, ...) case INTRINS_ ## id: return true; #include "llvm-intrinsics.h" default: break; diff --git a/src/mono/mono/mini/mini-llvm.c b/src/mono/mono/mini/mini-llvm.c index 30f53d428d3964..7ca649bc5bd95a 100644 --- a/src/mono/mono/mini/mini-llvm.c +++ b/src/mono/mono/mini/mini-llvm.c @@ -51,17 +51,6 @@ #error "The version of the mono llvm repository is too old." #endif -static LLVMTypeRef intrin_types [LLVM_VectorWidths][LLVM_ElementWidths]; - -static const llvm_ovr_tag_t intrin_arm64_ovr [] = { - #define INTRINS(sym, ...) 0, - #define INTRINS_OVR(sym, ...) 0, - #define INTRINS_OVR_2_ARG(sym, ...) 0, - #define INTRINS_OVR_3_ARG(sym, ...) 0, - #define INTRINS_OVR_TAG(sym, _, spec) spec, - #include "llvm-intrinsics.h" -}; - /* * Information associated by mono with LLVM modules. */ @@ -320,6 +309,34 @@ static void create_aot_info_var (MonoLLVMModule *module); static void set_invariant_load_flag (LLVMValueRef v); static void set_nonnull_load_flag (LLVMValueRef v); + +static LLVMTypeRef intrin_types [INTRIN_vectorwidths][INTRIN_elementwidths]; + +static const llvm_ovr_tag_t intrin_arm64_ovr [] = { + #define INTRINS(sym, ...) 0, + #define INTRINS_OVR(sym, ...) 0, + #define INTRINS_OVR_2_ARG(sym, ...) 0, + #define INTRINS_OVR_3_ARG(sym, ...) 0, + #define INTRINS_OVR_TAG(sym, _, spec) spec, + #define INTRINS_OVR_TAG_FTOI(sym, _, spec) spec, + #include "llvm-intrinsics.h" +}; + +static gboolean +is_ftoi_intrinsic(IntrinsicId id) { + switch (id) { + #define INTRINS(sym, ...) + #define INTRINS_OVR(sym, ...) + #define INTRINS_OVR_2_ARG(sym, ...) + #define INTRINS_OVR_3_ARG(sym, ...) + #define INTRINS_OVR_TAG(sym, _, spec) + #define INTRINS_OVR_TAG_FTOI(sym, _, spec) case INTRINS_ ## sym: + #include "llvm-intrinsics.h" + return TRUE; + } + return FALSE; +} + static inline void set_failure (EmitContext *ctx, const char *message) { @@ -7212,14 +7229,14 @@ process_bb (EmitContext *ctx, MonoBasicBlock *bb) LLVMValueRef rhs_int = convert (ctx, rhs, intermediate_t); LLVMValueRef result = NULL; switch (ins->inst_c0) { - case XBINOP_FORCEINT_Or: + case XBINOP_FORCEINT_or: result = LLVMBuildOr (builder, lhs_int, rhs_int, ""); break; - case XBINOP_FORCEINT_OrNot: + case XBINOP_FORCEINT_ornot: result = LLVMBuildNot (builder, rhs_int, ""); result = LLVMBuildOr (builder, result, lhs_int, ""); break; - case XBINOP_FORCEINT_Xor: + case XBINOP_FORCEINT_xor: result = LLVMBuildXor (builder, lhs_int, rhs_int, ""); break; } @@ -9380,6 +9397,58 @@ process_bb (EmitContext *ctx, MonoBasicBlock *bb) values [ins->dreg] = result; break; } + case OP_ARM64_UCVTF: + case OP_ARM64_SCVTF: + case OP_ARM64_UCVTF_SCALAR: + case OP_ARM64_SCVTF_SCALAR: { + LLVMTypeRef ret_t = simd_class_to_llvm_type (ctx, ins->klass); + gboolean scalar = FALSE; + gboolean is_unsigned = FALSE; + switch (ins->opcode) { + case OP_ARM64_UCVTF_SCALAR: scalar = TRUE; case OP_ARM64_UCVTF: is_unsigned = TRUE; break; + case OP_ARM64_SCVTF_SCALAR: scalar = TRUE; break; + } + LLVMValueRef result = lhs; + LLVMTypeRef cvt_t = ret_t; + if (scalar) { + result = scalar_from_vector (ctx, result); + cvt_t = LLVMGetElementType (ret_t); + } + if (is_unsigned) + result = LLVMBuildUIToFP (builder, result, cvt_t, "arm64_ucvtf"); + else + result = LLVMBuildSIToFP (builder, result, cvt_t, "arm64_scvtf"); + if (scalar) + result = vector_from_scalar_ty (ctx, ret_t, result); + values [ins->dreg] = result; + break; + } + case OP_ARM64_FCVTZS: + case OP_ARM64_FCVTZS_SCALAR: + case OP_ARM64_FCVTZU: + case OP_ARM64_FCVTZU_SCALAR: { + LLVMTypeRef ret_t = simd_class_to_llvm_type (ctx, ins->klass); + gboolean scalar = FALSE; + gboolean is_unsigned = FALSE; + switch (ins->opcode) { + case OP_ARM64_FCVTZU_SCALAR: scalar = TRUE; case OP_ARM64_FCVTZU: is_unsigned = TRUE; break; + case OP_ARM64_FCVTZS_SCALAR: scalar = TRUE; break; + } + LLVMValueRef result = lhs; + LLVMTypeRef cvt_t = ret_t; + if (scalar) { + result = scalar_from_vector (ctx, result); + cvt_t = LLVMGetElementType (ret_t); + } + if (is_unsigned) + result = LLVMBuildFPToUI (builder, result, cvt_t, "arm64_fcvtzu"); + else + result = LLVMBuildFPToSI (builder, result, cvt_t, "arm64_fcvtzs"); + if (scalar) + result = vector_from_scalar_ty (ctx, ret_t, result); + values [ins->dreg] = result; + break; + } case OP_ARM64_SELECT_SCALAR: { LLVMValueRef result = LLVMBuildExtractElement (builder, lhs, rhs, ""); LLVMTypeRef elem_t = LLVMTypeOf (result); @@ -9707,7 +9776,7 @@ process_bb (EmitContext *ctx, MonoBasicBlock *bb) LLVMValueRef val = lhs; if (high) val = extract_high_elements (ctx, val); - LLVMValueRef result = call_overloaded_intrins (ctx, INTRINS_AARCH64_ADV_SIMD_PMULL, LLVM_Vector128 | LLVM_Int16, &val, "arm64_pmull"); + LLVMValueRef result = call_intrins (ctx, INTRINS_AARCH64_ADV_SIMD_PMULL, &val, "arm64_pmull"); values [ins->dreg] = result; break; } @@ -10192,9 +10261,10 @@ process_bb (EmitContext *ctx, MonoBasicBlock *bb) args [i] = scalar_from_vector (ctx, args [i]); } LLVMValueRef result = call_overloaded_intrins (ctx, iid, ovr_tag, args, ""); - if (!arm64_fake_scalar_op) - result = vector_from_scalar (ctx, lhs, result); - else + if (!arm64_fake_scalar_op) { + LLVMTypeRef ret_t = simd_class_to_llvm_type (ctx, ins->klass); + result = vector_from_scalar_ty (ctx, ret_t, result); + } else result = keep_lowest_element (ctx, result); values [ins->dreg] = result; break; @@ -11518,13 +11588,16 @@ add_intrinsic (LLVMModuleRef module, int id) if (intrin_arm64_ovr [id] != 0) { llvm_ovr_tag_t spec = intrin_arm64_ovr [id]; - for (int vw = 0; vw < LLVM_VectorWidths; ++vw) { - for (int ew = 0; ew < LLVM_ElementWidths; ++ew) { - llvm_ovr_tag_t vec_bit = LLVM_Vector128 >> ((LLVM_VectorWidths - 1) - vw); - llvm_ovr_tag_t elem_bit = LLVM_Int8 << ew; + for (int vw = 0; vw < INTRIN_vectorwidths; ++vw) { + for (int ew = 0; ew < INTRIN_elementwidths; ++ew) { + llvm_ovr_tag_t vec_bit = INTRIN_vector128 >> ((INTRIN_vectorwidths - 1) - vw); + llvm_ovr_tag_t elem_bit = INTRIN_int8 << ew; llvm_ovr_tag_t test = vec_bit | elem_bit; if ((spec & test) == test) { - intrins = add_intrins1 (module, id, intrin_types [vw][ew]); + if (is_ftoi_intrinsic (id) && (elem_bit & (INTRIN_int32 | INTRIN_int64))) { + intrins = add_intrins2 (module, id, intrin_types [vw][ew], intrin_types [vw][ew + 2]); + } else + intrins = add_intrins1 (module, id, intrin_types [vw][ew]); int ovr_id = int_from_id_and_ovr_tag (id, test); g_hash_table_insert (intrins_id_to_intrins, GINT_TO_POINTER (ovr_id), intrins); } @@ -11540,6 +11613,7 @@ add_intrinsic (LLVMModuleRef module, int id) #define INTRINS_OVR_2_ARG(intrin_name, llvm_id, llvm_type1, llvm_type2) case INTRINS_ ## intrin_name: intrins = add_intrins2(module, id, llvm_type1, llvm_type2); break; #define INTRINS_OVR_3_ARG(intrin_name, llvm_id, llvm_type1, llvm_type2, llvm_type3) case INTRINS_ ## intrin_name: intrins = add_intrins3(module, id, llvm_type1, llvm_type2, llvm_type3); break; #define INTRINS_OVR_TAG(...) + #define INTRINS_OVR_TAG_FTOI(...) #include "llvm-intrinsics.h" default: diff --git a/src/mono/mono/mini/mini-ops.h b/src/mono/mono/mini/mini-ops.h index 8f8a44c678e322..5ca74725a2ba4c 100644 --- a/src/mono/mono/mini/mini-ops.h +++ b/src/mono/mono/mini/mini-ops.h @@ -1754,4 +1754,14 @@ MINI_OP(OP_EXTRACT_VAR_R4, "extract_var_r4", FREG, XREG, IREG) MINI_OP(OP_EXTRACT_VAR_R8, "extract_var_r8", FREG, XREG, IREG) MINI_OP(OP_EXTRACT_VAR_I8, "extract_var_i8", LREG, XREG, IREG) +MINI_OP(OP_ARM64_FCVTZU, "arm64_fcvtzu", XREG, XREG, NONE) +MINI_OP(OP_ARM64_FCVTZS, "arm64_fcvtzs", XREG, XREG, NONE) +MINI_OP(OP_ARM64_FCVTZU_SCALAR, "arm64_fcvtzu_scalar", XREG, XREG, NONE) +MINI_OP(OP_ARM64_FCVTZS_SCALAR, "arm64_fcvtzs_scalar", XREG, XREG, NONE) + +MINI_OP(OP_ARM64_UCVTF, "arm64_ucvtf", XREG, XREG, NONE) +MINI_OP(OP_ARM64_SCVTF, "arm64_scvtf", XREG, XREG, NONE) +MINI_OP(OP_ARM64_UCVTF_SCALAR, "arm64_ucvtf_scalar", XREG, XREG, NONE) +MINI_OP(OP_ARM64_SCVTF_SCALAR, "arm64_scvtf_scalar", XREG, XREG, NONE) + #endif // TARGET_ARM64 diff --git a/src/mono/mono/mini/mini.c b/src/mono/mono/mini/mini.c index 4c3a0ab09fb0ae..b3bb0feca073e6 100644 --- a/src/mono/mono/mini/mini.c +++ b/src/mono/mono/mini/mini.c @@ -4329,17 +4329,17 @@ ovr_tag_from_mono_vector_class (MonoClass *klass) { int size = mono_class_value_size (klass, NULL); llvm_ovr_tag_t ret = 0; switch (size) { - case 8: ret |= LLVM_Vector64; break; - case 16: ret |= LLVM_Vector128; break; + case 8: ret |= INTRIN_vector64; break; + case 16: ret |= INTRIN_vector128; break; } MonoType *etype = mono_class_get_context (klass)->class_inst->type_argv [0]; switch (etype->type) { - case MONO_TYPE_I1: case MONO_TYPE_U1: ret |= LLVM_Int8; break; - case MONO_TYPE_I2: case MONO_TYPE_U2: ret |= LLVM_Int16; break; - case MONO_TYPE_I4: case MONO_TYPE_U4: ret |= LLVM_Int32; break; - case MONO_TYPE_I8: case MONO_TYPE_U8: ret |= LLVM_Int64; break; - case MONO_TYPE_R4: ret |= LLVM_Float32; break; - case MONO_TYPE_R8: ret |= LLVM_Float64; break; + case MONO_TYPE_I1: case MONO_TYPE_U1: ret |= INTRIN_int8; break; + case MONO_TYPE_I2: case MONO_TYPE_U2: ret |= INTRIN_int16; break; + case MONO_TYPE_I4: case MONO_TYPE_U4: ret |= INTRIN_int32; break; + case MONO_TYPE_I8: case MONO_TYPE_U8: ret |= INTRIN_int64; break; + case MONO_TYPE_R4: ret |= INTRIN_float32; break; + case MONO_TYPE_R8: ret |= INTRIN_float64; break; } return ret; } diff --git a/src/mono/mono/mini/mini.h b/src/mono/mono/mini/mini.h index 91c3b4b3b49e89..bdcc412a48bb26 100644 --- a/src/mono/mono/mini/mini.h +++ b/src/mono/mono/mini/mini.h @@ -2999,42 +2999,27 @@ typedef enum { SIMD_OP_ARM64_PMULL64_UPPER, } SimdOp; -/* An intrinsic id. The lower 23 bits are used to store a mono-specific ID. The - * next 9 bits store overload tag bits. In the configuration of LLVM 9 we use, - * there are 7017 total intrinsics defined in IntrinsicEnums.inc, so only 13 - * bits are needed to label each intrinsic overload group. - */ -typedef enum { -#define INTRINS(id, llvm_id) INTRINS_ ## id, -#define INTRINS_OVR(id, llvm_id, ty) INTRINS_ ## id, -#define INTRINS_OVR_2_ARG(id, llvm_id, ty1, ty2) INTRINS_ ## id, -#define INTRINS_OVR_3_ARG(id, llvm_id, ty1, ty2, ty3) INTRINS_ ## id, -#define INTRINS_OVR_TAG(id, ...) INTRINS_ ## id, -#include "llvm-intrinsics.h" - INTRINS_NUM -} IntrinsicId; - -typedef enum { - XBINOP_FORCEINT_Not, - XBINOP_FORCEINT_Or, - XBINOP_FORCEINT_OrNot, - XBINOP_FORCEINT_Xor, -} XBinOpId; +enum { + XBINOP_FORCEINT_not, + XBINOP_FORCEINT_or, + XBINOP_FORCEINT_ornot, + XBINOP_FORCEINT_xor, +}; enum { - LLVM_Scalar = 1 << 0, - LLVM_Vector64 = 1 << 1, - LLVM_Vector128 = 1 << 2, - LLVM_VectorWidths = 3, - LLVM_VectorMask = 0x7, - - LLVM_Int8 = 1 << 3, - LLVM_Int16 = 1 << 4, - LLVM_Int32 = 1 << 5, - LLVM_Int64 = 1 << 6, - LLVM_Float32 = 1 << 7, - LLVM_Float64 = 1 << 8, - LLVM_ElementWidths = 6, + INTRIN_scalar = 1 << 0, + INTRIN_vector64 = 1 << 1, + INTRIN_vector128 = 1 << 2, + INTRIN_vectorwidths = 3, + INTRIN_vectormask = 0x7, + + INTRIN_int8 = 1 << 3, + INTRIN_int16 = 1 << 4, + INTRIN_int32 = 1 << 5, + INTRIN_int64 = 1 << 6, + INTRIN_float32 = 1 << 7, + INTRIN_float64 = 1 << 8, + INTRIN_elementwidths = 6, }; typedef uint16_t llvm_ovr_tag_t; @@ -3042,13 +3027,13 @@ typedef uint16_t llvm_ovr_tag_t; static inline llvm_ovr_tag_t ovr_tag_force_scalar (llvm_ovr_tag_t tag) { - return (tag & ~LLVM_VectorMask) | LLVM_Scalar; + return (tag & ~INTRIN_vectormask) | INTRIN_scalar; } static inline llvm_ovr_tag_t ovr_tag_smaller_vector (llvm_ovr_tag_t tag) { - return (tag & ~LLVM_VectorMask) | ((tag & LLVM_VectorMask) >> 1); + return (tag & ~INTRIN_vectormask) | ((tag & INTRIN_vectormask) >> 1); } llvm_ovr_tag_t @@ -3060,6 +3045,22 @@ int_from_id_and_ovr_tag (int id, llvm_ovr_tag_t ovr_tag) return (((int) ovr_tag) << 23) | id; } +/* An intrinsic id. The lower 23 bits are used to store a mono-specific ID. The + * next 8 bits store overload tag bits. In the configuration of LLVM 9 we use, + * there are 7017 total intrinsics defined in IntrinsicEnums.inc, so only 13 + * bits are needed to label each intrinsic overload group. + */ +typedef enum { +#define INTRINS(id, llvm_id) INTRINS_ ## id, +#define INTRINS_OVR(id, llvm_id, ty) INTRINS_ ## id, +#define INTRINS_OVR_2_ARG(id, llvm_id, ty1, ty2) INTRINS_ ## id, +#define INTRINS_OVR_3_ARG(id, llvm_id, ty1, ty2, ty3) INTRINS_ ## id, +#define INTRINS_OVR_TAG(id, ...) INTRINS_ ## id, +#define INTRINS_OVR_TAG_FTOI(id, ...) INTRINS_ ## id, +#include "llvm-intrinsics.h" + INTRINS_NUM +} IntrinsicId; + const char *mono_arch_xregname (int reg); MonoCPUFeatures mono_arch_get_cpu_features (void); diff --git a/src/mono/mono/mini/simd-intrinsics.c b/src/mono/mono/mini/simd-intrinsics.c index 10a1a19a0d0891..0cc2b51a0497e8 100644 --- a/src/mono/mono/mini/simd-intrinsics.c +++ b/src/mono/mono/mini/simd-intrinsics.c @@ -300,22 +300,6 @@ type_is_float (MonoType *type) { return FALSE; } -static int -type_to_extract_var_op (MonoTypeEnum type) -{ - switch (type) { - case MONO_TYPE_I1: return OP_EXTRACT_VAR_U1; - case MONO_TYPE_U1: return OP_EXTRACT_VAR_I1; - case MONO_TYPE_I2: return OP_EXTRACT_VAR_U2; - case MONO_TYPE_U2: return OP_EXTRACT_VAR_I2; - case MONO_TYPE_I4: case MONO_TYPE_U4: return OP_EXTRACT_VAR_I4; - case MONO_TYPE_I8: case MONO_TYPE_U8: return OP_EXTRACT_VAR_I8; - case MONO_TYPE_R4: return OP_EXTRACT_VAR_R4; - case MONO_TYPE_R8: return OP_EXTRACT_VAR_R8; - default: g_assert_not_reached (); - } -} - static int type_to_expand_op (MonoType *type) { @@ -970,6 +954,21 @@ emit_invalid_operation (MonoCompile *cfg, const char* message) #ifdef TARGET_ARM64 +static int +type_to_extract_var_op (MonoTypeEnum type) +{ + switch (type) { + case MONO_TYPE_I1: return OP_EXTRACT_VAR_U1; + case MONO_TYPE_U1: return OP_EXTRACT_VAR_I1; + case MONO_TYPE_I2: return OP_EXTRACT_VAR_U2; + case MONO_TYPE_U2: return OP_EXTRACT_VAR_I2; + case MONO_TYPE_I4: case MONO_TYPE_U4: return OP_EXTRACT_VAR_I4; + case MONO_TYPE_I8: case MONO_TYPE_U8: return OP_EXTRACT_VAR_I8; + case MONO_TYPE_R4: return OP_EXTRACT_VAR_R4; + case MONO_TYPE_R8: return OP_EXTRACT_VAR_R8; + default: g_assert_not_reached (); + } +} static SimdIntrinsic armbase_methods [] = { {SN_LeadingSignCount}, @@ -1027,6 +1026,28 @@ static SimdIntrinsic advsimd_methods [] = { {SN_AbsoluteCompareGreaterThanOrEqual}, {SN_AbsoluteCompareLessThan}, {SN_AbsoluteCompareLessThanOrEqual}, + {SN_ConvertToInt32RoundAwayFromZero, OP_XOP_OVR_X_X, INTRINS_AARCH64_ADV_SIMD_FCVTAS}, + {SN_ConvertToInt32RoundAwayFromZeroScalar, OP_XOP_OVR_SCALAR_X_X, INTRINS_AARCH64_ADV_SIMD_FCVTAS}, + {SN_ConvertToInt32RoundToEven, OP_XOP_OVR_X_X, INTRINS_AARCH64_ADV_SIMD_FCVTNS}, + {SN_ConvertToInt32RoundToEvenScalar, OP_XOP_OVR_SCALAR_X_X, INTRINS_AARCH64_ADV_SIMD_FCVTNS}, + {SN_ConvertToInt32RoundToNegativeInfinity, OP_XOP_OVR_X_X, INTRINS_AARCH64_ADV_SIMD_FCVTMS}, + {SN_ConvertToInt32RoundToNegativeInfinityScalar, OP_XOP_OVR_SCALAR_X_X, INTRINS_AARCH64_ADV_SIMD_FCVTMS}, + {SN_ConvertToInt32RoundToPositiveInfinity, OP_XOP_OVR_X_X, INTRINS_AARCH64_ADV_SIMD_FCVTPS}, + {SN_ConvertToInt32RoundToPositiveInfinityScalar, OP_XOP_OVR_SCALAR_X_X, INTRINS_AARCH64_ADV_SIMD_FCVTPS}, + {SN_ConvertToInt32RoundToZero, OP_ARM64_FCVTZS}, + {SN_ConvertToInt32RoundToZeroScalar, OP_ARM64_FCVTZS_SCALAR}, + {SN_ConvertToSingle, OP_ARM64_SCVTF, None, OP_ARM64_UCVTF}, + {SN_ConvertToSingleScalar, OP_ARM64_SCVTF_SCALAR, None, OP_ARM64_UCVTF_SCALAR}, + {SN_ConvertToUInt32RoundAwayFromZero, OP_XOP_OVR_X_X, INTRINS_AARCH64_ADV_SIMD_FCVTAU}, + {SN_ConvertToUInt32RoundAwayFromZeroScalar, OP_XOP_OVR_SCALAR_X_X, INTRINS_AARCH64_ADV_SIMD_FCVTAU}, + {SN_ConvertToUInt32RoundToEven, OP_XOP_OVR_X_X, INTRINS_AARCH64_ADV_SIMD_FCVTNU}, + {SN_ConvertToUInt32RoundToEvenScalar, OP_XOP_OVR_SCALAR_X_X, INTRINS_AARCH64_ADV_SIMD_FCVTNU}, + {SN_ConvertToUInt32RoundToNegativeInfinity, OP_XOP_OVR_X_X, INTRINS_AARCH64_ADV_SIMD_FCVTMU}, + {SN_ConvertToUInt32RoundToNegativeInfinityScalar, OP_XOP_OVR_SCALAR_X_X, INTRINS_AARCH64_ADV_SIMD_FCVTMU}, + {SN_ConvertToUInt32RoundToPositiveInfinity, OP_XOP_OVR_X_X, INTRINS_AARCH64_ADV_SIMD_FCVTPU}, + {SN_ConvertToUInt32RoundToPositiveInfinityScalar, OP_XOP_OVR_SCALAR_X_X, INTRINS_AARCH64_ADV_SIMD_FCVTPU}, + {SN_ConvertToUInt32RoundToZero, OP_ARM64_FCVTZU}, + {SN_ConvertToUInt32RoundToZeroScalar, OP_ARM64_FCVTZU_SCALAR}, {SN_Divide, OP_XBINOP, OP_FDIV}, {SN_DivideScalar, OP_XBINOP_SCALAR, OP_FDIV}, {SN_DuplicateSelectedScalarToVector128}, @@ -1122,9 +1143,9 @@ static SimdIntrinsic advsimd_methods [] = { {SN_NegateSaturate, OP_XOP_OVR_X_X, INTRINS_AARCH64_ADV_SIMD_SQNEG}, {SN_NegateSaturateScalar}, {SN_NegateScalar, OP_ARM64_XNEG_SCALAR}, - {SN_Not, OP_XBINOP_FORCEINT, XBINOP_FORCEINT_Not}, - {SN_Or, OP_XBINOP_FORCEINT, XBINOP_FORCEINT_Or}, - {SN_OrNot, OP_XBINOP_FORCEINT, XBINOP_FORCEINT_OrNot}, + {SN_Not, OP_XBINOP_FORCEINT, XBINOP_FORCEINT_not}, + {SN_Or, OP_XBINOP_FORCEINT, XBINOP_FORCEINT_or}, + {SN_OrNot, OP_XBINOP_FORCEINT, XBINOP_FORCEINT_ornot}, {SN_PolynomialMultiply, OP_XOP_OVR_X_X_X, INTRINS_AARCH64_ADV_SIMD_PMUL}, {SN_PolynomialMultiplyWideningLower, OP_ARM64_PMULL}, {SN_PolynomialMultiplyWideningUpper, OP_ARM64_PMULL2}, @@ -1237,7 +1258,7 @@ static SimdIntrinsic advsimd_methods [] = { {SN_UnzipOdd, OP_ARM64_UZP2}, {SN_VectorTableLookup, OP_XOP_OVR_X_X_X, INTRINS_AARCH64_ADV_SIMD_TBL1}, {SN_VectorTableLookupExtension, OP_XOP_OVR_X_X_X_X, INTRINS_AARCH64_ADV_SIMD_TBX1}, - {SN_Xor, OP_XBINOP_FORCEINT, XBINOP_FORCEINT_Xor}, + {SN_Xor, OP_XBINOP_FORCEINT, XBINOP_FORCEINT_xor}, {SN_ZeroExtendWideningLower, OP_ARM64_UXTL}, {SN_ZeroExtendWideningUpper, OP_ARM64_UXTL2}, {SN_ZipHigh, OP_ARM64_ZIP2}, @@ -1666,7 +1687,7 @@ static SimdIntrinsic sse_methods [] = { {SN_SubtractScalar, OP_SSE_SUBSS}, {SN_UnpackHigh, OP_SSE_UNPACKHI}, {SN_UnpackLow, OP_SSE_UNPACKLO}, - {SN_Xor, OP_XBINOP_FORCEINT, XBINOP_FORCEINT_Xor}, + {SN_Xor, OP_XBINOP_FORCEINT, XBINOP_FORCEINT_xor}, {SN_get_IsSupported} }; @@ -1778,7 +1799,7 @@ static SimdIntrinsic sse2_methods [] = { {SN_SumAbsoluteDifferences, OP_XOP_X_X_X, SIMD_OP_SSE_PSADBW}, {SN_UnpackHigh, OP_SSE_UNPACKHI}, {SN_UnpackLow, OP_SSE_UNPACKLO}, - {SN_Xor, OP_XBINOP_FORCEINT, XBINOP_FORCEINT_Xor}, + {SN_Xor, OP_XBINOP_FORCEINT, XBINOP_FORCEINT_xor}, {SN_get_IsSupported} }; From 069bebbcd68212155048767f69c5c59888358994 Mon Sep 17 00:00:00 2001 From: Imran Hameed Date: Sat, 6 Mar 2021 21:23:31 -0800 Subject: [PATCH 23/58] Add fp64->{u64,i64} --- src/mono/mono/mini/simd-intrinsics.c | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/src/mono/mono/mini/simd-intrinsics.c b/src/mono/mono/mini/simd-intrinsics.c index 0cc2b51a0497e8..434635fe3d522e 100644 --- a/src/mono/mono/mini/simd-intrinsics.c +++ b/src/mono/mono/mini/simd-intrinsics.c @@ -1036,6 +1036,16 @@ static SimdIntrinsic advsimd_methods [] = { {SN_ConvertToInt32RoundToPositiveInfinityScalar, OP_XOP_OVR_SCALAR_X_X, INTRINS_AARCH64_ADV_SIMD_FCVTPS}, {SN_ConvertToInt32RoundToZero, OP_ARM64_FCVTZS}, {SN_ConvertToInt32RoundToZeroScalar, OP_ARM64_FCVTZS_SCALAR}, + {SN_ConvertToInt64RoundAwayFromZero, OP_XOP_OVR_X_X, INTRINS_AARCH64_ADV_SIMD_FCVTAS}, + {SN_ConvertToInt64RoundAwayFromZeroScalar, OP_XOP_OVR_SCALAR_X_X, INTRINS_AARCH64_ADV_SIMD_FCVTAS}, + {SN_ConvertToInt64RoundToEven, OP_XOP_OVR_X_X, INTRINS_AARCH64_ADV_SIMD_FCVTNS}, + {SN_ConvertToInt64RoundToEvenScalar, OP_XOP_OVR_SCALAR_X_X, INTRINS_AARCH64_ADV_SIMD_FCVTNS}, + {SN_ConvertToInt64RoundToNegativeInfinity, OP_XOP_OVR_X_X, INTRINS_AARCH64_ADV_SIMD_FCVTMS}, + {SN_ConvertToInt64RoundToNegativeInfinityScalar, OP_XOP_OVR_SCALAR_X_X, INTRINS_AARCH64_ADV_SIMD_FCVTMS}, + {SN_ConvertToInt64RoundToPositiveInfinity, OP_XOP_OVR_X_X, INTRINS_AARCH64_ADV_SIMD_FCVTPS}, + {SN_ConvertToInt64RoundToPositiveInfinityScalar, OP_XOP_OVR_SCALAR_X_X, INTRINS_AARCH64_ADV_SIMD_FCVTPS}, + {SN_ConvertToInt64RoundToZero, OP_ARM64_FCVTZS}, + {SN_ConvertToInt64RoundToZeroScalar, OP_ARM64_FCVTZS_SCALAR}, {SN_ConvertToSingle, OP_ARM64_SCVTF, None, OP_ARM64_UCVTF}, {SN_ConvertToSingleScalar, OP_ARM64_SCVTF_SCALAR, None, OP_ARM64_UCVTF_SCALAR}, {SN_ConvertToUInt32RoundAwayFromZero, OP_XOP_OVR_X_X, INTRINS_AARCH64_ADV_SIMD_FCVTAU}, @@ -1048,6 +1058,16 @@ static SimdIntrinsic advsimd_methods [] = { {SN_ConvertToUInt32RoundToPositiveInfinityScalar, OP_XOP_OVR_SCALAR_X_X, INTRINS_AARCH64_ADV_SIMD_FCVTPU}, {SN_ConvertToUInt32RoundToZero, OP_ARM64_FCVTZU}, {SN_ConvertToUInt32RoundToZeroScalar, OP_ARM64_FCVTZU_SCALAR}, + {SN_ConvertToUInt64RoundAwayFromZero, OP_XOP_OVR_X_X, INTRINS_AARCH64_ADV_SIMD_FCVTAU}, + {SN_ConvertToUInt64RoundAwayFromZeroScalar, OP_XOP_OVR_SCALAR_X_X, INTRINS_AARCH64_ADV_SIMD_FCVTAU}, + {SN_ConvertToUInt64RoundToEven, OP_XOP_OVR_X_X, INTRINS_AARCH64_ADV_SIMD_FCVTNU}, + {SN_ConvertToUInt64RoundToEvenScalar, OP_XOP_OVR_SCALAR_X_X, INTRINS_AARCH64_ADV_SIMD_FCVTNU}, + {SN_ConvertToUInt64RoundToNegativeInfinity, OP_XOP_OVR_X_X, INTRINS_AARCH64_ADV_SIMD_FCVTMU}, + {SN_ConvertToUInt64RoundToNegativeInfinityScalar, OP_XOP_OVR_SCALAR_X_X, INTRINS_AARCH64_ADV_SIMD_FCVTMU}, + {SN_ConvertToUInt64RoundToPositiveInfinity, OP_XOP_OVR_X_X, INTRINS_AARCH64_ADV_SIMD_FCVTPU}, + {SN_ConvertToUInt64RoundToPositiveInfinityScalar, OP_XOP_OVR_SCALAR_X_X, INTRINS_AARCH64_ADV_SIMD_FCVTPU}, + {SN_ConvertToUInt64RoundToZero, OP_ARM64_FCVTZU}, + {SN_ConvertToUInt64RoundToZeroScalar, OP_ARM64_FCVTZU_SCALAR}, {SN_Divide, OP_XBINOP, OP_FDIV}, {SN_DivideScalar, OP_XBINOP_SCALAR, OP_FDIV}, {SN_DuplicateSelectedScalarToVector128}, From 9788f4a7fa408f87cbc4c8d5003bdfb25d61f72e Mon Sep 17 00:00:00 2001 From: Imran Hameed Date: Sat, 6 Mar 2021 22:55:09 -0800 Subject: [PATCH 24/58] arm64 fp conversions --- src/mono/mono/mini/llvm-intrinsics.h | 2 ++ src/mono/mono/mini/mini-llvm.c | 33 ++++++++++++++++++++++++++++ src/mono/mono/mini/mini-ops.h | 8 +++++++ src/mono/mono/mini/simd-intrinsics.c | 9 +++++++- 4 files changed, 51 insertions(+), 1 deletion(-) diff --git a/src/mono/mono/mini/llvm-intrinsics.h b/src/mono/mono/mini/llvm-intrinsics.h index c0bc985e31d390..57a7f57c76b8b6 100644 --- a/src/mono/mono/mini/llvm-intrinsics.h +++ b/src/mono/mono/mini/llvm-intrinsics.h @@ -307,6 +307,8 @@ INTRINS_OVR_2_ARG(AARCH64_ADV_SIMD_ABS_COMPARE_LT_DOUBLE, aarch64_neon_facgt, ss INTRINS_OVR_2_ARG(AARCH64_ADV_SIMD_ABS_COMPARE_LTE_FLOAT, aarch64_neon_facge, sse_i4_t, sse_r4_t) INTRINS_OVR_2_ARG(AARCH64_ADV_SIMD_ABS_COMPARE_LTE_DOUBLE, aarch64_neon_facge, sse_i4_t, sse_r8_t) +INTRINS_OVR_2_ARG(AARCH64_ADV_SIMD_FCVTXN, aarch64_neon_fcvtxn, v64_r4_t, v128_r8_t) + INTRINS_OVR_TAG_FTOI(AARCH64_ADV_SIMD_FCVTAS, aarch64_neon_fcvtas, Scalar | V64 | V128 | I4 | I8) INTRINS_OVR_TAG_FTOI(AARCH64_ADV_SIMD_FCVTNS, aarch64_neon_fcvtns, Scalar | V64 | V128 | I4 | I8) INTRINS_OVR_TAG_FTOI(AARCH64_ADV_SIMD_FCVTMS, aarch64_neon_fcvtms, Scalar | V64 | V128 | I4 | I8) diff --git a/src/mono/mono/mini/mini-llvm.c b/src/mono/mono/mini/mini-llvm.c index 7ca649bc5bd95a..bd178db863c11b 100644 --- a/src/mono/mono/mini/mini-llvm.c +++ b/src/mono/mono/mini/mini-llvm.c @@ -9397,6 +9397,39 @@ process_bb (EmitContext *ctx, MonoBasicBlock *bb) values [ins->dreg] = result; break; } + case OP_ARM64_FCVTL: + case OP_ARM64_FCVTL2: { + LLVMTypeRef ret_t = simd_class_to_llvm_type (ctx, ins->klass); + gboolean high = ins->opcode == OP_ARM64_FCVTL2; + LLVMValueRef result = lhs; + if (high) + result = extract_high_elements (ctx, result); + result = LLVMBuildFPExt (builder, result, ret_t, "arm64_fcvtl"); + values [ins->dreg] = result; + break; + } + case OP_ARM64_FCVTXN: + case OP_ARM64_FCVTXN2: + case OP_ARM64_FCVTN: + case OP_ARM64_FCVTN2: { + gboolean high = FALSE; + int iid = 0; + switch (ins->opcode) { + case OP_ARM64_FCVTXN2: high = TRUE; case OP_ARM64_FCVTXN: iid = INTRINS_AARCH64_ADV_SIMD_FCVTXN; break; + case OP_ARM64_FCVTN2: high = TRUE; break; + } + LLVMValueRef result = lhs; + if (high) + result = rhs; + if (iid) + result = call_intrins (ctx, iid, &result, ""); + else + result = LLVMBuildFPTrunc (builder, result, v64_r4_t, ""); + if (high) + result = concatenate_vectors (ctx, lhs, result); + values [ins->dreg] = result; + break; + } case OP_ARM64_UCVTF: case OP_ARM64_SCVTF: case OP_ARM64_UCVTF_SCALAR: diff --git a/src/mono/mono/mini/mini-ops.h b/src/mono/mono/mini/mini-ops.h index 5ca74725a2ba4c..00ce63530abc71 100644 --- a/src/mono/mono/mini/mini-ops.h +++ b/src/mono/mono/mini/mini-ops.h @@ -1764,4 +1764,12 @@ MINI_OP(OP_ARM64_SCVTF, "arm64_scvtf", XREG, XREG, NONE) MINI_OP(OP_ARM64_UCVTF_SCALAR, "arm64_ucvtf_scalar", XREG, XREG, NONE) MINI_OP(OP_ARM64_SCVTF_SCALAR, "arm64_scvtf_scalar", XREG, XREG, NONE) +MINI_OP(OP_ARM64_FCVTN, "arm64_fcvtn", XREG, XREG, NONE) +MINI_OP(OP_ARM64_FCVTN2, "arm64_fcvtn2", XREG, XREG, XREG) +MINI_OP(OP_ARM64_FCVTXN, "arm64_fcvtxn", XREG, XREG, NONE) +MINI_OP(OP_ARM64_FCVTXN2, "arm64_fcvtxn2", XREG, XREG, XREG) + +MINI_OP(OP_ARM64_FCVTL, "arm64_fcvtl", XREG, XREG, NONE) +MINI_OP(OP_ARM64_FCVTL2, "arm64_fcvtl2", XREG, XREG, NONE) + #endif // TARGET_ARM64 diff --git a/src/mono/mono/mini/simd-intrinsics.c b/src/mono/mono/mini/simd-intrinsics.c index 434635fe3d522e..86f941f7bfb525 100644 --- a/src/mono/mono/mini/simd-intrinsics.c +++ b/src/mono/mono/mini/simd-intrinsics.c @@ -129,7 +129,6 @@ lookup_intrins_info (SimdIntrinsic *intrinsics, int size, MonoMethod *cmethod) } } #endif - return (SimdIntrinsic *)mono_binary_search (cmethod->name, intrinsics, size / sizeof (SimdIntrinsic), sizeof (SimdIntrinsic), &simd_intrinsic_info_compare_by_name); } @@ -432,6 +431,7 @@ emit_hardware_intrinsics ( op = info->floating_op; c0 = info->floating_instc0; } + } support_probe_complete: if (id == SN_get_IsSupported) { @@ -1026,6 +1026,9 @@ static SimdIntrinsic advsimd_methods [] = { {SN_AbsoluteCompareGreaterThanOrEqual}, {SN_AbsoluteCompareLessThan}, {SN_AbsoluteCompareLessThanOrEqual}, + {SN_ConvertToDouble, OP_ARM64_SCVTF, None, OP_ARM64_UCVTF, None, OP_ARM64_FCVTL}, + {SN_ConvertToDoubleScalar, OP_ARM64_SCVTF_SCALAR, None, OP_ARM64_UCVTF_SCALAR}, + {SN_ConvertToDoubleUpper, OP_ARM64_FCVTL2}, {SN_ConvertToInt32RoundAwayFromZero, OP_XOP_OVR_X_X, INTRINS_AARCH64_ADV_SIMD_FCVTAS}, {SN_ConvertToInt32RoundAwayFromZeroScalar, OP_XOP_OVR_SCALAR_X_X, INTRINS_AARCH64_ADV_SIMD_FCVTAS}, {SN_ConvertToInt32RoundToEven, OP_XOP_OVR_X_X, INTRINS_AARCH64_ADV_SIMD_FCVTNS}, @@ -1047,7 +1050,11 @@ static SimdIntrinsic advsimd_methods [] = { {SN_ConvertToInt64RoundToZero, OP_ARM64_FCVTZS}, {SN_ConvertToInt64RoundToZeroScalar, OP_ARM64_FCVTZS_SCALAR}, {SN_ConvertToSingle, OP_ARM64_SCVTF, None, OP_ARM64_UCVTF}, + {SN_ConvertToSingleLower, OP_ARM64_FCVTN}, + {SN_ConvertToSingleRoundToOddLower, OP_ARM64_FCVTXN}, + {SN_ConvertToSingleRoundToOddUpper, OP_ARM64_FCVTXN2}, {SN_ConvertToSingleScalar, OP_ARM64_SCVTF_SCALAR, None, OP_ARM64_UCVTF_SCALAR}, + {SN_ConvertToSingleUpper, OP_ARM64_FCVTN2}, {SN_ConvertToUInt32RoundAwayFromZero, OP_XOP_OVR_X_X, INTRINS_AARCH64_ADV_SIMD_FCVTAU}, {SN_ConvertToUInt32RoundAwayFromZeroScalar, OP_XOP_OVR_SCALAR_X_X, INTRINS_AARCH64_ADV_SIMD_FCVTAU}, {SN_ConvertToUInt32RoundToEven, OP_XOP_OVR_X_X, INTRINS_AARCH64_ADV_SIMD_FCVTNU}, From fc6f37348e245d958969e758b050a9e0b93a6e58 Mon Sep 17 00:00:00 2001 From: Imran Hameed Date: Sun, 7 Mar 2021 00:20:42 -0800 Subject: [PATCH 25/58] Add comparisons --- src/mono/mono/mini/mini-llvm.c | 66 ++++++++++++++++++++++++++++ src/mono/mono/mini/mini-ops.h | 4 ++ src/mono/mono/mini/simd-intrinsics.c | 12 +++++ 3 files changed, 82 insertions(+) diff --git a/src/mono/mono/mini/mini-llvm.c b/src/mono/mono/mini/mini-llvm.c index bd178db863c11b..1cd77ec8aaab24 100644 --- a/src/mono/mono/mini/mini-llvm.c +++ b/src/mono/mono/mini/mini-llvm.c @@ -4763,6 +4763,25 @@ create_const_vector_2_i32 (int v0, int v1) return LLVMConstVector (mask, 2); } +static LLVMTypeRef +to_integral_vector_type (LLVMTypeRef t) +{ + unsigned int elems = LLVMGetVectorSize (t); + LLVMTypeRef elem_t = LLVMGetElementType (t); + unsigned int bits = mono_llvm_get_prim_size_bits (elem_t); + return LLVMVectorType (LLVMIntType (bits), elems); +} + +static LLVMValueRef +bitcast_to_integral (EmitContext *ctx, LLVMValueRef vec) +{ + LLVMTypeRef src_t = LLVMTypeOf (vec); + LLVMTypeRef dst_t = to_integral_vector_type (src_t); + if (dst_t != src_t) + return LLVMBuildBitCast (ctx->builder, vec, dst_t, "bc2i"); + return vec; +} + static LLVMValueRef extract_high_elements (EmitContext *ctx, LLVMValueRef src_vec) { @@ -9397,6 +9416,53 @@ process_bb (EmitContext *ctx, MonoBasicBlock *bb) values [ins->dreg] = result; break; } + case OP_XCOMPARE_FP_SCALAR: + case OP_XCOMPARE_FP: { + g_assert (LLVMTypeOf (lhs) == LLVMTypeOf (rhs)); + gboolean scalar = ins->opcode == OP_XCOMPARE_FP_SCALAR; + LLVMRealPredicate pred = fpcond_to_llvm_cond [ins->inst_c0]; + LLVMTypeRef ret_t = LLVMTypeOf (lhs); + LLVMTypeRef reti_t = to_integral_vector_type (ret_t); + LLVMValueRef args [] = { lhs, rhs }; + if (scalar) + for (int i = 0; i < 2; ++i) + args [i] = scalar_from_vector (ctx, args [i]); + LLVMValueRef result = LLVMBuildFCmp (builder, pred, args [0], args [1], "xcompare_fp"); + if (scalar) + result = vector_from_scalar_ty (ctx, LLVMVectorType (LLVMIntType (1), LLVMGetVectorSize (reti_t)), result); + result = LLVMBuildSExt (builder, result, reti_t, ""); + result = LLVMBuildBitCast (builder, result, ret_t, ""); + values [ins->dreg] = result; + break; + } + case OP_XCOMPARE_SCALAR: + case OP_XCOMPARE: { + g_assert (LLVMTypeOf (lhs) == LLVMTypeOf (rhs)); + gboolean scalar = ins->opcode == OP_XCOMPARE_SCALAR; + LLVMIntPredicate pred = cond_to_llvm_cond [ins->inst_c0]; + LLVMTypeRef ret_t = LLVMTypeOf (lhs); + LLVMValueRef args [] = { lhs, rhs }; + if (scalar) + for (int i = 0; i < 2; ++i) + args [i] = scalar_from_vector (ctx, args [i]); + LLVMValueRef result = LLVMBuildICmp (builder, pred, args [0], args [1], "xcompare"); + if (scalar) + result = vector_from_scalar_ty (ctx, LLVMVectorType (LLVMIntType (1), LLVMGetVectorSize (ret_t)), result); + values [ins->dreg] = LLVMBuildSExt (builder, result, ret_t, ""); + break; + } + case OP_ARM64_CMTST: { + LLVMTypeRef ret_t = simd_class_to_llvm_type (ctx, ins->klass); + LLVMValueRef l = bitcast_to_integral (ctx, lhs); + LLVMValueRef r = bitcast_to_integral (ctx, rhs); + LLVMValueRef result = LLVMBuildAnd (builder, l, r, "arm64_cmtst"); + LLVMTypeRef t = LLVMTypeOf (l); + result = LLVMBuildICmp (builder, LLVMIntNE, result, LLVMConstNull (t), ""); + result = LLVMBuildSExt (builder, result, t, ""); + result = convert (ctx, result, ret_t); + values [ins->dreg] = result; + break; + } case OP_ARM64_FCVTL: case OP_ARM64_FCVTL2: { LLVMTypeRef ret_t = simd_class_to_llvm_type (ctx, ins->klass); diff --git a/src/mono/mono/mini/mini-ops.h b/src/mono/mono/mini/mini-ops.h index 00ce63530abc71..efeb13425191ed 100644 --- a/src/mono/mono/mini/mini-ops.h +++ b/src/mono/mono/mini/mini-ops.h @@ -1528,7 +1528,9 @@ MINI_OP(OP_FILL_PROF_CALL_CTX, "fill_prof_call_ctx", NONE, IREG, NONE) MINI_OP(OP_XEQUAL, "xequal", IREG, XREG, XREG) /* Per element compate, inst_c0 contains a CompRelation */ MINI_OP(OP_XCOMPARE, "xcompare", XREG, XREG, XREG) +MINI_OP(OP_XCOMPARE_SCALAR, "xcompare_scalar", XREG, XREG, XREG) MINI_OP(OP_XCOMPARE_FP, "xcompare_fp", XREG, XREG, XREG) +MINI_OP(OP_XCOMPARE_FP_SCALAR, "xcompare_fp_scalar", XREG, XREG, XREG) /* * Generic SIMD operations, the rest of the JIT doesn't care about the exact operation. @@ -1772,4 +1774,6 @@ MINI_OP(OP_ARM64_FCVTXN2, "arm64_fcvtxn2", XREG, XREG, XREG) MINI_OP(OP_ARM64_FCVTL, "arm64_fcvtl", XREG, XREG, NONE) MINI_OP(OP_ARM64_FCVTL2, "arm64_fcvtl2", XREG, XREG, NONE) +MINI_OP(OP_ARM64_CMTST, "arm64_cmtst", XREG, XREG, XREG) + #endif // TARGET_ARM64 diff --git a/src/mono/mono/mini/simd-intrinsics.c b/src/mono/mono/mini/simd-intrinsics.c index 86f941f7bfb525..0d6930c9a609b5 100644 --- a/src/mono/mono/mini/simd-intrinsics.c +++ b/src/mono/mono/mini/simd-intrinsics.c @@ -1026,6 +1026,18 @@ static SimdIntrinsic advsimd_methods [] = { {SN_AbsoluteCompareGreaterThanOrEqual}, {SN_AbsoluteCompareLessThan}, {SN_AbsoluteCompareLessThanOrEqual}, + {SN_CompareEqual, OP_XCOMPARE, CMP_EQ, OP_XCOMPARE, CMP_EQ, OP_XCOMPARE_FP, CMP_EQ}, + {SN_CompareEqualScalar, OP_XCOMPARE, CMP_EQ, OP_XCOMPARE, CMP_EQ, OP_XCOMPARE_FP, CMP_EQ}, + {SN_CompareGreaterThan, OP_XCOMPARE, CMP_GT, OP_XCOMPARE, CMP_GT_UN, OP_XCOMPARE_FP, CMP_GT}, + {SN_CompareGreaterThanOrEqual, OP_XCOMPARE, CMP_GE, OP_XCOMPARE, CMP_GE_UN, OP_XCOMPARE_FP, CMP_GE}, + {SN_CompareGreaterThanOrEqualScalar, OP_XCOMPARE, CMP_GE, OP_XCOMPARE, CMP_GE_UN, OP_XCOMPARE_FP, CMP_GE}, + {SN_CompareGreaterThanScalar, OP_XCOMPARE, CMP_GT, OP_XCOMPARE, CMP_GT_UN, OP_XCOMPARE_FP, CMP_GT}, + {SN_CompareLessThan, OP_XCOMPARE, CMP_LT, OP_XCOMPARE, CMP_LT_UN, OP_XCOMPARE_FP, CMP_LT}, + {SN_CompareLessThanOrEqual, OP_XCOMPARE, CMP_LE, OP_XCOMPARE, CMP_LE_UN, OP_XCOMPARE_FP, CMP_LE}, + {SN_CompareLessThanOrEqualScalar, OP_XCOMPARE_SCALAR, CMP_LE, OP_XCOMPARE_SCALAR, CMP_LE_UN, OP_XCOMPARE_FP_SCALAR, CMP_LE}, + {SN_CompareLessThanScalar, OP_XCOMPARE_SCALAR, CMP_LT, OP_XCOMPARE_SCALAR, CMP_LT_UN, OP_XCOMPARE_FP_SCALAR, CMP_LT}, + {SN_CompareTest, OP_ARM64_CMTST}, + {SN_CompareTestScalar, OP_ARM64_CMTST}, {SN_ConvertToDouble, OP_ARM64_SCVTF, None, OP_ARM64_UCVTF, None, OP_ARM64_FCVTL}, {SN_ConvertToDoubleScalar, OP_ARM64_SCVTF_SCALAR, None, OP_ARM64_UCVTF_SCALAR}, {SN_ConvertToDoubleUpper, OP_ARM64_FCVTL2}, From 07a7bff8da99278846956f481e0aec35d661f888 Mon Sep 17 00:00:00 2001 From: Imran Hameed Date: Sun, 7 Mar 2021 00:31:01 -0800 Subject: [PATCH 26/58] Implement ceiling --- src/mono/mono/mini/simd-intrinsics.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/mono/mono/mini/simd-intrinsics.c b/src/mono/mono/mini/simd-intrinsics.c index 0d6930c9a609b5..2b346eae2ec52e 100644 --- a/src/mono/mono/mini/simd-intrinsics.c +++ b/src/mono/mono/mini/simd-intrinsics.c @@ -1026,6 +1026,8 @@ static SimdIntrinsic advsimd_methods [] = { {SN_AbsoluteCompareGreaterThanOrEqual}, {SN_AbsoluteCompareLessThan}, {SN_AbsoluteCompareLessThanOrEqual}, + {SN_Ceiling, OP_XOP_OVR_X_X, INTRINS_AARCH64_ADV_SIMD_FRINTP}, + {SN_CeilingScalar, OP_XOP_OVR_SCALAR_X_X, INTRINS_AARCH64_ADV_SIMD_FRINTP}, {SN_CompareEqual, OP_XCOMPARE, CMP_EQ, OP_XCOMPARE, CMP_EQ, OP_XCOMPARE_FP, CMP_EQ}, {SN_CompareEqualScalar, OP_XCOMPARE, CMP_EQ, OP_XCOMPARE, CMP_EQ, OP_XCOMPARE_FP, CMP_EQ}, {SN_CompareGreaterThan, OP_XCOMPARE, CMP_GT, OP_XCOMPARE, CMP_GT_UN, OP_XCOMPARE_FP, CMP_GT}, From f8594f7511715d94138fa91f0c52493b6bbe9d44 Mon Sep 17 00:00:00 2001 From: Imran Hameed Date: Sun, 7 Mar 2021 00:33:06 -0800 Subject: [PATCH 27/58] Revert some stray unrelated changes --- src/mono/mono/mini/mini-llvm.c | 8 ++++++++ src/mono/mono/mini/mini-ops.h | 7 ++++--- src/mono/mono/mini/simd-intrinsics.c | 4 ++-- 3 files changed, 14 insertions(+), 5 deletions(-) diff --git a/src/mono/mono/mini/mini-llvm.c b/src/mono/mono/mini/mini-llvm.c index 1cd77ec8aaab24..d1c749957f0a28 100644 --- a/src/mono/mono/mini/mini-llvm.c +++ b/src/mono/mono/mini/mini-llvm.c @@ -8161,6 +8161,14 @@ process_bb (EmitContext *ctx, MonoBasicBlock *bb) break; } + case OP_SSE_XOR: { + LLVMValueRef vec_lhs_i64 = convert (ctx, lhs, sse_i8_t); + LLVMValueRef vec_rhs_i64 = convert (ctx, rhs, sse_i8_t); + LLVMValueRef vec_and = LLVMBuildXor (builder, vec_lhs_i64, vec_rhs_i64, ""); + values [ins->dreg] = LLVMBuildBitCast (builder, vec_and, type_to_sse_type (ins->inst_c1), ""); + break; + } + case OP_SSE_AND: { LLVMValueRef vec_lhs_i64 = convert (ctx, lhs, sse_i8_t); LLVMValueRef vec_rhs_i64 = convert (ctx, rhs, sse_i8_t); diff --git a/src/mono/mono/mini/mini-ops.h b/src/mono/mono/mini/mini-ops.h index efeb13425191ed..6640c788f8fcc0 100644 --- a/src/mono/mono/mini/mini-ops.h +++ b/src/mono/mono/mini/mini-ops.h @@ -787,13 +787,13 @@ MINI_OP(OP_NOT_NULL, "not_null", NONE, IREG, NONE) #if defined(TARGET_X86) || defined(TARGET_AMD64) || defined(TARGET_WASM) || defined(TARGET_ARM64) +MINI_OP(OP_EXTRACT_I4, "extract_i4", IREG, XREG, NONE) MINI_OP(OP_ICONV_TO_R4_RAW, "iconv_to_r4_raw", FREG, IREG, NONE) -MINI_OP(OP_EXTRACT_I1, "extract_i1", IREG, XREG, NONE) -MINI_OP(OP_EXTRACT_U1, "extract_u1", IREG, XREG, NONE) MINI_OP(OP_EXTRACT_I2, "extract_i2", IREG, XREG, NONE) MINI_OP(OP_EXTRACT_U2, "extract_u2", IREG, XREG, NONE) -MINI_OP(OP_EXTRACT_I4, "extract_i4", IREG, XREG, NONE) +MINI_OP(OP_EXTRACT_I1, "extract_i1", IREG, XREG, NONE) +MINI_OP(OP_EXTRACT_U1, "extract_u1", IREG, XREG, NONE) MINI_OP(OP_EXTRACT_R4, "extract_r4", FREG, XREG, NONE) MINI_OP(OP_EXTRACT_R8, "extract_r8", FREG, XREG, NONE) MINI_OP(OP_EXTRACT_I8, "extract_i8", LREG, XREG, NONE) @@ -1022,6 +1022,7 @@ MINI_OP(OP_SSE_UNPACKHI, "sse_unpackhi", XREG, XREG, XREG) MINI_OP(OP_SSE_SHUFFLE, "sse_shuffle", XREG, XREG, XREG) MINI_OP(OP_SSE_AND, "sse_and", XREG, XREG, XREG) MINI_OP(OP_SSE_OR, "sse_or", XREG, XREG, XREG) +MINI_OP(OP_SSE_XOR, "sse_xor", XREG, XREG, XREG) MINI_OP(OP_SSE_ANDN, "sse_andn", XREG, XREG, XREG) MINI_OP(OP_SSE_ADDSS, "sse_addss", XREG, XREG, XREG) MINI_OP(OP_SSE_SUBSS, "sse_subss", XREG, XREG, XREG) diff --git a/src/mono/mono/mini/simd-intrinsics.c b/src/mono/mono/mini/simd-intrinsics.c index 2b346eae2ec52e..3f1f1089407684 100644 --- a/src/mono/mono/mini/simd-intrinsics.c +++ b/src/mono/mono/mini/simd-intrinsics.c @@ -1728,7 +1728,7 @@ static SimdIntrinsic sse_methods [] = { {SN_SubtractScalar, OP_SSE_SUBSS}, {SN_UnpackHigh, OP_SSE_UNPACKHI}, {SN_UnpackLow, OP_SSE_UNPACKLO}, - {SN_Xor, OP_XBINOP_FORCEINT, XBINOP_FORCEINT_xor}, + {SN_Xor, OP_SSE_XOR}, {SN_get_IsSupported} }; @@ -1840,7 +1840,7 @@ static SimdIntrinsic sse2_methods [] = { {SN_SumAbsoluteDifferences, OP_XOP_X_X_X, SIMD_OP_SSE_PSADBW}, {SN_UnpackHigh, OP_SSE_UNPACKHI}, {SN_UnpackLow, OP_SSE_UNPACKLO}, - {SN_Xor, OP_XBINOP_FORCEINT, XBINOP_FORCEINT_xor}, + {SN_Xor, OP_SSE_XOR}, {SN_get_IsSupported} }; From f37685d1b49046e56879b773ec44915c9d20285b Mon Sep 17 00:00:00 2001 From: Imran Hameed Date: Sun, 7 Mar 2021 00:53:30 -0800 Subject: [PATCH 28/58] Implement more bitwise operations --- src/mono/mono/mini/mini-llvm.c | 33 ++++++++++++++++++++++++++++ src/mono/mono/mini/mini-ops.h | 5 +++++ src/mono/mono/mini/mini.h | 2 +- src/mono/mono/mini/simd-intrinsics.c | 5 ++++- 4 files changed, 43 insertions(+), 2 deletions(-) diff --git a/src/mono/mono/mini/mini-llvm.c b/src/mono/mono/mini/mini-llvm.c index d1c749957f0a28..e551d2e221dadf 100644 --- a/src/mono/mono/mini/mini-llvm.c +++ b/src/mono/mono/mini/mini-llvm.c @@ -7248,6 +7248,9 @@ process_bb (EmitContext *ctx, MonoBasicBlock *bb) LLVMValueRef rhs_int = convert (ctx, rhs, intermediate_t); LLVMValueRef result = NULL; switch (ins->inst_c0) { + case XBINOP_FORCEINT_and: + result = LLVMBuildAnd (builder, lhs_int, rhs_int, ""); + break; case XBINOP_FORCEINT_or: result = LLVMBuildOr (builder, lhs_int, rhs_int, ""); break; @@ -9459,6 +9462,36 @@ process_bb (EmitContext *ctx, MonoBasicBlock *bb) values [ins->dreg] = LLVMBuildSExt (builder, result, ret_t, ""); break; } + case OP_ARM64_MVN: { + LLVMTypeRef ret_t = LLVMTypeOf (lhs); + LLVMValueRef result = bitcast_to_integral (ctx, lhs); + result = LLVMBuildNot (builder, result, "arm64_mvn"); + result = convert (ctx, result, ret_t); + values [ins->dreg] = result; + break; + } + case OP_ARM64_BIC: { + LLVMTypeRef ret_t = LLVMTypeOf (lhs); + LLVMValueRef result = bitcast_to_integral (ctx, lhs); + LLVMValueRef mask = bitcast_to_integral (ctx, rhs); + mask = LLVMBuildNot (builder, mask, ""); + result = LLVMBuildAnd (builder, mask, result, "arm64_bic"); + result = convert (ctx, result, ret_t); + values [ins->dreg] = result; + break; + } + case OP_ARM64_BSL: { + LLVMTypeRef ret_t = LLVMTypeOf (rhs); + LLVMValueRef select = bitcast_to_integral (ctx, lhs); + LLVMValueRef left = bitcast_to_integral (ctx, rhs); + LLVMValueRef right = bitcast_to_integral (ctx, arg3); + LLVMValueRef result1 = LLVMBuildAnd (builder, select, left, "arm64_bsl"); + LLVMValueRef result2 = LLVMBuildAnd (builder, LLVMBuildNot (builder, select, ""), right, ""); + LLVMValueRef result = LLVMBuildOr (builder, result1, result2, ""); + result = convert (ctx, result, ret_t); + values [ins->dreg] = result; + break; + } case OP_ARM64_CMTST: { LLVMTypeRef ret_t = simd_class_to_llvm_type (ctx, ins->klass); LLVMValueRef l = bitcast_to_integral (ctx, lhs); diff --git a/src/mono/mono/mini/mini-ops.h b/src/mono/mono/mini/mini-ops.h index 6640c788f8fcc0..b2e725d70789dd 100644 --- a/src/mono/mono/mini/mini-ops.h +++ b/src/mono/mono/mini/mini-ops.h @@ -1777,4 +1777,9 @@ MINI_OP(OP_ARM64_FCVTL2, "arm64_fcvtl2", XREG, XREG, NONE) MINI_OP(OP_ARM64_CMTST, "arm64_cmtst", XREG, XREG, XREG) +MINI_OP3(OP_ARM64_BSL, "arm64_bsl", XREG, XREG, XREG, XREG) +MINI_OP(OP_ARM64_BIC, "arm64_bic", XREG, XREG, XREG) + +MINI_OP(OP_ARM64_MVN, "arm64_mvn", XREG, XREG, NONE) + #endif // TARGET_ARM64 diff --git a/src/mono/mono/mini/mini.h b/src/mono/mono/mini/mini.h index bdcc412a48bb26..48fde7a348b87d 100644 --- a/src/mono/mono/mini/mini.h +++ b/src/mono/mono/mini/mini.h @@ -3000,7 +3000,7 @@ typedef enum { } SimdOp; enum { - XBINOP_FORCEINT_not, + XBINOP_FORCEINT_and, XBINOP_FORCEINT_or, XBINOP_FORCEINT_ornot, XBINOP_FORCEINT_xor, diff --git a/src/mono/mono/mini/simd-intrinsics.c b/src/mono/mono/mini/simd-intrinsics.c index 3f1f1089407684..579385b22327e2 100644 --- a/src/mono/mono/mini/simd-intrinsics.c +++ b/src/mono/mono/mini/simd-intrinsics.c @@ -1026,6 +1026,9 @@ static SimdIntrinsic advsimd_methods [] = { {SN_AbsoluteCompareGreaterThanOrEqual}, {SN_AbsoluteCompareLessThan}, {SN_AbsoluteCompareLessThanOrEqual}, + {SN_And, OP_XBINOP_FORCEINT, XBINOP_FORCEINT_and}, + {SN_BitwiseClear, OP_ARM64_BIC}, + {SN_BitwiseSelect, OP_ARM64_BSL}, {SN_Ceiling, OP_XOP_OVR_X_X, INTRINS_AARCH64_ADV_SIMD_FRINTP}, {SN_CeilingScalar, OP_XOP_OVR_SCALAR_X_X, INTRINS_AARCH64_ADV_SIMD_FRINTP}, {SN_CompareEqual, OP_XCOMPARE, CMP_EQ, OP_XCOMPARE, CMP_EQ, OP_XCOMPARE_FP, CMP_EQ}, @@ -1184,7 +1187,7 @@ static SimdIntrinsic advsimd_methods [] = { {SN_NegateSaturate, OP_XOP_OVR_X_X, INTRINS_AARCH64_ADV_SIMD_SQNEG}, {SN_NegateSaturateScalar}, {SN_NegateScalar, OP_ARM64_XNEG_SCALAR}, - {SN_Not, OP_XBINOP_FORCEINT, XBINOP_FORCEINT_not}, + {SN_Not, OP_ARM64_MVN}, {SN_Or, OP_XBINOP_FORCEINT, XBINOP_FORCEINT_or}, {SN_OrNot, OP_XBINOP_FORCEINT, XBINOP_FORCEINT_ornot}, {SN_PolynomialMultiply, OP_XOP_OVR_X_X_X, INTRINS_AARCH64_ADV_SIMD_PMUL}, From e666083daf1ee027c2b5db5f2c98aaba661f9bba Mon Sep 17 00:00:00 2001 From: Imran Hameed Date: Sun, 7 Mar 2021 03:47:28 -0800 Subject: [PATCH 29/58] More arithmetic --- src/mono/mono/mini/llvm-intrinsics.h | 53 ++++- src/mono/mono/mini/mini-llvm-cpp.cpp | 12 +- src/mono/mono/mini/mini-llvm.c | 340 +++++++++++++++++++++------ src/mono/mono/mini/mini-ops.h | 33 ++- src/mono/mono/mini/mini.c | 20 -- src/mono/mono/mini/mini.h | 43 +--- src/mono/mono/mini/simd-intrinsics.c | 38 +++ 7 files changed, 387 insertions(+), 152 deletions(-) diff --git a/src/mono/mono/mini/llvm-intrinsics.h b/src/mono/mono/mini/llvm-intrinsics.h index 57a7f57c76b8b6..27ae5b8f1b76be 100644 --- a/src/mono/mono/mini/llvm-intrinsics.h +++ b/src/mono/mono/mini/llvm-intrinsics.h @@ -21,6 +21,10 @@ #define I8 INTRIN_int64 #define R4 INTRIN_float32 #define R8 INTRIN_float64 +#define Ftoi INTRIN_kind_ftoi +#define Widen INTRIN_kind_widen +#define WidenAcross INTRIN_kind_widen_across +#define Across INTRIN_kind_across INTRINS_OVR_2_ARG(MEMSET, memset, LLVMPointerType (LLVMInt8Type (), 0), LLVMInt32Type ()) INTRINS_OVR_3_ARG(MEMCPY, memcpy, LLVMPointerType (LLVMInt8Type (), 0), LLVMPointerType (LLVMInt8Type (), 0), LLVMInt32Type () ) @@ -307,17 +311,36 @@ INTRINS_OVR_2_ARG(AARCH64_ADV_SIMD_ABS_COMPARE_LT_DOUBLE, aarch64_neon_facgt, ss INTRINS_OVR_2_ARG(AARCH64_ADV_SIMD_ABS_COMPARE_LTE_FLOAT, aarch64_neon_facge, sse_i4_t, sse_r4_t) INTRINS_OVR_2_ARG(AARCH64_ADV_SIMD_ABS_COMPARE_LTE_DOUBLE, aarch64_neon_facge, sse_i4_t, sse_r8_t) +INTRINS_OVR_TAG_KIND(AARCH64_ADV_SIMD_UADDLV, aarch64_neon_uaddlv, WidenAcross, V64 | V128 | I1 | I2 | I4) +INTRINS_OVR_TAG_KIND(AARCH64_ADV_SIMD_SADDLV, aarch64_neon_saddlv, WidenAcross, V64 | V128 | I1 | I2 | I4) + +INTRINS_OVR_TAG(AARCH64_ADV_SIMD_ADDP, aarch64_neon_addp, V64 | V128 | I1 | I2 | I4 | I8) +INTRINS_OVR_TAG(AARCH64_ADV_SIMD_FADDP, aarch64_neon_faddp, V64 | V128 | R4 | R8) + +INTRINS_OVR_TAG_KIND(AARCH64_ADV_SIMD_FMAXNMV, aarch64_neon_fmaxnmv, Across, V128 | R4) +INTRINS_OVR_TAG_KIND(AARCH64_ADV_SIMD_FMINNMV, aarch64_neon_fminnmv, Across, V128 | R4) + +INTRINS_OVR_TAG_KIND(AARCH64_ADV_SIMD_SADDV, aarch64_neon_saddv, Across, V64 | V128 | I1 | I2 | I4) +INTRINS_OVR_TAG_KIND(AARCH64_ADV_SIMD_UADDV, aarch64_neon_uaddv, Across, V64 | V128 | I1 | I2 | I4 | I8) +INTRINS_OVR_TAG_KIND(AARCH64_ADV_SIMD_SMAXV, aarch64_neon_smaxv, Across, V64 | V128 | I1 | I2 | I4) +INTRINS_OVR_TAG_KIND(AARCH64_ADV_SIMD_UMAXV, aarch64_neon_umaxv, Across, V64 | V128 | I1 | I2 | I4) +INTRINS_OVR_TAG_KIND(AARCH64_ADV_SIMD_SMINV, aarch64_neon_sminv, Across, V64 | V128 | I1 | I2 | I4) +INTRINS_OVR_TAG_KIND(AARCH64_ADV_SIMD_UMINV, aarch64_neon_uminv, Across, V64 | V128 | I1 | I2 | I4) + +INTRINS_OVR_TAG_KIND(AARCH64_ADV_SIMD_SADDLP, aarch64_neon_saddlp, Widen, V64 | V128 | I1 | I2 | I4 | I8) +INTRINS_OVR_TAG_KIND(AARCH64_ADV_SIMD_UADDLP, aarch64_neon_uaddlp, Widen, V64 | V128 | I1 | I2 | I4 | I8) + INTRINS_OVR_2_ARG(AARCH64_ADV_SIMD_FCVTXN, aarch64_neon_fcvtxn, v64_r4_t, v128_r8_t) -INTRINS_OVR_TAG_FTOI(AARCH64_ADV_SIMD_FCVTAS, aarch64_neon_fcvtas, Scalar | V64 | V128 | I4 | I8) -INTRINS_OVR_TAG_FTOI(AARCH64_ADV_SIMD_FCVTNS, aarch64_neon_fcvtns, Scalar | V64 | V128 | I4 | I8) -INTRINS_OVR_TAG_FTOI(AARCH64_ADV_SIMD_FCVTMS, aarch64_neon_fcvtms, Scalar | V64 | V128 | I4 | I8) -INTRINS_OVR_TAG_FTOI(AARCH64_ADV_SIMD_FCVTPS, aarch64_neon_fcvtps, Scalar | V64 | V128 | I4 | I8) +INTRINS_OVR_TAG_KIND(AARCH64_ADV_SIMD_FCVTAS, aarch64_neon_fcvtas, Ftoi, Scalar | V64 | V128 | I4 | I8) +INTRINS_OVR_TAG_KIND(AARCH64_ADV_SIMD_FCVTNS, aarch64_neon_fcvtns, Ftoi, Scalar | V64 | V128 | I4 | I8) +INTRINS_OVR_TAG_KIND(AARCH64_ADV_SIMD_FCVTMS, aarch64_neon_fcvtms, Ftoi, Scalar | V64 | V128 | I4 | I8) +INTRINS_OVR_TAG_KIND(AARCH64_ADV_SIMD_FCVTPS, aarch64_neon_fcvtps, Ftoi, Scalar | V64 | V128 | I4 | I8) -INTRINS_OVR_TAG_FTOI(AARCH64_ADV_SIMD_FCVTAU, aarch64_neon_fcvtau, Scalar | V64 | V128 | I4 | I8) -INTRINS_OVR_TAG_FTOI(AARCH64_ADV_SIMD_FCVTNU, aarch64_neon_fcvtnu, Scalar | V64 | V128 | I4 | I8) -INTRINS_OVR_TAG_FTOI(AARCH64_ADV_SIMD_FCVTMU, aarch64_neon_fcvtmu, Scalar | V64 | V128 | I4 | I8) -INTRINS_OVR_TAG_FTOI(AARCH64_ADV_SIMD_FCVTPU, aarch64_neon_fcvtpu, Scalar | V64 | V128 | I4 | I8) +INTRINS_OVR_TAG_KIND(AARCH64_ADV_SIMD_FCVTAU, aarch64_neon_fcvtau, Ftoi, Scalar | V64 | V128 | I4 | I8) +INTRINS_OVR_TAG_KIND(AARCH64_ADV_SIMD_FCVTNU, aarch64_neon_fcvtnu, Ftoi, Scalar | V64 | V128 | I4 | I8) +INTRINS_OVR_TAG_KIND(AARCH64_ADV_SIMD_FCVTMU, aarch64_neon_fcvtmu, Ftoi, Scalar | V64 | V128 | I4 | I8) +INTRINS_OVR_TAG_KIND(AARCH64_ADV_SIMD_FCVTPU, aarch64_neon_fcvtpu, Ftoi, Scalar | V64 | V128 | I4 | I8) INTRINS_OVR_TAG(AARCH64_ADV_SIMD_SQXTUN, aarch64_neon_sqxtun, V64 | I1 | I2 | I4) INTRINS_OVR_TAG(AARCH64_ADV_SIMD_SQXTN, aarch64_neon_sqxtn, V64 | I1 | I2 | I4) @@ -360,7 +383,6 @@ INTRINS_OVR_2_ARG(AARCH64_ADV_SIMD_FMINV_F64, aarch64_neon_fminv, r8_t, v128_r8_ INTRINS_OVR_TAG(AARCH64_ADV_SIMD_SQDMULH, aarch64_neon_sqdmulh, V64 | V128 | I2 | I4) -INTRINS_OVR_TAG(AARCH64_ADV_SIMD_SQADD, aarch64_neon_sqadd, V128 | I4 | I8) INTRINS_OVR_TAG(AARCH64_ADV_SIMD_SQDMULL, aarch64_neon_sqdmull, V128 | I4 | I8) INTRINS_OVR_TAG(AARCH64_ADV_SIMD_SQRDMULH, aarch64_neon_sqrdmulh, V64 | V128 | I2 | I4) @@ -390,10 +412,18 @@ INTRINS_OVR_TAG(AARCH64_ADV_SIMD_FRINTM, floor, Scalar | V64 | V128 | R4 | R8) INTRINS_OVR_TAG(AARCH64_ADV_SIMD_FRINTP, ceil, Scalar | V64 | V128 | R4 | R8) INTRINS_OVR_TAG(AARCH64_ADV_SIMD_FRINTZ, trunc, Scalar | V64 | V128 | R4 | R8) +INTRINS_OVR_TAG(AARCH64_ADV_SIMD_SUQADD, aarch64_neon_suqadd, Scalar | V64 | V128 | I1 | I2 | I4 | I8) +INTRINS_OVR_TAG(AARCH64_ADV_SIMD_USQADD, aarch64_neon_usqadd, Scalar | V64 | V128 | I1 | I2 | I4 | I8) +INTRINS_OVR_TAG(AARCH64_ADV_SIMD_UQADD, aarch64_neon_uqadd, Scalar | V64 | V128 | I1 | I2 | I4 | I8) +INTRINS_OVR_TAG(AARCH64_ADV_SIMD_SQADD, aarch64_neon_sqadd, Scalar | V64 | V128 | I1 | I2 | I4 | I8) INTRINS_OVR_TAG(AARCH64_ADV_SIMD_UQSUB, aarch64_neon_uqsub, Scalar | V64 | V128 | I1 | I2 | I4 | I8) INTRINS_OVR_TAG(AARCH64_ADV_SIMD_SQSUB, aarch64_neon_sqsub, Scalar | V64 | V128 | I1 | I2 | I4 | I8) + +INTRINS_OVR_TAG(AARCH64_ADV_SIMD_RADDHN, aarch64_neon_raddhn, V64 | I1 | I2 | I4) INTRINS_OVR_TAG(AARCH64_ADV_SIMD_RSUBHN, aarch64_neon_rsubhn, V64 | I1 | I2 | I4) + INTRINS_OVR_TAG(AARCH64_ADV_SIMD_FSQRT, sqrt, V64 | V128 | R4 | R8) + INTRINS_OVR_TAG(AARCH64_ADV_SIMD_UQSHRN, aarch64_neon_uqshrn, V64 | I1 | I2 | I4) // Constant shift INTRINS_OVR_TAG(AARCH64_ADV_SIMD_RSHRN, aarch64_neon_rshrn, V64 | I1 | I2 | I4) // Constant shift @@ -426,7 +456,7 @@ INTRINS_OVR_TAG(AARCH64_ADV_SIMD_TBL1, aarch64_neon_tbl1, V64 | V128 | I1) #undef INTRINS_OVR_2_ARG #undef INTRINS_OVR_3_ARG #undef INTRINS_OVR_TAG -#undef INTRINS_OVR_TAG_FTOI +#undef INTRINS_OVR_TAG_KIND #undef Scalar #undef V64 #undef V128 @@ -436,3 +466,6 @@ INTRINS_OVR_TAG(AARCH64_ADV_SIMD_TBL1, aarch64_neon_tbl1, V64 | V128 | I1) #undef I8 #undef R4 #undef R8 +#undef Ftoi +#undef WidenAcross +#undef Across diff --git a/src/mono/mono/mini/mini-llvm-cpp.cpp b/src/mono/mono/mini/mini-llvm-cpp.cpp index dc8b81f57568e6..0392e2f0dcf249 100644 --- a/src/mono/mono/mini/mini-llvm-cpp.cpp +++ b/src/mono/mono/mini/mini-llvm-cpp.cpp @@ -633,11 +633,11 @@ get_intrins_id (IntrinsicId id) Intrinsic::ID intrins_id = Intrinsic::ID::not_intrinsic; switch (id) { #define INTRINS(id, llvm_id) case INTRINS_ ## id: intrins_id = Intrinsic::ID::llvm_id; break; -#define INTRINS_OVR(id, llvm_id, ty) case INTRINS_ ## id: intrins_id = Intrinsic::ID::llvm_id; break; -#define INTRINS_OVR_2_ARG(id, llvm_id, ty1, ty2) case INTRINS_ ## id: intrins_id = Intrinsic::ID::llvm_id; break; -#define INTRINS_OVR_3_ARG(id, llvm_id, ty1, ty2, ty3) case INTRINS_ ## id: intrins_id = Intrinsic::ID::llvm_id; break; -#define INTRINS_OVR_TAG(id, llvm_id, ...) case INTRINS_ ## id: intrins_id = Intrinsic::ID::llvm_id; break; -#define INTRINS_OVR_TAG_FTOI(id, llvm_id, ...) case INTRINS_ ## id: intrins_id = Intrinsic::ID::llvm_id; break; +#define INTRINS_OVR(id, llvm_id, ty) INTRINS(id, llvm_id) +#define INTRINS_OVR_2_ARG(id, llvm_id, ty1, ty2) INTRINS(id, llvm_id) +#define INTRINS_OVR_3_ARG(id, llvm_id, ty1, ty2, ty3) INTRINS(id, llvm_id) +#define INTRINS_OVR_TAG(id, llvm_id, ...) INTRINS(id, llvm_id) +#define INTRINS_OVR_TAG_KIND(id, llvm_id, ...) INTRINS(id, llvm_id) #include "llvm-intrinsics.h" default: break; @@ -654,7 +654,7 @@ is_overloaded_intrins (IntrinsicId id) #define INTRINS_OVR_2_ARG(id, llvm_id, ty1, ty2) case INTRINS_ ## id: return true; #define INTRINS_OVR_3_ARG(id, llvm_id, ty1, ty2, ty3) case INTRINS_ ## id: return true; #define INTRINS_OVR_TAG(id, llvm_id, ...) case INTRINS_ ## id: return true; -#define INTRINS_OVR_TAG_FTOI(id, llvm_id, ...) case INTRINS_ ## id: return true; +#define INTRINS_OVR_TAG_KIND(id, llvm_id, ...) case INTRINS_ ## id: return true; #include "llvm-intrinsics.h" default: break; diff --git a/src/mono/mono/mini/mini-llvm.c b/src/mono/mono/mini/mini-llvm.c index e551d2e221dadf..f3920e65649460 100644 --- a/src/mono/mono/mini/mini-llvm.c +++ b/src/mono/mono/mini/mini-llvm.c @@ -309,6 +309,23 @@ static void create_aot_info_var (MonoLLVMModule *module); static void set_invariant_load_flag (LLVMValueRef v); static void set_nonnull_load_flag (LLVMValueRef v); +enum { + INTRIN_scalar = 1 << 0, + INTRIN_vector64 = 1 << 1, + INTRIN_vector128 = 1 << 2, + INTRIN_vectorwidths = 3, + INTRIN_vectormask = 0x7, + + INTRIN_int8 = 1 << 3, + INTRIN_int16 = 1 << 4, + INTRIN_int32 = 1 << 5, + INTRIN_int64 = 1 << 6, + INTRIN_float32 = 1 << 7, + INTRIN_float64 = 1 << 8, + INTRIN_elementwidths = 6, +}; + +typedef uint16_t llvm_ovr_tag_t; static LLVMTypeRef intrin_types [INTRIN_vectorwidths][INTRIN_elementwidths]; @@ -318,23 +335,92 @@ static const llvm_ovr_tag_t intrin_arm64_ovr [] = { #define INTRINS_OVR_2_ARG(sym, ...) 0, #define INTRINS_OVR_3_ARG(sym, ...) 0, #define INTRINS_OVR_TAG(sym, _, spec) spec, - #define INTRINS_OVR_TAG_FTOI(sym, _, spec) spec, + #define INTRINS_OVR_TAG_KIND(sym, _, kind, spec) spec, #include "llvm-intrinsics.h" }; -static gboolean -is_ftoi_intrinsic(IntrinsicId id) { - switch (id) { - #define INTRINS(sym, ...) - #define INTRINS_OVR(sym, ...) - #define INTRINS_OVR_2_ARG(sym, ...) - #define INTRINS_OVR_3_ARG(sym, ...) - #define INTRINS_OVR_TAG(sym, _, spec) - #define INTRINS_OVR_TAG_FTOI(sym, _, spec) case INTRINS_ ## sym: +enum { + INTRIN_kind_ftoi = 1, + INTRIN_kind_widen, + INTRIN_kind_widen_across, + INTRIN_kind_across, +}; + +static const uint8_t intrin_kind [] = { + #define INTRINS(sym, ...) 0, + #define INTRINS_OVR(sym, ...) 0, + #define INTRINS_OVR_2_ARG(sym, ...) 0, + #define INTRINS_OVR_3_ARG(sym, ...) 0, + #define INTRINS_OVR_TAG(sym, _, spec) 0, + #define INTRINS_OVR_TAG_KIND(sym, _, kind, spec) kind, #include "llvm-intrinsics.h" - return TRUE; +}; + +static inline llvm_ovr_tag_t +ovr_tag_force_scalar (llvm_ovr_tag_t tag) +{ + return (tag & ~INTRIN_vectormask) | INTRIN_scalar; +} + +static inline llvm_ovr_tag_t +ovr_tag_smaller_vector (llvm_ovr_tag_t tag) +{ + return (tag & ~INTRIN_vectormask) | ((tag & INTRIN_vectormask) >> 1); +} + +static int +int_from_id_and_ovr_tag (int id, llvm_ovr_tag_t ovr_tag) +{ + return (((int) ovr_tag) << 23) | id; +} + +static llvm_ovr_tag_t +ovr_tag_from_mono_vector_class (MonoClass *klass) { + int size = mono_class_value_size (klass, NULL); + llvm_ovr_tag_t ret = 0; + switch (size) { + case 8: ret |= INTRIN_vector64; break; + case 16: ret |= INTRIN_vector128; break; + } + MonoType *etype = mono_class_get_context (klass)->class_inst->type_argv [0]; + switch (etype->type) { + case MONO_TYPE_I1: case MONO_TYPE_U1: ret |= INTRIN_int8; break; + case MONO_TYPE_I2: case MONO_TYPE_U2: ret |= INTRIN_int16; break; + case MONO_TYPE_I4: case MONO_TYPE_U4: ret |= INTRIN_int32; break; + case MONO_TYPE_I8: case MONO_TYPE_U8: ret |= INTRIN_int64; break; + case MONO_TYPE_R4: ret |= INTRIN_float32; break; + case MONO_TYPE_R8: ret |= INTRIN_float64; break; + } + return ret; +} + +static llvm_ovr_tag_t +ovr_tag_from_llvm_type (LLVMTypeRef type) +{ + llvm_ovr_tag_t ret = 0; + LLVMTypeKind kind = LLVMGetTypeKind (type); + LLVMTypeRef elem_t = NULL; + switch (kind) { + case LLVMVectorTypeKind: { + elem_t = LLVMGetElementType (type); + unsigned int bits = mono_llvm_get_prim_size_bits (type); + switch (bits) { + case 64: ret |= INTRIN_vector64; break; + case 128: ret |= INTRIN_vector128; break; + default: g_assert_not_reached (); + } + break; } - return FALSE; + default: + g_assert_not_reached (); + } + if (elem_t == i1_t) ret |= INTRIN_int8; + if (elem_t == i2_t) ret |= INTRIN_int16; + if (elem_t == i4_t) ret |= INTRIN_int32; + if (elem_t == i8_t) ret |= INTRIN_int64; + if (elem_t == r4_t) ret |= INTRIN_float32; + if (elem_t == r8_t) ret |= INTRIN_float64; + return ret; } static inline void @@ -4705,28 +4791,6 @@ emit_landing_pad (EmitContext *ctx, int group_index, int group_size) return lpad_bb; } -static LLVMValueRef -broadcast_element (EmitContext *ctx, LLVMValueRef elem, int count) -{ - LLVMTypeRef t = LLVMTypeOf (elem); - LLVMTypeRef init_vec_t = LLVMVectorType (t, 1); - LLVMValueRef undef = LLVMGetUndef (init_vec_t); - LLVMValueRef vec = LLVMBuildInsertElement (ctx->builder, undef, elem, const_int32 (0), ""); - LLVMValueRef select_zero = LLVMConstNull (LLVMVectorType (LLVMInt32Type (), count)); - return LLVMBuildShuffleVector (ctx->builder, vec, undef, select_zero, "broadcast"); -} - -static LLVMValueRef -create_shift_vector (EmitContext *ctx, LLVMValueRef shiftarg, LLVMValueRef shiftamt) -{ - LLVMTypeRef t = LLVMTypeOf (shiftarg); - unsigned int elems = LLVMGetVectorSize (t); - LLVMTypeRef elem_t = LLVMGetElementType (t); - shiftamt = convert_full (ctx, shiftamt, elem_t, TRUE); - shiftamt = broadcast_element (ctx, shiftamt, elems); - return shiftamt; -} - static LLVMValueRef create_const_vector (LLVMTypeRef t, const int *vals, int count) { @@ -4763,6 +4827,37 @@ create_const_vector_2_i32 (int v0, int v1) return LLVMConstVector (mask, 2); } +static LLVMValueRef +broadcast_element (EmitContext *ctx, LLVMValueRef elem, int count) +{ + LLVMTypeRef t = LLVMTypeOf (elem); + LLVMTypeRef init_vec_t = LLVMVectorType (t, 1); + LLVMValueRef undef = LLVMGetUndef (init_vec_t); + LLVMValueRef vec = LLVMBuildInsertElement (ctx->builder, undef, elem, const_int32 (0), ""); + LLVMValueRef select_zero = LLVMConstNull (LLVMVectorType (LLVMInt32Type (), count)); + return LLVMBuildShuffleVector (ctx->builder, vec, undef, select_zero, "broadcast"); +} + +static LLVMValueRef +broadcast_constant (int const_val, LLVMTypeRef elem_t, int count) +{ + int vals [MAX_VECTOR_ELEMS]; + for (int i = 0; i < count; ++i) + vals [i] = const_val; + return create_const_vector (elem_t, vals, count); +} + +static LLVMValueRef +create_shift_vector (EmitContext *ctx, LLVMValueRef type_donor, LLVMValueRef shiftamt) +{ + LLVMTypeRef t = LLVMTypeOf (type_donor); + unsigned int elems = LLVMGetVectorSize (t); + LLVMTypeRef elem_t = LLVMGetElementType (t); + shiftamt = convert_full (ctx, shiftamt, elem_t, TRUE); + shiftamt = broadcast_element (ctx, shiftamt, elems); + return shiftamt; +} + static LLVMTypeRef to_integral_vector_type (LLVMTypeRef t) { @@ -10197,45 +10292,54 @@ process_bb (EmitContext *ctx, MonoBasicBlock *bb) mono_llvm_build_store (builder, val, lhs, FALSE, LLVM_BARRIER_NONE); break; } + case OP_ARM64_ADDHN: + case OP_ARM64_ADDHN2: case OP_ARM64_SUBHN: - case OP_ARM64_SUBHN2: { - LLVMValueRef args [2] = { lhs, rhs }; - gboolean high = ins->opcode == OP_ARM64_SUBHN2; - if (high) { - args [0] = rhs; - args [1] = arg3; - } - LLVMValueRef result = LLVMBuildSub (builder, args [0], args [1], ""); - LLVMTypeRef t = LLVMTypeOf (args [0]); - LLVMTypeRef elt_t = LLVMGetElementType (t); - unsigned int elem_bits = mono_llvm_get_prim_size_bits (elt_t); - unsigned int elems = LLVMGetVectorSize (t); - int shift_vals [MAX_VECTOR_ELEMS] = { 0 }; - for (unsigned int i = 0; i < elems; ++i) - shift_vals [i] = elem_bits / 2; - result = LLVMBuildLShr (builder, result, create_const_vector (elt_t, shift_vals, elems), ""); - result = LLVMBuildTrunc (builder, result, LLVMVectorType (LLVMIntType (elem_bits / 2), elems), ""); - if (high) - result = concatenate_vectors (ctx, lhs, result); - values [ins->dreg] = result; - break; - } + case OP_ARM64_SUBHN2: + case OP_ARM64_RADDHN: + case OP_ARM64_RADDHN2: case OP_ARM64_RSUBHN: case OP_ARM64_RSUBHN2: { LLVMValueRef args [2] = { lhs, rhs }; - gboolean high = ins->opcode == OP_ARM64_RSUBHN2; + gboolean high = FALSE; + gboolean subtract = FALSE; + int iid = 0; + switch (ins->opcode) { + case OP_ARM64_ADDHN2: high = TRUE; case OP_ARM64_ADDHN: break; + case OP_ARM64_SUBHN2: high = TRUE; case OP_ARM64_SUBHN: subtract = TRUE; break; + case OP_ARM64_RSUBHN2: high = TRUE; case OP_ARM64_RSUBHN: iid = INTRINS_AARCH64_ADV_SIMD_RSUBHN; break; + case OP_ARM64_RADDHN2: high = TRUE; case OP_ARM64_RADDHN: iid = INTRINS_AARCH64_ADV_SIMD_RADDHN; break; + } llvm_ovr_tag_t ovr_tag = ovr_tag_from_mono_vector_class (ins->klass); if (high) { args [0] = rhs; args [1] = arg3; ovr_tag = ovr_tag_smaller_vector (ovr_tag); } - LLVMValueRef result = call_overloaded_intrins (ctx, INTRINS_AARCH64_ADV_SIMD_RSUBHN, ovr_tag, args, ""); + LLVMValueRef result = NULL; + if (iid != 0) + result = call_overloaded_intrins (ctx, iid, ovr_tag, args, ""); + else { + LLVMTypeRef t = LLVMTypeOf (args [0]); + LLVMTypeRef elt_t = LLVMGetElementType (t); + unsigned int elems = LLVMGetVectorSize (t); + unsigned int elem_bits = mono_llvm_get_prim_size_bits (elt_t); + if (subtract) + result = LLVMBuildSub (builder, args [0], args [1], ""); + else + result = LLVMBuildAdd (builder, args [0], args [1], ""); + result = LLVMBuildLShr (builder, result, broadcast_constant (elem_bits / 2, elt_t, elems), ""); + result = LLVMBuildTrunc (builder, result, LLVMVectorType (LLVMIntType (elem_bits / 2), elems), ""); + } if (high) result = concatenate_vectors (ctx, lhs, result); values [ins->dreg] = result; break; } + case OP_ARM64_SADD: + case OP_ARM64_UADD: + case OP_ARM64_SADD2: + case OP_ARM64_UADD2: case OP_ARM64_SSUB: case OP_ARM64_USUB: case OP_ARM64_SSUB2: @@ -10243,12 +10347,14 @@ process_bb (EmitContext *ctx, MonoBasicBlock *bb) LLVMTypeRef ret_t = simd_class_to_llvm_type (ctx, ins->klass); gboolean is_unsigned = FALSE; gboolean high = FALSE; + gboolean subtract = FALSE; switch (ins->opcode) { - case OP_ARM64_USUB: is_unsigned = TRUE; break; - case OP_ARM64_SSUB2: high = TRUE; break; - case OP_ARM64_USUB2: high = TRUE; is_unsigned = TRUE; break; + case OP_ARM64_SADD2: high = TRUE; case OP_ARM64_SADD: break; + case OP_ARM64_UADD2: high = TRUE; case OP_ARM64_UADD: is_unsigned = TRUE; break; + case OP_ARM64_SSUB2: high = TRUE; case OP_ARM64_SSUB: subtract = TRUE; break; + case OP_ARM64_USUB2: high = TRUE; case OP_ARM64_USUB: subtract = TRUE; is_unsigned = TRUE; break; } - LLVMValueRef args [2] = { lhs, rhs }; + LLVMValueRef args [] = { lhs, rhs }; for (int i = 0; i < 2; ++i) { LLVMValueRef arg = args [i]; LLVMTypeRef arg_t = LLVMTypeOf (arg); @@ -10260,7 +10366,71 @@ process_bb (EmitContext *ctx, MonoBasicBlock *bb) arg = LLVMBuildSExt (builder, arg, ret_t, ""); args [i] = arg; } - values [ins->dreg] = LLVMBuildSub (builder, args [0], args [1], "arm64_sub"); + LLVMValueRef result = NULL; + if (subtract) + result = LLVMBuildSub (builder, args [0], args [1], "arm64_sub"); + else + result = LLVMBuildAdd (builder, args [0], args [1], "arm64_add"); + values [ins->dreg] = result; + break; + } + case OP_ARM64_XHORIZ: { + gboolean truncate = FALSE; + LLVMTypeRef arg_t = LLVMTypeOf (lhs); + LLVMTypeRef elem_t = LLVMGetElementType (arg_t); + LLVMTypeRef ret_t = simd_class_to_llvm_type (ctx, ins->klass); + llvm_ovr_tag_t ovr_tag = ovr_tag_from_llvm_type (arg_t); + if (elem_t == i1_t || elem_t == i2_t) + truncate = TRUE; + LLVMValueRef result = call_overloaded_intrins (ctx, ins->inst_c0, ovr_tag, &lhs, ""); + if (truncate) { + // @llvm.aarch64.neon.saddv.i32.v8i16 ought to return an i16, but doesn't in LLVM 9. + result = LLVMBuildTrunc (builder, result, elem_t, ""); + } + result = vector_from_scalar_ty (ctx, ret_t, result); + values [ins->dreg] = result; + break; + } + case OP_ARM64_SADDLV: + case OP_ARM64_UADDLV: { + LLVMTypeRef arg_t = LLVMTypeOf (lhs); + LLVMTypeRef elem_t = LLVMGetElementType (arg_t); + LLVMTypeRef ret_t = simd_class_to_llvm_type (ctx, ins->klass); + llvm_ovr_tag_t ovr_tag = ovr_tag_from_llvm_type (arg_t); + gboolean truncate = elem_t == i1_t; + int iid = ins->opcode == OP_ARM64_UADDLV ? INTRINS_AARCH64_ADV_SIMD_UADDLV : INTRINS_AARCH64_ADV_SIMD_SADDLV; + LLVMValueRef result = call_overloaded_intrins (ctx, iid, ovr_tag, &lhs, ""); + if (truncate) { + // @llvm.aarch64.neon.saddlv.i32.v16i8 ought to return an i16, but doesn't in LLVM 9. + result = LLVMBuildTrunc (builder, result, i2_t, ""); + } + result = vector_from_scalar_ty (ctx, ret_t, result); + values [ins->dreg] = result; + break; + } + case OP_ARM64_UADALP: + case OP_ARM64_SADALP: { + llvm_ovr_tag_t ovr_tag = ovr_tag_from_mono_vector_class (ins->klass); + int iid = ins->opcode == OP_ARM64_UADALP ? INTRINS_AARCH64_ADV_SIMD_UADDLP : INTRINS_AARCH64_ADV_SIMD_SADDLP; + LLVMValueRef result = call_overloaded_intrins (ctx, iid, ovr_tag, &rhs, ""); + result = LLVMBuildAdd (builder, result, lhs, ""); + values [ins->dreg] = result; + break; + } + case OP_ARM64_ADDP_SCALAR: { + llvm_ovr_tag_t ovr_tag = INTRIN_vector128 | INTRIN_int64; + LLVMValueRef result = call_overloaded_intrins (ctx, INTRINS_AARCH64_ADV_SIMD_UADDV, ovr_tag, &lhs, "arm64_addp_scalar"); + result = LLVMBuildInsertElement (builder, LLVMGetUndef (v64_i8_t), result, const_int32 (0), ""); + values [ins->dreg] = result; + break; + } + case OP_ARM64_FADDP_SCALAR: { + LLVMTypeRef ret_t = simd_class_to_llvm_type (ctx, ins->klass); + LLVMValueRef hi = LLVMBuildExtractElement (builder, lhs, const_int32 (0), ""); + LLVMValueRef lo = LLVMBuildExtractElement (builder, lhs, const_int32 (1), ""); + LLVMValueRef result = LLVMBuildFAdd (builder, hi, lo, "arm64_faddp_scalar"); + result = LLVMBuildInsertElement (builder, LLVMGetUndef (ret_t), result, const_int32 (0), ""); + values [ins->dreg] = result; break; } case OP_ARM64_SXTL: @@ -10271,8 +10441,7 @@ process_bb (EmitContext *ctx, MonoBasicBlock *bb) gboolean is_unsigned = FALSE; switch (ins->opcode) { case OP_ARM64_SXTL2: high = TRUE; break; - case OP_ARM64_UXTL: is_unsigned = TRUE; break; - case OP_ARM64_UXTL2: high = TRUE; is_unsigned = TRUE; break; + case OP_ARM64_UXTL2: high = TRUE; case OP_ARM64_UXTL: is_unsigned = TRUE; break; } LLVMTypeRef t = LLVMTypeOf (lhs); unsigned int elem_bits = LLVMGetIntTypeWidth (LLVMGetElementType (t)); @@ -11734,10 +11903,43 @@ add_intrinsic (LLVMModuleRef module, int id) llvm_ovr_tag_t elem_bit = INTRIN_int8 << ew; llvm_ovr_tag_t test = vec_bit | elem_bit; if ((spec & test) == test) { - if (is_ftoi_intrinsic (id) && (elem_bit & (INTRIN_int32 | INTRIN_int64))) { - intrins = add_intrins2 (module, id, intrin_types [vw][ew], intrin_types [vw][ew + 2]); + uint8_t kind = intrin_kind [id]; + LLVMTypeRef distinguishing_type = intrin_types [vw][ew]; + if (kind == INTRIN_kind_ftoi && (elem_bit & (INTRIN_int32 | INTRIN_int64))) { + /* + * @llvm.aarch64.neon.fcvtas.v4i32.v4f32 + * @llvm.aarch64.neon.fcvtas.v2i64.v2f64 + */ + intrins = add_intrins2 (module, id, distinguishing_type, intrin_types [vw][ew + 2]); + } else if (kind == INTRIN_kind_widen) { + /* + * @llvm.aarch64.neon.saddlp.v2i64.v4i32 + * @llvm.aarch64.neon.saddlp.v4i16.v8i8 + */ + intrins = add_intrins2 (module, id, distinguishing_type, intrin_types [vw][ew - 1]); + } else if (kind == INTRIN_kind_widen_across) { + /* + * @llvm.aarch64.neon.saddlv.i64.v4i32 + * @llvm.aarch64.neon.saddlv.i32.v8i16 + * @llvm.aarch64.neon.saddlv.i32.v16i8 + * i8/i16 return types for NEON intrinsics will make isel fail as of LLVM 9. + */ + int associated_prim = MAX(ew + 1, 2); + LLVMTypeRef associated_scalar_type = intrin_types [0][associated_prim]; + intrins = add_intrins2 (module, id, associated_scalar_type, distinguishing_type); + } else if (kind == INTRIN_kind_across) { + /* + * @llvm.aarch64.neon.uaddv.i64.v4i64 + * @llvm.aarch64.neon.uaddv.i32.v4i32 + * @llvm.aarch64.neon.uaddv.i32.v8i16 + * @llvm.aarch64.neon.uaddv.i32.v16i8 + * i8/i16 return types for NEON intrinsics will make isel fail as of LLVM 9. + */ + int associated_prim = MAX(ew, 2); + LLVMTypeRef associated_scalar_type = intrin_types [0][associated_prim]; + intrins = add_intrins2 (module, id, associated_scalar_type, distinguishing_type); } else - intrins = add_intrins1 (module, id, intrin_types [vw][ew]); + intrins = add_intrins1 (module, id, distinguishing_type); int ovr_id = int_from_id_and_ovr_tag (id, test); g_hash_table_insert (intrins_id_to_intrins, GINT_TO_POINTER (ovr_id), intrins); } @@ -11753,7 +11955,7 @@ add_intrinsic (LLVMModuleRef module, int id) #define INTRINS_OVR_2_ARG(intrin_name, llvm_id, llvm_type1, llvm_type2) case INTRINS_ ## intrin_name: intrins = add_intrins2(module, id, llvm_type1, llvm_type2); break; #define INTRINS_OVR_3_ARG(intrin_name, llvm_id, llvm_type1, llvm_type2, llvm_type3) case INTRINS_ ## intrin_name: intrins = add_intrins3(module, id, llvm_type1, llvm_type2, llvm_type3); break; #define INTRINS_OVR_TAG(...) - #define INTRINS_OVR_TAG_FTOI(...) + #define INTRINS_OVR_TAG_KIND(...) #include "llvm-intrinsics.h" default: diff --git a/src/mono/mono/mini/mini-ops.h b/src/mono/mono/mini/mini-ops.h index b2e725d70789dd..30f4890b17ed70 100644 --- a/src/mono/mono/mini/mini-ops.h +++ b/src/mono/mono/mini/mini-ops.h @@ -1608,13 +1608,9 @@ MINI_OP(OP_ARM64_SXTL2, "arm64_sxtl2", XREG, XREG, NONE) MINI_OP(OP_ARM64_SMULH, "arm64_smulh", LREG, LREG, LREG) MINI_OP(OP_ARM64_SQRT_SCALAR, "arm64_sqrt_scalar", XREG, XREG, NONE) -MINI_OP(OP_ARM64_SSUB, "arm64_ssub", XREG, XREG, XREG) -MINI_OP(OP_ARM64_SSUB2, "arm64_ssub2", XREG, XREG, XREG) MINI_OP(OP_ARM64_TRN1, "arm64_trn1", XREG, XREG, XREG) MINI_OP(OP_ARM64_TRN2, "arm64_trn2", XREG, XREG, XREG) MINI_OP(OP_ARM64_UMULH, "arm64_umulh", LREG, LREG, LREG) -MINI_OP(OP_ARM64_USUB, "arm64_usub", XREG, XREG, XREG) -MINI_OP(OP_ARM64_USUB2, "arm64_usub2", XREG, XREG, XREG) MINI_OP(OP_ARM64_UXTL, "arm64_uxtl", XREG, XREG, NONE) MINI_OP(OP_ARM64_UXTL2, "arm64_uxtl2", XREG, XREG, NONE) MINI_OP(OP_ARM64_UZP1, "arm64_uzp1", XREG, XREG, XREG) @@ -1628,11 +1624,16 @@ MINI_OP3(OP_ARM64_STNP_SCALAR, "arm64_stnp_scalar", NONE, IREG, XREG, XREG) MINI_OP3(OP_ARM64_STP, "arm64_stp", NONE, IREG, XREG, XREG) MINI_OP3(OP_ARM64_STP_SCALAR, "arm64_stp_scalar", NONE, IREG, XREG, XREG) -MINI_OP(OP_ARM64_RSUBHN, "arm64_rsubhn", XREG, XREG, XREG) +MINI_OP(OP_ARM64_ADDHN, "arm64_addhn", XREG, XREG, XREG) +MINI_OP3(OP_ARM64_ADDHN2, "arm64_addhn2", XREG, XREG, XREG, XREG) MINI_OP(OP_ARM64_SUBHN, "arm64_subhn", XREG, XREG, XREG) -MINI_OP3(OP_ARM64_RSUBHN2, "arm64_rsubhn2", XREG, XREG, XREG, XREG) MINI_OP3(OP_ARM64_SUBHN2, "arm64_subhn2", XREG, XREG, XREG, XREG) +MINI_OP(OP_ARM64_RADDHN, "arm64_raddhn", XREG, XREG, XREG) +MINI_OP3(OP_ARM64_RADDHN2, "arm64_raddhn2", XREG, XREG, XREG, XREG) +MINI_OP(OP_ARM64_RSUBHN, "arm64_rsubhn", XREG, XREG, XREG) +MINI_OP3(OP_ARM64_RSUBHN2, "arm64_rsubhn2", XREG, XREG, XREG, XREG) + MINI_OP(OP_ARM64_SHRN, "arm64_shrn", XREG, XREG, IREG) MINI_OP(OP_ARM64_UQSHRN, "arm64_uqshrn", XREG, XREG, IREG) MINI_OP3(OP_ARM64_SHRN2, "arm64_shrn2", XREG, XREG, XREG, IREG) @@ -1782,4 +1783,24 @@ MINI_OP(OP_ARM64_BIC, "arm64_bic", XREG, XREG, XREG) MINI_OP(OP_ARM64_MVN, "arm64_mvn", XREG, XREG, NONE) +MINI_OP(OP_ARM64_SADD, "arm64_sadd", XREG, XREG, XREG) +MINI_OP(OP_ARM64_SADD2, "arm64_sadd2", XREG, XREG, XREG) +MINI_OP(OP_ARM64_UADD, "arm64_uadd", XREG, XREG, XREG) +MINI_OP(OP_ARM64_UADD2, "arm64_uadd2", XREG, XREG, XREG) +MINI_OP(OP_ARM64_SSUB, "arm64_ssub", XREG, XREG, XREG) +MINI_OP(OP_ARM64_SSUB2, "arm64_ssub2", XREG, XREG, XREG) +MINI_OP(OP_ARM64_USUB, "arm64_usub", XREG, XREG, XREG) +MINI_OP(OP_ARM64_USUB2, "arm64_usub2", XREG, XREG, XREG) + +MINI_OP(OP_ARM64_ADDP_SCALAR, "arm64_addp_scalar", XREG, XREG, NONE) +MINI_OP(OP_ARM64_FADDP_SCALAR, "arm64_faddp_scalar", XREG, XREG, NONE) + +MINI_OP(OP_ARM64_SADALP, "arm64_sadalp_scalar", XREG, XREG, XREG) +MINI_OP(OP_ARM64_UADALP, "arm64_uadalp_scalar", XREG, XREG, XREG) + +MINI_OP(OP_ARM64_SADDLV, "arm64_saddlv", XREG, XREG, NONE) +MINI_OP(OP_ARM64_UADDLV, "arm64_uaddlv", XREG, XREG, NONE) + +MINI_OP(OP_ARM64_XHORIZ, "arm64_xhoriz", XREG, XREG, NONE) + #endif // TARGET_ARM64 diff --git a/src/mono/mono/mini/mini.c b/src/mono/mono/mini/mini.c index b3bb0feca073e6..e219d5d616de53 100644 --- a/src/mono/mono/mini/mini.c +++ b/src/mono/mono/mini/mini.c @@ -4323,23 +4323,3 @@ mini_get_cpu_features (MonoCompile* cfg) // apply parameters passed via -mattr return (features | mono_cpu_features_enabled) & ~mono_cpu_features_disabled; } - -llvm_ovr_tag_t -ovr_tag_from_mono_vector_class (MonoClass *klass) { - int size = mono_class_value_size (klass, NULL); - llvm_ovr_tag_t ret = 0; - switch (size) { - case 8: ret |= INTRIN_vector64; break; - case 16: ret |= INTRIN_vector128; break; - } - MonoType *etype = mono_class_get_context (klass)->class_inst->type_argv [0]; - switch (etype->type) { - case MONO_TYPE_I1: case MONO_TYPE_U1: ret |= INTRIN_int8; break; - case MONO_TYPE_I2: case MONO_TYPE_U2: ret |= INTRIN_int16; break; - case MONO_TYPE_I4: case MONO_TYPE_U4: ret |= INTRIN_int32; break; - case MONO_TYPE_I8: case MONO_TYPE_U8: ret |= INTRIN_int64; break; - case MONO_TYPE_R4: ret |= INTRIN_float32; break; - case MONO_TYPE_R8: ret |= INTRIN_float64; break; - } - return ret; -} diff --git a/src/mono/mono/mini/mini.h b/src/mono/mono/mini/mini.h index 48fde7a348b87d..f755d643107987 100644 --- a/src/mono/mono/mini/mini.h +++ b/src/mono/mono/mini/mini.h @@ -3006,47 +3006,8 @@ enum { XBINOP_FORCEINT_xor, }; -enum { - INTRIN_scalar = 1 << 0, - INTRIN_vector64 = 1 << 1, - INTRIN_vector128 = 1 << 2, - INTRIN_vectorwidths = 3, - INTRIN_vectormask = 0x7, - - INTRIN_int8 = 1 << 3, - INTRIN_int16 = 1 << 4, - INTRIN_int32 = 1 << 5, - INTRIN_int64 = 1 << 6, - INTRIN_float32 = 1 << 7, - INTRIN_float64 = 1 << 8, - INTRIN_elementwidths = 6, -}; - -typedef uint16_t llvm_ovr_tag_t; - -static inline llvm_ovr_tag_t -ovr_tag_force_scalar (llvm_ovr_tag_t tag) -{ - return (tag & ~INTRIN_vectormask) | INTRIN_scalar; -} - -static inline llvm_ovr_tag_t -ovr_tag_smaller_vector (llvm_ovr_tag_t tag) -{ - return (tag & ~INTRIN_vectormask) | ((tag & INTRIN_vectormask) >> 1); -} - -llvm_ovr_tag_t -ovr_tag_from_mono_vector_class (MonoClass *klass); - -static int -int_from_id_and_ovr_tag (int id, llvm_ovr_tag_t ovr_tag) -{ - return (((int) ovr_tag) << 23) | id; -} - /* An intrinsic id. The lower 23 bits are used to store a mono-specific ID. The - * next 8 bits store overload tag bits. In the configuration of LLVM 9 we use, + * next 9 bits store overload tag bits. In the configuration of LLVM 9 we use, * there are 7017 total intrinsics defined in IntrinsicEnums.inc, so only 13 * bits are needed to label each intrinsic overload group. */ @@ -3056,7 +3017,7 @@ typedef enum { #define INTRINS_OVR_2_ARG(id, llvm_id, ty1, ty2) INTRINS_ ## id, #define INTRINS_OVR_3_ARG(id, llvm_id, ty1, ty2, ty3) INTRINS_ ## id, #define INTRINS_OVR_TAG(id, ...) INTRINS_ ## id, -#define INTRINS_OVR_TAG_FTOI(id, ...) INTRINS_ ## id, +#define INTRINS_OVR_TAG_KIND(id, ...) INTRINS_ ## id, #include "llvm-intrinsics.h" INTRINS_NUM } IntrinsicId; diff --git a/src/mono/mono/mini/simd-intrinsics.c b/src/mono/mono/mini/simd-intrinsics.c index 579385b22327e2..cd140490f7b86e 100644 --- a/src/mono/mono/mini/simd-intrinsics.c +++ b/src/mono/mono/mini/simd-intrinsics.c @@ -1026,6 +1026,24 @@ static SimdIntrinsic advsimd_methods [] = { {SN_AbsoluteCompareGreaterThanOrEqual}, {SN_AbsoluteCompareLessThan}, {SN_AbsoluteCompareLessThanOrEqual}, + {SN_Add, OP_XBINOP, OP_IADD, None, None, OP_XBINOP_SCALAR, OP_FADD}, + {SN_AddAcross, OP_ARM64_XHORIZ, INTRINS_AARCH64_ADV_SIMD_SADDV, OP_ARM64_XHORIZ, INTRINS_AARCH64_ADV_SIMD_UADDV}, + {SN_AddAcrossWidening, OP_ARM64_SADDLV, None, OP_ARM64_UADDLV}, + {SN_AddHighNarrowingLower, OP_ARM64_ADDHN}, + {SN_AddHighNarrowingUpper, OP_ARM64_ADDHN2}, + {SN_AddPairwise, OP_XOP_OVR_X_X_X, INTRINS_AARCH64_ADV_SIMD_ADDP, None, None, OP_XOP_OVR_X_X_X, INTRINS_AARCH64_ADV_SIMD_FADDP}, + {SN_AddPairwiseScalar, OP_ARM64_ADDP_SCALAR, None, None, None, OP_ARM64_FADDP_SCALAR}, + {SN_AddPairwiseWidening, OP_XOP_OVR_X_X, INTRINS_AARCH64_ADV_SIMD_SADDLP, OP_XOP_OVR_X_X, INTRINS_AARCH64_ADV_SIMD_UADDLP}, + {SN_AddPairwiseWideningAndAdd, OP_ARM64_SADALP, None, OP_ARM64_UADALP}, + {SN_AddPairwiseWideningAndAddScalar, OP_ARM64_SADALP, None, OP_ARM64_UADALP}, + {SN_AddPairwiseWideningScalar, OP_XOP_OVR_X_X, INTRINS_AARCH64_ADV_SIMD_SADDLP, OP_XOP_OVR_X_X, INTRINS_AARCH64_ADV_SIMD_UADDLP}, + {SN_AddRoundedHighNarrowingLower, OP_ARM64_RADDHN}, + {SN_AddRoundedHighNarrowingUpper, OP_ARM64_RADDHN2}, + {SN_AddSaturate}, + {SN_AddSaturateScalar}, + {SN_AddScalar, OP_XBINOP_SCALAR, OP_IADD, None, None, OP_XBINOP_SCALAR, OP_FADD}, + {SN_AddWideningLower, OP_ARM64_SADD, None, OP_ARM64_UADD}, + {SN_AddWideningUpper, OP_ARM64_SADD2, None, OP_ARM64_UADD2}, {SN_And, OP_XBINOP_FORCEINT, XBINOP_FORCEINT_and}, {SN_BitwiseClear, OP_ARM64_BIC}, {SN_BitwiseSelect, OP_ARM64_BSL}, @@ -1126,11 +1144,15 @@ static SimdIntrinsic advsimd_methods [] = { {SN_LoadVector128, OP_ARM64_LD1}, {SN_LoadVector64, OP_ARM64_LD1}, {SN_Max, OP_XOP_OVR_X_X_X, INTRINS_AARCH64_ADV_SIMD_SMAX, OP_XOP_OVR_X_X_X, INTRINS_AARCH64_ADV_SIMD_UMAX, OP_XOP_OVR_X_X_X, INTRINS_AARCH64_ADV_SIMD_FMAX}, + {SN_MaxAcross, OP_ARM64_XHORIZ, INTRINS_AARCH64_ADV_SIMD_SMAXV, OP_ARM64_XHORIZ, INTRINS_AARCH64_ADV_SIMD_UMAXV}, {SN_MaxNumber, OP_XOP_OVR_X_X_X, INTRINS_AARCH64_ADV_SIMD_FMAXNM}, + {SN_MaxNumberAcross, OP_ARM64_XHORIZ, INTRINS_AARCH64_ADV_SIMD_FMAXNMV}, {SN_MaxNumberScalar, OP_XOP_OVR_SCALAR_X_X_X, INTRINS_AARCH64_ADV_SIMD_FMAXNM}, {SN_MaxPairwise, OP_XOP_OVR_X_X_X, INTRINS_AARCH64_ADV_SIMD_SMAXP, OP_XOP_OVR_X_X_X, INTRINS_AARCH64_ADV_SIMD_UMAXP, OP_XOP_OVR_X_X_X, INTRINS_AARCH64_ADV_SIMD_FMAXP}, {SN_Min, OP_XOP_OVR_X_X_X, INTRINS_AARCH64_ADV_SIMD_SMIN, OP_XOP_OVR_X_X_X, INTRINS_AARCH64_ADV_SIMD_UMIN, OP_XOP_OVR_X_X_X, INTRINS_AARCH64_ADV_SIMD_FMIN}, + {SN_MinAcross, OP_ARM64_XHORIZ, INTRINS_AARCH64_ADV_SIMD_SMINV, OP_ARM64_XHORIZ, INTRINS_AARCH64_ADV_SIMD_UMINV}, {SN_MinNumber, OP_XOP_OVR_X_X_X, INTRINS_AARCH64_ADV_SIMD_FMINNM}, + {SN_MinNumberAcross, OP_ARM64_XHORIZ, INTRINS_AARCH64_ADV_SIMD_FMINNMV}, {SN_MinNumberScalar, OP_XOP_OVR_SCALAR_X_X_X, INTRINS_AARCH64_ADV_SIMD_FMINNM}, {SN_MinPairwise, OP_XOP_OVR_X_X_X, INTRINS_AARCH64_ADV_SIMD_SMINP, OP_XOP_OVR_X_X_X, INTRINS_AARCH64_ADV_SIMD_UMINP, OP_XOP_OVR_X_X_X, INTRINS_AARCH64_ADV_SIMD_FMINP}, {SN_MinPairwiseScalar}, @@ -1470,6 +1492,22 @@ emit_arm64_intrinsics ( } return emit_simd_ins_for_sig (cfg, klass, OP_XOP_X_X, op, arg0_type, fsig, args); } + case SN_AddSaturate: + case SN_AddSaturateScalar: { + gboolean arg0_unsigned = type_is_unsigned (fsig->params [0]); + gboolean arg1_unsigned = type_is_unsigned (fsig->params [1]); + int iid = 0; + if (arg0_unsigned && arg1_unsigned) + iid = INTRINS_AARCH64_ADV_SIMD_UQADD; + else if (arg0_unsigned && !arg1_unsigned) + iid = INTRINS_AARCH64_ADV_SIMD_USQADD; + else if (!arg0_unsigned && arg1_unsigned) + iid = INTRINS_AARCH64_ADV_SIMD_SUQADD; + else + iid = INTRINS_AARCH64_ADV_SIMD_SQADD; + int op = id == SN_AddSaturateScalar ? OP_XOP_OVR_SCALAR_X_X_X : OP_XOP_OVR_X_X_X; + return emit_simd_ins_for_sig (cfg, klass, op, iid, arg0_type, fsig, args); + } case SN_DuplicateSelectedScalarToVector128: case SN_DuplicateSelectedScalarToVector64: case SN_DuplicateToVector64: From 5ebdb81303d72390d7c106ee265f57d6342187e5 Mon Sep 17 00:00:00 2001 From: Imran Hameed Date: Sun, 7 Mar 2021 07:28:26 -0800 Subject: [PATCH 30/58] Implement Abs.*, Absolute.* --- src/mono/mono/mini/llvm-intrinsics.h | 20 ++++----- src/mono/mono/mini/mini-llvm.c | 54 +++++++++++++++++++----- src/mono/mono/mini/mini-ops.h | 15 +++++++ src/mono/mono/mini/mini.h | 10 ----- src/mono/mono/mini/simd-intrinsics.c | 63 +++++----------------------- 5 files changed, 80 insertions(+), 82 deletions(-) diff --git a/src/mono/mono/mini/llvm-intrinsics.h b/src/mono/mono/mini/llvm-intrinsics.h index 27ae5b8f1b76be..4828d064c4cb29 100644 --- a/src/mono/mono/mini/llvm-intrinsics.h +++ b/src/mono/mono/mini/llvm-intrinsics.h @@ -292,16 +292,6 @@ INTRINS(AARCH64_SHA256SU1, aarch64_crypto_sha256su1) INTRINS(AARCH64_SHA256H, aarch64_crypto_sha256h) INTRINS(AARCH64_SHA256H2, aarch64_crypto_sha256h2) INTRINS(AARCH64_PMULL64, aarch64_neon_pmull64) -INTRINS_OVR(AARCH64_ADV_SIMD_ABS_FLOAT, fabs, sse_r4_t) -INTRINS_OVR(AARCH64_ADV_SIMD_ABS_DOUBLE, fabs, sse_r8_t) -INTRINS_OVR(AARCH64_ADV_SIMD_ABS_INT8, aarch64_neon_abs, sse_i1_t) -INTRINS_OVR(AARCH64_ADV_SIMD_ABS_INT16, aarch64_neon_abs, sse_i2_t) -INTRINS_OVR(AARCH64_ADV_SIMD_ABS_INT32, aarch64_neon_abs, sse_i4_t) -INTRINS_OVR(AARCH64_ADV_SIMD_ABS_INT64, aarch64_neon_abs, sse_i8_t) -INTRINS_OVR(AARCH64_ADV_SIMD_ABS_SATURATE_INT8, aarch64_neon_sqabs, sse_i1_t) -INTRINS_OVR(AARCH64_ADV_SIMD_ABS_SATURATE_INT16, aarch64_neon_sqabs, sse_i2_t) -INTRINS_OVR(AARCH64_ADV_SIMD_ABS_SATURATE_INT32, aarch64_neon_sqabs, sse_i4_t) -INTRINS_OVR(AARCH64_ADV_SIMD_ABS_SATURATE_INT64, aarch64_neon_sqabs, sse_i8_t) INTRINS_OVR_2_ARG(AARCH64_ADV_SIMD_ABS_COMPARE_GT_FLOAT, aarch64_neon_facgt, sse_i4_t, sse_r4_t) INTRINS_OVR_2_ARG(AARCH64_ADV_SIMD_ABS_COMPARE_GT_DOUBLE, aarch64_neon_facgt, sse_i4_t, sse_r8_t) INTRINS_OVR_2_ARG(AARCH64_ADV_SIMD_ABS_COMPARE_GTE_FLOAT, aarch64_neon_facge, sse_i4_t, sse_r4_t) @@ -311,6 +301,16 @@ INTRINS_OVR_2_ARG(AARCH64_ADV_SIMD_ABS_COMPARE_LT_DOUBLE, aarch64_neon_facgt, ss INTRINS_OVR_2_ARG(AARCH64_ADV_SIMD_ABS_COMPARE_LTE_FLOAT, aarch64_neon_facge, sse_i4_t, sse_r4_t) INTRINS_OVR_2_ARG(AARCH64_ADV_SIMD_ABS_COMPARE_LTE_DOUBLE, aarch64_neon_facge, sse_i4_t, sse_r8_t) +INTRINS_OVR_TAG(AARCH64_ADV_SIMD_FABD_SCALAR, aarch64_sisd_fabd, Scalar | R4 | R8) + +INTRINS_OVR_TAG(AARCH64_ADV_SIMD_FABD, aarch64_neon_fabd, V64 | V128 | R4 | R8) +INTRINS_OVR_TAG(AARCH64_ADV_SIMD_UABD, aarch64_neon_uabd, V64 | V128 | I1 | I2 | I4) +INTRINS_OVR_TAG(AARCH64_ADV_SIMD_SABD, aarch64_neon_sabd, V64 | V128 | I1 | I2 | I4) + +INTRINS_OVR_TAG(AARCH64_ADV_SIMD_SQABS, aarch64_neon_sqabs, Scalar | V64 | V128 | I1 | I2 | I4 | I8) +INTRINS_OVR_TAG(AARCH64_ADV_SIMD_FABS, fabs, Scalar | V64 | V128 | R4 | R8) +INTRINS_OVR_TAG(AARCH64_ADV_SIMD_ABS, aarch64_neon_abs, Scalar | V64 | V128 | I1 | I2 | I4 | I8) + INTRINS_OVR_TAG_KIND(AARCH64_ADV_SIMD_UADDLV, aarch64_neon_uaddlv, WidenAcross, V64 | V128 | I1 | I2 | I4) INTRINS_OVR_TAG_KIND(AARCH64_ADV_SIMD_SADDLV, aarch64_neon_saddlv, WidenAcross, V64 | V128 | I1 | I2 | I4) diff --git a/src/mono/mono/mini/mini-llvm.c b/src/mono/mono/mini/mini-llvm.c index f3920e65649460..ac86cf2ee8b524 100644 --- a/src/mono/mono/mini/mini-llvm.c +++ b/src/mono/mono/mini/mini-llvm.c @@ -9477,16 +9477,6 @@ process_bb (EmitContext *ctx, MonoBasicBlock *bb) switch (ins->inst_c0) { case SIMD_OP_AES_IMC: id = INTRINS_AARCH64_AESIMC; break; case SIMD_OP_ARM64_AES_AESMC: id = INTRINS_AARCH64_AESMC; break; - case SIMD_OP_ARM64_FABS: id = INTRINS_AARCH64_ADV_SIMD_ABS_FLOAT; break; - case SIMD_OP_ARM64_DABS: id = INTRINS_AARCH64_ADV_SIMD_ABS_DOUBLE; break; - case SIMD_OP_ARM64_I8ABS: id = INTRINS_AARCH64_ADV_SIMD_ABS_INT8; break; - case SIMD_OP_ARM64_I16ABS: id = INTRINS_AARCH64_ADV_SIMD_ABS_INT16; break; - case SIMD_OP_ARM64_I32ABS: id = INTRINS_AARCH64_ADV_SIMD_ABS_INT32; break; - case SIMD_OP_ARM64_I64ABS: id = INTRINS_AARCH64_ADV_SIMD_ABS_INT64; break; - case SIMD_OP_ARM64_I8ABS_SATURATE: id = INTRINS_AARCH64_ADV_SIMD_ABS_SATURATE_INT8; break; - case SIMD_OP_ARM64_I16ABS_SATURATE: id = INTRINS_AARCH64_ADV_SIMD_ABS_SATURATE_INT16; break; - case SIMD_OP_ARM64_I32ABS_SATURATE: id = INTRINS_AARCH64_ADV_SIMD_ABS_SATURATE_INT32; break; - case SIMD_OP_ARM64_I64ABS_SATURATE: id = INTRINS_AARCH64_ADV_SIMD_ABS_SATURATE_INT64; break; case SIMD_OP_ARM64_SHA1H: id = INTRINS_AARCH64_SHA1H; getLowerElement = TRUE; break; default: g_assert_not_reached (); break; } @@ -10374,6 +10364,50 @@ process_bb (EmitContext *ctx, MonoBasicBlock *bb) values [ins->dreg] = result; break; } + case OP_ARM64_SABAL: + case OP_ARM64_SABAL2: + case OP_ARM64_UABAL: + case OP_ARM64_UABAL2: + case OP_ARM64_SABDL: + case OP_ARM64_SABDL2: + case OP_ARM64_UABDL: + case OP_ARM64_UABDL2: + case OP_ARM64_SABA: + case OP_ARM64_UABA: + case OP_ARM64_SABD: + case OP_ARM64_UABD: { + LLVMTypeRef ret_t = simd_class_to_llvm_type (ctx, ins->klass); + gboolean is_unsigned = FALSE; + gboolean high = FALSE; + gboolean add = FALSE; + gboolean widen = FALSE; + switch (ins->opcode) { + case OP_ARM64_SABAL2: high = TRUE; case OP_ARM64_SABAL: widen = TRUE; add = TRUE; break; + case OP_ARM64_UABAL2: high = TRUE; case OP_ARM64_UABAL: widen = TRUE; add = TRUE; is_unsigned = TRUE; break; + case OP_ARM64_SABDL2: high = TRUE; case OP_ARM64_SABDL: widen = TRUE; break; + case OP_ARM64_UABDL2: high = TRUE; case OP_ARM64_UABDL: widen = TRUE; is_unsigned = TRUE; break; + case OP_ARM64_SABA: add = TRUE; break; + case OP_ARM64_UABA: add = TRUE; is_unsigned = TRUE; break; + case OP_ARM64_UABD: is_unsigned = TRUE; break; + } + LLVMValueRef args [] = { lhs, rhs }; + if (add) { + args [0] = rhs; + args [1] = arg3; + } + if (high) + for (int i = 0; i < 2; ++i) + args [i] = extract_high_elements (ctx, args [i]); + int iid = is_unsigned ? INTRINS_AARCH64_ADV_SIMD_UABD : INTRINS_AARCH64_ADV_SIMD_SABD; + llvm_ovr_tag_t ovr_tag = ovr_tag_from_llvm_type (LLVMTypeOf (args [0])); + LLVMValueRef result = call_overloaded_intrins (ctx, iid, ovr_tag, args, ""); + if (widen) + result = LLVMBuildZExt (builder, result, ret_t, ""); + if (add) + result = LLVMBuildAdd (builder, result, lhs, ""); + values [ins->dreg] = result; + break; + } case OP_ARM64_XHORIZ: { gboolean truncate = FALSE; LLVMTypeRef arg_t = LLVMTypeOf (lhs); diff --git a/src/mono/mono/mini/mini-ops.h b/src/mono/mono/mini/mini-ops.h index 30f4890b17ed70..f658140033db6d 100644 --- a/src/mono/mono/mini/mini-ops.h +++ b/src/mono/mono/mini/mini-ops.h @@ -1803,4 +1803,19 @@ MINI_OP(OP_ARM64_UADDLV, "arm64_uaddlv", XREG, XREG, NONE) MINI_OP(OP_ARM64_XHORIZ, "arm64_xhoriz", XREG, XREG, NONE) +MINI_OP3(OP_ARM64_SABAL, "arm64_sabal", XREG, XREG, XREG, XREG) +MINI_OP3(OP_ARM64_SABAL2, "arm64_sabal2", XREG, XREG, XREG, XREG) +MINI_OP3(OP_ARM64_UABAL, "arm64_uabal", XREG, XREG, XREG, XREG) +MINI_OP3(OP_ARM64_UABAL2, "arm64_uabal2", XREG, XREG, XREG, XREG) + +MINI_OP(OP_ARM64_SABDL, "arm64_sabdl", XREG, XREG, XREG) +MINI_OP(OP_ARM64_SABDL2, "arm64_sabdl2", XREG, XREG, XREG) +MINI_OP(OP_ARM64_UABDL, "arm64_uabdl", XREG, XREG, XREG) +MINI_OP(OP_ARM64_UABDL2, "arm64_uabdl2", XREG, XREG, XREG) + +MINI_OP3(OP_ARM64_SABA, "arm64_saba", XREG, XREG, XREG, XREG) +MINI_OP3(OP_ARM64_UABA, "arm64_uaba", XREG, XREG, XREG, XREG) +MINI_OP(OP_ARM64_SABD, "arm64_sabd", XREG, XREG, XREG) +MINI_OP(OP_ARM64_UABD, "arm64_uabd", XREG, XREG, XREG) + #endif // TARGET_ARM64 diff --git a/src/mono/mono/mini/mini.h b/src/mono/mono/mini/mini.h index f755d643107987..a678bf895478a5 100644 --- a/src/mono/mono/mini/mini.h +++ b/src/mono/mono/mini/mini.h @@ -2877,16 +2877,6 @@ enum { /* SIMD operations */ typedef enum { - SIMD_OP_ARM64_FABS, - SIMD_OP_ARM64_DABS, - SIMD_OP_ARM64_I8ABS, - SIMD_OP_ARM64_I16ABS, - SIMD_OP_ARM64_I32ABS, - SIMD_OP_ARM64_I64ABS, - SIMD_OP_ARM64_I8ABS_SATURATE, - SIMD_OP_ARM64_I16ABS_SATURATE, - SIMD_OP_ARM64_I32ABS_SATURATE, - SIMD_OP_ARM64_I64ABS_SATURATE, SIMD_OP_ARM64_FABSOLUTE_COMPARE_GREATER_THAN, SIMD_OP_ARM64_DABSOLUTE_COMPARE_GREATER_THAN, SIMD_OP_ARM64_FABSOLUTE_COMPARE_GREATER_THAN_OR_EQUAL, diff --git a/src/mono/mono/mini/simd-intrinsics.c b/src/mono/mono/mini/simd-intrinsics.c index cd140490f7b86e..31cd5781e73836 100644 --- a/src/mono/mono/mini/simd-intrinsics.c +++ b/src/mono/mono/mini/simd-intrinsics.c @@ -1019,13 +1019,21 @@ static SimdIntrinsic sha256_methods [] = { // In Vim you can use `sort /.*{[0-9A-z]*/ r` to sort this table. static SimdIntrinsic advsimd_methods [] = { - {SN_Abs}, - {SN_AbsSaturate}, - {SN_AbsScalar}, + {SN_Abs, OP_XOP_OVR_X_X, INTRINS_AARCH64_ADV_SIMD_ABS, None, None, OP_XOP_OVR_X_X, INTRINS_AARCH64_ADV_SIMD_FABS}, + {SN_AbsSaturate, OP_XOP_OVR_X_X, INTRINS_AARCH64_ADV_SIMD_SQABS}, + {SN_AbsSaturateScalar, OP_XOP_OVR_SCALAR_X_X, INTRINS_AARCH64_ADV_SIMD_SQABS}, + {SN_AbsScalar, OP_XOP_OVR_SCALAR_X_X, INTRINS_AARCH64_ADV_SIMD_ABS, None, None, OP_XOP_OVR_SCALAR_X_X, INTRINS_AARCH64_ADV_SIMD_FABS}, {SN_AbsoluteCompareGreaterThan}, {SN_AbsoluteCompareGreaterThanOrEqual}, {SN_AbsoluteCompareLessThan}, {SN_AbsoluteCompareLessThanOrEqual}, + {SN_AbsoluteDifference, OP_ARM64_SABD, None, OP_ARM64_UABD, None, OP_XOP_OVR_X_X_X, INTRINS_AARCH64_ADV_SIMD_FABD}, + {SN_AbsoluteDifferenceAdd, OP_ARM64_SABA, None, OP_ARM64_UABA}, + {SN_AbsoluteDifferenceScalar, OP_XOP_OVR_SCALAR_X_X_X, INTRINS_AARCH64_ADV_SIMD_FABD_SCALAR}, + {SN_AbsoluteDifferenceWideningLower, OP_ARM64_SABDL, None, OP_ARM64_UABDL}, + {SN_AbsoluteDifferenceWideningLowerAndAdd, OP_ARM64_SABAL, None, OP_ARM64_UABAL}, + {SN_AbsoluteDifferenceWideningUpper, OP_ARM64_SABDL2, None, OP_ARM64_UABDL2}, + {SN_AbsoluteDifferenceWideningUpperAndAdd, OP_ARM64_SABAL2, None, OP_ARM64_UABAL2}, {SN_Add, OP_XBINOP, OP_IADD, None, None, OP_XBINOP_SCALAR, OP_FADD}, {SN_AddAcross, OP_ARM64_XHORIZ, INTRINS_AARCH64_ADV_SIMD_SADDV, OP_ARM64_XHORIZ, INTRINS_AARCH64_ADV_SIMD_UADDV}, {SN_AddAcrossWidening, OP_ARM64_SADDLV, None, OP_ARM64_UADDLV}, @@ -1416,33 +1424,6 @@ emit_arm64_intrinsics ( SimdOp op = (SimdOp) 0; IntrinsicId iid = (IntrinsicId) -1; switch (id) { - case SN_Abs: { - // HACK: Temporary, while Vector64 support is completed - MonoClass *arg0_klass = mono_class_from_mono_type_internal (fsig->params [0]); - if (m_class_get_name (arg0_klass), "Vector64`1") - mono_emit_jit_icall (cfg, mono_throw_platform_not_supported, NULL); - - switch (arg0_type) { - case MONO_TYPE_R8: - op = SIMD_OP_ARM64_DABS; - break; - case MONO_TYPE_R4: - op = SIMD_OP_ARM64_FABS; - break; - case MONO_TYPE_I1: - op = SIMD_OP_ARM64_I8ABS; - break; - case MONO_TYPE_I2: - op = SIMD_OP_ARM64_I16ABS; - break; - case MONO_TYPE_I4: - op = SIMD_OP_ARM64_I32ABS; - break; - case MONO_TYPE_I8: - op = SIMD_OP_ARM64_I64ABS; - break; - } - } case SN_AbsoluteCompareGreaterThan: { return emit_absolute_compare (cfg, klass, fsig, arg0_type, args, SIMD_OP_ARM64_FABSOLUTE_COMPARE_GREATER_THAN, SIMD_OP_ARM64_DABSOLUTE_COMPARE_GREATER_THAN); @@ -1470,28 +1451,6 @@ emit_arm64_intrinsics ( return emit_absolute_compare (cfg, klass, fsig, arg0_type, args, SIMD_OP_ARM64_FABSOLUTE_COMPARE_LESS_THAN_OR_EQUAL, SIMD_OP_ARM64_DABSOLUTE_COMPARE_LESS_THAN_OR_EQUAL); } - case SN_AbsSaturate: { - switch (arg0_type) { - case MONO_TYPE_I1: op = SIMD_OP_ARM64_I8ABS_SATURATE; break; - case MONO_TYPE_I2: op = SIMD_OP_ARM64_I16ABS_SATURATE; break; - case MONO_TYPE_I4: op = SIMD_OP_ARM64_I32ABS_SATURATE; break; - case MONO_TYPE_I8: op = SIMD_OP_ARM64_I64ABS_SATURATE; break; - default: g_assert_not_reached (); - } - - return emit_simd_ins_for_sig (cfg, klass, OP_XOP_X_X, op, arg0_type, fsig, args); - } - - case SN_AbsScalar: { - switch (arg0_type) { - case MONO_TYPE_I1: op = SIMD_OP_ARM64_I8ABS_SATURATE; break; - case MONO_TYPE_I2: op = SIMD_OP_ARM64_I16ABS_SATURATE; break; - case MONO_TYPE_I4: op = SIMD_OP_ARM64_I32ABS_SATURATE; break; - case MONO_TYPE_I8: op = SIMD_OP_ARM64_I64ABS_SATURATE; break; - default: g_assert_not_reached (); - } - return emit_simd_ins_for_sig (cfg, klass, OP_XOP_X_X, op, arg0_type, fsig, args); - } case SN_AddSaturate: case SN_AddSaturateScalar: { gboolean arg0_unsigned = type_is_unsigned (fsig->params [0]); From b20631fb6f864201307ceb9edd455cdaedf71b95 Mon Sep 17 00:00:00 2001 From: Imran Hameed Date: Sun, 7 Mar 2021 08:07:58 -0800 Subject: [PATCH 31/58] Implement AbsoluteCompare.* --- src/mono/mono/mini/llvm-intrinsics.h | 11 ++--- src/mono/mono/mini/mini-llvm.c | 36 ++++++++++---- src/mono/mono/mini/mini-ops.h | 2 + src/mono/mono/mini/mini.h | 8 ---- src/mono/mono/mini/simd-intrinsics.c | 72 +++++++++++++--------------- 5 files changed, 65 insertions(+), 64 deletions(-) diff --git a/src/mono/mono/mini/llvm-intrinsics.h b/src/mono/mono/mini/llvm-intrinsics.h index 4828d064c4cb29..2f862b677ebe69 100644 --- a/src/mono/mono/mini/llvm-intrinsics.h +++ b/src/mono/mono/mini/llvm-intrinsics.h @@ -292,14 +292,9 @@ INTRINS(AARCH64_SHA256SU1, aarch64_crypto_sha256su1) INTRINS(AARCH64_SHA256H, aarch64_crypto_sha256h) INTRINS(AARCH64_SHA256H2, aarch64_crypto_sha256h2) INTRINS(AARCH64_PMULL64, aarch64_neon_pmull64) -INTRINS_OVR_2_ARG(AARCH64_ADV_SIMD_ABS_COMPARE_GT_FLOAT, aarch64_neon_facgt, sse_i4_t, sse_r4_t) -INTRINS_OVR_2_ARG(AARCH64_ADV_SIMD_ABS_COMPARE_GT_DOUBLE, aarch64_neon_facgt, sse_i4_t, sse_r8_t) -INTRINS_OVR_2_ARG(AARCH64_ADV_SIMD_ABS_COMPARE_GTE_FLOAT, aarch64_neon_facge, sse_i4_t, sse_r4_t) -INTRINS_OVR_2_ARG(AARCH64_ADV_SIMD_ABS_COMPARE_GTE_DOUBLE, aarch64_neon_facge, sse_i4_t, sse_r8_t) -INTRINS_OVR_2_ARG(AARCH64_ADV_SIMD_ABS_COMPARE_LT_FLOAT, aarch64_neon_facgt, sse_i4_t, sse_r4_t) -INTRINS_OVR_2_ARG(AARCH64_ADV_SIMD_ABS_COMPARE_LT_DOUBLE, aarch64_neon_facgt, sse_i4_t, sse_r8_t) -INTRINS_OVR_2_ARG(AARCH64_ADV_SIMD_ABS_COMPARE_LTE_FLOAT, aarch64_neon_facge, sse_i4_t, sse_r4_t) -INTRINS_OVR_2_ARG(AARCH64_ADV_SIMD_ABS_COMPARE_LTE_DOUBLE, aarch64_neon_facge, sse_i4_t, sse_r8_t) + +INTRINS_OVR_TAG_KIND(AARCH64_ADV_SIMD_FACGE, aarch64_neon_facge, Ftoi, Scalar | V64 | V128 | I4 | I8) +INTRINS_OVR_TAG_KIND(AARCH64_ADV_SIMD_FACGT, aarch64_neon_facgt, Ftoi, Scalar | V64 | V128 | I4 | I8) INTRINS_OVR_TAG(AARCH64_ADV_SIMD_FABD_SCALAR, aarch64_sisd_fabd, Scalar | R4 | R8) diff --git a/src/mono/mono/mini/mini-llvm.c b/src/mono/mono/mini/mini-llvm.c index ac86cf2ee8b524..f24023f2caa41f 100644 --- a/src/mono/mono/mini/mini-llvm.c +++ b/src/mono/mono/mini/mini-llvm.c @@ -368,6 +368,12 @@ ovr_tag_smaller_vector (llvm_ovr_tag_t tag) return (tag & ~INTRIN_vectormask) | ((tag & INTRIN_vectormask) >> 1); } +static inline llvm_ovr_tag_t +ovr_tag_corresponding_integer (llvm_ovr_tag_t tag) +{ + return ((tag & ~INTRIN_vectormask) >> 2) | (tag & INTRIN_vectormask); +} + static int int_from_id_and_ovr_tag (int id, llvm_ovr_tag_t ovr_tag) { @@ -9409,14 +9415,6 @@ process_bb (EmitContext *ctx, MonoBasicBlock *bb) case SIMD_OP_AES_ENC: id = INTRINS_AARCH64_AESE; break; case SIMD_OP_ARM64_SHA1SU1: id = INTRINS_AARCH64_SHA1SU1; break; case SIMD_OP_ARM64_SHA256SU0: id = INTRINS_AARCH64_SHA256SU0; break; - case SIMD_OP_ARM64_FABSOLUTE_COMPARE_GREATER_THAN: id = INTRINS_AARCH64_ADV_SIMD_ABS_COMPARE_GT_FLOAT; break; - case SIMD_OP_ARM64_DABSOLUTE_COMPARE_GREATER_THAN: id = INTRINS_AARCH64_ADV_SIMD_ABS_COMPARE_GT_DOUBLE; break; - case SIMD_OP_ARM64_FABSOLUTE_COMPARE_GREATER_THAN_OR_EQUAL: id = INTRINS_AARCH64_ADV_SIMD_ABS_COMPARE_GTE_FLOAT; break; - case SIMD_OP_ARM64_DABSOLUTE_COMPARE_GREATER_THAN_OR_EQUAL: id = INTRINS_AARCH64_ADV_SIMD_ABS_COMPARE_GTE_DOUBLE; break; - case SIMD_OP_ARM64_FABSOLUTE_COMPARE_LESS_THAN: id = INTRINS_AARCH64_ADV_SIMD_ABS_COMPARE_LT_FLOAT; break; - case SIMD_OP_ARM64_DABSOLUTE_COMPARE_LESS_THAN: id = INTRINS_AARCH64_ADV_SIMD_ABS_COMPARE_LT_DOUBLE; break; - case SIMD_OP_ARM64_FABSOLUTE_COMPARE_LESS_THAN_OR_EQUAL: id = INTRINS_AARCH64_ADV_SIMD_ABS_COMPARE_LTE_FLOAT; break; - case SIMD_OP_ARM64_DABSOLUTE_COMPARE_LESS_THAN_OR_EQUAL: id = INTRINS_AARCH64_ADV_SIMD_ABS_COMPARE_LTE_DOUBLE; break; case SIMD_OP_ARM64_PMULL64_LOWER: id = INTRINS_AARCH64_PMULL64; getElement = TRUE; @@ -10539,6 +10537,28 @@ process_bb (EmitContext *ctx, MonoBasicBlock *bb) values [ins->dreg] = LLVMBuildShuffleVector (builder, lhs, rhs, create_const_vector_i32 (mask, src_elems), "arm64_zip"); break; } + case OP_ARM64_ABSCOMPARE: { + IntrinsicId iid = (IntrinsicId) ins->inst_c0; + gboolean scalar = ins->inst_c1; + LLVMTypeRef ret_t = simd_class_to_llvm_type (ctx, ins->klass); + LLVMTypeRef elem_t = LLVMGetElementType (ret_t); + llvm_ovr_tag_t ovr_tag = ovr_tag_from_mono_vector_class (ins->klass); + ovr_tag = ovr_tag_corresponding_integer (ovr_tag); + LLVMValueRef args [] = { lhs, rhs }; + LLVMTypeRef result_t = ret_t; + if (scalar) { + ovr_tag = ovr_tag_force_scalar (ovr_tag); + result_t = elem_t; + for (int i = 0; i < 2; ++i) + args [i] = scalar_from_vector (ctx, args [i]); + } + LLVMValueRef result = call_overloaded_intrins (ctx, iid, ovr_tag, args, ""); + result = LLVMBuildBitCast (builder, result, result_t, ""); + if (scalar) + result = vector_from_scalar_ty (ctx, ret_t, result); + values [ins->dreg] = result; + break; + } case OP_XOP_OVR_X_X: { IntrinsicId iid = (IntrinsicId) ins->inst_c0; llvm_ovr_tag_t ovr_tag = ovr_tag_from_mono_vector_class (ins->klass); diff --git a/src/mono/mono/mini/mini-ops.h b/src/mono/mono/mini/mini-ops.h index f658140033db6d..739d425144a274 100644 --- a/src/mono/mono/mini/mini-ops.h +++ b/src/mono/mono/mini/mini-ops.h @@ -1818,4 +1818,6 @@ MINI_OP3(OP_ARM64_UABA, "arm64_uaba", XREG, XREG, XREG, XREG) MINI_OP(OP_ARM64_SABD, "arm64_sabd", XREG, XREG, XREG) MINI_OP(OP_ARM64_UABD, "arm64_uabd", XREG, XREG, XREG) +MINI_OP(OP_ARM64_ABSCOMPARE, "arm64_abscompare", XREG, XREG, XREG) + #endif // TARGET_ARM64 diff --git a/src/mono/mono/mini/mini.h b/src/mono/mono/mini/mini.h index a678bf895478a5..789b155a92ae57 100644 --- a/src/mono/mono/mini/mini.h +++ b/src/mono/mono/mini/mini.h @@ -2877,14 +2877,6 @@ enum { /* SIMD operations */ typedef enum { - SIMD_OP_ARM64_FABSOLUTE_COMPARE_GREATER_THAN, - SIMD_OP_ARM64_DABSOLUTE_COMPARE_GREATER_THAN, - SIMD_OP_ARM64_FABSOLUTE_COMPARE_GREATER_THAN_OR_EQUAL, - SIMD_OP_ARM64_DABSOLUTE_COMPARE_GREATER_THAN_OR_EQUAL, - SIMD_OP_ARM64_FABSOLUTE_COMPARE_LESS_THAN, - SIMD_OP_ARM64_DABSOLUTE_COMPARE_LESS_THAN, - SIMD_OP_ARM64_FABSOLUTE_COMPARE_LESS_THAN_OR_EQUAL, - SIMD_OP_ARM64_DABSOLUTE_COMPARE_LESS_THAN_OR_EQUAL, SIMD_OP_SSE_CVTSS2SI, SIMD_OP_SSE_CVTTSS2SI, SIMD_OP_SSE_CVTSS2SI64, diff --git a/src/mono/mono/mini/simd-intrinsics.c b/src/mono/mono/mini/simd-intrinsics.c index 31cd5781e73836..3d9a18a6b07e06 100644 --- a/src/mono/mono/mini/simd-intrinsics.c +++ b/src/mono/mono/mini/simd-intrinsics.c @@ -1025,8 +1025,12 @@ static SimdIntrinsic advsimd_methods [] = { {SN_AbsScalar, OP_XOP_OVR_SCALAR_X_X, INTRINS_AARCH64_ADV_SIMD_ABS, None, None, OP_XOP_OVR_SCALAR_X_X, INTRINS_AARCH64_ADV_SIMD_FABS}, {SN_AbsoluteCompareGreaterThan}, {SN_AbsoluteCompareGreaterThanOrEqual}, + {SN_AbsoluteCompareGreaterThanOrEqualScalar}, + {SN_AbsoluteCompareGreaterThanScalar}, {SN_AbsoluteCompareLessThan}, {SN_AbsoluteCompareLessThanOrEqual}, + {SN_AbsoluteCompareLessThanOrEqualScalar}, + {SN_AbsoluteCompareLessThanScalar}, {SN_AbsoluteDifference, OP_ARM64_SABD, None, OP_ARM64_UABD, None, OP_XOP_OVR_X_X_X, INTRINS_AARCH64_ADV_SIMD_FABD}, {SN_AbsoluteDifferenceAdd, OP_ARM64_SABA, None, OP_ARM64_UABA}, {SN_AbsoluteDifferenceScalar, OP_XOP_OVR_SCALAR_X_X_X, INTRINS_AARCH64_ADV_SIMD_FABD_SCALAR}, @@ -1340,25 +1344,6 @@ static SimdIntrinsic advsimd_methods [] = { {SN_get_IsSupported}, }; -static -MonoInst *emit_absolute_compare (MonoCompile *cfg, MonoClass *klass, MonoMethodSignature *fsig, MonoTypeEnum arg0_type, MonoInst **args, SimdOp op_for_r4, SimdOp op_for_r8) -{ - SimdOp op = (SimdOp)0; - - switch (get_underlying_type (fsig->params [0])) { - case MONO_TYPE_R4: - op = op_for_r4; - break; - case MONO_TYPE_R8: - op = op_for_r8; - break; - default: - g_assert_not_reached(); - } - - return emit_simd_ins_for_sig (cfg, klass, OP_XOP_X_X_X, op, arg0_type, fsig, args); -} - static const IntrinGroup supported_arm_intrinsics [] = { { "AdvSimd", MONO_CPU_ARM64_NEON, advsimd_methods, sizeof (advsimd_methods) }, { "Aes", MONO_CPU_ARM64_CRYPTO, crypto_aes_methods, sizeof (crypto_aes_methods) }, @@ -1425,30 +1410,37 @@ emit_arm64_intrinsics ( IntrinsicId iid = (IntrinsicId) -1; switch (id) { - case SN_AbsoluteCompareGreaterThan: { - return emit_absolute_compare (cfg, klass, fsig, arg0_type, args, SIMD_OP_ARM64_FABSOLUTE_COMPARE_GREATER_THAN, SIMD_OP_ARM64_DABSOLUTE_COMPARE_GREATER_THAN); - } - - case SN_AbsoluteCompareGreaterThanOrEqual: { - return emit_absolute_compare (cfg, klass, fsig, arg0_type, args, SIMD_OP_ARM64_FABSOLUTE_COMPARE_GREATER_THAN_OR_EQUAL, SIMD_OP_ARM64_DABSOLUTE_COMPARE_GREATER_THAN_OR_EQUAL); - } - - case SN_AbsoluteCompareLessThan: { - // Compare less than uses the same instructions as greater than, with arguments swapped. - MonoInst *temp_for_swap = args [0]; - args [0] = args [1]; - args [1] = temp_for_swap; + case SN_AbsoluteCompareGreaterThan: + case SN_AbsoluteCompareGreaterThanOrEqual: + case SN_AbsoluteCompareLessThan: + case SN_AbsoluteCompareLessThanOrEqual: + case SN_AbsoluteCompareGreaterThanScalar: + case SN_AbsoluteCompareGreaterThanOrEqualScalar: + case SN_AbsoluteCompareLessThanScalar: + case SN_AbsoluteCompareLessThanOrEqualScalar: { + gboolean reverse_args = FALSE; + gboolean use_geq = FALSE; + gboolean scalar = FALSE; + MonoInst *cmp_args [] = { args [0], args [1] }; + switch (id) { + case SN_AbsoluteCompareGreaterThanScalar: scalar = TRUE; + case SN_AbsoluteCompareGreaterThan: break; - return emit_absolute_compare (cfg, klass, fsig, arg0_type, args, SIMD_OP_ARM64_FABSOLUTE_COMPARE_LESS_THAN, SIMD_OP_ARM64_DABSOLUTE_COMPARE_LESS_THAN); - } + case SN_AbsoluteCompareGreaterThanOrEqualScalar: scalar = TRUE; + case SN_AbsoluteCompareGreaterThanOrEqual: use_geq = TRUE; break; - case SN_AbsoluteCompareLessThanOrEqual: { - // Compare less than uses the same instructions as greater than, with arguments swapped. - MonoInst *temp_for_swap = args [0]; - args [0] = args [1]; - args [1] = temp_for_swap; + case SN_AbsoluteCompareLessThanScalar: scalar = TRUE; + case SN_AbsoluteCompareLessThan: reverse_args = TRUE; break; - return emit_absolute_compare (cfg, klass, fsig, arg0_type, args, SIMD_OP_ARM64_FABSOLUTE_COMPARE_LESS_THAN_OR_EQUAL, SIMD_OP_ARM64_DABSOLUTE_COMPARE_LESS_THAN_OR_EQUAL); + case SN_AbsoluteCompareLessThanOrEqualScalar: scalar = TRUE; + case SN_AbsoluteCompareLessThanOrEqual: reverse_args = TRUE; use_geq = TRUE; break; + } + if (reverse_args) { + cmp_args [0] = args [1]; + cmp_args [1] = args [0]; + } + int iid = use_geq ? INTRINS_AARCH64_ADV_SIMD_FACGE : INTRINS_AARCH64_ADV_SIMD_FACGT; + return emit_simd_ins_for_sig (cfg, klass, OP_ARM64_ABSCOMPARE, iid, scalar, fsig, cmp_args); } case SN_AddSaturate: From 4a78abe4f607327711297737093a39b41709160d Mon Sep 17 00:00:00 2001 From: Imran Hameed Date: Sun, 7 Mar 2021 11:05:44 -0800 Subject: [PATCH 32/58] Misc fixes to shifts, math --- src/mono/mono/mini/llvm-intrinsics.h | 8 +- src/mono/mono/mini/mini-llvm.c | 124 ++++++++++++++------------- src/mono/mono/mini/mini-ops.h | 24 ++---- src/mono/mono/mini/simd-intrinsics.c | 92 +++++++------------- 4 files changed, 107 insertions(+), 141 deletions(-) diff --git a/src/mono/mono/mini/llvm-intrinsics.h b/src/mono/mono/mini/llvm-intrinsics.h index 2f862b677ebe69..2701a933dce6c1 100644 --- a/src/mono/mono/mini/llvm-intrinsics.h +++ b/src/mono/mono/mini/llvm-intrinsics.h @@ -384,12 +384,12 @@ INTRINS_OVR_TAG(AARCH64_ADV_SIMD_SQRDMULH, aarch64_neon_sqrdmulh, V64 | V128 | I INTRINS_OVR_TAG(AARCH64_ADV_SIMD_SMULL, aarch64_neon_smull, V128 | I2 | I4 | I8) INTRINS_OVR_TAG(AARCH64_ADV_SIMD_UMULL, aarch64_neon_umull, V128 | I2 | I4 | I8) -INTRINS_OVR_TAG(AARCH64_ADV_SIMD_SQNEG, aarch64_neon_sqneg, V64 | V128 | I1 | I2 | I4 | I8) +INTRINS_OVR_TAG(AARCH64_ADV_SIMD_SQNEG, aarch64_neon_sqneg, Scalar | V64 | V128 | I1 | I2 | I4 | I8) INTRINS_OVR_TAG(AARCH64_ADV_SIMD_PMUL, aarch64_neon_pmul, V64 | V128 | I1) INTRINS_OVR(AARCH64_ADV_SIMD_PMULL, aarch64_neon_pmull, v128_i2_t) -INTRINS_OVR_TAG(AARCH64_ADV_SIMD_CNT, ctpop, V64 | V128 | I8) +INTRINS_OVR_TAG(AARCH64_ADV_SIMD_CNT, ctpop, V64 | V128 | I1) INTRINS_OVR_TAG(AARCH64_ADV_SIMD_URECPE, aarch64_neon_urecpe, V64 | V128 | I4) INTRINS_OVR_TAG(AARCH64_ADV_SIMD_FRECPE, aarch64_neon_frecpe, Scalar | V64 | V128 | R4 | R8) @@ -417,7 +417,7 @@ INTRINS_OVR_TAG(AARCH64_ADV_SIMD_SQSUB, aarch64_neon_sqsub, Scalar | V64 | V128 INTRINS_OVR_TAG(AARCH64_ADV_SIMD_RADDHN, aarch64_neon_raddhn, V64 | I1 | I2 | I4) INTRINS_OVR_TAG(AARCH64_ADV_SIMD_RSUBHN, aarch64_neon_rsubhn, V64 | I1 | I2 | I4) -INTRINS_OVR_TAG(AARCH64_ADV_SIMD_FSQRT, sqrt, V64 | V128 | R4 | R8) +INTRINS_OVR_TAG(AARCH64_ADV_SIMD_FSQRT, sqrt, Scalar | V64 | V128 | R4 | R8) INTRINS_OVR_TAG(AARCH64_ADV_SIMD_UQSHRN, aarch64_neon_uqshrn, V64 | I1 | I2 | I4) // Constant shift @@ -426,7 +426,7 @@ INTRINS_OVR_TAG(AARCH64_ADV_SIMD_SQRSHRN, aarch64_neon_sqrshrn, V64 | I1 | I2 | INTRINS_OVR_TAG(AARCH64_ADV_SIMD_SQRSHRUN, aarch64_neon_sqrshrun, V64 | I1 | I2 | I4) // Constant shift INTRINS_OVR_TAG(AARCH64_ADV_SIMD_SQSHRN, aarch64_neon_sqshrn, V64 | I1 | I2 | I4) // Constant shift INTRINS_OVR_TAG(AARCH64_ADV_SIMD_SQSHRUN, aarch64_neon_sqshrun, V64 | I1 | I2 | I4) // Constant shift -INTRINS_OVR_TAG(AARCH64_ADV_SIMD_UQRSHRN, aarch64_neon_uqrshrn, V64 | I1 | I2 | I4) // Constant shift +INTRINS_OVR_TAG(AARCH64_ADV_SIMD_UQRSHRN, aarch64_neon_uqrshrn, Scalar | V64 | I1 | I2 | I4) // Constant shift INTRINS_OVR_TAG(AARCH64_ADV_SIMD_SQRSHL, aarch64_neon_sqrshl, V64 | V128 | I1 | I2 | I4 | I8) // Variable shift INTRINS_OVR_TAG(AARCH64_ADV_SIMD_SQSHL, aarch64_neon_sqshl, V64 | V128 | I1 | I2 | I4 | I8) // Variable shift diff --git a/src/mono/mono/mini/mini-llvm.c b/src/mono/mono/mini/mini-llvm.c index f24023f2caa41f..ce5320f5efad46 100644 --- a/src/mono/mono/mini/mini-llvm.c +++ b/src/mono/mono/mini/mini-llvm.c @@ -375,7 +375,7 @@ ovr_tag_corresponding_integer (llvm_ovr_tag_t tag) } static int -int_from_id_and_ovr_tag (int id, llvm_ovr_tag_t ovr_tag) +key_from_id_and_tag (int id, llvm_ovr_tag_t ovr_tag) { return (((int) ovr_tag) << 23) | id; } @@ -429,6 +429,21 @@ ovr_tag_from_llvm_type (LLVMTypeRef type) return ret; } +static inline gboolean +check_needs_fake_scalar_op (MonoTypeEnum type) +{ +#if defined(TARGET_ARM64) + switch (type) { + case MONO_TYPE_U1: + case MONO_TYPE_I1: + case MONO_TYPE_U2: + case MONO_TYPE_I2: + return TRUE; + } +#endif + return FALSE; +} + static inline void set_failure (EmitContext *ctx, const char *message) { @@ -5145,8 +5160,8 @@ get_float_const (MonoCompile *cfg, float val) static LLVMValueRef call_overloaded_intrins (EmitContext *ctx, int id, llvm_ovr_tag_t ovr_tag, LLVMValueRef *args, const char *name) { - int ovr_id = int_from_id_and_ovr_tag (id, ovr_tag); - LLVMValueRef intrins = get_intrins (ctx, ovr_id); + int key = key_from_id_and_tag (id, ovr_tag); + LLVMValueRef intrins = get_intrins (ctx, key); int nargs = LLVMCountParamTypes (LLVMGetElementType (LLVMTypeOf (intrins))); for (int i = 0; i < nargs; ++i) { LLVMTypeRef t1 = LLVMTypeOf (args [i]); @@ -9996,10 +10011,11 @@ process_bb (EmitContext *ctx, MonoBasicBlock *bb) case OP_ARM64_PMULL: case OP_ARM64_PMULL2: { gboolean high = ins->opcode == OP_ARM64_PMULL2; - LLVMValueRef val = lhs; + LLVMValueRef args [] = { lhs, rhs }; if (high) - val = extract_high_elements (ctx, val); - LLVMValueRef result = call_intrins (ctx, INTRINS_AARCH64_ADV_SIMD_PMULL, &val, "arm64_pmull"); + for (int i = 0; i < 2; ++i) + args [i] = extract_high_elements (ctx, args [i]); + LLVMValueRef result = call_intrins (ctx, INTRINS_AARCH64_ADV_SIMD_PMULL, args, "arm64_pmull"); values [ins->dreg] = result; break; } @@ -10069,29 +10085,18 @@ process_bb (EmitContext *ctx, MonoBasicBlock *bb) shiftarg = rhs; shift = arg3; } + LLVMTypeRef arg_t = LLVMTypeOf (shiftarg); + LLVMTypeRef elem_t = LLVMGetElementType (arg_t); + unsigned int elems = LLVMGetVectorSize (arg_t); + unsigned int bits = mono_llvm_get_prim_size_bits (elem_t); + LLVMTypeRef trunc_t = LLVMVectorType (LLVMIntType (bits / 2), elems); shift = create_shift_vector (ctx, shiftarg, shift); LLVMValueRef result = LLVMBuildLShr (builder, shiftarg, shift, "shrn"); - if (high) - result = concatenate_vectors (ctx, lhs, result); - values [ins->dreg] = result; - break; - } - case OP_ARM64_UQSHRN: - case OP_ARM64_UQSHRN2: { - // XXXih: TODO: unroll count/rhs/arg3 - llvm_ovr_tag_t ovr_tag = ovr_tag_from_mono_vector_class (ins->klass); - LLVMValueRef shiftarg = lhs; - LLVMValueRef shift = rhs; - gboolean high = ins->opcode == OP_ARM64_UQSHRN2; + result = LLVMBuildTrunc (builder, result, trunc_t, ""); if (high) { - shiftarg = rhs; - shift = arg3; - ovr_tag = ovr_tag_smaller_vector (ovr_tag); - } - LLVMValueRef args [] = { shiftarg, shift }; - LLVMValueRef result = call_overloaded_intrins (ctx, INTRINS_AARCH64_ADV_SIMD_UQSHRN, ovr_tag, args, ""); - if (high) result = concatenate_vectors (ctx, lhs, result); + } + values [ins->dreg] = result; break; } case OP_ARM64_SRSHR: @@ -10129,39 +10134,36 @@ process_bb (EmitContext *ctx, MonoBasicBlock *bb) values [ins->dreg] = result; break; } - case OP_ARM64_RSHRN: - case OP_ARM64_RSHRN2: - case OP_ARM64_SQRSHRN: - case OP_ARM64_SQRSHRN2: - case OP_ARM64_SQRSHRUN: - case OP_ARM64_SQRSHRUN2: - case OP_ARM64_SQSHRN: - case OP_ARM64_SQSHRN2: - case OP_ARM64_SQSHRUN: - case OP_ARM64_SQSHRUN2: - case OP_ARM64_UQRSHRN: - case OP_ARM64_UQRSHRN2: { + case OP_ARM64_XRSHIFT_SCALAR: + case OP_ARM64_XRSHIFT: + case OP_ARM64_XRSHIFT2: { // XXXih: TODO: unroll count/rhs/arg3 llvm_ovr_tag_t ovr_tag = ovr_tag_from_mono_vector_class (ins->klass); LLVMValueRef args [2] = { lhs, rhs }; gboolean high = FALSE; - int iid = 0; + gboolean scalar = FALSE; + int iid = ins->inst_c0; switch (ins->opcode) { - case OP_ARM64_RSHRN: iid = INTRINS_AARCH64_ADV_SIMD_RSHRN; case OP_ARM64_RSHRN2: high = TRUE; break; - case OP_ARM64_UQRSHRN: iid = INTRINS_AARCH64_ADV_SIMD_UQRSHRN; case OP_ARM64_UQRSHRN2: high = TRUE; break; - case OP_ARM64_SQRSHRN: iid = INTRINS_AARCH64_ADV_SIMD_SQRSHRN; case OP_ARM64_SQRSHRN2: high = TRUE; break; - case OP_ARM64_SQRSHRUN: iid = INTRINS_AARCH64_ADV_SIMD_SQRSHRUN; case OP_ARM64_SQRSHRUN2: high = TRUE; break; - case OP_ARM64_SQSHRN: iid = INTRINS_AARCH64_ADV_SIMD_SQSHRN; case OP_ARM64_SQSHRN2: high = TRUE; break; - case OP_ARM64_SQSHRUN: iid = INTRINS_AARCH64_ADV_SIMD_SQSHRUN; case OP_ARM64_SQSHRUN2: high = TRUE; break; + case OP_ARM64_XRSHIFT_SCALAR: scalar = TRUE; break; + case OP_ARM64_XRSHIFT2: high = TRUE; break; } if (high) { args [0] = rhs; args [1] = arg3; ovr_tag = ovr_tag_smaller_vector (ovr_tag); } + if (scalar) { + LLVMTypeRef arg_t = LLVMTypeOf (args [0]); + LLVMTypeRef elem_t = LLVMGetElementType (arg_t); + unsigned int elems = LLVMGetVectorSize (arg_t); + LLVMValueRef lo = scalar_from_vector (ctx, args [0]); + args [0] = vector_from_scalar_ty (ctx, LLVMVectorType (elem_t, elems * 2), lo); + } LLVMValueRef result = call_overloaded_intrins (ctx, iid, ovr_tag, args, ""); if (high) result = concatenate_vectors (ctx, lhs, result); + if (scalar) + result = keep_lowest_element (ctx, result); values [ins->dreg] = result; break; } @@ -10183,17 +10185,29 @@ process_bb (EmitContext *ctx, MonoBasicBlock *bb) llvm_ovr_tag_t ovr_tag = ovr_tag_from_mono_vector_class (ins->klass); LLVMValueRef shift = create_shift_vector (ctx, lhs, rhs); LLVMValueRef args [] = { lhs, shift }; - int iid = INTRINS_AARCH64_ADV_SIMD_SQSHLU; break; + int iid = INTRINS_AARCH64_ADV_SIMD_SQSHLU; values [ins->dreg] = call_overloaded_intrins (ctx, iid, ovr_tag, args, ""); break; } + case OP_ARM64_SSHLL: + case OP_ARM64_SSHLL2: case OP_ARM64_USHLL: case OP_ARM64_USHLL2: { - gboolean high = ins->opcode == OP_ARM64_USHLL2; + LLVMTypeRef ret_t = simd_class_to_llvm_type (ctx, ins->klass); + gboolean high = FALSE; + gboolean is_unsigned = FALSE; + switch (ins->opcode) { + case OP_ARM64_SSHLL2: high = TRUE; break; + case OP_ARM64_USHLL2: high = TRUE; case OP_ARM64_USHLL: is_unsigned = TRUE; break; + } LLVMValueRef result = lhs; if (high) result = extract_high_elements (ctx, result); - result = LLVMBuildShl (builder, result, create_shift_vector (ctx, result, rhs), "arm64_ushll"); + if (is_unsigned) + result = LLVMBuildZExt (builder, result, ret_t, "arm64_ushll"); + else + result = LLVMBuildSExt (builder, result, ret_t, "arm64_ushll"); + result = LLVMBuildShl (builder, result, create_shift_vector (ctx, result, rhs), ""); values [ins->dreg] = result; break; } @@ -10606,17 +10620,7 @@ process_bb (EmitContext *ctx, MonoBasicBlock *bb) * instruction selection. This is worked around by using a vector * operation and then explicitly clearing the upper bits of the register. */ - gboolean arm64_fake_scalar_op = FALSE; - switch (inst_c1_type (ins)) { - case MONO_TYPE_U1: - case MONO_TYPE_I1: - case MONO_TYPE_U2: - case MONO_TYPE_I2: - arm64_fake_scalar_op = TRUE; - } -#if !defined(TARGET_ARM64) - arm64_fake_scalar_op = FALSE; -#endif + gboolean arm64_fake_scalar_op = check_needs_fake_scalar_op (inst_c1_type (ins)); LLVMValueRef args [3] = { lhs, rhs, arg3 }; if (!arm64_fake_scalar_op) { ovr_tag = ovr_tag_force_scalar (ovr_tag); @@ -11994,8 +11998,8 @@ add_intrinsic (LLVMModuleRef module, int id) intrins = add_intrins2 (module, id, associated_scalar_type, distinguishing_type); } else intrins = add_intrins1 (module, id, distinguishing_type); - int ovr_id = int_from_id_and_ovr_tag (id, test); - g_hash_table_insert (intrins_id_to_intrins, GINT_TO_POINTER (ovr_id), intrins); + int key = key_from_id_and_tag (id, test); + g_hash_table_insert (intrins_id_to_intrins, GINT_TO_POINTER (key), intrins); } } } diff --git a/src/mono/mono/mini/mini-ops.h b/src/mono/mono/mini/mini-ops.h index 739d425144a274..eccfc5163fa7bf 100644 --- a/src/mono/mono/mini/mini-ops.h +++ b/src/mono/mono/mini/mini-ops.h @@ -1635,9 +1635,7 @@ MINI_OP(OP_ARM64_RSUBHN, "arm64_rsubhn", XREG, XREG, XREG) MINI_OP3(OP_ARM64_RSUBHN2, "arm64_rsubhn2", XREG, XREG, XREG, XREG) MINI_OP(OP_ARM64_SHRN, "arm64_shrn", XREG, XREG, IREG) -MINI_OP(OP_ARM64_UQSHRN, "arm64_uqshrn", XREG, XREG, IREG) MINI_OP3(OP_ARM64_SHRN2, "arm64_shrn2", XREG, XREG, XREG, IREG) -MINI_OP3(OP_ARM64_UQSHRN2, "arm64_uqshrn2", XREG, XREG, XREG, IREG) MINI_OP3(OP_ARM64_SLI, "arm64_sli", XREG, XREG, XREG, IREG) MINI_OP3(OP_ARM64_SRI, "arm64_sri", XREG, XREG, XREG, IREG) @@ -1655,19 +1653,13 @@ MINI_OP3(OP_ARM64_SSRA, "arm64_ssra", XREG, XREG, XREG, IREG) MINI_OP(OP_ARM64_USHLL, "arm64_ushll", XREG, XREG, IREG) MINI_OP(OP_ARM64_USHLL2, "arm64_ushll2", XREG, XREG, IREG) +MINI_OP(OP_ARM64_SSHLL, "arm64_sshll", XREG, XREG, IREG) +MINI_OP(OP_ARM64_SSHLL2, "arm64_sshll2", XREG, XREG, IREG) -MINI_OP(OP_ARM64_RSHRN, "arm64_rshrn", XREG, XREG, IREG) -MINI_OP(OP_ARM64_SQRSHRN, "arm64_sqrshrn", XREG, XREG, IREG) -MINI_OP(OP_ARM64_SQRSHRUN, "arm64_sqrshrun", XREG, XREG, IREG) -MINI_OP(OP_ARM64_SQSHRN, "arm64_sqshrn", XREG, XREG, IREG) -MINI_OP(OP_ARM64_SQSHRUN, "arm64_sqshrun", XREG, XREG, IREG) -MINI_OP(OP_ARM64_UQRSHRN, "arm64_uqrshrn", XREG, XREG, IREG) -MINI_OP3(OP_ARM64_RSHRN2, "arm64_rshrn2", XREG, XREG, XREG, IREG) -MINI_OP3(OP_ARM64_SQRSHRN2, "arm64_sqrshrn2", XREG, XREG, XREG, IREG) -MINI_OP3(OP_ARM64_SQRSHRUN2, "arm64_sqrshrun2", XREG, XREG, XREG, IREG) -MINI_OP3(OP_ARM64_SQSHRN2, "arm64_sqshrn2", XREG, XREG, XREG, IREG) -MINI_OP3(OP_ARM64_SQSHRUN2, "arm64_sqshrun2", XREG, XREG, XREG, IREG) -MINI_OP3(OP_ARM64_UQRSHRN2, "arm64_uqrshrn2", XREG, XREG, XREG, IREG) + +MINI_OP(OP_ARM64_XRSHIFT_SCALAR, "arm64_xrshift_scalar", XREG, XREG, IREG) +MINI_OP(OP_ARM64_XRSHIFT, "arm64_xrshift", XREG, XREG, IREG) +MINI_OP3(OP_ARM64_XRSHIFT2, "arm64_xrshift2", XREG, XREG, XREG, IREG) MINI_OP(OP_ARM64_UQSHL_IMM, "arm64_uqshl_imm", XREG, XREG, IREG) MINI_OP(OP_ARM64_SQSHL_IMM, "arm64_uqshl_imm", XREG, XREG, IREG) @@ -1678,8 +1670,8 @@ MINI_OP(OP_ARM64_REV16, "arm64_rev16", XREG, XREG, NONE) MINI_OP(OP_ARM64_REV32, "arm64_rev32", XREG, XREG, NONE) MINI_OP(OP_ARM64_REV64, "arm64_rev64", XREG, XREG, NONE) -MINI_OP(OP_ARM64_PMULL, "arm64_pmull", XREG, XREG, NONE) -MINI_OP(OP_ARM64_PMULL2, "arm64_pmull2", XREG, XREG, NONE) +MINI_OP(OP_ARM64_PMULL, "arm64_pmull", XREG, XREG, XREG) +MINI_OP(OP_ARM64_PMULL2, "arm64_pmull2", XREG, XREG, XREG) MINI_OP(OP_ARM64_XNEG, "arm64_xneg", XREG, XREG, NONE) MINI_OP(OP_ARM64_XNEG_SCALAR, "arm64_xneg_scalar", XREG, XREG, NONE) diff --git a/src/mono/mono/mini/simd-intrinsics.c b/src/mono/mono/mini/simd-intrinsics.c index 3d9a18a6b07e06..df9b19c3bc603d 100644 --- a/src/mono/mono/mini/simd-intrinsics.c +++ b/src/mono/mono/mini/simd-intrinsics.c @@ -1219,7 +1219,7 @@ static SimdIntrinsic advsimd_methods [] = { {SN_MultiplyWideningUpperAndSubtract, OP_ARM64_SMLSL2, None, OP_ARM64_UMLSL2}, {SN_Negate, OP_ARM64_XNEG}, {SN_NegateSaturate, OP_XOP_OVR_X_X, INTRINS_AARCH64_ADV_SIMD_SQNEG}, - {SN_NegateSaturateScalar}, + {SN_NegateSaturateScalar, OP_XOP_OVR_SCALAR_X_X, INTRINS_AARCH64_ADV_SIMD_SQNEG}, {SN_NegateScalar, OP_ARM64_XNEG_SCALAR}, {SN_Not, OP_ARM64_MVN}, {SN_Or, OP_XBINOP_FORCEINT, XBINOP_FORCEINT_or}, @@ -1228,13 +1228,15 @@ static SimdIntrinsic advsimd_methods [] = { {SN_PolynomialMultiplyWideningLower, OP_ARM64_PMULL}, {SN_PolynomialMultiplyWideningUpper, OP_ARM64_PMULL2}, {SN_PopCount, OP_XOP_OVR_SCALAR_X_X, INTRINS_AARCH64_ADV_SIMD_CNT}, - {SN_ReciprocalEstimate}, + {SN_ReciprocalEstimate, None, None, OP_XOP_OVR_X_X, INTRINS_AARCH64_ADV_SIMD_URECPE, OP_XOP_OVR_X_X, INTRINS_AARCH64_ADV_SIMD_FRECPE}, {SN_ReciprocalEstimateScalar, OP_XOP_OVR_SCALAR_X_X, INTRINS_AARCH64_ADV_SIMD_FRECPE}, {SN_ReciprocalExponentScalar, OP_XOP_OVR_SCALAR_X_X, INTRINS_AARCH64_ADV_SIMD_FRECPX}, - {SN_ReciprocalSquareRootEstimate}, + {SN_ReciprocalSquareRootEstimate, None, None, OP_XOP_OVR_X_X, INTRINS_AARCH64_ADV_SIMD_URSQRTE, OP_XOP_OVR_X_X, INTRINS_AARCH64_ADV_SIMD_FRSQRTE}, {SN_ReciprocalSquareRootEstimateScalar, OP_XOP_OVR_SCALAR_X_X, INTRINS_AARCH64_ADV_SIMD_FRSQRTE}, - {SN_ReciprocalStep, OP_XOP_OVR_X_X, INTRINS_AARCH64_ADV_SIMD_FRECPS}, - {SN_ReciprocalStepScalar, OP_XOP_OVR_SCALAR_X_X, INTRINS_AARCH64_ADV_SIMD_FRECPS}, + {SN_ReciprocalSquareRootStep, OP_XOP_OVR_X_X_X, INTRINS_AARCH64_ADV_SIMD_FRSQRTS}, + {SN_ReciprocalSquareRootStepScalar, OP_XOP_OVR_SCALAR_X_X_X, INTRINS_AARCH64_ADV_SIMD_FRSQRTS}, + {SN_ReciprocalStep, OP_XOP_OVR_X_X_X, INTRINS_AARCH64_ADV_SIMD_FRECPS}, + {SN_ReciprocalStepScalar, OP_XOP_OVR_SCALAR_X_X_X, INTRINS_AARCH64_ADV_SIMD_FRECPS}, {SN_ReverseElement16, OP_ARM64_REV32}, {SN_ReverseElement32, OP_ARM64_REV64}, {SN_ReverseElement8, OP_ARM64_REV16}, @@ -1260,13 +1262,13 @@ static SimdIntrinsic advsimd_methods [] = { {SN_ShiftLeftAndInsert, OP_ARM64_SRI}, {SN_ShiftLeftAndInsertScalar, OP_ARM64_SRI}, {SN_ShiftLeftLogical, OP_ARM64_SHL}, - {SN_ShiftLeftLogicalSaturate}, - {SN_ShiftLeftLogicalSaturateScalar}, + {SN_ShiftLeftLogicalSaturate, OP_ARM64_SQSHL_IMM, None, OP_ARM64_UQSHL_IMM}, + {SN_ShiftLeftLogicalSaturateScalar, OP_ARM64_SQSHL_IMM, None, OP_ARM64_UQSHL_IMM}, {SN_ShiftLeftLogicalSaturateUnsigned, OP_ARM64_SQSHLU}, {SN_ShiftLeftLogicalSaturateUnsignedScalar, OP_ARM64_SQSHLU}, {SN_ShiftLeftLogicalScalar, OP_ARM64_SHL}, - {SN_ShiftLeftLogicalWideningLower, OP_ARM64_USHLL}, - {SN_ShiftLeftLogicalWideningUpper, OP_ARM64_USHLL2}, + {SN_ShiftLeftLogicalWideningLower, OP_ARM64_SSHLL, None, OP_ARM64_USHLL}, + {SN_ShiftLeftLogicalWideningUpper, OP_ARM64_SSHLL2, None, OP_ARM64_USHLL2}, {SN_ShiftLogical, OP_XOP_OVR_X_X_X, INTRINS_AARCH64_ADV_SIMD_USHL}, {SN_ShiftLogicalRounded, OP_XOP_OVR_X_X_X, INTRINS_AARCH64_ADV_SIMD_URSHL}, {SN_ShiftLogicalRoundedSaturate, OP_XOP_OVR_X_X_X, INTRINS_AARCH64_ADV_SIMD_UQRSHL}, @@ -1280,40 +1282,44 @@ static SimdIntrinsic advsimd_methods [] = { {SN_ShiftRightArithmetic, OP_ARM64_SSHR}, {SN_ShiftRightArithmeticAdd, OP_ARM64_SSRA}, {SN_ShiftRightArithmeticAddScalar, OP_ARM64_SSRA}, - {SN_ShiftRightArithmeticNarrowingSaturateLower, OP_ARM64_SQSHRN}, - {SN_ShiftRightArithmeticNarrowingSaturateUnsignedLower, OP_ARM64_SQSHRUN}, - {SN_ShiftRightArithmeticNarrowingSaturateUnsignedUpper, OP_ARM64_SQSHRUN2}, - {SN_ShiftRightArithmeticNarrowingSaturateUpper, OP_ARM64_SQSHRN2}, + {SN_ShiftRightArithmeticNarrowingSaturateLower, OP_ARM64_XRSHIFT, INTRINS_AARCH64_ADV_SIMD_SQSHRN}, + {SN_ShiftRightArithmeticNarrowingSaturateUnsignedLower, OP_ARM64_XRSHIFT, INTRINS_AARCH64_ADV_SIMD_SQSHRUN}, + {SN_ShiftRightArithmeticNarrowingSaturateUnsignedScalar, OP_ARM64_XRSHIFT_SCALAR, INTRINS_AARCH64_ADV_SIMD_SQSHRUN}, + {SN_ShiftRightArithmeticNarrowingSaturateUnsignedUpper, OP_ARM64_XRSHIFT2, INTRINS_AARCH64_ADV_SIMD_SQSHRUN}, + {SN_ShiftRightArithmeticNarrowingSaturateUpper, OP_ARM64_XRSHIFT2, INTRINS_AARCH64_ADV_SIMD_SQSHRN}, {SN_ShiftRightArithmeticRounded, OP_ARM64_SRSHR}, {SN_ShiftRightArithmeticRoundedAdd, OP_ARM64_SRSRA}, {SN_ShiftRightArithmeticRoundedAddScalar, OP_ARM64_SRSRA}, - {SN_ShiftRightArithmeticRoundedNarrowingSaturateLower, OP_ARM64_SQRSHRN}, - {SN_ShiftRightArithmeticRoundedNarrowingSaturateUnsignedLower, OP_ARM64_SQRSHRUN}, - {SN_ShiftRightArithmeticRoundedNarrowingSaturateUnsignedUpper, OP_ARM64_SQRSHRUN2}, - {SN_ShiftRightArithmeticRoundedNarrowingSaturateUpper, OP_ARM64_SQRSHRN2}, + {SN_ShiftRightArithmeticRoundedNarrowingSaturateLower, OP_ARM64_XRSHIFT, INTRINS_AARCH64_ADV_SIMD_SQRSHRN}, + {SN_ShiftRightArithmeticRoundedNarrowingSaturateScalar, OP_ARM64_XRSHIFT_SCALAR, INTRINS_AARCH64_ADV_SIMD_SQRSHRN}, + {SN_ShiftRightArithmeticRoundedNarrowingSaturateUnsignedLower, OP_ARM64_XRSHIFT, INTRINS_AARCH64_ADV_SIMD_SQRSHRUN}, + {SN_ShiftRightArithmeticRoundedNarrowingSaturateUnsignedScalar, OP_ARM64_XRSHIFT_SCALAR, INTRINS_AARCH64_ADV_SIMD_SQRSHRUN}, + {SN_ShiftRightArithmeticRoundedNarrowingSaturateUnsignedUpper, OP_ARM64_XRSHIFT2, INTRINS_AARCH64_ADV_SIMD_SQRSHRUN}, + {SN_ShiftRightArithmeticRoundedNarrowingSaturateUpper, OP_ARM64_XRSHIFT2, INTRINS_AARCH64_ADV_SIMD_SQRSHRN}, {SN_ShiftRightArithmeticRoundedScalar, OP_ARM64_SRSHR}, {SN_ShiftRightArithmeticScalar, OP_ARM64_SSHR}, {SN_ShiftRightLogical, OP_ARM64_USHR}, {SN_ShiftRightLogicalAdd, OP_ARM64_USRA}, {SN_ShiftRightLogicalAddScalar, OP_ARM64_USRA}, {SN_ShiftRightLogicalNarrowingLower, OP_ARM64_SHRN}, - {SN_ShiftRightLogicalNarrowingSaturateLower, OP_ARM64_UQSHRN}, - {SN_ShiftRightLogicalNarrowingSaturateUpper, OP_ARM64_UQSHRN2}, + {SN_ShiftRightLogicalNarrowingSaturateLower, OP_ARM64_XRSHIFT, INTRINS_AARCH64_ADV_SIMD_UQSHRN}, + {SN_ShiftRightLogicalNarrowingSaturateScalar, OP_ARM64_XRSHIFT_SCALAR, INTRINS_AARCH64_ADV_SIMD_UQSHRN}, + {SN_ShiftRightLogicalNarrowingSaturateUpper, OP_ARM64_XRSHIFT2, INTRINS_AARCH64_ADV_SIMD_UQSHRN}, {SN_ShiftRightLogicalNarrowingUpper, OP_ARM64_SHRN2}, {SN_ShiftRightLogicalRounded, OP_ARM64_URSHR}, {SN_ShiftRightLogicalRoundedAdd, OP_ARM64_URSRA}, {SN_ShiftRightLogicalRoundedAddScalar, OP_ARM64_URSRA}, - {SN_ShiftRightLogicalRoundedNarrowingLower, OP_ARM64_RSHRN}, - {SN_ShiftRightLogicalRoundedNarrowingSaturateLower, OP_ARM64_UQRSHRN}, - {SN_ShiftRightLogicalRoundedNarrowingSaturateScalar}, - {SN_ShiftRightLogicalRoundedNarrowingSaturateUpper, OP_ARM64_UQRSHRN2}, - {SN_ShiftRightLogicalRoundedNarrowingUpper, OP_ARM64_RSHRN2}, + {SN_ShiftRightLogicalRoundedNarrowingLower, OP_ARM64_XRSHIFT, INTRINS_AARCH64_ADV_SIMD_RSHRN}, + {SN_ShiftRightLogicalRoundedNarrowingSaturateLower, OP_ARM64_XRSHIFT, INTRINS_AARCH64_ADV_SIMD_UQRSHRN}, + {SN_ShiftRightLogicalRoundedNarrowingSaturateScalar, OP_ARM64_XRSHIFT_SCALAR, INTRINS_AARCH64_ADV_SIMD_UQRSHRN}, + {SN_ShiftRightLogicalRoundedNarrowingSaturateUpper, OP_ARM64_XRSHIFT2, INTRINS_AARCH64_ADV_SIMD_UQRSHRN}, + {SN_ShiftRightLogicalRoundedNarrowingUpper, OP_ARM64_XRSHIFT2, INTRINS_AARCH64_ADV_SIMD_RSHRN}, {SN_ShiftRightLogicalRoundedScalar, OP_ARM64_URSHR}, {SN_ShiftRightLogicalScalar, OP_ARM64_USHR}, {SN_SignExtendWideningLower, OP_ARM64_SXTL}, {SN_SignExtendWideningUpper, OP_ARM64_SXTL2}, {SN_Sqrt, OP_XOP_OVR_X_X, INTRINS_AARCH64_ADV_SIMD_FSQRT}, - {SN_SqrtScalar, OP_ARM64_SQRT_SCALAR}, + {SN_SqrtScalar, OP_XOP_OVR_SCALAR_X_X, INTRINS_AARCH64_ADV_SIMD_FSQRT}, {SN_Store, OP_ARM64_ST1}, {SN_StorePair, OP_ARM64_STP}, {SN_StorePairNonTemporal, OP_ARM64_STNP}, @@ -1406,10 +1412,7 @@ emit_arm64_intrinsics ( } if (feature == MONO_CPU_ARM64_NEON) { - SimdOp op = (SimdOp) 0; - IntrinsicId iid = (IntrinsicId) -1; switch (id) { - case SN_AbsoluteCompareGreaterThan: case SN_AbsoluteCompareGreaterThanOrEqual: case SN_AbsoluteCompareLessThan: @@ -1442,7 +1445,6 @@ emit_arm64_intrinsics ( int iid = use_geq ? INTRINS_AARCH64_ADV_SIMD_FACGE : INTRINS_AARCH64_ADV_SIMD_FACGT; return emit_simd_ins_for_sig (cfg, klass, OP_ARM64_ABSCOMPARE, iid, scalar, fsig, cmp_args); } - case SN_AddSaturate: case SN_AddSaturateScalar: { gboolean arg0_unsigned = type_is_unsigned (fsig->params [0]); @@ -1591,38 +1593,6 @@ emit_arm64_intrinsics ( ret->sreg3 = scalar->dreg; return ret; } - case SN_NegateSaturateScalar: { - MonoInst *ret = emit_simd_ins_for_sig (cfg, klass, OP_XOP_OVR_X_X, INTRINS_AARCH64_ADV_SIMD_SQNEG, arg0_type, fsig, args); - ret = emit_simd_ins (cfg, klass, OP_ARM64_ZERO_UPPER, ret->dreg, -1); - return ret; - } - case SN_ReciprocalEstimate: - case SN_ReciprocalSquareRootEstimate: { - gboolean is_float = FALSE; - switch (arg0_type) { - case MONO_TYPE_R4: case MONO_TYPE_R8: is_float = TRUE; - } - switch (id) { - case SN_ReciprocalEstimate: - iid = is_float ? INTRINS_AARCH64_ADV_SIMD_FRECPE: INTRINS_AARCH64_ADV_SIMD_URECPE; - break; - case SN_ReciprocalSquareRootEstimate: - iid = is_float ? INTRINS_AARCH64_ADV_SIMD_FRSQRTE : INTRINS_AARCH64_ADV_SIMD_URSQRTE; - break; - } - return emit_simd_ins_for_sig (cfg, klass, OP_XOP_OVR_X_X, iid, arg0_type, fsig, args); - } - case SN_ShiftLeftLogicalSaturate: - case SN_ShiftLeftLogicalSaturateScalar: { - gboolean is_unsigned = type_is_unsigned (fsig->ret); - op = is_unsigned ? OP_ARM64_UQSHL_IMM : OP_ARM64_SQSHL_IMM; - return emit_simd_ins_for_sig (cfg, klass, op, 0, arg0_type, fsig, args); - } - case SN_ShiftRightLogicalRoundedNarrowingSaturateScalar: { - MonoInst *ret = emit_simd_ins_for_sig (cfg, klass, OP_ARM64_UQRSHRN, 0, arg0_type, fsig, args); - ret = emit_simd_ins (cfg, klass, OP_ARM64_ZERO_UPPER, ret->dreg, -1); - return ret; - } default: g_assert_not_reached (); } From d5ab5ec1d242bf365b921f1db03b4c184202cc48 Mon Sep 17 00:00:00 2001 From: Imran Hameed Date: Sun, 7 Mar 2021 18:42:55 -0800 Subject: [PATCH 33/58] Implement missing stuff from AdvSimd.Arm64 --- src/mono/mono/mini/llvm-intrinsics.h | 11 ++- src/mono/mono/mini/mini-llvm.c | 104 +++++++++++++++++++++++---- src/mono/mono/mini/mini-ops.h | 21 ++++-- src/mono/mono/mini/simd-intrinsics.c | 84 ++++++++++++++++------ 4 files changed, 175 insertions(+), 45 deletions(-) diff --git a/src/mono/mono/mini/llvm-intrinsics.h b/src/mono/mono/mini/llvm-intrinsics.h index 2701a933dce6c1..d752093ed5d362 100644 --- a/src/mono/mono/mini/llvm-intrinsics.h +++ b/src/mono/mono/mini/llvm-intrinsics.h @@ -312,8 +312,8 @@ INTRINS_OVR_TAG_KIND(AARCH64_ADV_SIMD_SADDLV, aarch64_neon_saddlv, WidenAcross, INTRINS_OVR_TAG(AARCH64_ADV_SIMD_ADDP, aarch64_neon_addp, V64 | V128 | I1 | I2 | I4 | I8) INTRINS_OVR_TAG(AARCH64_ADV_SIMD_FADDP, aarch64_neon_faddp, V64 | V128 | R4 | R8) -INTRINS_OVR_TAG_KIND(AARCH64_ADV_SIMD_FMAXNMV, aarch64_neon_fmaxnmv, Across, V128 | R4) -INTRINS_OVR_TAG_KIND(AARCH64_ADV_SIMD_FMINNMV, aarch64_neon_fminnmv, Across, V128 | R4) +INTRINS_OVR_TAG_KIND(AARCH64_ADV_SIMD_FMAXNMV, aarch64_neon_fmaxnmv, Across, V64 | V128 | R4 | R8) +INTRINS_OVR_TAG_KIND(AARCH64_ADV_SIMD_FMINNMV, aarch64_neon_fminnmv, Across, V64 | V128 | R4 | R8) INTRINS_OVR_TAG_KIND(AARCH64_ADV_SIMD_SADDV, aarch64_neon_saddv, Across, V64 | V128 | I1 | I2 | I4) INTRINS_OVR_TAG_KIND(AARCH64_ADV_SIMD_UADDV, aarch64_neon_uaddv, Across, V64 | V128 | I1 | I2 | I4 | I8) @@ -371,6 +371,9 @@ INTRINS_OVR_TAG(AARCH64_ADV_SIMD_UMINP, aarch64_neon_uminp, V64 | V128 | I1 | I2 INTRINS_OVR_TAG(AARCH64_ADV_SIMD_FMAXNM, aarch64_neon_fmaxnm, Scalar | V64 | V128 | R4 | R8) INTRINS_OVR_TAG(AARCH64_ADV_SIMD_FMINNM, aarch64_neon_fminnm, Scalar | V64 | V128 | R4 | R8) +INTRINS_OVR_TAG(AARCH64_ADV_SIMD_FMAXNMP, aarch64_neon_fmaxnmp, V64 | V128 | R4 | R8) +INTRINS_OVR_TAG(AARCH64_ADV_SIMD_FMINNMP, aarch64_neon_fminnmp, V64 | V128 | R4 | R8) + INTRINS_OVR_2_ARG(AARCH64_ADV_SIMD_FMAXV_F32, aarch64_neon_fminv, r4_t, v64_r4_t) INTRINS_OVR_2_ARG(AARCH64_ADV_SIMD_FMAXV_F64, aarch64_neon_fminv, r8_t, v128_r8_t) INTRINS_OVR_2_ARG(AARCH64_ADV_SIMD_FMINV_F32, aarch64_neon_fminv, r4_t, v64_r4_t) @@ -378,7 +381,7 @@ INTRINS_OVR_2_ARG(AARCH64_ADV_SIMD_FMINV_F64, aarch64_neon_fminv, r8_t, v128_r8_ INTRINS_OVR_TAG(AARCH64_ADV_SIMD_SQDMULH, aarch64_neon_sqdmulh, V64 | V128 | I2 | I4) -INTRINS_OVR_TAG(AARCH64_ADV_SIMD_SQDMULL, aarch64_neon_sqdmull, V128 | I4 | I8) +INTRINS_OVR_TAG(AARCH64_ADV_SIMD_SQDMULL, aarch64_neon_sqdmull, V64 | V128 | I2 | I4 | I8) INTRINS_OVR_TAG(AARCH64_ADV_SIMD_SQRDMULH, aarch64_neon_sqrdmulh, V64 | V128 | I2 | I4) INTRINS_OVR_TAG(AARCH64_ADV_SIMD_SMULL, aarch64_neon_smull, V128 | I2 | I4 | I8) @@ -389,6 +392,8 @@ INTRINS_OVR_TAG(AARCH64_ADV_SIMD_SQNEG, aarch64_neon_sqneg, Scalar | V64 | V128 INTRINS_OVR_TAG(AARCH64_ADV_SIMD_PMUL, aarch64_neon_pmul, V64 | V128 | I1) INTRINS_OVR(AARCH64_ADV_SIMD_PMULL, aarch64_neon_pmull, v128_i2_t) +INTRINS_OVR_TAG(AARCH64_ADV_SIMD_FMULX, aarch64_neon_fmulx, Scalar | V64 | V128 | R4 | R8) + INTRINS_OVR_TAG(AARCH64_ADV_SIMD_CNT, ctpop, V64 | V128 | I1) INTRINS_OVR_TAG(AARCH64_ADV_SIMD_URECPE, aarch64_neon_urecpe, V64 | V128 | I4) diff --git a/src/mono/mono/mini/mini-llvm.c b/src/mono/mono/mini/mini-llvm.c index ce5320f5efad46..11cecdf0299e64 100644 --- a/src/mono/mono/mini/mini-llvm.c +++ b/src/mono/mono/mini/mini-llvm.c @@ -368,6 +368,12 @@ ovr_tag_smaller_vector (llvm_ovr_tag_t tag) return (tag & ~INTRIN_vectormask) | ((tag & INTRIN_vectormask) >> 1); } +static inline llvm_ovr_tag_t +ovr_tag_double_width (llvm_ovr_tag_t tag) +{ + return ((tag & ~INTRIN_vectormask) << 1) | ((tag & INTRIN_vectormask) << 1); +} + static inline llvm_ovr_tag_t ovr_tag_corresponding_integer (llvm_ovr_tag_t tag) { @@ -4957,6 +4963,12 @@ vector_from_scalar (EmitContext *ctx, LLVMValueRef type_donor, LLVMValueRef x) return vector_from_scalar_ty (ctx, LLVMTypeOf (type_donor), x); } +static LLVMValueRef +undef_upper_elements (EmitContext *ctx, LLVMTypeRef type, LLVMValueRef x) +{ + return vector_from_scalar_ty (ctx, type, scalar_from_vector (ctx, x)); +} + static void emit_llvmonly_handler_start (EmitContext *ctx, MonoBasicBlock *bb, LLVMBasicBlockRef cbb) { @@ -9813,29 +9825,29 @@ process_bb (EmitContext *ctx, MonoBasicBlock *bb) break; } case OP_ARM64_SQDMULL: - case OP_ARM64_SQDMULL_SCALAR: + case OP_ARM64_SQDMULL_BYSCALAR: case OP_ARM64_SQDMULL2: - case OP_ARM64_SQDMULL2_SCALAR: + case OP_ARM64_SQDMULL2_BYSCALAR: case OP_ARM64_SQDMLAL: - case OP_ARM64_SQDMLAL_SCALAR: - case OP_ARM64_SQDMLSL: - case OP_ARM64_SQDMLSL_SCALAR: + case OP_ARM64_SQDMLAL_BYSCALAR: case OP_ARM64_SQDMLAL2: - case OP_ARM64_SQDMLAL2_SCALAR: + case OP_ARM64_SQDMLAL2_BYSCALAR: + case OP_ARM64_SQDMLSL: + case OP_ARM64_SQDMLSL_BYSCALAR: case OP_ARM64_SQDMLSL2: - case OP_ARM64_SQDMLSL2_SCALAR: { + case OP_ARM64_SQDMLSL2_BYSCALAR: { llvm_ovr_tag_t ovr_tag = ovr_tag_from_mono_vector_class (ins->klass); gboolean scalar = FALSE; gboolean add = FALSE; gboolean subtract = FALSE; gboolean high = FALSE; switch (ins->opcode) { - case OP_ARM64_SQDMULL_SCALAR: scalar = TRUE; case OP_ARM64_SQDMULL: break; - case OP_ARM64_SQDMULL2_SCALAR: scalar = TRUE; case OP_ARM64_SQDMULL2: high = TRUE; break; - case OP_ARM64_SQDMLAL_SCALAR: scalar = TRUE; case OP_ARM64_SQDMLAL: add = TRUE; break; - case OP_ARM64_SQDMLSL_SCALAR: scalar = TRUE; case OP_ARM64_SQDMLSL: subtract = TRUE; break; - case OP_ARM64_SQDMLAL2_SCALAR: scalar = TRUE; case OP_ARM64_SQDMLAL2: high = TRUE; add = TRUE; break; - case OP_ARM64_SQDMLSL2_SCALAR: scalar = TRUE; case OP_ARM64_SQDMLSL2: high = TRUE; subtract = TRUE; break; + case OP_ARM64_SQDMULL_BYSCALAR: scalar = TRUE; case OP_ARM64_SQDMULL: break; + case OP_ARM64_SQDMULL2_BYSCALAR: scalar = TRUE; case OP_ARM64_SQDMULL2: high = TRUE; break; + case OP_ARM64_SQDMLAL_BYSCALAR: scalar = TRUE; case OP_ARM64_SQDMLAL: add = TRUE; break; + case OP_ARM64_SQDMLAL2_BYSCALAR: scalar = TRUE; case OP_ARM64_SQDMLAL2: high = TRUE; add = TRUE; break; + case OP_ARM64_SQDMLSL_BYSCALAR: scalar = TRUE; case OP_ARM64_SQDMLSL: subtract = TRUE; break; + case OP_ARM64_SQDMLSL2_BYSCALAR: scalar = TRUE; case OP_ARM64_SQDMLSL2: high = TRUE; subtract = TRUE; break; } int iid = 0; if (add) @@ -9864,9 +9876,68 @@ process_bb (EmitContext *ctx, MonoBasicBlock *bb) values [ins->dreg] = result; break; } + case OP_ARM64_SQDMULH_SCALAR: + case OP_ARM64_SQDMULL_SCALAR: + case OP_ARM64_SQDMLAL_SCALAR: + case OP_ARM64_SQDMLSL_SCALAR: { + /* + * define dso_local i32 @__vqdmlslh_lane_s16(i32, i16, <4 x i16>, i32) local_unnamed_addr #0 { + * %5 = insertelement <4 x i16> undef, i16 %1, i64 0 + * %6 = shufflevector <4 x i16> %2, <4 x i16> undef, <4 x i32> + * %7 = tail call <4 x i32> @llvm.aarch64.neon.sqdmull.v4i32(<4 x i16> %5, <4 x i16> %6) + * %8 = extractelement <4 x i32> %7, i64 0 + * %9 = tail call i32 @llvm.aarch64.neon.sqsub.i32(i32 %0, i32 %8) + * ret i32 %9 + * } + */ + gboolean widening = TRUE; + int mulid = INTRINS_AARCH64_ADV_SIMD_SQDMULL; + int iid = 0; + switch (ins->opcode) { + case OP_ARM64_SQDMULH_SCALAR: mulid = INTRINS_AARCH64_ADV_SIMD_SQDMULH; widening = FALSE; break; + case OP_ARM64_SQDMLAL_SCALAR: iid = INTRINS_AARCH64_ADV_SIMD_SQADD; break; + case OP_ARM64_SQDMLSL_SCALAR: iid = INTRINS_AARCH64_ADV_SIMD_SQSUB; break; + } + LLVMTypeRef ret_t = simd_class_to_llvm_type (ctx, ins->klass); + LLVMValueRef mularg = lhs; + LLVMValueRef selected_scalar = rhs; + if (iid != 0) { + mularg = rhs; + selected_scalar = arg3; + } + LLVMTypeRef mularg_t = LLVMTypeOf (mularg); + llvm_ovr_tag_t multag = ovr_tag_from_llvm_type (mularg_t); + llvm_ovr_tag_t iidtag = ovr_tag_force_scalar (ovr_tag_from_llvm_type (ret_t)); + if (widening) + multag = ovr_tag_double_width (multag); + + mularg = undef_upper_elements (ctx, mularg_t, mularg); + selected_scalar = undef_upper_elements (ctx, mularg_t, selected_scalar); + + LLVMValueRef mulargs [] = { mularg, selected_scalar }; + LLVMValueRef result = call_overloaded_intrins (ctx, mulid, multag, mulargs, "arm64_sqdmlsl"); + result = scalar_from_vector (ctx, result); + + if (iid != 0) { + LLVMValueRef minuend = scalar_from_vector (ctx, lhs); + LLVMValueRef subargs [] = { minuend, result }; + result = call_overloaded_intrins (ctx, iid, iidtag, subargs, "arm64_sqdmlsl"); + } + result = vector_from_scalar_ty (ctx, ret_t, result); + values [ins->dreg] = result; + break; + } + case OP_ARM64_SQRDMULH_BYSCALAR: case OP_ARM64_SQRDMULH_SCALAR: + case OP_ARM64_SQRDMULH_SCALAR_SEL: case OP_ARM64_SQRDMULH_SEL: { - gboolean sel = ins->opcode == OP_ARM64_SQRDMULH_SEL; + gboolean sel = FALSE; + gboolean scalar = FALSE; + switch (ins->opcode) { + case OP_ARM64_SQRDMULH_SCALAR: scalar = TRUE; break; + case OP_ARM64_SQRDMULH_SCALAR_SEL: sel = TRUE; scalar = TRUE; break; + case OP_ARM64_SQRDMULH_SEL: sel = TRUE; break; + } // XXXih: TODO: unroll arg3, bounds checks for arg3 llvm_ovr_tag_t ovr_tag = ovr_tag_from_mono_vector_class (ins->klass); LLVMValueRef lane = arg3; @@ -9875,7 +9946,10 @@ process_bb (EmitContext *ctx, MonoBasicBlock *bb) LLVMTypeRef t = LLVMTypeOf (lhs); unsigned int elems = LLVMGetVectorSize (t); LLVMValueRef arg = LLVMBuildExtractElement (builder, rhs, lane, ""); - arg = broadcast_element (ctx, arg, elems); + if (scalar) + arg = vector_from_scalar_ty (ctx, t, arg); + else + arg = broadcast_element (ctx, arg, elems); LLVMValueRef args [] = { lhs, arg }; LLVMValueRef result = call_overloaded_intrins (ctx, INTRINS_AARCH64_ADV_SIMD_SQRDMULH, ovr_tag, args, ""); values [ins->dreg] = result; diff --git a/src/mono/mono/mini/mini-ops.h b/src/mono/mono/mini/mini-ops.h index eccfc5163fa7bf..28fc3051ba54a7 100644 --- a/src/mono/mono/mini/mini-ops.h +++ b/src/mono/mono/mini/mini-ops.h @@ -1708,21 +1708,28 @@ MINI_OP3(OP_ARM64_MLA_SCALAR, "arm64_mla_scalar", XREG, XREG, XREG, XREG) MINI_OP3(OP_ARM64_FMUL_SEL, "arm64_fmul_sel", XREG, XREG, XREG, IREG) -MINI_OP3(OP_ARM64_SQRDMULH_SEL, "arm64_sqrdmulh_sel", XREG, XREG, XREG, IREG) +MINI_OP(OP_ARM64_SQDMULH_SCALAR, "arm64_sqdmulh_scalar", XREG, XREG, XREG) +MINI_OP(OP_ARM64_SQDMULL_SCALAR, "arm64_sqdmull_scalar", XREG, XREG, XREG) +MINI_OP3(OP_ARM64_SQDMLSL_SCALAR, "arm64_sqdmlsl_scalar", XREG, XREG, XREG, XREG) +MINI_OP3(OP_ARM64_SQDMLAL_SCALAR, "arm64_sqdmlal_scalar", XREG, XREG, XREG, XREG) + MINI_OP(OP_ARM64_SQRDMULH_SCALAR, "arm64_sqrdmulh_scalar", XREG, XREG, XREG) +MINI_OP3(OP_ARM64_SQRDMULH_SCALAR_SEL, "arm64_sqrdmulh_scalar_sel", XREG, XREG, XREG, IREG) +MINI_OP3(OP_ARM64_SQRDMULH_SEL, "arm64_sqrdmulh_sel", XREG, XREG, XREG, IREG) +MINI_OP(OP_ARM64_SQRDMULH_BYSCALAR, "arm64_sqrdmulh_scalar", XREG, XREG, XREG) MINI_OP(OP_ARM64_SQDMULL, "arm64_sqdmull", XREG, XREG, XREG) -MINI_OP(OP_ARM64_SQDMULL_SCALAR, "arm64_sqdmull", XREG, XREG, XREG) +MINI_OP(OP_ARM64_SQDMULL_BYSCALAR, "arm64_sqdmull", XREG, XREG, XREG) MINI_OP(OP_ARM64_SQDMULL2, "arm64_sqdmull2", XREG, XREG, XREG) -MINI_OP(OP_ARM64_SQDMULL2_SCALAR, "arm64_sqdmull2", XREG, XREG, XREG) +MINI_OP(OP_ARM64_SQDMULL2_BYSCALAR, "arm64_sqdmull2", XREG, XREG, XREG) MINI_OP3(OP_ARM64_SQDMLSL, "arm64_sqdmlsl", XREG, XREG, XREG, XREG) -MINI_OP3(OP_ARM64_SQDMLSL_SCALAR, "arm64_sqdmlsl", XREG, XREG, XREG, XREG) +MINI_OP3(OP_ARM64_SQDMLSL_BYSCALAR, "arm64_sqdmlsl", XREG, XREG, XREG, XREG) MINI_OP3(OP_ARM64_SQDMLAL, "arm64_sqdmlal", XREG, XREG, XREG, XREG) -MINI_OP3(OP_ARM64_SQDMLAL_SCALAR, "arm64_sqdmlal_scalar", XREG, XREG, XREG, XREG) +MINI_OP3(OP_ARM64_SQDMLAL_BYSCALAR, "arm64_sqdmlal", XREG, XREG, XREG, XREG) MINI_OP3(OP_ARM64_SQDMLSL2, "arm64_sqdmlsl2", XREG, XREG, XREG, XREG) -MINI_OP3(OP_ARM64_SQDMLSL2_SCALAR, "arm64_sqdmlsl2_scalar", XREG, XREG, XREG, XREG) +MINI_OP3(OP_ARM64_SQDMLSL2_BYSCALAR, "arm64_sqdmlsl2", XREG, XREG, XREG, XREG) MINI_OP3(OP_ARM64_SQDMLAL2, "arm64_sqdmlal2", XREG, XREG, XREG, XREG) -MINI_OP3(OP_ARM64_SQDMLAL2_SCALAR, "arm64_sqdmlal2_scalar", XREG, XREG, XREG, XREG) +MINI_OP3(OP_ARM64_SQDMLAL2_BYSCALAR, "arm64_sqdmlal2", XREG, XREG, XREG, XREG) MINI_OP3(OP_ARM64_FMSUB, "arm64_fmsub_scalar", XREG, XREG, XREG, XREG) MINI_OP3(OP_ARM64_FMSUB_SCALAR, "arm64_fmsub_scalar", XREG, XREG, XREG, XREG) diff --git a/src/mono/mono/mini/simd-intrinsics.c b/src/mono/mono/mini/simd-intrinsics.c index df9b19c3bc603d..942d6082c9ac03 100644 --- a/src/mono/mono/mini/simd-intrinsics.c +++ b/src/mono/mono/mini/simd-intrinsics.c @@ -1148,6 +1148,7 @@ static SimdIntrinsic advsimd_methods [] = { {SN_FusedSubtractHalving, OP_XOP_OVR_X_X_X, INTRINS_AARCH64_ADV_SIMD_SHSUB, OP_XOP_OVR_X_X_X, INTRINS_AARCH64_ADV_SIMD_UHSUB}, {SN_Insert}, {SN_InsertScalar}, + {SN_InsertSelectedScalar}, {SN_LeadingSignCount, OP_XOP_OVR_X_X, INTRINS_AARCH64_ADV_SIMD_CLS}, {SN_LeadingZeroCount, OP_ARM64_CLZ}, {SN_LoadAndInsertScalar, OP_ARM64_LD1_INSERT}, @@ -1159,16 +1160,21 @@ static SimdIntrinsic advsimd_methods [] = { {SN_MaxAcross, OP_ARM64_XHORIZ, INTRINS_AARCH64_ADV_SIMD_SMAXV, OP_ARM64_XHORIZ, INTRINS_AARCH64_ADV_SIMD_UMAXV}, {SN_MaxNumber, OP_XOP_OVR_X_X_X, INTRINS_AARCH64_ADV_SIMD_FMAXNM}, {SN_MaxNumberAcross, OP_ARM64_XHORIZ, INTRINS_AARCH64_ADV_SIMD_FMAXNMV}, + {SN_MaxNumberPairwise, OP_XOP_OVR_X_X_X, INTRINS_AARCH64_ADV_SIMD_FMAXNMP}, + {SN_MaxNumberPairwiseScalar, OP_ARM64_XHORIZ, INTRINS_AARCH64_ADV_SIMD_FMAXNMV}, {SN_MaxNumberScalar, OP_XOP_OVR_SCALAR_X_X_X, INTRINS_AARCH64_ADV_SIMD_FMAXNM}, {SN_MaxPairwise, OP_XOP_OVR_X_X_X, INTRINS_AARCH64_ADV_SIMD_SMAXP, OP_XOP_OVR_X_X_X, INTRINS_AARCH64_ADV_SIMD_UMAXP, OP_XOP_OVR_X_X_X, INTRINS_AARCH64_ADV_SIMD_FMAXP}, + {SN_MaxPairwiseScalar}, + {SN_MaxScalar, OP_XOP_OVR_SCALAR_X_X_X, INTRINS_AARCH64_ADV_SIMD_FMAX}, {SN_Min, OP_XOP_OVR_X_X_X, INTRINS_AARCH64_ADV_SIMD_SMIN, OP_XOP_OVR_X_X_X, INTRINS_AARCH64_ADV_SIMD_UMIN, OP_XOP_OVR_X_X_X, INTRINS_AARCH64_ADV_SIMD_FMIN}, {SN_MinAcross, OP_ARM64_XHORIZ, INTRINS_AARCH64_ADV_SIMD_SMINV, OP_ARM64_XHORIZ, INTRINS_AARCH64_ADV_SIMD_UMINV}, {SN_MinNumber, OP_XOP_OVR_X_X_X, INTRINS_AARCH64_ADV_SIMD_FMINNM}, {SN_MinNumberAcross, OP_ARM64_XHORIZ, INTRINS_AARCH64_ADV_SIMD_FMINNMV}, + {SN_MinNumberPairwise, OP_XOP_OVR_X_X_X, INTRINS_AARCH64_ADV_SIMD_FMINNMP}, + {SN_MinNumberPairwiseScalar, OP_ARM64_XHORIZ, INTRINS_AARCH64_ADV_SIMD_FMINNMV}, {SN_MinNumberScalar, OP_XOP_OVR_SCALAR_X_X_X, INTRINS_AARCH64_ADV_SIMD_FMINNM}, {SN_MinPairwise, OP_XOP_OVR_X_X_X, INTRINS_AARCH64_ADV_SIMD_SMINP, OP_XOP_OVR_X_X_X, INTRINS_AARCH64_ADV_SIMD_UMINP, OP_XOP_OVR_X_X_X, INTRINS_AARCH64_ADV_SIMD_FMINP}, {SN_MinPairwiseScalar}, - {SN_MinPairwiseScalar}, {SN_MinScalar, OP_XOP_OVR_SCALAR_X_X_X, INTRINS_AARCH64_ADV_SIMD_FMIN}, {SN_Multiply, OP_XBINOP, OP_IMUL, None, None, OP_XBINOP, OP_FMUL}, {SN_MultiplyAdd, OP_ARM64_MLA}, @@ -1185,27 +1191,42 @@ static SimdIntrinsic advsimd_methods [] = { {SN_MultiplyDoublingByScalarSaturateHigh, OP_XOP_OVR_BYSCALAR_X_X_X, INTRINS_AARCH64_ADV_SIMD_SQDMULH}, {SN_MultiplyDoublingBySelectedScalarSaturateHigh}, {SN_MultiplyDoublingSaturateHigh, OP_XOP_OVR_X_X_X, INTRINS_AARCH64_ADV_SIMD_SQDMULH}, + {SN_MultiplyDoublingSaturateHighScalar, OP_ARM64_SQDMULH_SCALAR}, + {SN_MultiplyDoublingScalarBySelectedScalarSaturateHigh}, + {SN_MultiplyDoublingWideningAndAddSaturateScalar, OP_ARM64_SQDMLAL_SCALAR}, + {SN_MultiplyDoublingWideningAndSubtractSaturateScalar, OP_ARM64_SQDMLSL_SCALAR}, {SN_MultiplyDoublingWideningLowerAndAddSaturate, OP_ARM64_SQDMLAL}, - {SN_MultiplyDoublingWideningLowerAndSubtractSaturate, OP_ARM64_SQDMLSL}, - {SN_MultiplyDoublingWideningLowerByScalarAndAddSaturate, OP_ARM64_SQDMLAL_SCALAR}, - {SN_MultiplyDoublingWideningLowerByScalarAndSubtractSaturate, OP_ARM64_SQDMLSL_SCALAR}, + {SN_MultiplyDoublingWideningLowerAndSubtractSaturate, OP_ARM64_SQDMLSL_BYSCALAR}, + {SN_MultiplyDoublingWideningLowerByScalarAndAddSaturate, OP_ARM64_SQDMLAL_BYSCALAR}, + {SN_MultiplyDoublingWideningLowerByScalarAndSubtractSaturate, OP_ARM64_SQDMLSL_BYSCALAR}, {SN_MultiplyDoublingWideningLowerBySelectedScalarAndAddSaturate}, {SN_MultiplyDoublingWideningLowerBySelectedScalarAndSubtractSaturate}, {SN_MultiplyDoublingWideningSaturateLower, OP_ARM64_SQDMULL}, - {SN_MultiplyDoublingWideningSaturateLowerByScalar, OP_ARM64_SQDMULL_SCALAR}, + {SN_MultiplyDoublingWideningSaturateLowerByScalar, OP_ARM64_SQDMULL_BYSCALAR}, {SN_MultiplyDoublingWideningSaturateLowerBySelectedScalar}, + {SN_MultiplyDoublingWideningSaturateScalar, OP_ARM64_SQDMULL_SCALAR}, + {SN_MultiplyDoublingWideningSaturateScalarBySelectedScalar}, {SN_MultiplyDoublingWideningSaturateUpper, OP_ARM64_SQDMULL2}, - {SN_MultiplyDoublingWideningSaturateUpperByScalar, OP_ARM64_SQDMULL2_SCALAR}, + {SN_MultiplyDoublingWideningSaturateUpperByScalar, OP_ARM64_SQDMULL2_BYSCALAR}, {SN_MultiplyDoublingWideningSaturateUpperBySelectedScalar}, + {SN_MultiplyDoublingWideningScalarBySelectedScalarAndAddSaturate}, + {SN_MultiplyDoublingWideningScalarBySelectedScalarAndSubtractSaturate}, {SN_MultiplyDoublingWideningUpperAndAddSaturate, OP_ARM64_SQDMLAL2}, {SN_MultiplyDoublingWideningUpperAndSubtractSaturate, OP_ARM64_SQDMLSL2}, - {SN_MultiplyDoublingWideningUpperByScalarAndAddSaturate, OP_ARM64_SQDMLAL2_SCALAR}, - {SN_MultiplyDoublingWideningUpperByScalarAndSubtractSaturate, OP_ARM64_SQDMLSL2_SCALAR}, + {SN_MultiplyDoublingWideningUpperByScalarAndAddSaturate, OP_ARM64_SQDMLAL2_BYSCALAR}, + {SN_MultiplyDoublingWideningUpperByScalarAndSubtractSaturate, OP_ARM64_SQDMLSL2_BYSCALAR}, {SN_MultiplyDoublingWideningUpperBySelectedScalarAndAddSaturate}, {SN_MultiplyDoublingWideningUpperBySelectedScalarAndSubtractSaturate}, - {SN_MultiplyRoundedDoublingByScalarSaturateHigh, OP_ARM64_SQRDMULH_SCALAR}, + {SN_MultiplyExtended, OP_XOP_OVR_X_X_X, INTRINS_AARCH64_ADV_SIMD_FMULX}, + {SN_MultiplyExtendedByScalar, OP_XOP_OVR_BYSCALAR_X_X_X, INTRINS_AARCH64_ADV_SIMD_FMULX}, + {SN_MultiplyExtendedBySelectedScalar}, + {SN_MultiplyExtendedScalar, OP_XOP_OVR_SCALAR_X_X_X, INTRINS_AARCH64_ADV_SIMD_FMULX}, + {SN_MultiplyExtendedScalarBySelectedScalar}, + {SN_MultiplyRoundedDoublingByScalarSaturateHigh, OP_ARM64_SQRDMULH_BYSCALAR}, {SN_MultiplyRoundedDoublingBySelectedScalarSaturateHigh, OP_ARM64_SQRDMULH_SEL}, {SN_MultiplyRoundedDoublingSaturateHigh, OP_XOP_OVR_X_X_X, INTRINS_AARCH64_ADV_SIMD_SQRDMULH}, + {SN_MultiplyRoundedDoublingSaturateHighScalar, OP_ARM64_SQRDMULH_SCALAR}, + {SN_MultiplyRoundedDoublingScalarBySelectedScalarSaturateHigh, OP_ARM64_SQRDMULH_SCALAR_SEL}, {SN_MultiplyScalar, OP_XBINOP_SCALAR, OP_FMUL}, {SN_MultiplyScalarBySelectedScalar, OP_ARM64_FMUL_SEL}, {SN_MultiplySubtract, OP_ARM64_MLS}, @@ -1484,6 +1505,7 @@ emit_arm64_intrinsics ( ins->inst_c1 = arg0_type; return ins; } + case SN_InsertSelectedScalar: case SN_InsertScalar: case SN_Insert: { int insert_op = 0; @@ -1500,10 +1522,18 @@ emit_arm64_intrinsics ( default: g_assert_not_reached (); } int val_src_reg = args [2]->dreg; - if (id == SN_InsertScalar) { + switch (id) { + case SN_InsertSelectedScalar: { + MonoInst *scalar = emit_simd_ins (cfg, klass, OP_ARM64_SELECT_SCALAR, args [2]->dreg, args [3]->dreg); + val_src_reg = scalar->dreg; + // fallthrough + } + case SN_InsertScalar: { MonoInst *ins = emit_simd_ins (cfg, klass, extract_op, val_src_reg, -1); ins->inst_c0 = 0; val_src_reg = ins->dreg; + break; + } } MonoInst *ins = emit_simd_ins (cfg, klass, insert_op, args [0]->dreg, val_src_reg); ins->sreg3 = args [1]->dreg; @@ -1523,23 +1553,32 @@ emit_arm64_intrinsics ( } return emit_simd_ins_for_sig (cfg, klass, OP_XOP_SX_X, iid, arg0_type, fsig, args); } + case SN_MultiplyDoublingScalarBySelectedScalarSaturateHigh: + case SN_MultiplyDoublingWideningSaturateScalarBySelectedScalar: + case SN_MultiplyExtendedBySelectedScalar: + case SN_MultiplyExtendedScalarBySelectedScalar: case SN_MultiplyBySelectedScalar: case SN_MultiplyBySelectedScalarWideningLower: case SN_MultiplyBySelectedScalarWideningUpper: case SN_MultiplyDoublingBySelectedScalarSaturateHigh: case SN_MultiplyDoublingWideningSaturateLowerBySelectedScalar: case SN_MultiplyDoublingWideningSaturateUpperBySelectedScalar: { + MonoClass *ret_klass = mono_class_from_mono_type_internal (fsig->ret); gboolean is_unsigned = type_is_unsigned (fsig->ret); gboolean is_float = type_is_float (fsig->ret); int opcode = 0; int c0 = 0; switch (id) { + case SN_MultiplyDoublingScalarBySelectedScalarSaturateHigh: opcode = OP_ARM64_SQDMULH_SCALAR; break; + case SN_MultiplyDoublingWideningSaturateScalarBySelectedScalar: opcode = OP_ARM64_SQDMULL_SCALAR; break; + case SN_MultiplyExtendedBySelectedScalar: opcode = OP_XOP_OVR_BYSCALAR_X_X_X; c0 = INTRINS_AARCH64_ADV_SIMD_FMULX; break; + case SN_MultiplyExtendedScalarBySelectedScalar: opcode = OP_XOP_OVR_SCALAR_X_X_X; c0 = INTRINS_AARCH64_ADV_SIMD_FMULX; break; case SN_MultiplyBySelectedScalar: opcode = OP_XBINOP_BYSCALAR; c0 = OP_IMUL; break; case SN_MultiplyBySelectedScalarWideningLower: opcode = OP_ARM64_SMULL_SCALAR; break; case SN_MultiplyBySelectedScalarWideningUpper: opcode = OP_ARM64_SMULL2_SCALAR; break; case SN_MultiplyDoublingBySelectedScalarSaturateHigh: opcode = OP_XOP_OVR_BYSCALAR_X_X_X; c0 = INTRINS_AARCH64_ADV_SIMD_SQDMULH; break; - case SN_MultiplyDoublingWideningSaturateLowerBySelectedScalar: opcode = OP_ARM64_SQDMULL_SCALAR; break; - case SN_MultiplyDoublingWideningSaturateUpperBySelectedScalar: opcode = OP_ARM64_SQDMULL2_SCALAR; break; + case SN_MultiplyDoublingWideningSaturateLowerBySelectedScalar: opcode = OP_ARM64_SQDMULL_BYSCALAR; break; + case SN_MultiplyDoublingWideningSaturateUpperBySelectedScalar: opcode = OP_ARM64_SQDMULL2_BYSCALAR; break; default: g_assert_not_reached(); } if (is_unsigned) @@ -1551,11 +1590,13 @@ emit_arm64_intrinsics ( switch (opcode) { case OP_XBINOP_BYSCALAR: c0 = OP_FMUL; } - MonoInst *scalar = emit_simd_ins (cfg, klass, OP_ARM64_SELECT_SCALAR, args [1]->dreg, args [2]->dreg); - MonoInst *ret = emit_simd_ins (cfg, klass, opcode, args [0]->dreg, scalar->dreg); + MonoInst *scalar = emit_simd_ins (cfg, ret_klass, OP_ARM64_SELECT_SCALAR, args [1]->dreg, args [2]->dreg); + MonoInst *ret = emit_simd_ins (cfg, ret_klass, opcode, args [0]->dreg, scalar->dreg); ret->inst_c0 = c0; return ret; } + case SN_MultiplyDoublingWideningScalarBySelectedScalarAndAddSaturate: + case SN_MultiplyDoublingWideningScalarBySelectedScalarAndSubtractSaturate: case SN_MultiplyAddBySelectedScalar: case SN_MultiplySubtractBySelectedScalar: case SN_MultiplyBySelectedScalarWideningLowerAndAdd: @@ -1566,19 +1607,22 @@ emit_arm64_intrinsics ( case SN_MultiplyDoublingWideningLowerBySelectedScalarAndSubtractSaturate: case SN_MultiplyDoublingWideningUpperBySelectedScalarAndAddSaturate: case SN_MultiplyDoublingWideningUpperBySelectedScalarAndSubtractSaturate: { + MonoClass *ret_klass = mono_class_from_mono_type_internal (fsig->ret); gboolean is_unsigned = type_is_unsigned (fsig->ret); int opcode = 0; switch (id) { + case SN_MultiplyDoublingWideningScalarBySelectedScalarAndAddSaturate: opcode = OP_ARM64_SQDMLAL_SCALAR; break; + case SN_MultiplyDoublingWideningScalarBySelectedScalarAndSubtractSaturate: opcode = OP_ARM64_SQDMLSL_SCALAR; break; case SN_MultiplyAddBySelectedScalar: opcode = OP_ARM64_MLA_SCALAR; break; case SN_MultiplySubtractBySelectedScalar: opcode = OP_ARM64_MLS_SCALAR; break; case SN_MultiplyBySelectedScalarWideningLowerAndAdd: opcode = OP_ARM64_SMLAL_SCALAR; break; case SN_MultiplyBySelectedScalarWideningLowerAndSubtract: opcode = OP_ARM64_SMLSL_SCALAR; break; case SN_MultiplyBySelectedScalarWideningUpperAndAdd: opcode = OP_ARM64_SMLAL2_SCALAR; break; case SN_MultiplyBySelectedScalarWideningUpperAndSubtract: opcode = OP_ARM64_SMLSL2_SCALAR; break; - case SN_MultiplyDoublingWideningLowerBySelectedScalarAndAddSaturate: opcode = OP_ARM64_SQDMLAL_SCALAR; break; - case SN_MultiplyDoublingWideningLowerBySelectedScalarAndSubtractSaturate: opcode = OP_ARM64_SQDMLSL_SCALAR; break; - case SN_MultiplyDoublingWideningUpperBySelectedScalarAndAddSaturate: opcode = OP_ARM64_SQDMLAL2_SCALAR; break; - case SN_MultiplyDoublingWideningUpperBySelectedScalarAndSubtractSaturate: opcode = OP_ARM64_SQDMLSL2_SCALAR; break; + case SN_MultiplyDoublingWideningLowerBySelectedScalarAndAddSaturate: opcode = OP_ARM64_SQDMLAL_BYSCALAR; break; + case SN_MultiplyDoublingWideningLowerBySelectedScalarAndSubtractSaturate: opcode = OP_ARM64_SQDMLSL_BYSCALAR; break; + case SN_MultiplyDoublingWideningUpperBySelectedScalarAndAddSaturate: opcode = OP_ARM64_SQDMLAL2_BYSCALAR; break; + case SN_MultiplyDoublingWideningUpperBySelectedScalarAndSubtractSaturate: opcode = OP_ARM64_SQDMLSL2_BYSCALAR; break; default: g_assert_not_reached(); } if (is_unsigned) @@ -1588,8 +1632,8 @@ emit_arm64_intrinsics ( case OP_ARM64_SMLAL2_SCALAR: opcode = OP_ARM64_UMLAL2_SCALAR; break; case OP_ARM64_SMLSL2_SCALAR: opcode = OP_ARM64_UMLSL2_SCALAR; break; } - MonoInst *scalar = emit_simd_ins (cfg, klass, OP_ARM64_SELECT_SCALAR, args [2]->dreg, args [3]->dreg); - MonoInst *ret = emit_simd_ins (cfg, klass, opcode, args [0]->dreg, args [1]->dreg); + MonoInst *scalar = emit_simd_ins (cfg, ret_klass, OP_ARM64_SELECT_SCALAR, args [2]->dreg, args [3]->dreg); + MonoInst *ret = emit_simd_ins (cfg, ret_klass, opcode, args [0]->dreg, args [1]->dreg); ret->sreg3 = scalar->dreg; return ret; } From 9e9adab5c5637eaa0faec610c7125fc0edf83148 Mon Sep 17 00:00:00 2001 From: Imran Hameed Date: Sun, 7 Mar 2021 19:17:17 -0800 Subject: [PATCH 34/58] Implement the AdvSimd.Arm64 parts of fma --- src/mono/mono/mini/mini-llvm.c | 9 +++++++++ src/mono/mono/mini/mini-ops.h | 6 ++++-- src/mono/mono/mini/simd-intrinsics.c | 13 +++++++++++++ 3 files changed, 26 insertions(+), 2 deletions(-) diff --git a/src/mono/mono/mini/mini-llvm.c b/src/mono/mono/mini/mini-llvm.c index 11cecdf0299e64..72e757369bd080 100644 --- a/src/mono/mono/mini/mini-llvm.c +++ b/src/mono/mono/mini/mini-llvm.c @@ -9788,25 +9788,34 @@ process_bb (EmitContext *ctx, MonoBasicBlock *bb) break; } case OP_ARM64_FMSUB: + case OP_ARM64_FMSUB_BYSCALAR: case OP_ARM64_FMSUB_SCALAR: case OP_ARM64_FNMSUB_SCALAR: case OP_ARM64_FMADD: + case OP_ARM64_FMADD_BYSCALAR: case OP_ARM64_FMADD_SCALAR: case OP_ARM64_FNMADD_SCALAR: { llvm_ovr_tag_t ovr_tag = ovr_tag_from_mono_vector_class (ins->klass); gboolean scalar = FALSE; gboolean negate = FALSE; gboolean subtract = FALSE; + gboolean byscalar = FALSE; switch (ins->opcode) { case OP_ARM64_FMSUB: subtract = TRUE; break; + case OP_ARM64_FMSUB_BYSCALAR: byscalar = TRUE; break; case OP_ARM64_FMSUB_SCALAR: subtract = TRUE; scalar = TRUE; break; case OP_ARM64_FNMSUB_SCALAR: subtract = TRUE; scalar = TRUE; negate = TRUE; break; case OP_ARM64_FMADD: break; + case OP_ARM64_FMADD_BYSCALAR: byscalar = TRUE; break; case OP_ARM64_FMADD_SCALAR: scalar = TRUE; break; case OP_ARM64_FNMADD_SCALAR: scalar = TRUE; negate = TRUE; break; } // llvm.fma argument order: mulop1, mulop2, addend LLVMValueRef args [] = { rhs, arg3, lhs }; + if (byscalar) { + unsigned int elems = LLVMGetVectorSize (LLVMTypeOf (args [0])); + args [1] = broadcast_element (ctx, scalar_from_vector (ctx, args [1]), elems); + } if (scalar) { ovr_tag = ovr_tag_force_scalar (ovr_tag); for (int i = 0; i < 3; ++i) diff --git a/src/mono/mono/mini/mini-ops.h b/src/mono/mono/mini/mini-ops.h index 28fc3051ba54a7..092b832643d46e 100644 --- a/src/mono/mono/mini/mini-ops.h +++ b/src/mono/mono/mini/mini-ops.h @@ -1731,11 +1731,13 @@ MINI_OP3(OP_ARM64_SQDMLSL2_BYSCALAR, "arm64_sqdmlsl2", XREG, XREG, XREG, XREG) MINI_OP3(OP_ARM64_SQDMLAL2, "arm64_sqdmlal2", XREG, XREG, XREG, XREG) MINI_OP3(OP_ARM64_SQDMLAL2_BYSCALAR, "arm64_sqdmlal2", XREG, XREG, XREG, XREG) -MINI_OP3(OP_ARM64_FMSUB, "arm64_fmsub_scalar", XREG, XREG, XREG, XREG) +MINI_OP3(OP_ARM64_FMSUB, "arm64_fmsub", XREG, XREG, XREG, XREG) +MINI_OP3(OP_ARM64_FMSUB_BYSCALAR, "arm64_fmsub_byscalar", XREG, XREG, XREG, XREG) MINI_OP3(OP_ARM64_FMSUB_SCALAR, "arm64_fmsub_scalar", XREG, XREG, XREG, XREG) MINI_OP3(OP_ARM64_FNMSUB_SCALAR, "arm64_fnmsub_scalar", XREG, XREG, XREG, XREG) -MINI_OP3(OP_ARM64_FMADD, "arm64_fmadd_scalar", XREG, XREG, XREG, XREG) +MINI_OP3(OP_ARM64_FMADD, "arm64_fmadd", XREG, XREG, XREG, XREG) +MINI_OP3(OP_ARM64_FMADD_BYSCALAR, "arm64_fmadd_byscalar", XREG, XREG, XREG, XREG) MINI_OP3(OP_ARM64_FMADD_SCALAR, "arm64_fmadd_scalar", XREG, XREG, XREG, XREG) MINI_OP3(OP_ARM64_FNMADD_SCALAR, "arm64_fnmadd_scalar", XREG, XREG, XREG, XREG) diff --git a/src/mono/mono/mini/simd-intrinsics.c b/src/mono/mono/mini/simd-intrinsics.c index 942d6082c9ac03..8d65e4d855d75f 100644 --- a/src/mono/mono/mini/simd-intrinsics.c +++ b/src/mono/mono/mini/simd-intrinsics.c @@ -1140,11 +1140,16 @@ static SimdIntrinsic advsimd_methods [] = { {SN_FusedAddHalving, OP_XOP_OVR_X_X_X, INTRINS_AARCH64_ADV_SIMD_SHADD, OP_XOP_OVR_X_X_X, INTRINS_AARCH64_ADV_SIMD_UHADD}, {SN_FusedAddRoundedHalving, OP_XOP_OVR_X_X_X, INTRINS_AARCH64_ADV_SIMD_SRHADD, OP_XOP_OVR_X_X_X, INTRINS_AARCH64_ADV_SIMD_URHADD}, {SN_FusedMultiplyAdd, OP_ARM64_FMADD}, + {SN_FusedMultiplyAddBySelectedScalar}, {SN_FusedMultiplyAddNegatedScalar, OP_ARM64_FNMADD_SCALAR}, {SN_FusedMultiplyAddScalar, OP_ARM64_FMADD_SCALAR}, + {SN_FusedMultiplyAddScalarBySelectedScalar}, {SN_FusedMultiplySubtract, OP_ARM64_FMSUB}, + {SN_FusedMultiplySubtractByScalar, OP_ARM64_FMSUB_BYSCALAR}, + {SN_FusedMultiplySubtractBySelectedScalar}, {SN_FusedMultiplySubtractNegatedScalar, OP_ARM64_FNMSUB_SCALAR}, {SN_FusedMultiplySubtractScalar, OP_ARM64_FMSUB_SCALAR}, + {SN_FusedMultiplySubtractScalarBySelectedScalar}, {SN_FusedSubtractHalving, OP_XOP_OVR_X_X_X, INTRINS_AARCH64_ADV_SIMD_SHSUB, OP_XOP_OVR_X_X_X, INTRINS_AARCH64_ADV_SIMD_UHSUB}, {SN_Insert}, {SN_InsertScalar}, @@ -1595,6 +1600,10 @@ emit_arm64_intrinsics ( ret->inst_c0 = c0; return ret; } + case SN_FusedMultiplyAddBySelectedScalar: + case SN_FusedMultiplyAddScalarBySelectedScalar: + case SN_FusedMultiplySubtractBySelectedScalar: + case SN_FusedMultiplySubtractScalarBySelectedScalar: case SN_MultiplyDoublingWideningScalarBySelectedScalarAndAddSaturate: case SN_MultiplyDoublingWideningScalarBySelectedScalarAndSubtractSaturate: case SN_MultiplyAddBySelectedScalar: @@ -1611,6 +1620,10 @@ emit_arm64_intrinsics ( gboolean is_unsigned = type_is_unsigned (fsig->ret); int opcode = 0; switch (id) { + case SN_FusedMultiplyAddBySelectedScalar: opcode = OP_ARM64_FMADD_BYSCALAR; break; + case SN_FusedMultiplyAddScalarBySelectedScalar: opcode = OP_ARM64_FMADD_SCALAR; break; + case SN_FusedMultiplySubtractBySelectedScalar: opcode = OP_ARM64_FMSUB_BYSCALAR; break; + case SN_FusedMultiplySubtractScalarBySelectedScalar: opcode = OP_ARM64_FMSUB_SCALAR; break; case SN_MultiplyDoublingWideningScalarBySelectedScalarAndAddSaturate: opcode = OP_ARM64_SQDMLAL_SCALAR; break; case SN_MultiplyDoublingWideningScalarBySelectedScalarAndSubtractSaturate: opcode = OP_ARM64_SQDMLSL_SCALAR; break; case SN_MultiplyAddBySelectedScalar: opcode = OP_ARM64_MLA_SCALAR; break; From 6560bd2e8c09e3652a2c841599aa7e68378de5d0 Mon Sep 17 00:00:00 2001 From: Imran Hameed Date: Sun, 7 Mar 2021 20:38:34 -0800 Subject: [PATCH 35/58] More AdvSimd.Arm64 --- src/mono/mono/mini/llvm-intrinsics.h | 3 +++ src/mono/mono/mini/mini-llvm.c | 29 +++++++++++++++++++++++++++- src/mono/mono/mini/mini-ops.h | 2 ++ src/mono/mono/mini/simd-intrinsics.c | 4 ++++ 4 files changed, 37 insertions(+), 1 deletion(-) diff --git a/src/mono/mono/mini/llvm-intrinsics.h b/src/mono/mono/mini/llvm-intrinsics.h index d752093ed5d362..0efa64ca955d0d 100644 --- a/src/mono/mono/mini/llvm-intrinsics.h +++ b/src/mono/mono/mini/llvm-intrinsics.h @@ -337,6 +337,9 @@ INTRINS_OVR_TAG_KIND(AARCH64_ADV_SIMD_FCVTNU, aarch64_neon_fcvtnu, Ftoi, Scalar INTRINS_OVR_TAG_KIND(AARCH64_ADV_SIMD_FCVTMU, aarch64_neon_fcvtmu, Ftoi, Scalar | V64 | V128 | I4 | I8) INTRINS_OVR_TAG_KIND(AARCH64_ADV_SIMD_FCVTPU, aarch64_neon_fcvtpu, Ftoi, Scalar | V64 | V128 | I4 | I8) +INTRINS_OVR_2_ARG(AARCH64_ADV_SIMD_SCALAR_SQXTUN, aarch64_neon_scalar_sqxtun, i4_t, i8_t) +INTRINS_OVR_2_ARG(AARCH64_ADV_SIMD_SCALAR_SQXTN, aarch64_neon_scalar_sqxtn, i4_t, i8_t) +INTRINS_OVR_2_ARG(AARCH64_ADV_SIMD_SCALAR_UQXTN, aarch64_neon_scalar_uqxtn, i4_t, i8_t) INTRINS_OVR_TAG(AARCH64_ADV_SIMD_SQXTUN, aarch64_neon_sqxtun, V64 | I1 | I2 | I4) INTRINS_OVR_TAG(AARCH64_ADV_SIMD_SQXTN, aarch64_neon_sqxtn, V64 | I1 | I2 | I4) INTRINS_OVR_TAG(AARCH64_ADV_SIMD_UQXTN, aarch64_neon_uqxtn, V64 | I1 | I2 | I4) diff --git a/src/mono/mono/mini/mini-llvm.c b/src/mono/mono/mini/mini-llvm.c index 72e757369bd080..97c5e11b8fbb57 100644 --- a/src/mono/mono/mini/mini-llvm.c +++ b/src/mono/mono/mini/mini-llvm.c @@ -9747,6 +9747,34 @@ process_bb (EmitContext *ctx, MonoBasicBlock *bb) values [ins->dreg] = LLVMBuildTrunc (builder, hi64, LLVMInt64Type (), ""); break; } + case OP_ARM64_XNARROW_SCALAR: { + // Unfortunately, @llvm.aarch64.neon.scalar.sqxtun isn't available for i8 or i16. + LLVMTypeRef ret_t = simd_class_to_llvm_type (ctx, ins->klass); + llvm_ovr_tag_t ovr_tag = ovr_tag_from_llvm_type (ret_t); + LLVMTypeRef elem_t = LLVMGetElementType (ret_t); + LLVMValueRef result = NULL; + int iid = ins->inst_c0; + int scalar_iid = 0; + switch (iid) { + case INTRINS_AARCH64_ADV_SIMD_SQXTUN: scalar_iid = INTRINS_AARCH64_ADV_SIMD_SCALAR_SQXTUN; break; + case INTRINS_AARCH64_ADV_SIMD_SQXTN: scalar_iid = INTRINS_AARCH64_ADV_SIMD_SCALAR_SQXTN; break; + case INTRINS_AARCH64_ADV_SIMD_UQXTN: scalar_iid = INTRINS_AARCH64_ADV_SIMD_SCALAR_UQXTN; break; + default: g_assert_not_reached (); + } + if (elem_t == i4_t) { + LLVMValueRef arg = scalar_from_vector (ctx, lhs); + result = call_intrins (ctx, scalar_iid, &arg, "arm64_xnarrow_scalar"); + result = vector_from_scalar_ty (ctx, ret_t, result); + } else { + LLVMTypeRef arg_t = LLVMTypeOf (lhs); + LLVMTypeRef argelem_t = LLVMGetElementType (arg_t); + unsigned int argelems = LLVMGetVectorSize (arg_t); + LLVMValueRef arg = undef_upper_elements (ctx, LLVMVectorType (argelem_t, argelems * 2), lhs); + result = call_overloaded_intrins (ctx, iid, ovr_tag, &arg, "arm64_xnarrow_scalar"); + } + values [ins->dreg] = result; + break; + } case OP_ARM64_SQXTUN2: case OP_ARM64_UQXTN2: case OP_ARM64_SQXTN2: @@ -9947,7 +9975,6 @@ process_bb (EmitContext *ctx, MonoBasicBlock *bb) case OP_ARM64_SQRDMULH_SCALAR_SEL: sel = TRUE; scalar = TRUE; break; case OP_ARM64_SQRDMULH_SEL: sel = TRUE; break; } - // XXXih: TODO: unroll arg3, bounds checks for arg3 llvm_ovr_tag_t ovr_tag = ovr_tag_from_mono_vector_class (ins->klass); LLVMValueRef lane = arg3; if (!sel) diff --git a/src/mono/mono/mini/mini-ops.h b/src/mono/mono/mini/mini-ops.h index 092b832643d46e..5b4bdc0ced87d0 100644 --- a/src/mono/mono/mini/mini-ops.h +++ b/src/mono/mono/mini/mini-ops.h @@ -1821,4 +1821,6 @@ MINI_OP(OP_ARM64_UABD, "arm64_uabd", XREG, XREG, XREG) MINI_OP(OP_ARM64_ABSCOMPARE, "arm64_abscompare", XREG, XREG, XREG) +MINI_OP(OP_ARM64_XNARROW_SCALAR, "arm64_xnarrow_scalar", XREG, XREG, NONE) + #endif // TARGET_ARM64 diff --git a/src/mono/mono/mini/simd-intrinsics.c b/src/mono/mono/mini/simd-intrinsics.c index 8d65e4d855d75f..f6eaf1a0df8893 100644 --- a/src/mono/mono/mini/simd-intrinsics.c +++ b/src/mono/mono/mini/simd-intrinsics.c @@ -1131,7 +1131,9 @@ static SimdIntrinsic advsimd_methods [] = { {SN_Extract}, {SN_ExtractNarrowingLower, OP_ARM64_XTN}, {SN_ExtractNarrowingSaturateLower, OP_XOP_OVR_X_X, INTRINS_AARCH64_ADV_SIMD_SQXTN, OP_XOP_OVR_X_X, INTRINS_AARCH64_ADV_SIMD_UQXTN}, + {SN_ExtractNarrowingSaturateScalar, OP_ARM64_XNARROW_SCALAR, INTRINS_AARCH64_ADV_SIMD_SQXTN, OP_ARM64_XNARROW_SCALAR, INTRINS_AARCH64_ADV_SIMD_UQXTN}, {SN_ExtractNarrowingSaturateUnsignedLower, OP_XOP_OVR_X_X, INTRINS_AARCH64_ADV_SIMD_SQXTUN}, + {SN_ExtractNarrowingSaturateUnsignedScalar, OP_ARM64_XNARROW_SCALAR, INTRINS_AARCH64_ADV_SIMD_SQXTUN}, {SN_ExtractNarrowingSaturateUnsignedUpper, OP_ARM64_SQXTUN2}, {SN_ExtractNarrowingSaturateUpper, OP_ARM64_SQXTN2, None, OP_ARM64_UQXTN2}, {SN_ExtractNarrowingUpper, OP_ARM64_XTN2}, @@ -1140,6 +1142,7 @@ static SimdIntrinsic advsimd_methods [] = { {SN_FusedAddHalving, OP_XOP_OVR_X_X_X, INTRINS_AARCH64_ADV_SIMD_SHADD, OP_XOP_OVR_X_X_X, INTRINS_AARCH64_ADV_SIMD_UHADD}, {SN_FusedAddRoundedHalving, OP_XOP_OVR_X_X_X, INTRINS_AARCH64_ADV_SIMD_SRHADD, OP_XOP_OVR_X_X_X, INTRINS_AARCH64_ADV_SIMD_URHADD}, {SN_FusedMultiplyAdd, OP_ARM64_FMADD}, + {SN_FusedMultiplyAddByScalar, OP_ARM64_FMADD_BYSCALAR}, {SN_FusedMultiplyAddBySelectedScalar}, {SN_FusedMultiplyAddNegatedScalar, OP_ARM64_FNMADD_SCALAR}, {SN_FusedMultiplyAddScalar, OP_ARM64_FMADD_SCALAR}, @@ -1309,6 +1312,7 @@ static SimdIntrinsic advsimd_methods [] = { {SN_ShiftRightArithmeticAdd, OP_ARM64_SSRA}, {SN_ShiftRightArithmeticAddScalar, OP_ARM64_SSRA}, {SN_ShiftRightArithmeticNarrowingSaturateLower, OP_ARM64_XRSHIFT, INTRINS_AARCH64_ADV_SIMD_SQSHRN}, + {SN_ShiftRightArithmeticNarrowingSaturateScalar, OP_ARM64_XRSHIFT_SCALAR, INTRINS_AARCH64_ADV_SIMD_SQSHRN}, {SN_ShiftRightArithmeticNarrowingSaturateUnsignedLower, OP_ARM64_XRSHIFT, INTRINS_AARCH64_ADV_SIMD_SQSHRUN}, {SN_ShiftRightArithmeticNarrowingSaturateUnsignedScalar, OP_ARM64_XRSHIFT_SCALAR, INTRINS_AARCH64_ADV_SIMD_SQSHRUN}, {SN_ShiftRightArithmeticNarrowingSaturateUnsignedUpper, OP_ARM64_XRSHIFT2, INTRINS_AARCH64_ADV_SIMD_SQSHRUN}, From 5c60c8f830ead37dc0dd4e4e68310630f013b0e8 Mon Sep 17 00:00:00 2001 From: Imran Hameed Date: Mon, 8 Mar 2021 09:31:42 -0800 Subject: [PATCH 36/58] Implement ExtractVector64/ExtractVector128 --- src/mono/mono/mini/mini-llvm.c | 39 ++++++++++++++++++++++++++++ src/mono/mono/mini/mini-ops.h | 2 ++ src/mono/mono/mini/simd-intrinsics.c | 2 ++ 3 files changed, 43 insertions(+) diff --git a/src/mono/mono/mini/mini-llvm.c b/src/mono/mono/mini/mini-llvm.c index 97c5e11b8fbb57..2e6322e26c860e 100644 --- a/src/mono/mono/mini/mini-llvm.c +++ b/src/mono/mono/mini/mini-llvm.c @@ -9572,6 +9572,45 @@ process_bb (EmitContext *ctx, MonoBasicBlock *bb) values [ins->dreg] = LLVMBuildSExt (builder, result, ret_t, ""); break; } + case OP_ARM64_EXT: { + LLVMTypeRef ret_t = LLVMTypeOf (lhs); + unsigned int elems = LLVMGetVectorSize (ret_t); + const int unrolled_mask [MAX_VECTOR_ELEMS] = { + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, + 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31 + }; + LLVMValueRef index = arg3; + enum { ARM64_EXT_MAX_INDEX = MAX_VECTOR_ELEMS / 2 }; + const int max_index = elems; + LLVMBasicBlockRef cases [ARM64_EXT_MAX_INDEX] = { 0 }; + LLVMValueRef case_values [ARM64_EXT_MAX_INDEX] = { 0 }; + + LLVMBasicBlockRef default_case = gen_bb (ctx, "arm64_ext_default"); + LLVMValueRef default_value = lhs; + + LLVMValueRef llvmswitch = LLVMBuildSwitch (builder, index, default_case, ARM64_EXT_MAX_INDEX); + cbb = gen_bb (ctx, "arm64_ext_unroll_continue"); + for (int i = 0; i < max_index; ++i) { + LLVMBasicBlockRef llvmcase = gen_bb (ctx, "arm64_ext_case"); + LLVMAddCase (llvmswitch, const_int32 (i), llvmcase); + LLVMPositionBuilderAtEnd (builder, llvmcase); + LLVMValueRef mask = create_const_vector_i32 (&unrolled_mask [i], elems); + LLVMValueRef result = LLVMBuildShuffleVector (builder, lhs, rhs, mask, "arm64_ext"); + LLVMBuildBr (builder, cbb); + cases [i] = llvmcase; + case_values [i] = result; + } + LLVMPositionBuilderAtEnd (builder, default_case); + LLVMBuildBr (builder, cbb); + + LLVMPositionBuilderAtEnd (builder, cbb); + LLVMValueRef phi = LLVMBuildPhi (builder, ret_t, ""); + LLVMAddIncoming (phi, case_values, cases, max_index); + LLVMAddIncoming (phi, &default_value, &default_case, 1); + ctx->bblocks [bb->block_num].end_bblock = cbb; + values [ins->dreg] = phi; + break; + } case OP_ARM64_MVN: { LLVMTypeRef ret_t = LLVMTypeOf (lhs); LLVMValueRef result = bitcast_to_integral (ctx, lhs); diff --git a/src/mono/mono/mini/mini-ops.h b/src/mono/mono/mini/mini-ops.h index 5b4bdc0ced87d0..014e51dcb2effb 100644 --- a/src/mono/mono/mini/mini-ops.h +++ b/src/mono/mono/mini/mini-ops.h @@ -1823,4 +1823,6 @@ MINI_OP(OP_ARM64_ABSCOMPARE, "arm64_abscompare", XREG, XREG, XREG) MINI_OP(OP_ARM64_XNARROW_SCALAR, "arm64_xnarrow_scalar", XREG, XREG, NONE) +MINI_OP3(OP_ARM64_EXT, "arm64_ext", XREG, XREG, XREG, IREG) + #endif // TARGET_ARM64 diff --git a/src/mono/mono/mini/simd-intrinsics.c b/src/mono/mono/mini/simd-intrinsics.c index f6eaf1a0df8893..975c2c70748218 100644 --- a/src/mono/mono/mini/simd-intrinsics.c +++ b/src/mono/mono/mini/simd-intrinsics.c @@ -1137,6 +1137,8 @@ static SimdIntrinsic advsimd_methods [] = { {SN_ExtractNarrowingSaturateUnsignedUpper, OP_ARM64_SQXTUN2}, {SN_ExtractNarrowingSaturateUpper, OP_ARM64_SQXTN2, None, OP_ARM64_UQXTN2}, {SN_ExtractNarrowingUpper, OP_ARM64_XTN2}, + {SN_ExtractVector128, OP_ARM64_EXT}, + {SN_ExtractVector64, OP_ARM64_EXT}, {SN_Floor, OP_XOP_OVR_X_X, INTRINS_AARCH64_ADV_SIMD_FRINTM}, {SN_FloorScalar, OP_XOP_OVR_SCALAR_X_X, INTRINS_AARCH64_ADV_SIMD_FRINTM}, {SN_FusedAddHalving, OP_XOP_OVR_X_X_X, INTRINS_AARCH64_ADV_SIMD_SHADD, OP_XOP_OVR_X_X_X, INTRINS_AARCH64_ADV_SIMD_UHADD}, From 16a4f7ca20481b500aa5e2666f43b85b7e641758 Mon Sep 17 00:00:00 2001 From: Imran Hameed Date: Mon, 8 Mar 2021 12:20:47 -0800 Subject: [PATCH 37/58] Unroll instructions with immediate constants and no fallback support via LLVM --- src/mono/mono/mini/mini-llvm.c | 253 ++++++++++++++++++++------- src/mono/mono/mini/mini-ops.h | 8 +- src/mono/mono/mini/simd-intrinsics.c | 44 ++--- 3 files changed, 219 insertions(+), 86 deletions(-) diff --git a/src/mono/mono/mini/mini-llvm.c b/src/mono/mono/mini/mini-llvm.c index 2e6322e26c860e..c505cabf092952 100644 --- a/src/mono/mono/mini/mini-llvm.c +++ b/src/mono/mono/mini/mini-llvm.c @@ -380,6 +380,22 @@ ovr_tag_corresponding_integer (llvm_ovr_tag_t tag) return ((tag & ~INTRIN_vectormask) >> 2) | (tag & INTRIN_vectormask); } +static LLVMTypeRef +ovr_tag_to_llvm_type (llvm_ovr_tag_t tag) +{ + int vw = 0; + int ew = 0; + if (tag & INTRIN_vector64) vw = 1; + else if (tag & INTRIN_vector128) vw = 2; + + if (tag & INTRIN_int16) ew = 1; + else if (tag & INTRIN_int32) ew = 2; + else if (tag & INTRIN_int64) ew = 3; + else if (tag & INTRIN_float32) ew = 4; + else if (tag & INTRIN_float64) ew = 5; + return intrin_types [vw][ew]; +} + static int key_from_id_and_tag (int id, llvm_ovr_tag_t ovr_tag) { @@ -1305,6 +1321,88 @@ gen_bb (EmitContext *ctx, const char *prefix) return LLVMAppendBasicBlock (ctx->lmethod, bb_name); } +typedef struct { + EmitContext *ctx; + MonoBasicBlock *bb; + LLVMBasicBlockRef continuation; + LLVMValueRef phi; + LLVMValueRef switch_ins; + LLVMBasicBlockRef tmp_block; + LLVMBasicBlockRef default_case; + LLVMTypeRef switch_index_type; + const char *name; + int max_cases; + int i; +} ImmediateUnrollCtx; + + +static ImmediateUnrollCtx +immediate_unroll_begin ( + EmitContext *ctx, MonoBasicBlock *bb, int max_cases, + LLVMValueRef switch_index, LLVMTypeRef return_type, const char *name) +{ + LLVMBasicBlockRef default_case = gen_bb (ctx, name); + LLVMBasicBlockRef continuation = gen_bb (ctx, name); + LLVMValueRef switch_ins = LLVMBuildSwitch (ctx->builder, switch_index, default_case, max_cases); + LLVMPositionBuilderAtEnd (ctx->builder, continuation); + LLVMValueRef phi = LLVMBuildPhi (ctx->builder, return_type, name); + ImmediateUnrollCtx ictx = { 0 }; + ictx.ctx = ctx; + ictx.bb = bb; + ictx.continuation = continuation; + ictx.phi = phi; + ictx.switch_ins = switch_ins; + ictx.default_case = default_case; + ictx.switch_index_type = LLVMTypeOf (switch_index); + ictx.name = name; + ictx.max_cases = max_cases; + return ictx; +} + +static gboolean +immediate_unroll_next (ImmediateUnrollCtx *ictx, int *i) +{ + if (ictx->i >= ictx->max_cases) + return FALSE; + ictx->tmp_block = gen_bb (ictx->ctx, ictx->name); + LLVMPositionBuilderAtEnd (ictx->ctx->builder, ictx->tmp_block); + *i = ictx->i; + ++ictx->i; + return TRUE; +} + +static void +immediate_unroll_commit (ImmediateUnrollCtx *ictx, int switch_const, LLVMValueRef value) +{ + LLVMBuildBr (ictx->ctx->builder, ictx->continuation); + LLVMAddCase (ictx->switch_ins, LLVMConstInt (ictx->switch_index_type, switch_const, FALSE), ictx->tmp_block); + LLVMAddIncoming (ictx->phi, &value, &ictx->tmp_block, 1); +} + +static void +immediate_unroll_default (ImmediateUnrollCtx *ictx) +{ + LLVMPositionBuilderAtEnd (ictx->ctx->builder, ictx->default_case); +} + +static void +immediate_unroll_commit_default (ImmediateUnrollCtx *ictx, LLVMValueRef value) +{ + LLVMBuildBr (ictx->ctx->builder, ictx->continuation); + LLVMAddIncoming (ictx->phi, &value, &ictx->default_case, 1); +} + +static LLVMValueRef +immediate_unroll_end (ImmediateUnrollCtx *ictx, LLVMBasicBlockRef *continuation) +{ + EmitContext *ctx = ictx->ctx; + LLVMBuilderRef builder = ctx->builder; + LLVMPositionBuilderAtEnd (builder, ictx->continuation); + *continuation = ictx->continuation; + ctx->bblocks [ictx->bb->block_num].end_bblock = ictx->continuation; + return ictx->phi; +} + /* * resolve_patch: * @@ -9579,36 +9677,20 @@ process_bb (EmitContext *ctx, MonoBasicBlock *bb) 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31 }; - LLVMValueRef index = arg3; enum { ARM64_EXT_MAX_INDEX = MAX_VECTOR_ELEMS / 2 }; - const int max_index = elems; - LLVMBasicBlockRef cases [ARM64_EXT_MAX_INDEX] = { 0 }; - LLVMValueRef case_values [ARM64_EXT_MAX_INDEX] = { 0 }; - - LLVMBasicBlockRef default_case = gen_bb (ctx, "arm64_ext_default"); + int max_index = elems; + LLVMValueRef index = arg3; LLVMValueRef default_value = lhs; - - LLVMValueRef llvmswitch = LLVMBuildSwitch (builder, index, default_case, ARM64_EXT_MAX_INDEX); - cbb = gen_bb (ctx, "arm64_ext_unroll_continue"); - for (int i = 0; i < max_index; ++i) { - LLVMBasicBlockRef llvmcase = gen_bb (ctx, "arm64_ext_case"); - LLVMAddCase (llvmswitch, const_int32 (i), llvmcase); - LLVMPositionBuilderAtEnd (builder, llvmcase); + ImmediateUnrollCtx ictx = immediate_unroll_begin (ctx, bb, max_index, index, ret_t, "arm64_ext"); + int i = 0; + while (immediate_unroll_next (&ictx, &i)) { LLVMValueRef mask = create_const_vector_i32 (&unrolled_mask [i], elems); LLVMValueRef result = LLVMBuildShuffleVector (builder, lhs, rhs, mask, "arm64_ext"); - LLVMBuildBr (builder, cbb); - cases [i] = llvmcase; - case_values [i] = result; + immediate_unroll_commit (&ictx, i, result); } - LLVMPositionBuilderAtEnd (builder, default_case); - LLVMBuildBr (builder, cbb); - - LLVMPositionBuilderAtEnd (builder, cbb); - LLVMValueRef phi = LLVMBuildPhi (builder, ret_t, ""); - LLVMAddIncoming (phi, case_values, cases, max_index); - LLVMAddIncoming (phi, &default_value, &default_case, 1); - ctx->bblocks [bb->block_num].end_bblock = cbb; - values [ins->dreg] = phi; + immediate_unroll_default (&ictx); + immediate_unroll_commit_default (&ictx, default_value); + values [ins->dreg] = immediate_unroll_end (&ictx, &cbb); break; } case OP_ARM64_MVN: { @@ -10283,32 +10365,52 @@ process_bb (EmitContext *ctx, MonoBasicBlock *bb) values [ins->dreg] = result; break; } - case OP_ARM64_XRSHIFT_SCALAR: - case OP_ARM64_XRSHIFT: - case OP_ARM64_XRSHIFT2: { - // XXXih: TODO: unroll count/rhs/arg3 - llvm_ovr_tag_t ovr_tag = ovr_tag_from_mono_vector_class (ins->klass); - LLVMValueRef args [2] = { lhs, rhs }; + case OP_ARM64_XNSHIFT_SCALAR: + case OP_ARM64_XNSHIFT: + case OP_ARM64_XNSHIFT2: { + LLVMTypeRef intrin_result_t = simd_class_to_llvm_type (ctx, ins->klass); + llvm_ovr_tag_t ovr_tag = ovr_tag_from_llvm_type (intrin_result_t); + LLVMValueRef shift_arg = lhs; + LLVMValueRef shift_amount = rhs; gboolean high = FALSE; gboolean scalar = FALSE; int iid = ins->inst_c0; switch (ins->opcode) { - case OP_ARM64_XRSHIFT_SCALAR: scalar = TRUE; break; - case OP_ARM64_XRSHIFT2: high = TRUE; break; + case OP_ARM64_XNSHIFT_SCALAR: scalar = TRUE; break; + case OP_ARM64_XNSHIFT2: high = TRUE; break; } if (high) { - args [0] = rhs; - args [1] = arg3; + shift_arg = rhs; + shift_amount = arg3; ovr_tag = ovr_tag_smaller_vector (ovr_tag); + intrin_result_t = ovr_tag_to_llvm_type (ovr_tag); } + LLVMTypeRef shift_arg_t = LLVMTypeOf (shift_arg); + LLVMTypeRef shift_arg_elem_t = LLVMGetElementType (shift_arg_t); + unsigned int element_bits = mono_llvm_get_prim_size_bits (shift_arg_elem_t); + int range_min = 1; + int range_max = element_bits / 2; if (scalar) { - LLVMTypeRef arg_t = LLVMTypeOf (args [0]); - LLVMTypeRef elem_t = LLVMGetElementType (arg_t); - unsigned int elems = LLVMGetVectorSize (arg_t); - LLVMValueRef lo = scalar_from_vector (ctx, args [0]); - args [0] = vector_from_scalar_ty (ctx, LLVMVectorType (elem_t, elems * 2), lo); - } - LLVMValueRef result = call_overloaded_intrins (ctx, iid, ovr_tag, args, ""); + unsigned int elems = LLVMGetVectorSize (shift_arg_t); + LLVMValueRef lo = scalar_from_vector (ctx, shift_arg); + shift_arg = vector_from_scalar_ty (ctx, LLVMVectorType (shift_arg_elem_t, elems * 2), lo); + } + int max_index = range_max - range_min + 1; + ImmediateUnrollCtx ictx = immediate_unroll_begin (ctx, bb, max_index, shift_amount, intrin_result_t, "arm64_xnshift"); + int i = 0; + while (immediate_unroll_next (&ictx, &i)) { + int shift_const = i + range_min; + LLVMValueRef intrin_args [] = { shift_arg, const_int32 (shift_const) }; + LLVMValueRef result = call_overloaded_intrins (ctx, iid, ovr_tag, intrin_args, ""); + immediate_unroll_commit (&ictx, shift_const, result); + } + { + immediate_unroll_default (&ictx); + LLVMValueRef intrin_args [] = { shift_arg, const_int32 (range_max) }; + LLVMValueRef result = call_overloaded_intrins (ctx, iid, ovr_tag, intrin_args, ""); + immediate_unroll_commit_default (&ictx, result); + } + LLVMValueRef result = immediate_unroll_end (&ictx, &cbb); if (high) result = concatenate_vectors (ctx, lhs, result); if (scalar) @@ -10330,12 +10432,23 @@ process_bb (EmitContext *ctx, MonoBasicBlock *bb) break; } case OP_ARM64_SQSHLU: { - // XXXih: TODO: unroll count/rhs - llvm_ovr_tag_t ovr_tag = ovr_tag_from_mono_vector_class (ins->klass); - LLVMValueRef shift = create_shift_vector (ctx, lhs, rhs); - LLVMValueRef args [] = { lhs, shift }; + LLVMTypeRef intrin_result_t = simd_class_to_llvm_type (ctx, ins->klass); + llvm_ovr_tag_t ovr_tag = ovr_tag_from_llvm_type (intrin_result_t); + unsigned int element_bits = mono_llvm_get_prim_size_bits (LLVMGetElementType (intrin_result_t)); + int max_index = element_bits; int iid = INTRINS_AARCH64_ADV_SIMD_SQSHLU; - values [ins->dreg] = call_overloaded_intrins (ctx, iid, ovr_tag, args, ""); + ImmediateUnrollCtx ictx = immediate_unroll_begin (ctx, bb, max_index, rhs, intrin_result_t, "arm64_sqshlu"); + LLVMValueRef args [] = { lhs, NULL }; + int i = 0; + while (immediate_unroll_next (&ictx, &i)) { + int shift_const = i; + args [1] = create_shift_vector (ctx, lhs, const_int32 (shift_const)); + LLVMValueRef result = call_overloaded_intrins (ctx, iid, ovr_tag, args, ""); + immediate_unroll_commit (&ictx, shift_const, result); + } + immediate_unroll_default (&ictx); + immediate_unroll_commit_default (&ictx, lhs); + values [ins->dreg] = immediate_unroll_end (&ictx, &cbb); break; } case OP_ARM64_SSHLL: @@ -10362,11 +10475,30 @@ process_bb (EmitContext *ctx, MonoBasicBlock *bb) } case OP_ARM64_SLI: case OP_ARM64_SRI: { - // XXXih: TODO: unroll count/arg3 - LLVMValueRef args [3] = { lhs, rhs, arg3 }; - llvm_ovr_tag_t ovr_tag = ovr_tag_from_mono_vector_class (ins->klass); - int iid = ins->opcode == OP_ARM64_SLI ? INTRINS_AARCH64_ADV_SIMD_SLI : INTRINS_AARCH64_ADV_SIMD_SRI; - values [ins->dreg] = call_overloaded_intrins (ctx, iid, ovr_tag, args, ""); + LLVMTypeRef intrin_result_t = simd_class_to_llvm_type (ctx, ins->klass); + llvm_ovr_tag_t ovr_tag = ovr_tag_from_llvm_type (intrin_result_t); + unsigned int element_bits = mono_llvm_get_prim_size_bits (LLVMGetElementType (intrin_result_t)); + int range_min = 0; + int range_max = element_bits - 1; + if (ins->opcode == OP_ARM64_SRI) { + ++range_min; + ++range_max; + } + int iid = ins->opcode == OP_ARM64_SRI ? INTRINS_AARCH64_ADV_SIMD_SRI : INTRINS_AARCH64_ADV_SIMD_SLI; + int max_index = range_max - range_min + 1; + ImmediateUnrollCtx ictx = immediate_unroll_begin (ctx, bb, max_index, arg3, intrin_result_t, "arm64_ext"); + LLVMValueRef intrin_args [3] = { lhs, rhs, arg3 }; + int i = 0; + while (immediate_unroll_next (&ictx, &i)) { + int shift_const = i + range_min; + intrin_args [2] = const_int32 (shift_const); + LLVMValueRef result = call_overloaded_intrins (ctx, iid, ovr_tag, intrin_args, ""); + immediate_unroll_commit (&ictx, shift_const, result); + } + immediate_unroll_default (&ictx); + immediate_unroll_commit_default (&ictx, lhs); + LLVMValueRef result = immediate_unroll_end (&ictx, &cbb); + values [ins->dreg] = result; break; } case OP_ARM64_SQRT_SCALAR: { @@ -10407,9 +10539,9 @@ process_bb (EmitContext *ctx, MonoBasicBlock *bb) } case OP_ARM64_LD1_INSERT: { LLVMTypeRef ret_t = simd_class_to_llvm_type (ctx, ins->klass); - unsigned int bytes = mono_llvm_get_prim_size_bits (ret_t) / 8; + unsigned int alignment = mono_llvm_get_prim_size_bits (ret_t) / 8; LLVMValueRef address = arg3; - LLVMValueRef result = mono_llvm_build_aligned_load (builder, address, "arm64_ld1_insert", FALSE, bytes); + LLVMValueRef result = mono_llvm_build_aligned_load (builder, address, "arm64_ld1_insert", FALSE, alignment); result = LLVMBuildInsertElement (builder, lhs, result, rhs, "arm64_ld1_insert"); values [ins->dreg] = result; break; @@ -10418,11 +10550,11 @@ process_bb (EmitContext *ctx, MonoBasicBlock *bb) case OP_ARM64_LD1: { gboolean replicate = ins->opcode == OP_ARM64_LD1R; LLVMTypeRef ret_t = simd_class_to_llvm_type (ctx, ins->klass); - unsigned int bytes = mono_llvm_get_prim_size_bits (ret_t) / 8; + unsigned int alignment = mono_llvm_get_prim_size_bits (ret_t) / 8; LLVMValueRef address = lhs; if (!replicate) address = convert (ctx, address, LLVMPointerType (ret_t, 0)); - LLVMValueRef result = mono_llvm_build_aligned_load (builder, address, "arm64_ld1", FALSE, bytes); + LLVMValueRef result = mono_llvm_build_aligned_load (builder, address, "arm64_ld1", FALSE, alignment); if (replicate) { unsigned int elems = LLVMGetVectorSize (ret_t); result = broadcast_element (ctx, result, elems); @@ -10433,14 +10565,15 @@ process_bb (EmitContext *ctx, MonoBasicBlock *bb) case OP_ARM64_ST1: { LLVMTypeRef t = LLVMTypeOf (rhs); LLVMValueRef address = convert (ctx, lhs, LLVMPointerType (t, 0)); - unsigned int bytes = mono_llvm_get_prim_size_bits (t) / 8; - mono_llvm_build_aligned_store (builder, rhs, address, FALSE, bytes); + unsigned int alignment = mono_llvm_get_prim_size_bits (t) / 8; + mono_llvm_build_aligned_store (builder, rhs, address, FALSE, alignment); break; } case OP_ARM64_ST1_SCALAR: { - // XXXih: TODO: unroll arg3 + LLVMTypeRef t = LLVMGetElementType (LLVMTypeOf (rhs)); + unsigned int alignment = mono_llvm_get_prim_size_bits (t) / 8; LLVMValueRef val = LLVMBuildExtractElement (builder, rhs, arg3, "arm64_st1_scalar"); - mono_llvm_build_store (builder, val, lhs, FALSE, LLVM_BARRIER_NONE); + mono_llvm_build_aligned_store (builder, val, lhs, FALSE, alignment); break; } case OP_ARM64_ADDHN: diff --git a/src/mono/mono/mini/mini-ops.h b/src/mono/mono/mini/mini-ops.h index 014e51dcb2effb..b5224c3c88e61c 100644 --- a/src/mono/mono/mini/mini-ops.h +++ b/src/mono/mono/mini/mini-ops.h @@ -1656,10 +1656,10 @@ MINI_OP(OP_ARM64_USHLL2, "arm64_ushll2", XREG, XREG, IREG) MINI_OP(OP_ARM64_SSHLL, "arm64_sshll", XREG, XREG, IREG) MINI_OP(OP_ARM64_SSHLL2, "arm64_sshll2", XREG, XREG, IREG) - -MINI_OP(OP_ARM64_XRSHIFT_SCALAR, "arm64_xrshift_scalar", XREG, XREG, IREG) -MINI_OP(OP_ARM64_XRSHIFT, "arm64_xrshift", XREG, XREG, IREG) -MINI_OP3(OP_ARM64_XRSHIFT2, "arm64_xrshift2", XREG, XREG, XREG, IREG) +/* Narrowing arm64 shifts that aren't decomposed into urshl or srshl. */ +MINI_OP(OP_ARM64_XNSHIFT_SCALAR, "arm64_xrshift_scalar", XREG, XREG, IREG) +MINI_OP(OP_ARM64_XNSHIFT, "arm64_xnshift", XREG, XREG, IREG) +MINI_OP3(OP_ARM64_XNSHIFT2, "arm64_xnshift2", XREG, XREG, XREG, IREG) MINI_OP(OP_ARM64_UQSHL_IMM, "arm64_uqshl_imm", XREG, XREG, IREG) MINI_OP(OP_ARM64_SQSHL_IMM, "arm64_uqshl_imm", XREG, XREG, IREG) diff --git a/src/mono/mono/mini/simd-intrinsics.c b/src/mono/mono/mini/simd-intrinsics.c index 975c2c70748218..99ad7d58ba3df1 100644 --- a/src/mono/mono/mini/simd-intrinsics.c +++ b/src/mono/mono/mini/simd-intrinsics.c @@ -1290,8 +1290,8 @@ static SimdIntrinsic advsimd_methods [] = { {SN_ShiftArithmeticSaturate, OP_XOP_OVR_X_X_X, INTRINS_AARCH64_ADV_SIMD_SQSHL}, {SN_ShiftArithmeticSaturateScalar, OP_XOP_OVR_X_X_X, INTRINS_AARCH64_ADV_SIMD_SQSHL}, {SN_ShiftArithmeticScalar, OP_XOP_OVR_X_X_X, INTRINS_AARCH64_ADV_SIMD_SSHL}, - {SN_ShiftLeftAndInsert, OP_ARM64_SRI}, - {SN_ShiftLeftAndInsertScalar, OP_ARM64_SRI}, + {SN_ShiftLeftAndInsert, OP_ARM64_SLI}, + {SN_ShiftLeftAndInsertScalar, OP_ARM64_SLI}, {SN_ShiftLeftLogical, OP_ARM64_SHL}, {SN_ShiftLeftLogicalSaturate, OP_ARM64_SQSHL_IMM, None, OP_ARM64_UQSHL_IMM}, {SN_ShiftLeftLogicalSaturateScalar, OP_ARM64_SQSHL_IMM, None, OP_ARM64_UQSHL_IMM}, @@ -1313,39 +1313,39 @@ static SimdIntrinsic advsimd_methods [] = { {SN_ShiftRightArithmetic, OP_ARM64_SSHR}, {SN_ShiftRightArithmeticAdd, OP_ARM64_SSRA}, {SN_ShiftRightArithmeticAddScalar, OP_ARM64_SSRA}, - {SN_ShiftRightArithmeticNarrowingSaturateLower, OP_ARM64_XRSHIFT, INTRINS_AARCH64_ADV_SIMD_SQSHRN}, - {SN_ShiftRightArithmeticNarrowingSaturateScalar, OP_ARM64_XRSHIFT_SCALAR, INTRINS_AARCH64_ADV_SIMD_SQSHRN}, - {SN_ShiftRightArithmeticNarrowingSaturateUnsignedLower, OP_ARM64_XRSHIFT, INTRINS_AARCH64_ADV_SIMD_SQSHRUN}, - {SN_ShiftRightArithmeticNarrowingSaturateUnsignedScalar, OP_ARM64_XRSHIFT_SCALAR, INTRINS_AARCH64_ADV_SIMD_SQSHRUN}, - {SN_ShiftRightArithmeticNarrowingSaturateUnsignedUpper, OP_ARM64_XRSHIFT2, INTRINS_AARCH64_ADV_SIMD_SQSHRUN}, - {SN_ShiftRightArithmeticNarrowingSaturateUpper, OP_ARM64_XRSHIFT2, INTRINS_AARCH64_ADV_SIMD_SQSHRN}, + {SN_ShiftRightArithmeticNarrowingSaturateLower, OP_ARM64_XNSHIFT, INTRINS_AARCH64_ADV_SIMD_SQSHRN}, + {SN_ShiftRightArithmeticNarrowingSaturateScalar, OP_ARM64_XNSHIFT_SCALAR, INTRINS_AARCH64_ADV_SIMD_SQSHRN}, + {SN_ShiftRightArithmeticNarrowingSaturateUnsignedLower, OP_ARM64_XNSHIFT, INTRINS_AARCH64_ADV_SIMD_SQSHRUN}, + {SN_ShiftRightArithmeticNarrowingSaturateUnsignedScalar, OP_ARM64_XNSHIFT_SCALAR, INTRINS_AARCH64_ADV_SIMD_SQSHRUN}, + {SN_ShiftRightArithmeticNarrowingSaturateUnsignedUpper, OP_ARM64_XNSHIFT2, INTRINS_AARCH64_ADV_SIMD_SQSHRUN}, + {SN_ShiftRightArithmeticNarrowingSaturateUpper, OP_ARM64_XNSHIFT2, INTRINS_AARCH64_ADV_SIMD_SQSHRN}, {SN_ShiftRightArithmeticRounded, OP_ARM64_SRSHR}, {SN_ShiftRightArithmeticRoundedAdd, OP_ARM64_SRSRA}, {SN_ShiftRightArithmeticRoundedAddScalar, OP_ARM64_SRSRA}, - {SN_ShiftRightArithmeticRoundedNarrowingSaturateLower, OP_ARM64_XRSHIFT, INTRINS_AARCH64_ADV_SIMD_SQRSHRN}, - {SN_ShiftRightArithmeticRoundedNarrowingSaturateScalar, OP_ARM64_XRSHIFT_SCALAR, INTRINS_AARCH64_ADV_SIMD_SQRSHRN}, - {SN_ShiftRightArithmeticRoundedNarrowingSaturateUnsignedLower, OP_ARM64_XRSHIFT, INTRINS_AARCH64_ADV_SIMD_SQRSHRUN}, - {SN_ShiftRightArithmeticRoundedNarrowingSaturateUnsignedScalar, OP_ARM64_XRSHIFT_SCALAR, INTRINS_AARCH64_ADV_SIMD_SQRSHRUN}, - {SN_ShiftRightArithmeticRoundedNarrowingSaturateUnsignedUpper, OP_ARM64_XRSHIFT2, INTRINS_AARCH64_ADV_SIMD_SQRSHRUN}, - {SN_ShiftRightArithmeticRoundedNarrowingSaturateUpper, OP_ARM64_XRSHIFT2, INTRINS_AARCH64_ADV_SIMD_SQRSHRN}, + {SN_ShiftRightArithmeticRoundedNarrowingSaturateLower, OP_ARM64_XNSHIFT, INTRINS_AARCH64_ADV_SIMD_SQRSHRN}, + {SN_ShiftRightArithmeticRoundedNarrowingSaturateScalar, OP_ARM64_XNSHIFT_SCALAR, INTRINS_AARCH64_ADV_SIMD_SQRSHRN}, + {SN_ShiftRightArithmeticRoundedNarrowingSaturateUnsignedLower, OP_ARM64_XNSHIFT, INTRINS_AARCH64_ADV_SIMD_SQRSHRUN}, + {SN_ShiftRightArithmeticRoundedNarrowingSaturateUnsignedScalar, OP_ARM64_XNSHIFT_SCALAR, INTRINS_AARCH64_ADV_SIMD_SQRSHRUN}, + {SN_ShiftRightArithmeticRoundedNarrowingSaturateUnsignedUpper, OP_ARM64_XNSHIFT2, INTRINS_AARCH64_ADV_SIMD_SQRSHRUN}, + {SN_ShiftRightArithmeticRoundedNarrowingSaturateUpper, OP_ARM64_XNSHIFT2, INTRINS_AARCH64_ADV_SIMD_SQRSHRN}, {SN_ShiftRightArithmeticRoundedScalar, OP_ARM64_SRSHR}, {SN_ShiftRightArithmeticScalar, OP_ARM64_SSHR}, {SN_ShiftRightLogical, OP_ARM64_USHR}, {SN_ShiftRightLogicalAdd, OP_ARM64_USRA}, {SN_ShiftRightLogicalAddScalar, OP_ARM64_USRA}, {SN_ShiftRightLogicalNarrowingLower, OP_ARM64_SHRN}, - {SN_ShiftRightLogicalNarrowingSaturateLower, OP_ARM64_XRSHIFT, INTRINS_AARCH64_ADV_SIMD_UQSHRN}, - {SN_ShiftRightLogicalNarrowingSaturateScalar, OP_ARM64_XRSHIFT_SCALAR, INTRINS_AARCH64_ADV_SIMD_UQSHRN}, - {SN_ShiftRightLogicalNarrowingSaturateUpper, OP_ARM64_XRSHIFT2, INTRINS_AARCH64_ADV_SIMD_UQSHRN}, + {SN_ShiftRightLogicalNarrowingSaturateLower, OP_ARM64_XNSHIFT, INTRINS_AARCH64_ADV_SIMD_UQSHRN}, + {SN_ShiftRightLogicalNarrowingSaturateScalar, OP_ARM64_XNSHIFT_SCALAR, INTRINS_AARCH64_ADV_SIMD_UQSHRN}, + {SN_ShiftRightLogicalNarrowingSaturateUpper, OP_ARM64_XNSHIFT2, INTRINS_AARCH64_ADV_SIMD_UQSHRN}, {SN_ShiftRightLogicalNarrowingUpper, OP_ARM64_SHRN2}, {SN_ShiftRightLogicalRounded, OP_ARM64_URSHR}, {SN_ShiftRightLogicalRoundedAdd, OP_ARM64_URSRA}, {SN_ShiftRightLogicalRoundedAddScalar, OP_ARM64_URSRA}, - {SN_ShiftRightLogicalRoundedNarrowingLower, OP_ARM64_XRSHIFT, INTRINS_AARCH64_ADV_SIMD_RSHRN}, - {SN_ShiftRightLogicalRoundedNarrowingSaturateLower, OP_ARM64_XRSHIFT, INTRINS_AARCH64_ADV_SIMD_UQRSHRN}, - {SN_ShiftRightLogicalRoundedNarrowingSaturateScalar, OP_ARM64_XRSHIFT_SCALAR, INTRINS_AARCH64_ADV_SIMD_UQRSHRN}, - {SN_ShiftRightLogicalRoundedNarrowingSaturateUpper, OP_ARM64_XRSHIFT2, INTRINS_AARCH64_ADV_SIMD_UQRSHRN}, - {SN_ShiftRightLogicalRoundedNarrowingUpper, OP_ARM64_XRSHIFT2, INTRINS_AARCH64_ADV_SIMD_RSHRN}, + {SN_ShiftRightLogicalRoundedNarrowingLower, OP_ARM64_XNSHIFT, INTRINS_AARCH64_ADV_SIMD_RSHRN}, + {SN_ShiftRightLogicalRoundedNarrowingSaturateLower, OP_ARM64_XNSHIFT, INTRINS_AARCH64_ADV_SIMD_UQRSHRN}, + {SN_ShiftRightLogicalRoundedNarrowingSaturateScalar, OP_ARM64_XNSHIFT_SCALAR, INTRINS_AARCH64_ADV_SIMD_UQRSHRN}, + {SN_ShiftRightLogicalRoundedNarrowingSaturateUpper, OP_ARM64_XNSHIFT2, INTRINS_AARCH64_ADV_SIMD_UQRSHRN}, + {SN_ShiftRightLogicalRoundedNarrowingUpper, OP_ARM64_XNSHIFT2, INTRINS_AARCH64_ADV_SIMD_RSHRN}, {SN_ShiftRightLogicalRoundedScalar, OP_ARM64_URSHR}, {SN_ShiftRightLogicalScalar, OP_ARM64_USHR}, {SN_SignExtendWideningLower, OP_ARM64_SXTL}, From 643d6a4f503722980ad54c4a029c624f45c0cddc Mon Sep 17 00:00:00 2001 From: Imran Hameed Date: Mon, 8 Mar 2021 17:53:03 -0800 Subject: [PATCH 38/58] ... Fix MinPairwiseScalar/MinAcross/MaxPairwiseScalar/MaxAcross --- src/mono/mono/mini/llvm-intrinsics.h | 7 ++----- src/mono/mono/mini/mini-llvm.c | 15 +-------------- src/mono/mono/mini/mini-ops.h | 3 --- src/mono/mono/mini/simd-intrinsics.c | 21 ++++----------------- 4 files changed, 7 insertions(+), 39 deletions(-) diff --git a/src/mono/mono/mini/llvm-intrinsics.h b/src/mono/mono/mini/llvm-intrinsics.h index 0efa64ca955d0d..a9ec67532d62a1 100644 --- a/src/mono/mono/mini/llvm-intrinsics.h +++ b/src/mono/mono/mini/llvm-intrinsics.h @@ -321,6 +321,8 @@ INTRINS_OVR_TAG_KIND(AARCH64_ADV_SIMD_SMAXV, aarch64_neon_smaxv, Across, V64 | V INTRINS_OVR_TAG_KIND(AARCH64_ADV_SIMD_UMAXV, aarch64_neon_umaxv, Across, V64 | V128 | I1 | I2 | I4) INTRINS_OVR_TAG_KIND(AARCH64_ADV_SIMD_SMINV, aarch64_neon_sminv, Across, V64 | V128 | I1 | I2 | I4) INTRINS_OVR_TAG_KIND(AARCH64_ADV_SIMD_UMINV, aarch64_neon_uminv, Across, V64 | V128 | I1 | I2 | I4) +INTRINS_OVR_TAG_KIND(AARCH64_ADV_SIMD_FMAXV, aarch64_neon_fmaxv, Across, V64 | V128 | R4 | R8) +INTRINS_OVR_TAG_KIND(AARCH64_ADV_SIMD_FMINV, aarch64_neon_fminv, Across, V64 | V128 | R4 | R8) INTRINS_OVR_TAG_KIND(AARCH64_ADV_SIMD_SADDLP, aarch64_neon_saddlp, Widen, V64 | V128 | I1 | I2 | I4 | I8) INTRINS_OVR_TAG_KIND(AARCH64_ADV_SIMD_UADDLP, aarch64_neon_uaddlp, Widen, V64 | V128 | I1 | I2 | I4 | I8) @@ -377,11 +379,6 @@ INTRINS_OVR_TAG(AARCH64_ADV_SIMD_FMINNM, aarch64_neon_fminnm, Scalar | V64 | V12 INTRINS_OVR_TAG(AARCH64_ADV_SIMD_FMAXNMP, aarch64_neon_fmaxnmp, V64 | V128 | R4 | R8) INTRINS_OVR_TAG(AARCH64_ADV_SIMD_FMINNMP, aarch64_neon_fminnmp, V64 | V128 | R4 | R8) -INTRINS_OVR_2_ARG(AARCH64_ADV_SIMD_FMAXV_F32, aarch64_neon_fminv, r4_t, v64_r4_t) -INTRINS_OVR_2_ARG(AARCH64_ADV_SIMD_FMAXV_F64, aarch64_neon_fminv, r8_t, v128_r8_t) -INTRINS_OVR_2_ARG(AARCH64_ADV_SIMD_FMINV_F32, aarch64_neon_fminv, r4_t, v64_r4_t) -INTRINS_OVR_2_ARG(AARCH64_ADV_SIMD_FMINV_F64, aarch64_neon_fminv, r8_t, v128_r8_t) - INTRINS_OVR_TAG(AARCH64_ADV_SIMD_SQDMULH, aarch64_neon_sqdmulh, V64 | V128 | I2 | I4) INTRINS_OVR_TAG(AARCH64_ADV_SIMD_SQDMULL, aarch64_neon_sqdmull, V64 | V128 | I2 | I4 | I8) diff --git a/src/mono/mono/mini/mini-llvm.c b/src/mono/mono/mini/mini-llvm.c index c505cabf092952..0d5eb5eaac621d 100644 --- a/src/mono/mono/mini/mini-llvm.c +++ b/src/mono/mono/mini/mini-llvm.c @@ -1335,7 +1335,6 @@ typedef struct { int i; } ImmediateUnrollCtx; - static ImmediateUnrollCtx immediate_unroll_begin ( EmitContext *ctx, MonoBasicBlock *bb, int max_cases, @@ -9586,16 +9585,10 @@ process_bb (EmitContext *ctx, MonoBasicBlock *bb) values [ins->dreg] = call_intrins (ctx, id, args, ""); break; } - case OP_XOP_SX_X: case OP_XOP_X_X: { IntrinsicId id = (IntrinsicId)0; - gboolean pack_result = FALSE; gboolean getLowerElement = FALSE; switch (ins->opcode) { - case OP_XOP_SX_X: - pack_result = TRUE; - id = ins->inst_c0; - break; default: switch (ins->inst_c0) { case SIMD_OP_AES_IMC: id = INTRINS_AARCH64_AESIMC; break; @@ -9604,16 +9597,10 @@ process_bb (EmitContext *ctx, MonoBasicBlock *bb) default: g_assert_not_reached (); break; } } - if (getLowerElement) - pack_result = TRUE; LLVMValueRef arg0 = lhs; if (getLowerElement) arg0 = LLVMBuildExtractElement (ctx->builder, arg0, const_int32 (0), ""); LLVMValueRef result = call_intrins (ctx, id, &arg0, ""); - if (pack_result) { - LLVMTypeRef t = simd_class_to_llvm_type (ctx, ins->klass); - result = vector_zero_from_scalar (ctx, t, result); - } values [ins->dreg] = result; break; } @@ -10269,7 +10256,7 @@ process_bb (EmitContext *ctx, MonoBasicBlock *bb) mask [i] = i + 1; mask [i + 1] = i; } - LLVMValueRef result = LLVMBuildShuffleVector(builder, tmp, LLVMGetUndef (tmp_t), create_const_vector_i32 (mask, tmp_elements), ""); + LLVMValueRef result = LLVMBuildShuffleVector (builder, tmp, LLVMGetUndef (tmp_t), create_const_vector_i32 (mask, tmp_elements), ""); result = LLVMBuildBitCast (builder, result, t, ""); values [ins->dreg] = result; break; diff --git a/src/mono/mono/mini/mini-ops.h b/src/mono/mono/mini/mini-ops.h index b5224c3c88e61c..736dca59c764a9 100644 --- a/src/mono/mono/mini/mini-ops.h +++ b/src/mono/mono/mini/mini-ops.h @@ -1557,9 +1557,6 @@ MINI_OP(OP_XOP_I4_I4_I4, "xop_i4_i4_i4", IREG, IREG, IREG) MINI_OP(OP_XOP_I4_I4_I8, "xop_i4_i4_i8", IREG, IREG, LREG) MINI_OP3(OP_XOP_X_X_X_X, "xop_x_x_x_x", XREG, XREG, XREG, XREG) -/* SX: an LLVM scalar that will be wrapped in a vector */ -MINI_OP(OP_XOP_SX_X, "xop_sx_x", XREG, XREG, NONE) - MINI_OP(OP_XOP_OVR_X_X, "xop_ovr_x_x", XREG, XREG, NONE) MINI_OP(OP_XOP_OVR_X_X_X, "xop_ovr_x_x_x", XREG, XREG, XREG) MINI_OP3(OP_XOP_OVR_X_X_X_X, "xop_ovr_x_x_x_x", XREG, XREG, XREG, XREG) diff --git a/src/mono/mono/mini/simd-intrinsics.c b/src/mono/mono/mini/simd-intrinsics.c index 99ad7d58ba3df1..d1d986bd1c0bb9 100644 --- a/src/mono/mono/mini/simd-intrinsics.c +++ b/src/mono/mono/mini/simd-intrinsics.c @@ -1167,24 +1167,24 @@ static SimdIntrinsic advsimd_methods [] = { {SN_LoadVector128, OP_ARM64_LD1}, {SN_LoadVector64, OP_ARM64_LD1}, {SN_Max, OP_XOP_OVR_X_X_X, INTRINS_AARCH64_ADV_SIMD_SMAX, OP_XOP_OVR_X_X_X, INTRINS_AARCH64_ADV_SIMD_UMAX, OP_XOP_OVR_X_X_X, INTRINS_AARCH64_ADV_SIMD_FMAX}, - {SN_MaxAcross, OP_ARM64_XHORIZ, INTRINS_AARCH64_ADV_SIMD_SMAXV, OP_ARM64_XHORIZ, INTRINS_AARCH64_ADV_SIMD_UMAXV}, + {SN_MaxAcross, OP_ARM64_XHORIZ, INTRINS_AARCH64_ADV_SIMD_SMAXV, OP_ARM64_XHORIZ, INTRINS_AARCH64_ADV_SIMD_UMAXV, OP_ARM64_XHORIZ, INTRINS_AARCH64_ADV_SIMD_FMAXV}, {SN_MaxNumber, OP_XOP_OVR_X_X_X, INTRINS_AARCH64_ADV_SIMD_FMAXNM}, {SN_MaxNumberAcross, OP_ARM64_XHORIZ, INTRINS_AARCH64_ADV_SIMD_FMAXNMV}, {SN_MaxNumberPairwise, OP_XOP_OVR_X_X_X, INTRINS_AARCH64_ADV_SIMD_FMAXNMP}, {SN_MaxNumberPairwiseScalar, OP_ARM64_XHORIZ, INTRINS_AARCH64_ADV_SIMD_FMAXNMV}, {SN_MaxNumberScalar, OP_XOP_OVR_SCALAR_X_X_X, INTRINS_AARCH64_ADV_SIMD_FMAXNM}, {SN_MaxPairwise, OP_XOP_OVR_X_X_X, INTRINS_AARCH64_ADV_SIMD_SMAXP, OP_XOP_OVR_X_X_X, INTRINS_AARCH64_ADV_SIMD_UMAXP, OP_XOP_OVR_X_X_X, INTRINS_AARCH64_ADV_SIMD_FMAXP}, - {SN_MaxPairwiseScalar}, + {SN_MaxPairwiseScalar, OP_ARM64_XHORIZ, INTRINS_AARCH64_ADV_SIMD_FMAXV}, {SN_MaxScalar, OP_XOP_OVR_SCALAR_X_X_X, INTRINS_AARCH64_ADV_SIMD_FMAX}, {SN_Min, OP_XOP_OVR_X_X_X, INTRINS_AARCH64_ADV_SIMD_SMIN, OP_XOP_OVR_X_X_X, INTRINS_AARCH64_ADV_SIMD_UMIN, OP_XOP_OVR_X_X_X, INTRINS_AARCH64_ADV_SIMD_FMIN}, - {SN_MinAcross, OP_ARM64_XHORIZ, INTRINS_AARCH64_ADV_SIMD_SMINV, OP_ARM64_XHORIZ, INTRINS_AARCH64_ADV_SIMD_UMINV}, + {SN_MinAcross, OP_ARM64_XHORIZ, INTRINS_AARCH64_ADV_SIMD_SMINV, OP_ARM64_XHORIZ, INTRINS_AARCH64_ADV_SIMD_UMINV, OP_ARM64_XHORIZ, INTRINS_AARCH64_ADV_SIMD_FMINV}, {SN_MinNumber, OP_XOP_OVR_X_X_X, INTRINS_AARCH64_ADV_SIMD_FMINNM}, {SN_MinNumberAcross, OP_ARM64_XHORIZ, INTRINS_AARCH64_ADV_SIMD_FMINNMV}, {SN_MinNumberPairwise, OP_XOP_OVR_X_X_X, INTRINS_AARCH64_ADV_SIMD_FMINNMP}, {SN_MinNumberPairwiseScalar, OP_ARM64_XHORIZ, INTRINS_AARCH64_ADV_SIMD_FMINNMV}, {SN_MinNumberScalar, OP_XOP_OVR_SCALAR_X_X_X, INTRINS_AARCH64_ADV_SIMD_FMINNM}, {SN_MinPairwise, OP_XOP_OVR_X_X_X, INTRINS_AARCH64_ADV_SIMD_SMINP, OP_XOP_OVR_X_X_X, INTRINS_AARCH64_ADV_SIMD_UMINP, OP_XOP_OVR_X_X_X, INTRINS_AARCH64_ADV_SIMD_FMINP}, - {SN_MinPairwiseScalar}, + {SN_MinPairwiseScalar, OP_ARM64_XHORIZ, INTRINS_AARCH64_ADV_SIMD_FMINV}, {SN_MinScalar, OP_XOP_OVR_SCALAR_X_X_X, INTRINS_AARCH64_ADV_SIMD_FMIN}, {SN_Multiply, OP_XBINOP, OP_IMUL, None, None, OP_XBINOP, OP_FMUL}, {SN_MultiplyAdd, OP_ARM64_MLA}, @@ -1551,19 +1551,6 @@ emit_arm64_intrinsics ( ins->inst_c1 = arg0_type; return ins; } - case SN_MaxPairwiseScalar: - case SN_MinPairwiseScalar: { - int iid = 0; - switch (id) { - case SN_MaxPairwiseScalar: - iid = arg0_type == MONO_TYPE_R4 ? INTRINS_AARCH64_ADV_SIMD_FMAXV_F32 : INTRINS_AARCH64_ADV_SIMD_FMAXV_F64; - break; - case SN_MinPairwiseScalar: - iid = arg0_type == MONO_TYPE_R4 ? INTRINS_AARCH64_ADV_SIMD_FMINV_F32 : INTRINS_AARCH64_ADV_SIMD_FMINV_F64; - break; - } - return emit_simd_ins_for_sig (cfg, klass, OP_XOP_SX_X, iid, arg0_type, fsig, args); - } case SN_MultiplyDoublingScalarBySelectedScalarSaturateHigh: case SN_MultiplyDoublingWideningSaturateScalarBySelectedScalar: case SN_MultiplyExtendedBySelectedScalar: From 76d24b6b53f9f3942b3d882e9071a4a8a5c56a18 Mon Sep 17 00:00:00 2001 From: Imran Hameed Date: Mon, 8 Mar 2021 18:04:25 -0800 Subject: [PATCH 39/58] Fix brain-os for ld1/st1 --- src/mono/mono/mini/mini-llvm.c | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/src/mono/mono/mini/mini-llvm.c b/src/mono/mono/mini/mini-llvm.c index 0d5eb5eaac621d..3216949284caad 100644 --- a/src/mono/mono/mini/mini-llvm.c +++ b/src/mono/mono/mini/mini-llvm.c @@ -10518,16 +10518,17 @@ process_bb (EmitContext *ctx, MonoBasicBlock *bb) dst_t = LLVMPointerType (LLVMVectorType (rhs_elt_t, rhs_elems * 2), 0); val = concatenate_vectors (ctx, rhs, arg3); } - LLVMValueRef addr = convert (ctx, lhs, dst_t); - LLVMValueRef store = mono_llvm_build_store (builder, val, addr, FALSE, LLVM_BARRIER_NONE); + LLVMValueRef address = convert (ctx, lhs, dst_t); + LLVMValueRef store = mono_llvm_build_store (builder, val, address, FALSE, LLVM_BARRIER_NONE); if (nontemporal) set_nontemporal_flag (store); break; } case OP_ARM64_LD1_INSERT: { LLVMTypeRef ret_t = simd_class_to_llvm_type (ctx, ins->klass); + LLVMTypeRef elem_t = LLVMGetElementType (ret_t); + LLVMValueRef address = convert (ctx, arg3, LLVMPointerType (elem_t, 0)); unsigned int alignment = mono_llvm_get_prim_size_bits (ret_t) / 8; - LLVMValueRef address = arg3; LLVMValueRef result = mono_llvm_build_aligned_load (builder, address, "arm64_ld1_insert", FALSE, alignment); result = LLVMBuildInsertElement (builder, lhs, result, rhs, "arm64_ld1_insert"); values [ins->dreg] = result; @@ -10558,9 +10559,10 @@ process_bb (EmitContext *ctx, MonoBasicBlock *bb) } case OP_ARM64_ST1_SCALAR: { LLVMTypeRef t = LLVMGetElementType (LLVMTypeOf (rhs)); - unsigned int alignment = mono_llvm_get_prim_size_bits (t) / 8; LLVMValueRef val = LLVMBuildExtractElement (builder, rhs, arg3, "arm64_st1_scalar"); - mono_llvm_build_aligned_store (builder, val, lhs, FALSE, alignment); + LLVMValueRef address = convert (ctx, lhs, LLVMPointerType (t, 0)); + unsigned int alignment = mono_llvm_get_prim_size_bits (t) / 8; + mono_llvm_build_aligned_store (builder, val, address, FALSE, alignment); break; } case OP_ARM64_ADDHN: From aa657a84d55528b58203eefe3e8f50d334db62a7 Mon Sep 17 00:00:00 2001 From: Imran Hameed Date: Mon, 8 Mar 2021 21:42:44 -0800 Subject: [PATCH 40/58] ??? --- src/mono/mono/mini/mini-llvm.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/mono/mono/mini/mini-llvm.c b/src/mono/mono/mini/mini-llvm.c index 3216949284caad..3d9d8c7feea9e9 100644 --- a/src/mono/mono/mini/mini-llvm.c +++ b/src/mono/mono/mini/mini-llvm.c @@ -9550,6 +9550,7 @@ process_bb (EmitContext *ctx, MonoBasicBlock *bb) getElement = TRUE; element_idx = 1; bitcast_result = TRUE; + break; default: g_assert_not_reached (); break; } LLVMValueRef arg1 = rhs; From e526dc66c3dea0dea128761957138013b669c6f7 Mon Sep 17 00:00:00 2001 From: Imran Hameed Date: Tue, 9 Mar 2021 07:14:44 -0800 Subject: [PATCH 41/58] Fix floating point Add/Subtract --- src/mono/mono/mini/simd-intrinsics.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/mono/mono/mini/simd-intrinsics.c b/src/mono/mono/mini/simd-intrinsics.c index d1d986bd1c0bb9..71ae6c843a0ac4 100644 --- a/src/mono/mono/mini/simd-intrinsics.c +++ b/src/mono/mono/mini/simd-intrinsics.c @@ -1038,7 +1038,7 @@ static SimdIntrinsic advsimd_methods [] = { {SN_AbsoluteDifferenceWideningLowerAndAdd, OP_ARM64_SABAL, None, OP_ARM64_UABAL}, {SN_AbsoluteDifferenceWideningUpper, OP_ARM64_SABDL2, None, OP_ARM64_UABDL2}, {SN_AbsoluteDifferenceWideningUpperAndAdd, OP_ARM64_SABAL2, None, OP_ARM64_UABAL2}, - {SN_Add, OP_XBINOP, OP_IADD, None, None, OP_XBINOP_SCALAR, OP_FADD}, + {SN_Add, OP_XBINOP, OP_IADD, None, None, OP_XBINOP, OP_FADD}, {SN_AddAcross, OP_ARM64_XHORIZ, INTRINS_AARCH64_ADV_SIMD_SADDV, OP_ARM64_XHORIZ, INTRINS_AARCH64_ADV_SIMD_UADDV}, {SN_AddAcrossWidening, OP_ARM64_SADDLV, None, OP_ARM64_UADDLV}, {SN_AddHighNarrowingLower, OP_ARM64_ADDHN}, @@ -1358,7 +1358,7 @@ static SimdIntrinsic advsimd_methods [] = { {SN_StorePairScalar, OP_ARM64_STP_SCALAR}, {SN_StorePairScalarNonTemporal, OP_ARM64_STNP_SCALAR}, {SN_StoreSelectedScalar, OP_ARM64_ST1_SCALAR}, - {SN_Subtract, OP_XBINOP, OP_ISUB, None, None, OP_XBINOP_SCALAR, OP_FSUB}, + {SN_Subtract, OP_XBINOP, OP_ISUB, None, None, OP_XBINOP, OP_FSUB}, {SN_SubtractHighNarrowingLower, OP_ARM64_SUBHN}, {SN_SubtractHighNarrowingUpper, OP_ARM64_SUBHN2}, {SN_SubtractRoundedHighNarrowingLower, OP_ARM64_RSUBHN}, From e95199ddb0b501acfab86986dc562c4b7cf85cff Mon Sep 17 00:00:00 2001 From: Imran Hameed Date: Tue, 9 Mar 2021 07:49:21 -0800 Subject: [PATCH 42/58] Fix CompareEqualScalar/CompareGreaterThanOrEqualScalar/CompareGreaterThanScalar --- src/mono/mono/mini/simd-intrinsics.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/mono/mono/mini/simd-intrinsics.c b/src/mono/mono/mini/simd-intrinsics.c index 71ae6c843a0ac4..03f1ba6684a746 100644 --- a/src/mono/mono/mini/simd-intrinsics.c +++ b/src/mono/mono/mini/simd-intrinsics.c @@ -1062,11 +1062,11 @@ static SimdIntrinsic advsimd_methods [] = { {SN_Ceiling, OP_XOP_OVR_X_X, INTRINS_AARCH64_ADV_SIMD_FRINTP}, {SN_CeilingScalar, OP_XOP_OVR_SCALAR_X_X, INTRINS_AARCH64_ADV_SIMD_FRINTP}, {SN_CompareEqual, OP_XCOMPARE, CMP_EQ, OP_XCOMPARE, CMP_EQ, OP_XCOMPARE_FP, CMP_EQ}, - {SN_CompareEqualScalar, OP_XCOMPARE, CMP_EQ, OP_XCOMPARE, CMP_EQ, OP_XCOMPARE_FP, CMP_EQ}, + {SN_CompareEqualScalar, OP_XCOMPARE_SCALAR, CMP_EQ, OP_XCOMPARE_SCALAR, CMP_EQ, OP_XCOMPARE_FP_SCALAR, CMP_EQ}, {SN_CompareGreaterThan, OP_XCOMPARE, CMP_GT, OP_XCOMPARE, CMP_GT_UN, OP_XCOMPARE_FP, CMP_GT}, {SN_CompareGreaterThanOrEqual, OP_XCOMPARE, CMP_GE, OP_XCOMPARE, CMP_GE_UN, OP_XCOMPARE_FP, CMP_GE}, - {SN_CompareGreaterThanOrEqualScalar, OP_XCOMPARE, CMP_GE, OP_XCOMPARE, CMP_GE_UN, OP_XCOMPARE_FP, CMP_GE}, - {SN_CompareGreaterThanScalar, OP_XCOMPARE, CMP_GT, OP_XCOMPARE, CMP_GT_UN, OP_XCOMPARE_FP, CMP_GT}, + {SN_CompareGreaterThanOrEqualScalar, OP_XCOMPARE_SCALAR, CMP_GE, OP_XCOMPARE_SCALAR, CMP_GE_UN, OP_XCOMPARE_FP_SCALAR, CMP_GE}, + {SN_CompareGreaterThanScalar, OP_XCOMPARE_SCALAR, CMP_GT, OP_XCOMPARE_SCALAR, CMP_GT_UN, OP_XCOMPARE_FP_SCALAR, CMP_GT}, {SN_CompareLessThan, OP_XCOMPARE, CMP_LT, OP_XCOMPARE, CMP_LT_UN, OP_XCOMPARE_FP, CMP_LT}, {SN_CompareLessThanOrEqual, OP_XCOMPARE, CMP_LE, OP_XCOMPARE, CMP_LE_UN, OP_XCOMPARE_FP, CMP_LE}, {SN_CompareLessThanOrEqualScalar, OP_XCOMPARE_SCALAR, CMP_LE, OP_XCOMPARE_SCALAR, CMP_LE_UN, OP_XCOMPARE_FP_SCALAR, CMP_LE}, From c94c07b553c008be0ad6b64ba207f930d2073964 Mon Sep 17 00:00:00 2001 From: Imran Hameed Date: Tue, 9 Mar 2021 08:06:03 -0800 Subject: [PATCH 43/58] Fix ShiftRightArithmeticRounded, ShiftRightArithmeticRoundedScalar, ShiftRightArithmeticRoundedAddScalar --- src/mono/mono/mini/mini-llvm.c | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/src/mono/mono/mini/mini-llvm.c b/src/mono/mono/mini/mini-llvm.c index 3d9d8c7feea9e9..a76ac6915b03dc 100644 --- a/src/mono/mono/mini/mini-llvm.c +++ b/src/mono/mono/mini/mini-llvm.c @@ -10328,16 +10328,13 @@ process_bb (EmitContext *ctx, MonoBasicBlock *bb) gboolean right = FALSE; gboolean add = FALSE; switch (ins->opcode) { - case OP_ARM64_URSHR: right = TRUE; break; - case OP_ARM64_URSRA: right = TRUE; add = TRUE; break; - case OP_ARM64_SRSRA: add = TRUE; break; + case OP_ARM64_URSRA: add = TRUE; case OP_ARM64_URSHR: right = TRUE; break; + case OP_ARM64_SRSRA: add = TRUE; case OP_ARM64_SRSHR: right = TRUE; break; } int iid = 0; switch (ins->opcode) { - case OP_ARM64_URSHR: - case OP_ARM64_URSRA: iid = INTRINS_AARCH64_ADV_SIMD_URSHL; break; - case OP_ARM64_SRSHR: - case OP_ARM64_SRSRA: iid = INTRINS_AARCH64_ADV_SIMD_SRSHL; break; + case OP_ARM64_URSRA: case OP_ARM64_URSHR: iid = INTRINS_AARCH64_ADV_SIMD_URSHL; break; + case OP_ARM64_SRSRA: case OP_ARM64_SRSHR: iid = INTRINS_AARCH64_ADV_SIMD_SRSHL; break; } if (add) { shiftarg = rhs; From 3d81808f84f01c6760f161d1a1c44b40cc86e635 Mon Sep 17 00:00:00 2001 From: Imran Hameed Date: Tue, 9 Mar 2021 08:49:29 -0800 Subject: [PATCH 44/58] Fix MultiplyDoublingWideningLowerAndSubtractSaturate, FusedMultiplySubtractBySelectedScalar, FusedMultiplySubtractByScalar --- src/mono/mono/mini/mini-llvm.c | 2 +- src/mono/mono/mini/simd-intrinsics.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/mono/mono/mini/mini-llvm.c b/src/mono/mono/mini/mini-llvm.c index a76ac6915b03dc..05abe3adba9aa8 100644 --- a/src/mono/mono/mini/mini-llvm.c +++ b/src/mono/mono/mini/mini-llvm.c @@ -9939,7 +9939,7 @@ process_bb (EmitContext *ctx, MonoBasicBlock *bb) gboolean byscalar = FALSE; switch (ins->opcode) { case OP_ARM64_FMSUB: subtract = TRUE; break; - case OP_ARM64_FMSUB_BYSCALAR: byscalar = TRUE; break; + case OP_ARM64_FMSUB_BYSCALAR: subtract = TRUE; byscalar = TRUE; break; case OP_ARM64_FMSUB_SCALAR: subtract = TRUE; scalar = TRUE; break; case OP_ARM64_FNMSUB_SCALAR: subtract = TRUE; scalar = TRUE; negate = TRUE; break; case OP_ARM64_FMADD: break; diff --git a/src/mono/mono/mini/simd-intrinsics.c b/src/mono/mono/mini/simd-intrinsics.c index 03f1ba6684a746..a6ed9a58725f53 100644 --- a/src/mono/mono/mini/simd-intrinsics.c +++ b/src/mono/mono/mini/simd-intrinsics.c @@ -1206,7 +1206,7 @@ static SimdIntrinsic advsimd_methods [] = { {SN_MultiplyDoublingWideningAndAddSaturateScalar, OP_ARM64_SQDMLAL_SCALAR}, {SN_MultiplyDoublingWideningAndSubtractSaturateScalar, OP_ARM64_SQDMLSL_SCALAR}, {SN_MultiplyDoublingWideningLowerAndAddSaturate, OP_ARM64_SQDMLAL}, - {SN_MultiplyDoublingWideningLowerAndSubtractSaturate, OP_ARM64_SQDMLSL_BYSCALAR}, + {SN_MultiplyDoublingWideningLowerAndSubtractSaturate, OP_ARM64_SQDMLSL}, {SN_MultiplyDoublingWideningLowerByScalarAndAddSaturate, OP_ARM64_SQDMLAL_BYSCALAR}, {SN_MultiplyDoublingWideningLowerByScalarAndSubtractSaturate, OP_ARM64_SQDMLSL_BYSCALAR}, {SN_MultiplyDoublingWideningLowerBySelectedScalarAndAddSaturate}, From d81b1f0c970c0df8cbe902db4201457268a73969 Mon Sep 17 00:00:00 2001 From: Imran Hameed Date: Tue, 9 Mar 2021 10:25:36 -0800 Subject: [PATCH 45/58] Fix ShiftLogicalSaturateScalar, ShiftArithmeticRoundedSaturateScalar, ShiftArithmeticSaturateScalar, ShiftLeftLogicalSaturate and ShiftLeftLogicalSaturateScalar Fix ShiftLeftLogicalSaturate and ShiftLeftLogicalSaturateScalar: decompose it into a promotion of the second argument into a vector followed by an overloaded invocation of @llvm.aarch64.neon.uqshl or @llvm.aarch64.neon.sqshl --- src/mono/mono/mini/llvm-intrinsics.h | 4 +-- src/mono/mono/mini/mini-llvm.c | 40 +++++++++------------------- src/mono/mono/mini/mini-ops.h | 3 --- src/mono/mono/mini/simd-intrinsics.c | 26 ++++++++++++++---- 4 files changed, 36 insertions(+), 37 deletions(-) diff --git a/src/mono/mono/mini/llvm-intrinsics.h b/src/mono/mono/mini/llvm-intrinsics.h index a9ec67532d62a1..1565f069cd696d 100644 --- a/src/mono/mono/mini/llvm-intrinsics.h +++ b/src/mono/mono/mini/llvm-intrinsics.h @@ -434,11 +434,11 @@ INTRINS_OVR_TAG(AARCH64_ADV_SIMD_SQSHRUN, aarch64_neon_sqshrun, V64 | I1 | I2 | INTRINS_OVR_TAG(AARCH64_ADV_SIMD_UQRSHRN, aarch64_neon_uqrshrn, Scalar | V64 | I1 | I2 | I4) // Constant shift INTRINS_OVR_TAG(AARCH64_ADV_SIMD_SQRSHL, aarch64_neon_sqrshl, V64 | V128 | I1 | I2 | I4 | I8) // Variable shift -INTRINS_OVR_TAG(AARCH64_ADV_SIMD_SQSHL, aarch64_neon_sqshl, V64 | V128 | I1 | I2 | I4 | I8) // Variable shift +INTRINS_OVR_TAG(AARCH64_ADV_SIMD_SQSHL, aarch64_neon_sqshl, Scalar | V64 | V128 | I1 | I2 | I4 | I8) // Variable shift INTRINS_OVR_TAG(AARCH64_ADV_SIMD_SRSHL, aarch64_neon_srshl, V64 | V128 | I1 | I2 | I4 | I8) // Variable shift INTRINS_OVR_TAG(AARCH64_ADV_SIMD_SSHL, aarch64_neon_sshl, V64 | V128 | I1 | I2 | I4 | I8) // Variable shift INTRINS_OVR_TAG(AARCH64_ADV_SIMD_UQRSHL, aarch64_neon_uqrshl, V64 | V128 | I1 | I2 | I4 | I8) // Variable shift -INTRINS_OVR_TAG(AARCH64_ADV_SIMD_UQSHL, aarch64_neon_uqshl, V64 | V128 | I1 | I2 | I4 | I8) // Variable shift +INTRINS_OVR_TAG(AARCH64_ADV_SIMD_UQSHL, aarch64_neon_uqshl, Scalar | V64 | V128 | I1 | I2 | I4 | I8) // Variable shift INTRINS_OVR_TAG(AARCH64_ADV_SIMD_URSHL, aarch64_neon_urshl, V64 | V128 | I1 | I2 | I4 | I8) // Variable shift INTRINS_OVR_TAG(AARCH64_ADV_SIMD_USHL, aarch64_neon_ushl, V64 | V128 | I1 | I2 | I4 | I8) // Variable shift diff --git a/src/mono/mono/mini/mini-llvm.c b/src/mono/mono/mini/mini-llvm.c index 05abe3adba9aa8..915301d63e416d 100644 --- a/src/mono/mono/mini/mini-llvm.c +++ b/src/mono/mono/mini/mini-llvm.c @@ -7490,6 +7490,17 @@ process_bb (EmitContext *ctx, MonoBasicBlock *bb) values [ins->dreg] = LLVMBuildBitCast (builder, result, t, ""); break; } + case OP_CREATE_SCALAR: + case OP_CREATE_SCALAR_UNSAFE: { + MonoTypeEnum primty = inst_c1_type (ins); + LLVMTypeRef type = simd_class_to_llvm_type (ctx, ins->klass); + // use undef vector (most likely empty but may contain garbage values) for OP_CREATE_SCALAR_UNSAFE + // and zero one for OP_CREATE_SCALAR + LLVMValueRef vector = (ins->opcode == OP_CREATE_SCALAR) ? LLVMConstNull (type) : LLVMGetUndef (type); + LLVMValueRef val = convert_full (ctx, lhs, primitive_type_to_llvm_type (primty), primitive_type_is_unsigned (primty)); + values [ins->dreg] = LLVMBuildInsertElement (builder, vector, val, const_int32 (0), ""); + break; + } #endif // defined(TARGET_X86) || defined(TARGET_AMD64) || defined(TARGET_ARM64) || defined(TARGET_WASM) #if defined(TARGET_X86) || defined(TARGET_AMD64) || defined(TARGET_WASM) @@ -9057,18 +9068,6 @@ process_bb (EmitContext *ctx, MonoBasicBlock *bb) break; } - case OP_CREATE_SCALAR: - case OP_CREATE_SCALAR_UNSAFE: { - MonoTypeEnum primty = inst_c1_type (ins); - LLVMTypeRef type = simd_class_to_llvm_type (ctx, ins->klass); - // use undef vector (most likely empty but may contain garbage values) for OP_CREATE_SCALAR_UNSAFE - // and zero one for OP_CREATE_SCALAR - LLVMValueRef vector = (ins->opcode == OP_CREATE_SCALAR) ? LLVMConstNull (type) : LLVMGetUndef (type); - LLVMValueRef insert_pos = LLVMConstInt (LLVMInt32Type (), 0, FALSE); - LLVMValueRef val = convert_full (ctx, lhs, primitive_type_to_llvm_type (primty), primitive_type_is_unsigned (primty)); - values [ins->dreg] = LLVMBuildInsertElement (builder, vector, val, insert_pos, ""); - break; - } case OP_SSE41_ROUNDP: { LLVMValueRef args [] = { lhs, LLVMConstInt (LLVMInt32Type (), ins->inst_c0, FALSE) }; values [ins->dreg] = call_intrins (ctx, ins->inst_c1 == MONO_TYPE_R4 ? INTRINS_SSE_ROUNDPS : INTRINS_SSE_ROUNDPD, args, dname); @@ -10403,19 +10402,6 @@ process_bb (EmitContext *ctx, MonoBasicBlock *bb) values [ins->dreg] = result; break; } - case OP_ARM64_UQSHL_IMM: - case OP_ARM64_SQSHL_IMM: { - llvm_ovr_tag_t ovr_tag = ovr_tag_from_mono_vector_class (ins->klass); - LLVMValueRef shift = create_shift_vector (ctx, lhs, rhs); - LLVMValueRef args [] = { lhs, shift }; - int iid = 0; - switch (ins->opcode) { - case OP_ARM64_UQSHL_IMM: iid = INTRINS_AARCH64_ADV_SIMD_UQSHL; break; - case OP_ARM64_SQSHL_IMM: iid = INTRINS_AARCH64_ADV_SIMD_SQSHL; break; - } - values [ins->dreg] = call_overloaded_intrins (ctx, iid, ovr_tag, args, ""); - break; - } case OP_ARM64_SQSHLU: { LLVMTypeRef intrin_result_t = simd_class_to_llvm_type (ctx, ins->klass); llvm_ovr_tag_t ovr_tag = ovr_tag_from_llvm_type (intrin_result_t); @@ -10876,13 +10862,13 @@ process_bb (EmitContext *ctx, MonoBasicBlock *bb) case OP_XOP_OVR_SCALAR_X_X_X: case OP_XOP_OVR_SCALAR_X_X_X_X: { int num_args = 0; + IntrinsicId iid = (IntrinsicId) ins->inst_c0; + llvm_ovr_tag_t ovr_tag = ovr_tag_from_mono_vector_class (ins->klass); switch (ins->opcode) { case OP_XOP_OVR_SCALAR_X_X: num_args = 1; break; case OP_XOP_OVR_SCALAR_X_X_X: num_args = 2; break; case OP_XOP_OVR_SCALAR_X_X_X_X: num_args = 3; break; } - IntrinsicId iid = (IntrinsicId) ins->inst_c0; - llvm_ovr_tag_t ovr_tag = ovr_tag_from_mono_vector_class (ins->klass); /* LLVM 9 NEON intrinsic functions have scalar overloads. Unfortunately * only overloads for 32 and 64-bit integers and floating point types are * supported. 8 and 16-bit integers are unsupported, and will fail during diff --git a/src/mono/mono/mini/mini-ops.h b/src/mono/mono/mini/mini-ops.h index 736dca59c764a9..925895eef6ce9d 100644 --- a/src/mono/mono/mini/mini-ops.h +++ b/src/mono/mono/mini/mini-ops.h @@ -1658,9 +1658,6 @@ MINI_OP(OP_ARM64_XNSHIFT_SCALAR, "arm64_xrshift_scalar", XREG, XREG, IREG) MINI_OP(OP_ARM64_XNSHIFT, "arm64_xnshift", XREG, XREG, IREG) MINI_OP3(OP_ARM64_XNSHIFT2, "arm64_xnshift2", XREG, XREG, XREG, IREG) -MINI_OP(OP_ARM64_UQSHL_IMM, "arm64_uqshl_imm", XREG, XREG, IREG) -MINI_OP(OP_ARM64_SQSHL_IMM, "arm64_uqshl_imm", XREG, XREG, IREG) - MINI_OP(OP_ARM64_SQSHLU, "arm64_sqshlu", XREG, XREG, IREG) MINI_OP(OP_ARM64_REV16, "arm64_rev16", XREG, XREG, NONE) diff --git a/src/mono/mono/mini/simd-intrinsics.c b/src/mono/mono/mini/simd-intrinsics.c index a6ed9a58725f53..5f339d04ac3937 100644 --- a/src/mono/mono/mini/simd-intrinsics.c +++ b/src/mono/mono/mini/simd-intrinsics.c @@ -1285,16 +1285,16 @@ static SimdIntrinsic advsimd_methods [] = { {SN_ShiftArithmetic, OP_XOP_OVR_X_X_X, INTRINS_AARCH64_ADV_SIMD_SSHL}, {SN_ShiftArithmeticRounded, OP_XOP_OVR_X_X_X, INTRINS_AARCH64_ADV_SIMD_SRSHL}, {SN_ShiftArithmeticRoundedSaturate, OP_XOP_OVR_X_X_X, INTRINS_AARCH64_ADV_SIMD_SQRSHL}, - {SN_ShiftArithmeticRoundedSaturateScalar, OP_XOP_OVR_X_X_X, INTRINS_AARCH64_ADV_SIMD_SQRSHL}, + {SN_ShiftArithmeticRoundedSaturateScalar, OP_XOP_OVR_SCALAR_X_X_X, INTRINS_AARCH64_ADV_SIMD_SQRSHL}, {SN_ShiftArithmeticRoundedScalar, OP_XOP_OVR_X_X_X, INTRINS_AARCH64_ADV_SIMD_SRSHL}, {SN_ShiftArithmeticSaturate, OP_XOP_OVR_X_X_X, INTRINS_AARCH64_ADV_SIMD_SQSHL}, - {SN_ShiftArithmeticSaturateScalar, OP_XOP_OVR_X_X_X, INTRINS_AARCH64_ADV_SIMD_SQSHL}, + {SN_ShiftArithmeticSaturateScalar, OP_XOP_OVR_SCALAR_X_X_X, INTRINS_AARCH64_ADV_SIMD_SQSHL}, {SN_ShiftArithmeticScalar, OP_XOP_OVR_X_X_X, INTRINS_AARCH64_ADV_SIMD_SSHL}, {SN_ShiftLeftAndInsert, OP_ARM64_SLI}, {SN_ShiftLeftAndInsertScalar, OP_ARM64_SLI}, {SN_ShiftLeftLogical, OP_ARM64_SHL}, - {SN_ShiftLeftLogicalSaturate, OP_ARM64_SQSHL_IMM, None, OP_ARM64_UQSHL_IMM}, - {SN_ShiftLeftLogicalSaturateScalar, OP_ARM64_SQSHL_IMM, None, OP_ARM64_UQSHL_IMM}, + {SN_ShiftLeftLogicalSaturate}, + {SN_ShiftLeftLogicalSaturateScalar}, {SN_ShiftLeftLogicalSaturateUnsigned, OP_ARM64_SQSHLU}, {SN_ShiftLeftLogicalSaturateUnsignedScalar, OP_ARM64_SQSHLU}, {SN_ShiftLeftLogicalScalar, OP_ARM64_SHL}, @@ -1306,7 +1306,7 @@ static SimdIntrinsic advsimd_methods [] = { {SN_ShiftLogicalRoundedSaturateScalar, OP_XOP_OVR_X_X_X, INTRINS_AARCH64_ADV_SIMD_UQRSHL}, {SN_ShiftLogicalRoundedScalar, OP_XOP_OVR_X_X_X, INTRINS_AARCH64_ADV_SIMD_URSHL}, {SN_ShiftLogicalSaturate, OP_XOP_OVR_X_X_X, INTRINS_AARCH64_ADV_SIMD_UQSHL}, - {SN_ShiftLogicalSaturateScalar, OP_XOP_OVR_X_X_X, INTRINS_AARCH64_ADV_SIMD_UQSHL}, + {SN_ShiftLogicalSaturateScalar, OP_XOP_OVR_SCALAR_X_X_X, INTRINS_AARCH64_ADV_SIMD_UQSHL}, {SN_ShiftLogicalScalar, OP_XOP_OVR_X_X_X, INTRINS_AARCH64_ADV_SIMD_USHL}, {SN_ShiftRightAndInsert, OP_ARM64_SRI}, {SN_ShiftRightAndInsertScalar, OP_ARM64_SRI}, @@ -1551,6 +1551,22 @@ emit_arm64_intrinsics ( ins->inst_c1 = arg0_type; return ins; } + case SN_ShiftLeftLogicalSaturate: + case SN_ShiftLeftLogicalSaturateScalar: { + MonoClass *ret_klass = mono_class_from_mono_type_internal (fsig->ret); + MonoType *etype = get_vector_t_elem_type (fsig->ret); + gboolean is_unsigned = type_is_unsigned (fsig->ret); + gboolean scalar = id == SN_ShiftLeftLogicalSaturateScalar; + int s2v = scalar ? OP_CREATE_SCALAR_UNSAFE : type_to_expand_op (etype); + int xop = scalar ? OP_XOP_OVR_SCALAR_X_X_X : OP_XOP_OVR_X_X_X; + int iid = is_unsigned ? INTRINS_AARCH64_ADV_SIMD_UQSHL : INTRINS_AARCH64_ADV_SIMD_SQSHL; + MonoInst *shift_vector = emit_simd_ins (cfg, ret_klass, s2v, args [1]->dreg, -1); + shift_vector->inst_c1 = etype->type; + MonoInst *ret = emit_simd_ins (cfg, ret_klass, xop, args [0]->dreg, shift_vector->dreg); + ret->inst_c0 = iid; + ret->inst_c1 = etype->type; + return ret; + } case SN_MultiplyDoublingScalarBySelectedScalarSaturateHigh: case SN_MultiplyDoublingWideningSaturateScalarBySelectedScalar: case SN_MultiplyExtendedBySelectedScalar: From e69386dc70689510ecf659fc83523037ad9d05d2 Mon Sep 17 00:00:00 2001 From: Imran Hameed Date: Tue, 9 Mar 2021 11:45:57 -0800 Subject: [PATCH 46/58] Fix ShiftLeftLogicalSaturateUnsignedScalar, ShiftLogicalRoundedSaturateScalar ShiftLeftLogicalSaturateUnsignedScalar: move scalar-op-from-vector-op code into shared functions --- src/mono/mono/mini/llvm-intrinsics.h | 6 +- src/mono/mono/mini/mini-llvm.c | 265 +++++++++++++++------------ src/mono/mono/mini/mini-ops.h | 1 + src/mono/mono/mini/simd-intrinsics.c | 4 +- 4 files changed, 156 insertions(+), 120 deletions(-) diff --git a/src/mono/mono/mini/llvm-intrinsics.h b/src/mono/mono/mini/llvm-intrinsics.h index 1565f069cd696d..29c38323e7c952 100644 --- a/src/mono/mono/mini/llvm-intrinsics.h +++ b/src/mono/mono/mini/llvm-intrinsics.h @@ -433,16 +433,16 @@ INTRINS_OVR_TAG(AARCH64_ADV_SIMD_SQSHRN, aarch64_neon_sqshrn, V64 | I1 | I2 | I4 INTRINS_OVR_TAG(AARCH64_ADV_SIMD_SQSHRUN, aarch64_neon_sqshrun, V64 | I1 | I2 | I4) // Constant shift INTRINS_OVR_TAG(AARCH64_ADV_SIMD_UQRSHRN, aarch64_neon_uqrshrn, Scalar | V64 | I1 | I2 | I4) // Constant shift -INTRINS_OVR_TAG(AARCH64_ADV_SIMD_SQRSHL, aarch64_neon_sqrshl, V64 | V128 | I1 | I2 | I4 | I8) // Variable shift +INTRINS_OVR_TAG(AARCH64_ADV_SIMD_SQRSHL, aarch64_neon_sqrshl, Scalar | V64 | V128 | I1 | I2 | I4 | I8) // Variable shift INTRINS_OVR_TAG(AARCH64_ADV_SIMD_SQSHL, aarch64_neon_sqshl, Scalar | V64 | V128 | I1 | I2 | I4 | I8) // Variable shift INTRINS_OVR_TAG(AARCH64_ADV_SIMD_SRSHL, aarch64_neon_srshl, V64 | V128 | I1 | I2 | I4 | I8) // Variable shift INTRINS_OVR_TAG(AARCH64_ADV_SIMD_SSHL, aarch64_neon_sshl, V64 | V128 | I1 | I2 | I4 | I8) // Variable shift -INTRINS_OVR_TAG(AARCH64_ADV_SIMD_UQRSHL, aarch64_neon_uqrshl, V64 | V128 | I1 | I2 | I4 | I8) // Variable shift +INTRINS_OVR_TAG(AARCH64_ADV_SIMD_UQRSHL, aarch64_neon_uqrshl, Scalar | V64 | V128 | I1 | I2 | I4 | I8) // Variable shift INTRINS_OVR_TAG(AARCH64_ADV_SIMD_UQSHL, aarch64_neon_uqshl, Scalar | V64 | V128 | I1 | I2 | I4 | I8) // Variable shift INTRINS_OVR_TAG(AARCH64_ADV_SIMD_URSHL, aarch64_neon_urshl, V64 | V128 | I1 | I2 | I4 | I8) // Variable shift INTRINS_OVR_TAG(AARCH64_ADV_SIMD_USHL, aarch64_neon_ushl, V64 | V128 | I1 | I2 | I4 | I8) // Variable shift -INTRINS_OVR_TAG(AARCH64_ADV_SIMD_SQSHLU, aarch64_neon_sqshlu, V64 | V128 | I1 | I2 | I4 | I8) // Constant shift +INTRINS_OVR_TAG(AARCH64_ADV_SIMD_SQSHLU, aarch64_neon_sqshlu, Scalar | V64 | V128 | I1 | I2 | I4 | I8) // Constant shift INTRINS_OVR_TAG(AARCH64_ADV_SIMD_SLI, aarch64_neon_vsli, V64 | V128 | I1 | I2 | I4 | I8) // Constant shift INTRINS_OVR_TAG(AARCH64_ADV_SIMD_SRI, aarch64_neon_vsri, V64 | V128 | I1 | I2 | I4 | I8) // Constant shift diff --git a/src/mono/mono/mini/mini-llvm.c b/src/mono/mono/mini/mini-llvm.c index 915301d63e416d..88e786784d1386 100644 --- a/src/mono/mono/mini/mini-llvm.c +++ b/src/mono/mono/mini/mini-llvm.c @@ -451,21 +451,6 @@ ovr_tag_from_llvm_type (LLVMTypeRef type) return ret; } -static inline gboolean -check_needs_fake_scalar_op (MonoTypeEnum type) -{ -#if defined(TARGET_ARM64) - switch (type) { - case MONO_TYPE_U1: - case MONO_TYPE_I1: - case MONO_TYPE_U2: - case MONO_TYPE_I2: - return TRUE; - } -#endif - return FALSE; -} - static inline void set_failure (EmitContext *ctx, const char *message) { @@ -1321,87 +1306,6 @@ gen_bb (EmitContext *ctx, const char *prefix) return LLVMAppendBasicBlock (ctx->lmethod, bb_name); } -typedef struct { - EmitContext *ctx; - MonoBasicBlock *bb; - LLVMBasicBlockRef continuation; - LLVMValueRef phi; - LLVMValueRef switch_ins; - LLVMBasicBlockRef tmp_block; - LLVMBasicBlockRef default_case; - LLVMTypeRef switch_index_type; - const char *name; - int max_cases; - int i; -} ImmediateUnrollCtx; - -static ImmediateUnrollCtx -immediate_unroll_begin ( - EmitContext *ctx, MonoBasicBlock *bb, int max_cases, - LLVMValueRef switch_index, LLVMTypeRef return_type, const char *name) -{ - LLVMBasicBlockRef default_case = gen_bb (ctx, name); - LLVMBasicBlockRef continuation = gen_bb (ctx, name); - LLVMValueRef switch_ins = LLVMBuildSwitch (ctx->builder, switch_index, default_case, max_cases); - LLVMPositionBuilderAtEnd (ctx->builder, continuation); - LLVMValueRef phi = LLVMBuildPhi (ctx->builder, return_type, name); - ImmediateUnrollCtx ictx = { 0 }; - ictx.ctx = ctx; - ictx.bb = bb; - ictx.continuation = continuation; - ictx.phi = phi; - ictx.switch_ins = switch_ins; - ictx.default_case = default_case; - ictx.switch_index_type = LLVMTypeOf (switch_index); - ictx.name = name; - ictx.max_cases = max_cases; - return ictx; -} - -static gboolean -immediate_unroll_next (ImmediateUnrollCtx *ictx, int *i) -{ - if (ictx->i >= ictx->max_cases) - return FALSE; - ictx->tmp_block = gen_bb (ictx->ctx, ictx->name); - LLVMPositionBuilderAtEnd (ictx->ctx->builder, ictx->tmp_block); - *i = ictx->i; - ++ictx->i; - return TRUE; -} - -static void -immediate_unroll_commit (ImmediateUnrollCtx *ictx, int switch_const, LLVMValueRef value) -{ - LLVMBuildBr (ictx->ctx->builder, ictx->continuation); - LLVMAddCase (ictx->switch_ins, LLVMConstInt (ictx->switch_index_type, switch_const, FALSE), ictx->tmp_block); - LLVMAddIncoming (ictx->phi, &value, &ictx->tmp_block, 1); -} - -static void -immediate_unroll_default (ImmediateUnrollCtx *ictx) -{ - LLVMPositionBuilderAtEnd (ictx->ctx->builder, ictx->default_case); -} - -static void -immediate_unroll_commit_default (ImmediateUnrollCtx *ictx, LLVMValueRef value) -{ - LLVMBuildBr (ictx->ctx->builder, ictx->continuation); - LLVMAddIncoming (ictx->phi, &value, &ictx->default_case, 1); -} - -static LLVMValueRef -immediate_unroll_end (ImmediateUnrollCtx *ictx, LLVMBasicBlockRef *continuation) -{ - EmitContext *ctx = ictx->ctx; - LLVMBuilderRef builder = ctx->builder; - LLVMPositionBuilderAtEnd (builder, ictx->continuation); - *continuation = ictx->continuation; - ctx->bblocks [ictx->bb->block_num].end_bblock = ictx->continuation; - return ictx->phi; -} - /* * resolve_patch: * @@ -5066,6 +4970,138 @@ undef_upper_elements (EmitContext *ctx, LLVMTypeRef type, LLVMValueRef x) return vector_from_scalar_ty (ctx, type, scalar_from_vector (ctx, x)); } +typedef struct { + EmitContext *ctx; + MonoBasicBlock *bb; + LLVMBasicBlockRef continuation; + LLVMValueRef phi; + LLVMValueRef switch_ins; + LLVMBasicBlockRef tmp_block; + LLVMBasicBlockRef default_case; + LLVMTypeRef switch_index_type; + const char *name; + int max_cases; + int i; +} ImmediateUnrollCtx; + +static ImmediateUnrollCtx +immediate_unroll_begin ( + EmitContext *ctx, MonoBasicBlock *bb, int max_cases, + LLVMValueRef switch_index, LLVMTypeRef return_type, const char *name) +{ + LLVMBasicBlockRef default_case = gen_bb (ctx, name); + LLVMBasicBlockRef continuation = gen_bb (ctx, name); + LLVMValueRef switch_ins = LLVMBuildSwitch (ctx->builder, switch_index, default_case, max_cases); + LLVMPositionBuilderAtEnd (ctx->builder, continuation); + LLVMValueRef phi = LLVMBuildPhi (ctx->builder, return_type, name); + ImmediateUnrollCtx ictx = { 0 }; + ictx.ctx = ctx; + ictx.bb = bb; + ictx.continuation = continuation; + ictx.phi = phi; + ictx.switch_ins = switch_ins; + ictx.default_case = default_case; + ictx.switch_index_type = LLVMTypeOf (switch_index); + ictx.name = name; + ictx.max_cases = max_cases; + return ictx; +} + +static gboolean +immediate_unroll_next (ImmediateUnrollCtx *ictx, int *i) +{ + if (ictx->i >= ictx->max_cases) + return FALSE; + ictx->tmp_block = gen_bb (ictx->ctx, ictx->name); + LLVMPositionBuilderAtEnd (ictx->ctx->builder, ictx->tmp_block); + *i = ictx->i; + ++ictx->i; + return TRUE; +} + +static void +immediate_unroll_commit (ImmediateUnrollCtx *ictx, int switch_const, LLVMValueRef value) +{ + LLVMBuildBr (ictx->ctx->builder, ictx->continuation); + LLVMAddCase (ictx->switch_ins, LLVMConstInt (ictx->switch_index_type, switch_const, FALSE), ictx->tmp_block); + LLVMAddIncoming (ictx->phi, &value, &ictx->tmp_block, 1); +} + +static void +immediate_unroll_default (ImmediateUnrollCtx *ictx) +{ + LLVMPositionBuilderAtEnd (ictx->ctx->builder, ictx->default_case); +} + +static void +immediate_unroll_commit_default (ImmediateUnrollCtx *ictx, LLVMValueRef value) +{ + LLVMBuildBr (ictx->ctx->builder, ictx->continuation); + LLVMAddIncoming (ictx->phi, &value, &ictx->default_case, 1); +} + +static LLVMValueRef +immediate_unroll_end (ImmediateUnrollCtx *ictx, LLVMBasicBlockRef *continuation) +{ + EmitContext *ctx = ictx->ctx; + LLVMBuilderRef builder = ctx->builder; + LLVMPositionBuilderAtEnd (builder, ictx->continuation); + *continuation = ictx->continuation; + ctx->bblocks [ictx->bb->block_num].end_bblock = ictx->continuation; + return ictx->phi; +} + +typedef struct { + EmitContext *ctx; + LLVMTypeRef return_type; + gboolean needs_fake_scalar_op; +} ScalarOpFromVectorOpCtx; + +static inline gboolean +check_needs_fake_scalar_op (MonoTypeEnum type) +{ +#if defined(TARGET_ARM64) + switch (type) { + case MONO_TYPE_U1: + case MONO_TYPE_I1: + case MONO_TYPE_U2: + case MONO_TYPE_I2: + return TRUE; + } +#endif + return FALSE; +} + +static ScalarOpFromVectorOpCtx +scalar_op_from_vector_op (EmitContext *ctx, LLVMTypeRef return_type, MonoInst *ins) +{ + ScalarOpFromVectorOpCtx ret = { 0 }; + ret.ctx = ctx; + ret.return_type = return_type; + ret.needs_fake_scalar_op = check_needs_fake_scalar_op (inst_c1_type (ins)); + return ret; +} + +static llvm_ovr_tag_t +scalar_op_from_vector_op_process_args (ScalarOpFromVectorOpCtx *sctx, LLVMValueRef *args, int num_args) +{ + llvm_ovr_tag_t ovr_tag = ovr_tag_from_llvm_type (sctx->return_type); + if (!sctx->needs_fake_scalar_op) { + ovr_tag = ovr_tag_force_scalar (ovr_tag); + for (int i = 0; i < num_args; ++i) + args [i] = scalar_from_vector (sctx->ctx, args [i]); + } + return ovr_tag; +} + +static LLVMValueRef +scalar_op_from_vector_op_process_result (ScalarOpFromVectorOpCtx *sctx, LLVMValueRef result) +{ + if (!sctx->needs_fake_scalar_op) + return vector_from_scalar_ty (sctx->ctx, sctx->return_type, result); + return keep_lowest_element (sctx->ctx, result); +} + static void emit_llvmonly_handler_start (EmitContext *ctx, MonoBasicBlock *bb, LLVMBasicBlockRef cbb) { @@ -10402,24 +10438,31 @@ process_bb (EmitContext *ctx, MonoBasicBlock *bb) values [ins->dreg] = result; break; } - case OP_ARM64_SQSHLU: { + case OP_ARM64_SQSHLU: + case OP_ARM64_SQSHLU_SCALAR: { + gboolean scalar = ins->opcode == OP_ARM64_SQSHLU_SCALAR; LLVMTypeRef intrin_result_t = simd_class_to_llvm_type (ctx, ins->klass); + LLVMTypeRef elem_t = LLVMGetElementType (intrin_result_t); + unsigned int element_bits = mono_llvm_get_prim_size_bits (elem_t); llvm_ovr_tag_t ovr_tag = ovr_tag_from_llvm_type (intrin_result_t); - unsigned int element_bits = mono_llvm_get_prim_size_bits (LLVMGetElementType (intrin_result_t)); int max_index = element_bits; - int iid = INTRINS_AARCH64_ADV_SIMD_SQSHLU; + ScalarOpFromVectorOpCtx sctx = scalar_op_from_vector_op (ctx, intrin_result_t, ins); ImmediateUnrollCtx ictx = immediate_unroll_begin (ctx, bb, max_index, rhs, intrin_result_t, "arm64_sqshlu"); - LLVMValueRef args [] = { lhs, NULL }; int i = 0; while (immediate_unroll_next (&ictx, &i)) { int shift_const = i; - args [1] = create_shift_vector (ctx, lhs, const_int32 (shift_const)); - LLVMValueRef result = call_overloaded_intrins (ctx, iid, ovr_tag, args, ""); + LLVMValueRef args [2] = { lhs, create_shift_vector (ctx, lhs, const_int32 (shift_const)) }; + if (scalar) + ovr_tag = scalar_op_from_vector_op_process_args (&sctx, args, 2); + LLVMValueRef result = call_overloaded_intrins (ctx, INTRINS_AARCH64_ADV_SIMD_SQSHLU, ovr_tag, args, ""); immediate_unroll_commit (&ictx, shift_const, result); } immediate_unroll_default (&ictx); immediate_unroll_commit_default (&ictx, lhs); - values [ins->dreg] = immediate_unroll_end (&ictx, &cbb); + LLVMValueRef result = immediate_unroll_end (&ictx, &cbb); + if (scalar) + result = scalar_op_from_vector_op_process_result (&sctx, result); + values [ins->dreg] = result; break; } case OP_ARM64_SSHLL: @@ -10863,7 +10906,7 @@ process_bb (EmitContext *ctx, MonoBasicBlock *bb) case OP_XOP_OVR_SCALAR_X_X_X_X: { int num_args = 0; IntrinsicId iid = (IntrinsicId) ins->inst_c0; - llvm_ovr_tag_t ovr_tag = ovr_tag_from_mono_vector_class (ins->klass); + LLVMTypeRef ret_t = simd_class_to_llvm_type (ctx, ins->klass); switch (ins->opcode) { case OP_XOP_OVR_SCALAR_X_X: num_args = 1; break; case OP_XOP_OVR_SCALAR_X_X_X: num_args = 2; break; @@ -10875,19 +10918,11 @@ process_bb (EmitContext *ctx, MonoBasicBlock *bb) * instruction selection. This is worked around by using a vector * operation and then explicitly clearing the upper bits of the register. */ - gboolean arm64_fake_scalar_op = check_needs_fake_scalar_op (inst_c1_type (ins)); + ScalarOpFromVectorOpCtx sctx = scalar_op_from_vector_op (ctx, ret_t, ins); LLVMValueRef args [3] = { lhs, rhs, arg3 }; - if (!arm64_fake_scalar_op) { - ovr_tag = ovr_tag_force_scalar (ovr_tag); - for (int i = 0; i < num_args; ++i) - args [i] = scalar_from_vector (ctx, args [i]); - } + llvm_ovr_tag_t ovr_tag = scalar_op_from_vector_op_process_args (&sctx, args, num_args); LLVMValueRef result = call_overloaded_intrins (ctx, iid, ovr_tag, args, ""); - if (!arm64_fake_scalar_op) { - LLVMTypeRef ret_t = simd_class_to_llvm_type (ctx, ins->klass); - result = vector_from_scalar_ty (ctx, ret_t, result); - } else - result = keep_lowest_element (ctx, result); + result = scalar_op_from_vector_op_process_result (&sctx, result); values [ins->dreg] = result; break; } diff --git a/src/mono/mono/mini/mini-ops.h b/src/mono/mono/mini/mini-ops.h index 925895eef6ce9d..6915d34654a93a 100644 --- a/src/mono/mono/mini/mini-ops.h +++ b/src/mono/mono/mini/mini-ops.h @@ -1659,6 +1659,7 @@ MINI_OP(OP_ARM64_XNSHIFT, "arm64_xnshift", XREG, XREG, IREG) MINI_OP3(OP_ARM64_XNSHIFT2, "arm64_xnshift2", XREG, XREG, XREG, IREG) MINI_OP(OP_ARM64_SQSHLU, "arm64_sqshlu", XREG, XREG, IREG) +MINI_OP(OP_ARM64_SQSHLU_SCALAR, "arm64_sqshlu_scalar", XREG, XREG, IREG) MINI_OP(OP_ARM64_REV16, "arm64_rev16", XREG, XREG, NONE) MINI_OP(OP_ARM64_REV32, "arm64_rev32", XREG, XREG, NONE) diff --git a/src/mono/mono/mini/simd-intrinsics.c b/src/mono/mono/mini/simd-intrinsics.c index 5f339d04ac3937..2da5525864e2c1 100644 --- a/src/mono/mono/mini/simd-intrinsics.c +++ b/src/mono/mono/mini/simd-intrinsics.c @@ -1296,14 +1296,14 @@ static SimdIntrinsic advsimd_methods [] = { {SN_ShiftLeftLogicalSaturate}, {SN_ShiftLeftLogicalSaturateScalar}, {SN_ShiftLeftLogicalSaturateUnsigned, OP_ARM64_SQSHLU}, - {SN_ShiftLeftLogicalSaturateUnsignedScalar, OP_ARM64_SQSHLU}, + {SN_ShiftLeftLogicalSaturateUnsignedScalar, OP_ARM64_SQSHLU_SCALAR}, {SN_ShiftLeftLogicalScalar, OP_ARM64_SHL}, {SN_ShiftLeftLogicalWideningLower, OP_ARM64_SSHLL, None, OP_ARM64_USHLL}, {SN_ShiftLeftLogicalWideningUpper, OP_ARM64_SSHLL2, None, OP_ARM64_USHLL2}, {SN_ShiftLogical, OP_XOP_OVR_X_X_X, INTRINS_AARCH64_ADV_SIMD_USHL}, {SN_ShiftLogicalRounded, OP_XOP_OVR_X_X_X, INTRINS_AARCH64_ADV_SIMD_URSHL}, {SN_ShiftLogicalRoundedSaturate, OP_XOP_OVR_X_X_X, INTRINS_AARCH64_ADV_SIMD_UQRSHL}, - {SN_ShiftLogicalRoundedSaturateScalar, OP_XOP_OVR_X_X_X, INTRINS_AARCH64_ADV_SIMD_UQRSHL}, + {SN_ShiftLogicalRoundedSaturateScalar, OP_XOP_OVR_SCALAR_X_X_X, INTRINS_AARCH64_ADV_SIMD_UQRSHL}, {SN_ShiftLogicalRoundedScalar, OP_XOP_OVR_X_X_X, INTRINS_AARCH64_ADV_SIMD_URSHL}, {SN_ShiftLogicalSaturate, OP_XOP_OVR_X_X_X, INTRINS_AARCH64_ADV_SIMD_UQSHL}, {SN_ShiftLogicalSaturateScalar, OP_XOP_OVR_SCALAR_X_X_X, INTRINS_AARCH64_ADV_SIMD_UQSHL}, From 5b4945654d2e8b0d2392440cefbe7aeaba7072c2 Mon Sep 17 00:00:00 2001 From: Imran Hameed Date: Tue, 9 Mar 2021 11:51:08 -0800 Subject: [PATCH 47/58] Fix PopCount --- src/mono/mono/mini/simd-intrinsics.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/mono/mono/mini/simd-intrinsics.c b/src/mono/mono/mini/simd-intrinsics.c index 2da5525864e2c1..d788c699cb838b 100644 --- a/src/mono/mono/mini/simd-intrinsics.c +++ b/src/mono/mono/mini/simd-intrinsics.c @@ -1258,7 +1258,7 @@ static SimdIntrinsic advsimd_methods [] = { {SN_PolynomialMultiply, OP_XOP_OVR_X_X_X, INTRINS_AARCH64_ADV_SIMD_PMUL}, {SN_PolynomialMultiplyWideningLower, OP_ARM64_PMULL}, {SN_PolynomialMultiplyWideningUpper, OP_ARM64_PMULL2}, - {SN_PopCount, OP_XOP_OVR_SCALAR_X_X, INTRINS_AARCH64_ADV_SIMD_CNT}, + {SN_PopCount, OP_XOP_OVR_X_X, INTRINS_AARCH64_ADV_SIMD_CNT}, {SN_ReciprocalEstimate, None, None, OP_XOP_OVR_X_X, INTRINS_AARCH64_ADV_SIMD_URECPE, OP_XOP_OVR_X_X, INTRINS_AARCH64_ADV_SIMD_FRECPE}, {SN_ReciprocalEstimateScalar, OP_XOP_OVR_SCALAR_X_X, INTRINS_AARCH64_ADV_SIMD_FRECPE}, {SN_ReciprocalExponentScalar, OP_XOP_OVR_SCALAR_X_X, INTRINS_AARCH64_ADV_SIMD_FRECPX}, From f31aec10a5d375fb54cd649923a8575209d9bc15 Mon Sep 17 00:00:00 2001 From: Imran Hameed Date: Tue, 9 Mar 2021 14:18:08 -0800 Subject: [PATCH 48/58] Fix ReverseElement8, ReverseElement16, ReverseElement32 --- src/mono/mono/mini/mini-llvm.c | 38 +++++++++++++++------------- src/mono/mono/mini/mini-ops.h | 4 +-- src/mono/mono/mini/simd-intrinsics.c | 6 ++--- 3 files changed, 24 insertions(+), 24 deletions(-) diff --git a/src/mono/mono/mini/mini-llvm.c b/src/mono/mono/mini/mini-llvm.c index 88e786784d1386..98110145a454b2 100644 --- a/src/mono/mono/mini/mini-llvm.c +++ b/src/mono/mono/mini/mini-llvm.c @@ -255,6 +255,7 @@ mini_llvm_ins_info[] = { enum { MAX_VECTOR_ELEMS = 32, // 2 vectors * 128 bits per vector / 8 bits per element + ARM64_MAX_VECTOR_ELEMS = 16, }; static LLVMIntPredicate cond_to_llvm_cond [] = { @@ -10273,26 +10274,27 @@ process_bb (EmitContext *ctx, MonoBasicBlock *bb) values [ins->dreg] = result; break; } - case OP_ARM64_REV16: - case OP_ARM64_REV32: - case OP_ARM64_REV64: { - unsigned int tmp_bits = 0; - switch (ins->opcode) { - case OP_ARM64_REV16: tmp_bits = 8; break; - case OP_ARM64_REV32: tmp_bits = 16; break; - case OP_ARM64_REV64: tmp_bits = 32; break; - } + case OP_ARM64_REVN: { LLVMTypeRef t = LLVMTypeOf (lhs); - unsigned int t_bits = mono_llvm_get_prim_size_bits (t); - unsigned int tmp_elements = t_bits / tmp_bits; - LLVMTypeRef tmp_t = LLVMVectorType (LLVMIntType (tmp_bits), tmp_elements); - LLVMValueRef tmp = LLVMBuildBitCast (builder, lhs, tmp_t, "arm64_rev"); - int mask [MAX_VECTOR_ELEMS] = { 0 }; - for (unsigned int i = 0; i < tmp_elements; i += 2) { - mask [i] = i + 1; - mask [i + 1] = i; + LLVMTypeRef elem_t = LLVMGetElementType (t); + unsigned int group_bits = mono_llvm_get_prim_size_bits (elem_t); + unsigned int vec_bits = mono_llvm_get_prim_size_bits (t); + unsigned int tmp_bits = ins->inst_c0; + unsigned int tmp_elements = vec_bits / tmp_bits; + const int cycle8 [] = { 7, 6, 5, 4, 3, 2, 1, 0, 15, 14, 13, 12, 11, 10, 9, 8 }; + const int cycle4 [] = { 3, 2, 1, 0, 7, 6, 5, 4, 11, 10, 9, 8, 15, 14, 13, 12 }; + const int cycle2 [] = { 1, 0, 3, 2, 5, 4, 7, 6, 9, 8, 11, 10, 13, 12, 15, 14 }; + const int *cycle = NULL; + switch (group_bits / tmp_bits) { + case 2: cycle = cycle2; break; + case 4: cycle = cycle4; break; + case 8: cycle = cycle8; break; + default: g_assert_not_reached (); } - LLVMValueRef result = LLVMBuildShuffleVector (builder, tmp, LLVMGetUndef (tmp_t), create_const_vector_i32 (mask, tmp_elements), ""); + g_assert (tmp_elements <= ARM64_MAX_VECTOR_ELEMS); + LLVMTypeRef tmp_t = LLVMVectorType (LLVMIntType (tmp_bits), tmp_elements); + LLVMValueRef tmp = LLVMBuildBitCast (builder, lhs, tmp_t, "arm64_revn"); + LLVMValueRef result = LLVMBuildShuffleVector (builder, tmp, LLVMGetUndef (tmp_t), create_const_vector_i32 (cycle, tmp_elements), ""); result = LLVMBuildBitCast (builder, result, t, ""); values [ins->dreg] = result; break; diff --git a/src/mono/mono/mini/mini-ops.h b/src/mono/mono/mini/mini-ops.h index 6915d34654a93a..bcbe9bbf17c51a 100644 --- a/src/mono/mono/mini/mini-ops.h +++ b/src/mono/mono/mini/mini-ops.h @@ -1661,9 +1661,7 @@ MINI_OP3(OP_ARM64_XNSHIFT2, "arm64_xnshift2", XREG, XREG, XREG, IREG) MINI_OP(OP_ARM64_SQSHLU, "arm64_sqshlu", XREG, XREG, IREG) MINI_OP(OP_ARM64_SQSHLU_SCALAR, "arm64_sqshlu_scalar", XREG, XREG, IREG) -MINI_OP(OP_ARM64_REV16, "arm64_rev16", XREG, XREG, NONE) -MINI_OP(OP_ARM64_REV32, "arm64_rev32", XREG, XREG, NONE) -MINI_OP(OP_ARM64_REV64, "arm64_rev64", XREG, XREG, NONE) +MINI_OP(OP_ARM64_REVN, "arm64_revn", XREG, XREG, NONE) MINI_OP(OP_ARM64_PMULL, "arm64_pmull", XREG, XREG, XREG) MINI_OP(OP_ARM64_PMULL2, "arm64_pmull2", XREG, XREG, XREG) diff --git a/src/mono/mono/mini/simd-intrinsics.c b/src/mono/mono/mini/simd-intrinsics.c index d788c699cb838b..09206b8e6c8baf 100644 --- a/src/mono/mono/mini/simd-intrinsics.c +++ b/src/mono/mono/mini/simd-intrinsics.c @@ -1268,9 +1268,9 @@ static SimdIntrinsic advsimd_methods [] = { {SN_ReciprocalSquareRootStepScalar, OP_XOP_OVR_SCALAR_X_X_X, INTRINS_AARCH64_ADV_SIMD_FRSQRTS}, {SN_ReciprocalStep, OP_XOP_OVR_X_X_X, INTRINS_AARCH64_ADV_SIMD_FRECPS}, {SN_ReciprocalStepScalar, OP_XOP_OVR_SCALAR_X_X_X, INTRINS_AARCH64_ADV_SIMD_FRECPS}, - {SN_ReverseElement16, OP_ARM64_REV32}, - {SN_ReverseElement32, OP_ARM64_REV64}, - {SN_ReverseElement8, OP_ARM64_REV16}, + {SN_ReverseElement16, OP_ARM64_REVN, 16}, + {SN_ReverseElement32, OP_ARM64_REVN, 32}, + {SN_ReverseElement8, OP_ARM64_REVN, 8}, {SN_ReverseElementBits, OP_XOP_OVR_X_X, INTRINS_AARCH64_ADV_SIMD_RBIT}, {SN_RoundAwayFromZero, OP_XOP_OVR_X_X, INTRINS_AARCH64_ADV_SIMD_FRINTA}, {SN_RoundAwayFromZeroScalar, OP_XOP_OVR_SCALAR_X_X, INTRINS_AARCH64_ADV_SIMD_FRINTA}, From 12fd7c9ae171cb813d2b9cd897b49bebc001a6d6 Mon Sep 17 00:00:00 2001 From: Imran Hameed Date: Tue, 9 Mar 2021 14:31:42 -0800 Subject: [PATCH 49/58] Fix ExtractNarrowingSaturateScalar, ExtractNarrowingSaturateUnsignedScalar --- src/mono/mono/mini/mini-llvm.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/mono/mono/mini/mini-llvm.c b/src/mono/mono/mini/mini-llvm.c index 98110145a454b2..d4f495da5695df 100644 --- a/src/mono/mono/mini/mini-llvm.c +++ b/src/mono/mono/mini/mini-llvm.c @@ -5098,9 +5098,9 @@ scalar_op_from_vector_op_process_args (ScalarOpFromVectorOpCtx *sctx, LLVMValueR static LLVMValueRef scalar_op_from_vector_op_process_result (ScalarOpFromVectorOpCtx *sctx, LLVMValueRef result) { - if (!sctx->needs_fake_scalar_op) - return vector_from_scalar_ty (sctx->ctx, sctx->return_type, result); - return keep_lowest_element (sctx->ctx, result); + if (sctx->needs_fake_scalar_op) + return keep_lowest_element (sctx->ctx, result); + return vector_from_scalar_ty (sctx->ctx, sctx->return_type, result); } static void @@ -9916,6 +9916,7 @@ process_bb (EmitContext *ctx, MonoBasicBlock *bb) unsigned int argelems = LLVMGetVectorSize (arg_t); LLVMValueRef arg = undef_upper_elements (ctx, LLVMVectorType (argelem_t, argelems * 2), lhs); result = call_overloaded_intrins (ctx, iid, ovr_tag, &arg, "arm64_xnarrow_scalar"); + result = keep_lowest_element (ctx, result); } values [ins->dreg] = result; break; From d2299445d36910f4356c44d21b673b5f882489b3 Mon Sep 17 00:00:00 2001 From: Imran Hameed Date: Tue, 9 Mar 2021 17:05:11 -0800 Subject: [PATCH 50/58] More test fixes: MultiplyDoublingSaturateHighScalar MultiplyDoublingScalarBySelectedScalarSaturateHigh MultiplyDoublingWideningSaturateScalarBySelectedScalar MultiplyDoublingWideningScalarBySelectedScalarAndAddSaturate MultiplyDoublingWideningScalarBySelectedScalarAndSubtractSaturate MultiplyRoundedDoublingByScalarSaturateHigh MultiplyRoundedDoublingBySelectedScalarSaturateHigh MultiplyRoundedDoublingSaturateHighScalar MultiplyRoundedDoublingScalarBySelectedScalarSaturateHigh - remove unnecessary special cases MultiplyDoublingWideningSaturateScalar - add support for the special-case scalar LLVM intrinsic for sqdmull --- src/mono/mono/mini/llvm-intrinsics.h | 5 +- src/mono/mono/mini/mini-llvm.c | 89 +++++++++++++--------------- src/mono/mono/mini/mini-ops.h | 6 -- src/mono/mono/mini/simd-intrinsics.c | 17 ++++-- 4 files changed, 56 insertions(+), 61 deletions(-) diff --git a/src/mono/mono/mini/llvm-intrinsics.h b/src/mono/mono/mini/llvm-intrinsics.h index 29c38323e7c952..8c8d0a89112877 100644 --- a/src/mono/mono/mini/llvm-intrinsics.h +++ b/src/mono/mono/mini/llvm-intrinsics.h @@ -379,10 +379,11 @@ INTRINS_OVR_TAG(AARCH64_ADV_SIMD_FMINNM, aarch64_neon_fminnm, Scalar | V64 | V12 INTRINS_OVR_TAG(AARCH64_ADV_SIMD_FMAXNMP, aarch64_neon_fmaxnmp, V64 | V128 | R4 | R8) INTRINS_OVR_TAG(AARCH64_ADV_SIMD_FMINNMP, aarch64_neon_fminnmp, V64 | V128 | R4 | R8) -INTRINS_OVR_TAG(AARCH64_ADV_SIMD_SQDMULH, aarch64_neon_sqdmulh, V64 | V128 | I2 | I4) +INTRINS_OVR_TAG(AARCH64_ADV_SIMD_SQDMULH, aarch64_neon_sqdmulh, Scalar | V64 | V128 | I2 | I4) +INTRINS(AARCH64_ADV_SIMD_SQDMULL_SCALAR, aarch64_neon_sqdmulls_scalar) INTRINS_OVR_TAG(AARCH64_ADV_SIMD_SQDMULL, aarch64_neon_sqdmull, V64 | V128 | I2 | I4 | I8) -INTRINS_OVR_TAG(AARCH64_ADV_SIMD_SQRDMULH, aarch64_neon_sqrdmulh, V64 | V128 | I2 | I4) +INTRINS_OVR_TAG(AARCH64_ADV_SIMD_SQRDMULH, aarch64_neon_sqrdmulh, Scalar | V64 | V128 | I2 | I4) INTRINS_OVR_TAG(AARCH64_ADV_SIMD_SMULL, aarch64_neon_smull, V128 | I2 | I4 | I8) INTRINS_OVR_TAG(AARCH64_ADV_SIMD_UMULL, aarch64_neon_umull, V128 | I2 | I4 | I8) diff --git a/src/mono/mono/mini/mini-llvm.c b/src/mono/mono/mini/mini-llvm.c index d4f495da5695df..64262fb5a3c018 100644 --- a/src/mono/mono/mini/mini-llvm.c +++ b/src/mono/mono/mini/mini-llvm.c @@ -369,12 +369,6 @@ ovr_tag_smaller_vector (llvm_ovr_tag_t tag) return (tag & ~INTRIN_vectormask) | ((tag & INTRIN_vectormask) >> 1); } -static inline llvm_ovr_tag_t -ovr_tag_double_width (llvm_ovr_tag_t tag) -{ - return ((tag & ~INTRIN_vectormask) << 1) | ((tag & INTRIN_vectormask) << 1); -} - static inline llvm_ovr_tag_t ovr_tag_corresponding_integer (llvm_ovr_tag_t tag) { @@ -10059,7 +10053,6 @@ process_bb (EmitContext *ctx, MonoBasicBlock *bb) values [ins->dreg] = result; break; } - case OP_ARM64_SQDMULH_SCALAR: case OP_ARM64_SQDMULL_SCALAR: case OP_ARM64_SQDMLAL_SCALAR: case OP_ARM64_SQDMLSL_SCALAR: { @@ -10072,12 +10065,18 @@ process_bb (EmitContext *ctx, MonoBasicBlock *bb) * %9 = tail call i32 @llvm.aarch64.neon.sqsub.i32(i32 %0, i32 %8) * ret i32 %9 * } + * + * define dso_local i64 @__vqdmlals_s32(i64, i32, i32) local_unnamed_addr #0 { + * %4 = tail call i64 @llvm.aarch64.neon.sqdmulls.scalar(i32 %1, i32 %2) #2 + * %5 = tail call i64 @llvm.aarch64.neon.sqadd.i64(i64 %0, i64 %4) #2 + * ret i64 %5 + * } */ - gboolean widening = TRUE; int mulid = INTRINS_AARCH64_ADV_SIMD_SQDMULL; int iid = 0; + gboolean scalar_mul_result = FALSE; + gboolean scalar_acc_result = FALSE; switch (ins->opcode) { - case OP_ARM64_SQDMULH_SCALAR: mulid = INTRINS_AARCH64_ADV_SIMD_SQDMULH; widening = FALSE; break; case OP_ARM64_SQDMLAL_SCALAR: iid = INTRINS_AARCH64_ADV_SIMD_SQADD; break; case OP_ARM64_SQDMLSL_SCALAR: iid = INTRINS_AARCH64_ADV_SIMD_SQSUB; break; } @@ -10091,49 +10090,45 @@ process_bb (EmitContext *ctx, MonoBasicBlock *bb) LLVMTypeRef mularg_t = LLVMTypeOf (mularg); llvm_ovr_tag_t multag = ovr_tag_from_llvm_type (mularg_t); llvm_ovr_tag_t iidtag = ovr_tag_force_scalar (ovr_tag_from_llvm_type (ret_t)); - if (widening) - multag = ovr_tag_double_width (multag); - - mularg = undef_upper_elements (ctx, mularg_t, mularg); - selected_scalar = undef_upper_elements (ctx, mularg_t, selected_scalar); + if (multag & INTRIN_int32) { + /* The (i32, i32) -> i64 variant of aarch64_neon_sqdmull has + * a unique, non-overloaded name. + */ + mulid = INTRINS_AARCH64_ADV_SIMD_SQDMULL_SCALAR; + multag = 0; + iidtag = INTRIN_int64 | INTRIN_scalar; + scalar_mul_result = TRUE; + scalar_acc_result = TRUE; + } else if (multag & INTRIN_int16) { + /* We were passed a (<4 x i16>, <4 x i16>) but the + * widening multiplication intrinsic will yield a <4 x i32>. + */ + multag = INTRIN_int32 | INTRIN_vector128; + } else + g_assert_not_reached (); + if (scalar_mul_result) { + mularg = scalar_from_vector (ctx, mularg); + selected_scalar = scalar_from_vector (ctx, selected_scalar); + } else { + mularg = undef_upper_elements (ctx, mularg_t, mularg); + selected_scalar = undef_upper_elements (ctx, mularg_t, selected_scalar); + } LLVMValueRef mulargs [] = { mularg, selected_scalar }; - LLVMValueRef result = call_overloaded_intrins (ctx, mulid, multag, mulargs, "arm64_sqdmlsl"); - result = scalar_from_vector (ctx, result); + LLVMValueRef result = call_overloaded_intrins (ctx, mulid, multag, mulargs, "arm64_sqdmull_scalar"); if (iid != 0) { - LLVMValueRef minuend = scalar_from_vector (ctx, lhs); - LLVMValueRef subargs [] = { minuend, result }; - result = call_overloaded_intrins (ctx, iid, iidtag, subargs, "arm64_sqdmlsl"); - } - result = vector_from_scalar_ty (ctx, ret_t, result); - values [ins->dreg] = result; - break; - } - case OP_ARM64_SQRDMULH_BYSCALAR: - case OP_ARM64_SQRDMULH_SCALAR: - case OP_ARM64_SQRDMULH_SCALAR_SEL: - case OP_ARM64_SQRDMULH_SEL: { - gboolean sel = FALSE; - gboolean scalar = FALSE; - switch (ins->opcode) { - case OP_ARM64_SQRDMULH_SCALAR: scalar = TRUE; break; - case OP_ARM64_SQRDMULH_SCALAR_SEL: sel = TRUE; scalar = TRUE; break; - case OP_ARM64_SQRDMULH_SEL: sel = TRUE; break; - } - llvm_ovr_tag_t ovr_tag = ovr_tag_from_mono_vector_class (ins->klass); - LLVMValueRef lane = arg3; - if (!sel) - lane = const_int32 (0); - LLVMTypeRef t = LLVMTypeOf (lhs); - unsigned int elems = LLVMGetVectorSize (t); - LLVMValueRef arg = LLVMBuildExtractElement (builder, rhs, lane, ""); - if (scalar) - arg = vector_from_scalar_ty (ctx, t, arg); + LLVMValueRef acc = scalar_from_vector (ctx, lhs); + if (!scalar_mul_result) + result = scalar_from_vector (ctx, result); + LLVMValueRef subargs [] = { acc, result }; + result = call_overloaded_intrins (ctx, iid, iidtag, subargs, "arm64_sqdmlxl_scalar"); + scalar_acc_result = TRUE; + } + if (scalar_acc_result) + result = vector_from_scalar_ty (ctx, ret_t, result); else - arg = broadcast_element (ctx, arg, elems); - LLVMValueRef args [] = { lhs, arg }; - LLVMValueRef result = call_overloaded_intrins (ctx, INTRINS_AARCH64_ADV_SIMD_SQRDMULH, ovr_tag, args, ""); + result = keep_lowest_element (ctx, result); values [ins->dreg] = result; break; } diff --git a/src/mono/mono/mini/mini-ops.h b/src/mono/mono/mini/mini-ops.h index bcbe9bbf17c51a..861a9fc1bf14b2 100644 --- a/src/mono/mono/mini/mini-ops.h +++ b/src/mono/mono/mini/mini-ops.h @@ -1701,16 +1701,10 @@ MINI_OP3(OP_ARM64_MLA_SCALAR, "arm64_mla_scalar", XREG, XREG, XREG, XREG) MINI_OP3(OP_ARM64_FMUL_SEL, "arm64_fmul_sel", XREG, XREG, XREG, IREG) -MINI_OP(OP_ARM64_SQDMULH_SCALAR, "arm64_sqdmulh_scalar", XREG, XREG, XREG) MINI_OP(OP_ARM64_SQDMULL_SCALAR, "arm64_sqdmull_scalar", XREG, XREG, XREG) MINI_OP3(OP_ARM64_SQDMLSL_SCALAR, "arm64_sqdmlsl_scalar", XREG, XREG, XREG, XREG) MINI_OP3(OP_ARM64_SQDMLAL_SCALAR, "arm64_sqdmlal_scalar", XREG, XREG, XREG, XREG) -MINI_OP(OP_ARM64_SQRDMULH_SCALAR, "arm64_sqrdmulh_scalar", XREG, XREG, XREG) -MINI_OP3(OP_ARM64_SQRDMULH_SCALAR_SEL, "arm64_sqrdmulh_scalar_sel", XREG, XREG, XREG, IREG) -MINI_OP3(OP_ARM64_SQRDMULH_SEL, "arm64_sqrdmulh_sel", XREG, XREG, XREG, IREG) -MINI_OP(OP_ARM64_SQRDMULH_BYSCALAR, "arm64_sqrdmulh_scalar", XREG, XREG, XREG) - MINI_OP(OP_ARM64_SQDMULL, "arm64_sqdmull", XREG, XREG, XREG) MINI_OP(OP_ARM64_SQDMULL_BYSCALAR, "arm64_sqdmull", XREG, XREG, XREG) MINI_OP(OP_ARM64_SQDMULL2, "arm64_sqdmull2", XREG, XREG, XREG) diff --git a/src/mono/mono/mini/simd-intrinsics.c b/src/mono/mono/mini/simd-intrinsics.c index 09206b8e6c8baf..01efc88c309588 100644 --- a/src/mono/mono/mini/simd-intrinsics.c +++ b/src/mono/mono/mini/simd-intrinsics.c @@ -1201,7 +1201,7 @@ static SimdIntrinsic advsimd_methods [] = { {SN_MultiplyDoublingByScalarSaturateHigh, OP_XOP_OVR_BYSCALAR_X_X_X, INTRINS_AARCH64_ADV_SIMD_SQDMULH}, {SN_MultiplyDoublingBySelectedScalarSaturateHigh}, {SN_MultiplyDoublingSaturateHigh, OP_XOP_OVR_X_X_X, INTRINS_AARCH64_ADV_SIMD_SQDMULH}, - {SN_MultiplyDoublingSaturateHighScalar, OP_ARM64_SQDMULH_SCALAR}, + {SN_MultiplyDoublingSaturateHighScalar, OP_XOP_OVR_SCALAR_X_X_X, INTRINS_AARCH64_ADV_SIMD_SQDMULH}, {SN_MultiplyDoublingScalarBySelectedScalarSaturateHigh}, {SN_MultiplyDoublingWideningAndAddSaturateScalar, OP_ARM64_SQDMLAL_SCALAR}, {SN_MultiplyDoublingWideningAndSubtractSaturateScalar, OP_ARM64_SQDMLSL_SCALAR}, @@ -1232,11 +1232,11 @@ static SimdIntrinsic advsimd_methods [] = { {SN_MultiplyExtendedBySelectedScalar}, {SN_MultiplyExtendedScalar, OP_XOP_OVR_SCALAR_X_X_X, INTRINS_AARCH64_ADV_SIMD_FMULX}, {SN_MultiplyExtendedScalarBySelectedScalar}, - {SN_MultiplyRoundedDoublingByScalarSaturateHigh, OP_ARM64_SQRDMULH_BYSCALAR}, - {SN_MultiplyRoundedDoublingBySelectedScalarSaturateHigh, OP_ARM64_SQRDMULH_SEL}, + {SN_MultiplyRoundedDoublingByScalarSaturateHigh, OP_XOP_OVR_BYSCALAR_X_X_X, INTRINS_AARCH64_ADV_SIMD_SQRDMULH}, + {SN_MultiplyRoundedDoublingBySelectedScalarSaturateHigh}, {SN_MultiplyRoundedDoublingSaturateHigh, OP_XOP_OVR_X_X_X, INTRINS_AARCH64_ADV_SIMD_SQRDMULH}, - {SN_MultiplyRoundedDoublingSaturateHighScalar, OP_ARM64_SQRDMULH_SCALAR}, - {SN_MultiplyRoundedDoublingScalarBySelectedScalarSaturateHigh, OP_ARM64_SQRDMULH_SCALAR_SEL}, + {SN_MultiplyRoundedDoublingSaturateHighScalar, OP_XOP_OVR_SCALAR_X_X_X, INTRINS_AARCH64_ADV_SIMD_SQRDMULH}, + {SN_MultiplyRoundedDoublingScalarBySelectedScalarSaturateHigh}, {SN_MultiplyScalar, OP_XBINOP_SCALAR, OP_FMUL}, {SN_MultiplyScalarBySelectedScalar, OP_ARM64_FMUL_SEL}, {SN_MultiplySubtract, OP_ARM64_MLS}, @@ -1567,6 +1567,8 @@ emit_arm64_intrinsics ( ret->inst_c1 = etype->type; return ret; } + case SN_MultiplyRoundedDoublingBySelectedScalarSaturateHigh: + case SN_MultiplyRoundedDoublingScalarBySelectedScalarSaturateHigh: case SN_MultiplyDoublingScalarBySelectedScalarSaturateHigh: case SN_MultiplyDoublingWideningSaturateScalarBySelectedScalar: case SN_MultiplyExtendedBySelectedScalar: @@ -1583,7 +1585,9 @@ emit_arm64_intrinsics ( int opcode = 0; int c0 = 0; switch (id) { - case SN_MultiplyDoublingScalarBySelectedScalarSaturateHigh: opcode = OP_ARM64_SQDMULH_SCALAR; break; + case SN_MultiplyRoundedDoublingBySelectedScalarSaturateHigh: opcode = OP_XOP_OVR_BYSCALAR_X_X_X; c0 = INTRINS_AARCH64_ADV_SIMD_SQRDMULH; break; + case SN_MultiplyRoundedDoublingScalarBySelectedScalarSaturateHigh: opcode = OP_XOP_OVR_SCALAR_X_X_X; c0 = INTRINS_AARCH64_ADV_SIMD_SQRDMULH; break; + case SN_MultiplyDoublingScalarBySelectedScalarSaturateHigh: opcode = OP_XOP_OVR_SCALAR_X_X_X; c0 = INTRINS_AARCH64_ADV_SIMD_SQDMULH; break; case SN_MultiplyDoublingWideningSaturateScalarBySelectedScalar: opcode = OP_ARM64_SQDMULL_SCALAR; break; case SN_MultiplyExtendedBySelectedScalar: opcode = OP_XOP_OVR_BYSCALAR_X_X_X; c0 = INTRINS_AARCH64_ADV_SIMD_FMULX; break; case SN_MultiplyExtendedScalarBySelectedScalar: opcode = OP_XOP_OVR_SCALAR_X_X_X; c0 = INTRINS_AARCH64_ADV_SIMD_FMULX; break; @@ -1607,6 +1611,7 @@ emit_arm64_intrinsics ( MonoInst *scalar = emit_simd_ins (cfg, ret_klass, OP_ARM64_SELECT_SCALAR, args [1]->dreg, args [2]->dreg); MonoInst *ret = emit_simd_ins (cfg, ret_klass, opcode, args [0]->dreg, scalar->dreg); ret->inst_c0 = c0; + ret->inst_c1 = arg0_type; return ret; } case SN_FusedMultiplyAddBySelectedScalar: From 5baa928d31ad6810ab98d19356990dd5ddbfcca7 Mon Sep 17 00:00:00 2001 From: Imran Hameed Date: Tue, 9 Mar 2021 17:27:15 -0800 Subject: [PATCH 51/58] LoadAndReplicateToVector: coerce the source pointer to the element type when loading a single element --- src/mono/mono/mini/mini-llvm.c | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/mono/mono/mini/mini-llvm.c b/src/mono/mono/mini/mini-llvm.c index 64262fb5a3c018..b00a022b0f5ea9 100644 --- a/src/mono/mono/mini/mini-llvm.c +++ b/src/mono/mono/mini/mini-llvm.c @@ -10565,8 +10565,12 @@ process_bb (EmitContext *ctx, MonoBasicBlock *bb) LLVMTypeRef ret_t = simd_class_to_llvm_type (ctx, ins->klass); unsigned int alignment = mono_llvm_get_prim_size_bits (ret_t) / 8; LLVMValueRef address = lhs; - if (!replicate) - address = convert (ctx, address, LLVMPointerType (ret_t, 0)); + LLVMTypeRef address_t = LLVMPointerType (ret_t, 0); + if (replicate) { + LLVMTypeRef elem_t = LLVMGetElementType (ret_t); + address_t = LLVMPointerType (elem_t, 0); + } + address = convert (ctx, address, address_t); LLVMValueRef result = mono_llvm_build_aligned_load (builder, address, "arm64_ld1", FALSE, alignment); if (replicate) { unsigned int elems = LLVMGetVectorSize (ret_t); From 0d67d57b88a1bc3dad826e76cf5e1559a8a0d65c Mon Sep 17 00:00:00 2001 From: Imran Hameed Date: Tue, 9 Mar 2021 17:42:10 -0800 Subject: [PATCH 52/58] Move OP_INSERT_* and OP_XCAST to a shared arm64/amd64 region --- src/mono/mono/mini/mini-llvm.c | 48 +++++++++++++++++----------------- 1 file changed, 24 insertions(+), 24 deletions(-) diff --git a/src/mono/mono/mini/mini-llvm.c b/src/mono/mono/mini/mini-llvm.c index b00a022b0f5ea9..c4cadd6022d94d 100644 --- a/src/mono/mono/mini/mini-llvm.c +++ b/src/mono/mono/mini/mini-llvm.c @@ -7532,6 +7532,30 @@ process_bb (EmitContext *ctx, MonoBasicBlock *bb) values [ins->dreg] = LLVMBuildInsertElement (builder, vector, val, const_int32 (0), ""); break; } + case OP_INSERT_I1: + values [ins->dreg] = LLVMBuildInsertElement (builder, values [ins->sreg1], convert (ctx, values [ins->sreg2], LLVMInt8Type ()), LLVMConstInt (LLVMInt32Type (), ins->inst_c0, FALSE), dname); + break; + case OP_INSERT_I2: + values [ins->dreg] = LLVMBuildInsertElement (builder, values [ins->sreg1], convert (ctx, values [ins->sreg2], LLVMInt16Type ()), LLVMConstInt (LLVMInt32Type (), ins->inst_c0, FALSE), dname); + break; + case OP_INSERT_I4: + values [ins->dreg] = LLVMBuildInsertElement (builder, values [ins->sreg1], convert (ctx, values [ins->sreg2], LLVMInt32Type ()), LLVMConstInt (LLVMInt32Type (), ins->inst_c0, FALSE), dname); + break; + case OP_INSERT_I8: + values [ins->dreg] = LLVMBuildInsertElement (builder, values [ins->sreg1], convert (ctx, values [ins->sreg2], LLVMInt64Type ()), LLVMConstInt (LLVMInt32Type (), ins->inst_c0, FALSE), dname); + break; + case OP_INSERT_R4: + values [ins->dreg] = LLVMBuildInsertElement (builder, values [ins->sreg1], convert (ctx, values [ins->sreg2], LLVMFloatType ()), LLVMConstInt (LLVMInt32Type (), ins->inst_c0, FALSE), dname); + break; + case OP_INSERT_R8: + values [ins->dreg] = LLVMBuildInsertElement (builder, values [ins->sreg1], convert (ctx, values [ins->sreg2], LLVMDoubleType ()), LLVMConstInt (LLVMInt32Type (), ins->inst_c0, FALSE), dname); + break; + case OP_XCAST: { + LLVMTypeRef t = simd_class_to_llvm_type (ctx, ins->klass); + + values [ins->dreg] = LLVMBuildBitCast (builder, lhs, t, ""); + break; + } #endif // defined(TARGET_X86) || defined(TARGET_AMD64) || defined(TARGET_ARM64) || defined(TARGET_WASM) #if defined(TARGET_X86) || defined(TARGET_AMD64) || defined(TARGET_WASM) @@ -7769,24 +7793,6 @@ process_bb (EmitContext *ctx, MonoBasicBlock *bb) values [ins->dreg] = LLVMBuildZExt (builder, values [ins->dreg], LLVMInt32Type (), ""); break; } - case OP_INSERT_I1: - values [ins->dreg] = LLVMBuildInsertElement (builder, values [ins->sreg1], convert (ctx, values [ins->sreg2], LLVMInt8Type ()), LLVMConstInt (LLVMInt32Type (), ins->inst_c0, FALSE), dname); - break; - case OP_INSERT_I2: - values [ins->dreg] = LLVMBuildInsertElement (builder, values [ins->sreg1], convert (ctx, values [ins->sreg2], LLVMInt16Type ()), LLVMConstInt (LLVMInt32Type (), ins->inst_c0, FALSE), dname); - break; - case OP_INSERT_I4: - values [ins->dreg] = LLVMBuildInsertElement (builder, values [ins->sreg1], convert (ctx, values [ins->sreg2], LLVMInt32Type ()), LLVMConstInt (LLVMInt32Type (), ins->inst_c0, FALSE), dname); - break; - case OP_INSERT_I8: - values [ins->dreg] = LLVMBuildInsertElement (builder, values [ins->sreg1], convert (ctx, values [ins->sreg2], LLVMInt64Type ()), LLVMConstInt (LLVMInt32Type (), ins->inst_c0, FALSE), dname); - break; - case OP_INSERT_R4: - values [ins->dreg] = LLVMBuildInsertElement (builder, values [ins->sreg1], convert (ctx, values [ins->sreg2], LLVMFloatType ()), LLVMConstInt (LLVMInt32Type (), ins->inst_c0, FALSE), dname); - break; - case OP_INSERT_R8: - values [ins->dreg] = LLVMBuildInsertElement (builder, values [ins->sreg1], convert (ctx, values [ins->sreg2], LLVMDoubleType ()), LLVMConstInt (LLVMInt32Type (), ins->inst_c0, FALSE), dname); - break; case OP_XINSERT_I2: { LLVMBasicBlockRef bbs [64]; LLVMValueRef switch_ins; @@ -9294,12 +9300,6 @@ process_bb (EmitContext *ctx, MonoBasicBlock *bb) } #endif - case OP_XCAST: { - LLVMTypeRef t = simd_class_to_llvm_type (ctx, ins->klass); - - values [ins->dreg] = LLVMBuildBitCast (builder, lhs, t, ""); - break; - } case OP_XCOMPARE_FP: { LLVMRealPredicate pred = fpcond_to_llvm_cond [ins->inst_c0]; LLVMValueRef cmp = LLVMBuildFCmp (builder, pred, lhs, rhs, ""); From b14fc9ad21434837e5f4bed00c86417aa8c223bc Mon Sep 17 00:00:00 2001 From: Imran Hameed Date: Tue, 9 Mar 2021 17:57:26 -0800 Subject: [PATCH 53/58] Address feedback: move IntrinsicId (and another LLVM-only anonymous enum) to a separate header --- src/mono/mono/mini/CMakeLists.txt | 1 + src/mono/mono/mini/llvm-intrinsics-types.h | 27 ++++++++++++++++++++++ src/mono/mono/mini/mini-llvm-cpp.h | 2 +- src/mono/mono/mini/mini.h | 23 ------------------ src/mono/mono/mini/simd-intrinsics.c | 1 + 5 files changed, 30 insertions(+), 24 deletions(-) create mode 100644 src/mono/mono/mini/llvm-intrinsics-types.h diff --git a/src/mono/mono/mini/CMakeLists.txt b/src/mono/mono/mini/CMakeLists.txt index 98c8d1b166a369..cf3ca5b5de6c07 100644 --- a/src/mono/mono/mini/CMakeLists.txt +++ b/src/mono/mono/mini/CMakeLists.txt @@ -143,6 +143,7 @@ set(mini_common_sources arch-stubs.c llvm-runtime.h llvm-intrinsics.h + llvm-intrinsics-types.h type-checking.c lldb.h lldb.c diff --git a/src/mono/mono/mini/llvm-intrinsics-types.h b/src/mono/mono/mini/llvm-intrinsics-types.h new file mode 100644 index 00000000000000..f070ed982c912e --- /dev/null +++ b/src/mono/mono/mini/llvm-intrinsics-types.h @@ -0,0 +1,27 @@ +#ifndef __MONO_MINI_LLVM_INTRINSICS_TYPES_H__ +#define __MONO_MINI_LLVM_INTRINSICS_TYPES_H__ + +/* An intrinsic id. The lower 23 bits are used to store a mono-specific ID. The + * next 9 bits store overload tag bits. In the configuration of LLVM 9 we use, + * there are 7017 total intrinsics defined in IntrinsicEnums.inc, so only 13 + * bits are needed to label each intrinsic overload group. + */ +typedef enum { +#define INTRINS(id, llvm_id) INTRINS_ ## id, +#define INTRINS_OVR(id, llvm_id, ty) INTRINS_ ## id, +#define INTRINS_OVR_2_ARG(id, llvm_id, ty1, ty2) INTRINS_ ## id, +#define INTRINS_OVR_3_ARG(id, llvm_id, ty1, ty2, ty3) INTRINS_ ## id, +#define INTRINS_OVR_TAG(id, ...) INTRINS_ ## id, +#define INTRINS_OVR_TAG_KIND(id, ...) INTRINS_ ## id, +#include "llvm-intrinsics.h" + INTRINS_NUM +} IntrinsicId; + +enum { + XBINOP_FORCEINT_and, + XBINOP_FORCEINT_or, + XBINOP_FORCEINT_ornot, + XBINOP_FORCEINT_xor, +}; + +#endif /* __MONO_MINI_LLVM_INTRINSICS_TYPES_H__ */ diff --git a/src/mono/mono/mini/mini-llvm-cpp.h b/src/mono/mono/mini/mini-llvm-cpp.h index 82badd50d01bdd..baa90441be9094 100644 --- a/src/mono/mono/mini/mini-llvm-cpp.h +++ b/src/mono/mono/mini/mini-llvm-cpp.h @@ -16,7 +16,7 @@ #include "llvm-c/Core.h" #include "llvm-c/ExecutionEngine.h" -#include "mini-llvm.h" +#include "llvm-intrinsics-types.h" #ifdef HAVE_UNWIND_H #include diff --git a/src/mono/mono/mini/mini.h b/src/mono/mono/mini/mini.h index 789b155a92ae57..6d2f0fcf819aef 100644 --- a/src/mono/mono/mini/mini.h +++ b/src/mono/mono/mini/mini.h @@ -2981,29 +2981,6 @@ typedef enum { SIMD_OP_ARM64_PMULL64_UPPER, } SimdOp; -enum { - XBINOP_FORCEINT_and, - XBINOP_FORCEINT_or, - XBINOP_FORCEINT_ornot, - XBINOP_FORCEINT_xor, -}; - -/* An intrinsic id. The lower 23 bits are used to store a mono-specific ID. The - * next 9 bits store overload tag bits. In the configuration of LLVM 9 we use, - * there are 7017 total intrinsics defined in IntrinsicEnums.inc, so only 13 - * bits are needed to label each intrinsic overload group. - */ -typedef enum { -#define INTRINS(id, llvm_id) INTRINS_ ## id, -#define INTRINS_OVR(id, llvm_id, ty) INTRINS_ ## id, -#define INTRINS_OVR_2_ARG(id, llvm_id, ty1, ty2) INTRINS_ ## id, -#define INTRINS_OVR_3_ARG(id, llvm_id, ty1, ty2, ty3) INTRINS_ ## id, -#define INTRINS_OVR_TAG(id, ...) INTRINS_ ## id, -#define INTRINS_OVR_TAG_KIND(id, ...) INTRINS_ ## id, -#include "llvm-intrinsics.h" - INTRINS_NUM -} IntrinsicId; - const char *mono_arch_xregname (int reg); MonoCPUFeatures mono_arch_get_cpu_features (void); diff --git a/src/mono/mono/mini/simd-intrinsics.c b/src/mono/mono/mini/simd-intrinsics.c index 01efc88c309588..a7da1df3d41c6a 100644 --- a/src/mono/mono/mini/simd-intrinsics.c +++ b/src/mono/mono/mini/simd-intrinsics.c @@ -9,6 +9,7 @@ #include "mini.h" #include "mini-runtime.h" #include "ir-emit.h" +#include "llvm-intrinsics-types.h" #ifdef ENABLE_LLVM #include "mini-llvm.h" #include "mini-llvm-cpp.h" From 059bce5e8b442ea0eb0da9b7940fcde3a636e0f4 Mon Sep 17 00:00:00 2001 From: Imran Hameed Date: Wed, 10 Mar 2021 07:17:47 -0800 Subject: [PATCH 54/58] Don't attempt to scalarize a non-scalar sqshlu --- src/mono/mono/mini/mini-llvm.c | 33 ++++++++++++++++++++++----------- 1 file changed, 22 insertions(+), 11 deletions(-) diff --git a/src/mono/mono/mini/mini-llvm.c b/src/mono/mono/mini/mini-llvm.c index c4cadd6022d94d..fe0f673df200c7 100644 --- a/src/mono/mono/mini/mini-llvm.c +++ b/src/mono/mono/mini/mini-llvm.c @@ -5048,8 +5048,10 @@ immediate_unroll_end (ImmediateUnrollCtx *ictx, LLVMBasicBlockRef *continuation) typedef struct { EmitContext *ctx; + LLVMTypeRef intermediate_type; LLVMTypeRef return_type; gboolean needs_fake_scalar_op; + llvm_ovr_tag_t ovr_tag; } ScalarOpFromVectorOpCtx; static inline gboolean @@ -5072,21 +5074,23 @@ scalar_op_from_vector_op (EmitContext *ctx, LLVMTypeRef return_type, MonoInst *i { ScalarOpFromVectorOpCtx ret = { 0 }; ret.ctx = ctx; + ret.intermediate_type = return_type; ret.return_type = return_type; ret.needs_fake_scalar_op = check_needs_fake_scalar_op (inst_c1_type (ins)); + ret.ovr_tag = ovr_tag_from_llvm_type (return_type); + if (!ret.needs_fake_scalar_op) { + ret.ovr_tag = ovr_tag_force_scalar (ret.ovr_tag); + ret.intermediate_type = ovr_tag_to_llvm_type (ret.ovr_tag); + } return ret; } -static llvm_ovr_tag_t +static void scalar_op_from_vector_op_process_args (ScalarOpFromVectorOpCtx *sctx, LLVMValueRef *args, int num_args) { - llvm_ovr_tag_t ovr_tag = ovr_tag_from_llvm_type (sctx->return_type); - if (!sctx->needs_fake_scalar_op) { - ovr_tag = ovr_tag_force_scalar (ovr_tag); + if (!sctx->needs_fake_scalar_op) for (int i = 0; i < num_args; ++i) args [i] = scalar_from_vector (sctx->ctx, args [i]); - } - return ovr_tag; } static LLVMValueRef @@ -10445,18 +10449,25 @@ process_bb (EmitContext *ctx, MonoBasicBlock *bb) llvm_ovr_tag_t ovr_tag = ovr_tag_from_llvm_type (intrin_result_t); int max_index = element_bits; ScalarOpFromVectorOpCtx sctx = scalar_op_from_vector_op (ctx, intrin_result_t, ins); + intrin_result_t = scalar ? sctx.intermediate_type : intrin_result_t; + ovr_tag = scalar ? sctx.ovr_tag : ovr_tag; ImmediateUnrollCtx ictx = immediate_unroll_begin (ctx, bb, max_index, rhs, intrin_result_t, "arm64_sqshlu"); int i = 0; while (immediate_unroll_next (&ictx, &i)) { int shift_const = i; LLVMValueRef args [2] = { lhs, create_shift_vector (ctx, lhs, const_int32 (shift_const)) }; if (scalar) - ovr_tag = scalar_op_from_vector_op_process_args (&sctx, args, 2); + scalar_op_from_vector_op_process_args (&sctx, args, 2); LLVMValueRef result = call_overloaded_intrins (ctx, INTRINS_AARCH64_ADV_SIMD_SQSHLU, ovr_tag, args, ""); immediate_unroll_commit (&ictx, shift_const, result); } - immediate_unroll_default (&ictx); - immediate_unroll_commit_default (&ictx, lhs); + { + immediate_unroll_default (&ictx); + LLVMValueRef srcarg = lhs; + if (scalar) + scalar_op_from_vector_op_process_args (&sctx, &srcarg, 1); + immediate_unroll_commit_default (&ictx, srcarg); + } LLVMValueRef result = immediate_unroll_end (&ictx, &cbb); if (scalar) result = scalar_op_from_vector_op_process_result (&sctx, result); @@ -10922,8 +10933,8 @@ process_bb (EmitContext *ctx, MonoBasicBlock *bb) */ ScalarOpFromVectorOpCtx sctx = scalar_op_from_vector_op (ctx, ret_t, ins); LLVMValueRef args [3] = { lhs, rhs, arg3 }; - llvm_ovr_tag_t ovr_tag = scalar_op_from_vector_op_process_args (&sctx, args, num_args); - LLVMValueRef result = call_overloaded_intrins (ctx, iid, ovr_tag, args, ""); + scalar_op_from_vector_op_process_args (&sctx, args, num_args); + LLVMValueRef result = call_overloaded_intrins (ctx, iid, sctx.ovr_tag, args, ""); result = scalar_op_from_vector_op_process_result (&sctx, result); values [ins->dreg] = result; break; From bd2e5b259b8eb7c8740e1797ab79152b516c3b41 Mon Sep 17 00:00:00 2001 From: Imran Hameed Date: Wed, 10 Mar 2021 09:53:11 -0800 Subject: [PATCH 55/58] MultiplyDoublingWideningSaturateScalar etc.: consistently place the scalar or scalar-in-vector return value in a Vector64 Remove OP_ARM64_ZERO_UPPER, which is unused --- src/mono/mono/mini/mini-llvm.c | 27 +++++++++++++++------------ src/mono/mono/mini/mini-ops.h | 1 - 2 files changed, 15 insertions(+), 13 deletions(-) diff --git a/src/mono/mono/mini/mini-llvm.c b/src/mono/mono/mini/mini-llvm.c index fe0f673df200c7..cb17a3d8e798ca 100644 --- a/src/mono/mono/mini/mini-llvm.c +++ b/src/mono/mono/mini/mini-llvm.c @@ -369,6 +369,12 @@ ovr_tag_smaller_vector (llvm_ovr_tag_t tag) return (tag & ~INTRIN_vectormask) | ((tag & INTRIN_vectormask) >> 1); } +static inline llvm_ovr_tag_t +ovr_tag_smaller_elements (llvm_ovr_tag_t tag) +{ + return ((tag & ~INTRIN_vectormask) >> 1) | (tag & INTRIN_vectormask); +} + static inline llvm_ovr_tag_t ovr_tag_corresponding_integer (llvm_ovr_tag_t tag) { @@ -4913,11 +4919,12 @@ extract_high_elements (EmitContext *ctx, LLVMValueRef src_vec) } static LLVMValueRef -keep_lowest_element (EmitContext *ctx, LLVMValueRef vec) +keep_lowest_element (EmitContext *ctx, LLVMTypeRef dst_t, LLVMValueRef vec) { int mask [MAX_VECTOR_ELEMS] = { 0 }; LLVMTypeRef t = LLVMTypeOf (vec); - unsigned int elems = LLVMGetVectorSize (t); + g_assert (LLVMGetElementType (dst_t) == LLVMGetElementType (t)); + unsigned int elems = LLVMGetVectorSize (dst_t); mask [0] = 0; for (unsigned int i = 1; i < elems; ++i) mask [i] = elems + i; @@ -5097,7 +5104,7 @@ static LLVMValueRef scalar_op_from_vector_op_process_result (ScalarOpFromVectorOpCtx *sctx, LLVMValueRef result) { if (sctx->needs_fake_scalar_op) - return keep_lowest_element (sctx->ctx, result); + return keep_lowest_element (sctx->ctx, LLVMTypeOf (result), result); return vector_from_scalar_ty (sctx->ctx, sctx->return_type, result); } @@ -9914,7 +9921,7 @@ process_bb (EmitContext *ctx, MonoBasicBlock *bb) unsigned int argelems = LLVMGetVectorSize (arg_t); LLVMValueRef arg = undef_upper_elements (ctx, LLVMVectorType (argelem_t, argelems * 2), lhs); result = call_overloaded_intrins (ctx, iid, ovr_tag, &arg, "arm64_xnarrow_scalar"); - result = keep_lowest_element (ctx, result); + result = keep_lowest_element (ctx, LLVMTypeOf (result), result); } values [ins->dreg] = result; break; @@ -10091,9 +10098,9 @@ process_bb (EmitContext *ctx, MonoBasicBlock *bb) mularg = rhs; selected_scalar = arg3; } - LLVMTypeRef mularg_t = LLVMTypeOf (mularg); - llvm_ovr_tag_t multag = ovr_tag_from_llvm_type (mularg_t); + llvm_ovr_tag_t multag = ovr_tag_smaller_elements (ovr_tag_from_llvm_type (ret_t)); llvm_ovr_tag_t iidtag = ovr_tag_force_scalar (ovr_tag_from_llvm_type (ret_t)); + LLVMTypeRef mularg_t = ovr_tag_to_llvm_type (multag); if (multag & INTRIN_int32) { /* The (i32, i32) -> i64 variant of aarch64_neon_sqdmull has * a unique, non-overloaded name. @@ -10132,7 +10139,7 @@ process_bb (EmitContext *ctx, MonoBasicBlock *bb) if (scalar_acc_result) result = vector_from_scalar_ty (ctx, ret_t, result); else - result = keep_lowest_element (ctx, result); + result = keep_lowest_element (ctx, ret_t, result); values [ins->dreg] = result; break; } @@ -10436,7 +10443,7 @@ process_bb (EmitContext *ctx, MonoBasicBlock *bb) if (high) result = concatenate_vectors (ctx, lhs, result); if (scalar) - result = keep_lowest_element (ctx, result); + result = keep_lowest_element (ctx, LLVMTypeOf (result), result); values [ins->dreg] = result; break; } @@ -10939,10 +10946,6 @@ process_bb (EmitContext *ctx, MonoBasicBlock *bb) values [ins->dreg] = result; break; } - case OP_ARM64_ZERO_UPPER: { - values [ins->dreg] = keep_lowest_element (ctx, lhs); - break; - } #endif case OP_DUMMY_USE: diff --git a/src/mono/mono/mini/mini-ops.h b/src/mono/mono/mini/mini-ops.h index 861a9fc1bf14b2..4bd23c5e803b99 100644 --- a/src/mono/mono/mini/mini-ops.h +++ b/src/mono/mono/mini/mini-ops.h @@ -1612,7 +1612,6 @@ MINI_OP(OP_ARM64_UXTL, "arm64_uxtl", XREG, XREG, NONE) MINI_OP(OP_ARM64_UXTL2, "arm64_uxtl2", XREG, XREG, NONE) MINI_OP(OP_ARM64_UZP1, "arm64_uzp1", XREG, XREG, XREG) MINI_OP(OP_ARM64_UZP2, "arm64_uzp2", XREG, XREG, XREG) -MINI_OP(OP_ARM64_ZERO_UPPER, "arm64_zero_upper", XREG, XREG, NONE) MINI_OP(OP_ARM64_ZIP1, "arm64_zip1", XREG, XREG, XREG) MINI_OP(OP_ARM64_ZIP2, "arm64_zip2", XREG, XREG, XREG) MINI_OP3(OP_ARM64_ST1_SCALAR, "arm64_st1_scalar", NONE, IREG, XREG, IREG) From ca728df557e5ad7ab397b3399a5ce03481cc02b4 Mon Sep 17 00:00:00 2001 From: Imran Hameed Date: Wed, 10 Mar 2021 10:09:59 -0800 Subject: [PATCH 56/58] Explicitly zero out the unused bits in scalar ops built out of vector ops undef can apparently pass through intrinsic functions during optimization, so bias towards slightly worse but correct codegen for now --- src/mono/mono/mini/mini-llvm.c | 65 +++++++++++++--------------------- 1 file changed, 24 insertions(+), 41 deletions(-) diff --git a/src/mono/mono/mini/mini-llvm.c b/src/mono/mono/mini/mini-llvm.c index cb17a3d8e798ca..5d27e263997aa5 100644 --- a/src/mono/mono/mini/mini-llvm.c +++ b/src/mono/mono/mini/mini-llvm.c @@ -4921,13 +4921,14 @@ extract_high_elements (EmitContext *ctx, LLVMValueRef src_vec) static LLVMValueRef keep_lowest_element (EmitContext *ctx, LLVMTypeRef dst_t, LLVMValueRef vec) { - int mask [MAX_VECTOR_ELEMS] = { 0 }; LLVMTypeRef t = LLVMTypeOf (vec); g_assert (LLVMGetElementType (dst_t) == LLVMGetElementType (t)); unsigned int elems = LLVMGetVectorSize (dst_t); + unsigned int src_elems = LLVMGetVectorSize (t); + int mask [MAX_VECTOR_ELEMS] = { 0 }; mask [0] = 0; for (unsigned int i = 1; i < elems; ++i) - mask [i] = elems + i; + mask [i] = src_elems; return LLVMBuildShuffleVector (ctx->builder, vec, LLVMConstNull (t), create_const_vector_i32 (mask, elems), "keep_lowest"); } @@ -4949,27 +4950,9 @@ scalar_from_vector (EmitContext *ctx, LLVMValueRef xs) } static LLVMValueRef -vector_zero_from_scalar (EmitContext *ctx, LLVMTypeRef type, LLVMValueRef x) -{ - return LLVMBuildInsertElement (ctx->builder, LLVMConstNull (type), x, const_int32 (0), "s2vz"); -} - -static LLVMValueRef -vector_from_scalar_ty (EmitContext *ctx, LLVMTypeRef type, LLVMValueRef x) -{ - return LLVMBuildInsertElement (ctx->builder, LLVMGetUndef (type), x, const_int32 (0), "s2v"); -} - -static LLVMValueRef -vector_from_scalar (EmitContext *ctx, LLVMValueRef type_donor, LLVMValueRef x) -{ - return vector_from_scalar_ty (ctx, LLVMTypeOf (type_donor), x); -} - -static LLVMValueRef -undef_upper_elements (EmitContext *ctx, LLVMTypeRef type, LLVMValueRef x) +vector_from_scalar (EmitContext *ctx, LLVMTypeRef type, LLVMValueRef x) { - return vector_from_scalar_ty (ctx, type, scalar_from_vector (ctx, x)); + return LLVMBuildInsertElement (ctx->builder, LLVMConstNull (type), x, const_int32 (0), "s2v"); } typedef struct { @@ -5105,7 +5088,7 @@ scalar_op_from_vector_op_process_result (ScalarOpFromVectorOpCtx *sctx, LLVMValu { if (sctx->needs_fake_scalar_op) return keep_lowest_element (sctx->ctx, LLVMTypeOf (result), result); - return vector_from_scalar_ty (sctx->ctx, sctx->return_type, result); + return vector_from_scalar (sctx->ctx, sctx->return_type, result); } static void @@ -7500,7 +7483,7 @@ process_bb (EmitContext *ctx, MonoBasicBlock *bb) g_assert_not_reached (); } if (scalar) - result = vector_from_scalar (ctx, lhs, result); + result = vector_from_scalar (ctx, LLVMTypeOf (lhs), result); values [ins->dreg] = result; break; } @@ -9677,7 +9660,7 @@ process_bb (EmitContext *ctx, MonoBasicBlock *bb) args [i] = scalar_from_vector (ctx, args [i]); LLVMValueRef result = LLVMBuildFCmp (builder, pred, args [0], args [1], "xcompare_fp"); if (scalar) - result = vector_from_scalar_ty (ctx, LLVMVectorType (LLVMIntType (1), LLVMGetVectorSize (reti_t)), result); + result = vector_from_scalar (ctx, LLVMVectorType (LLVMIntType (1), LLVMGetVectorSize (reti_t)), result); result = LLVMBuildSExt (builder, result, reti_t, ""); result = LLVMBuildBitCast (builder, result, ret_t, ""); values [ins->dreg] = result; @@ -9695,7 +9678,7 @@ process_bb (EmitContext *ctx, MonoBasicBlock *bb) args [i] = scalar_from_vector (ctx, args [i]); LLVMValueRef result = LLVMBuildICmp (builder, pred, args [0], args [1], "xcompare"); if (scalar) - result = vector_from_scalar_ty (ctx, LLVMVectorType (LLVMIntType (1), LLVMGetVectorSize (ret_t)), result); + result = vector_from_scalar (ctx, LLVMVectorType (LLVMIntType (1), LLVMGetVectorSize (ret_t)), result); values [ins->dreg] = LLVMBuildSExt (builder, result, ret_t, ""); break; } @@ -9819,7 +9802,7 @@ process_bb (EmitContext *ctx, MonoBasicBlock *bb) else result = LLVMBuildSIToFP (builder, result, cvt_t, "arm64_scvtf"); if (scalar) - result = vector_from_scalar_ty (ctx, ret_t, result); + result = vector_from_scalar (ctx, ret_t, result); values [ins->dreg] = result; break; } @@ -9845,7 +9828,7 @@ process_bb (EmitContext *ctx, MonoBasicBlock *bb) else result = LLVMBuildFPToSI (builder, result, cvt_t, "arm64_fcvtzs"); if (scalar) - result = vector_from_scalar_ty (ctx, ret_t, result); + result = vector_from_scalar (ctx, ret_t, result); values [ins->dreg] = result; break; } @@ -9854,7 +9837,7 @@ process_bb (EmitContext *ctx, MonoBasicBlock *bb) LLVMTypeRef elem_t = LLVMTypeOf (result); unsigned int elem_bits = mono_llvm_get_prim_size_bits (elem_t); LLVMTypeRef t = LLVMVectorType (elem_t, 64 / elem_bits); - result = vector_from_scalar_ty (ctx, t, result); + result = vector_from_scalar (ctx, t, result); values [ins->dreg] = result; break; } @@ -9914,12 +9897,12 @@ process_bb (EmitContext *ctx, MonoBasicBlock *bb) if (elem_t == i4_t) { LLVMValueRef arg = scalar_from_vector (ctx, lhs); result = call_intrins (ctx, scalar_iid, &arg, "arm64_xnarrow_scalar"); - result = vector_from_scalar_ty (ctx, ret_t, result); + result = vector_from_scalar (ctx, ret_t, result); } else { LLVMTypeRef arg_t = LLVMTypeOf (lhs); LLVMTypeRef argelem_t = LLVMGetElementType (arg_t); unsigned int argelems = LLVMGetVectorSize (arg_t); - LLVMValueRef arg = undef_upper_elements (ctx, LLVMVectorType (argelem_t, argelems * 2), lhs); + LLVMValueRef arg = keep_lowest_element (ctx, LLVMVectorType (argelem_t, argelems * 2), lhs); result = call_overloaded_intrins (ctx, iid, ovr_tag, &arg, "arm64_xnarrow_scalar"); result = keep_lowest_element (ctx, LLVMTypeOf (result), result); } @@ -10008,7 +9991,7 @@ process_bb (EmitContext *ctx, MonoBasicBlock *bb) } LLVMValueRef result = call_overloaded_intrins (ctx, INTRINS_AARCH64_ADV_SIMD_FMA, ovr_tag, args, "arm64_fma"); if (scalar) - result = vector_from_scalar (ctx, lhs, result); + result = vector_from_scalar (ctx, LLVMTypeOf (lhs), result); values [ins->dreg] = result; break; } @@ -10122,8 +10105,8 @@ process_bb (EmitContext *ctx, MonoBasicBlock *bb) mularg = scalar_from_vector (ctx, mularg); selected_scalar = scalar_from_vector (ctx, selected_scalar); } else { - mularg = undef_upper_elements (ctx, mularg_t, mularg); - selected_scalar = undef_upper_elements (ctx, mularg_t, selected_scalar); + mularg = keep_lowest_element (ctx, mularg_t, mularg); + selected_scalar = keep_lowest_element (ctx, mularg_t, selected_scalar); } LLVMValueRef mulargs [] = { mularg, selected_scalar }; LLVMValueRef result = call_overloaded_intrins (ctx, mulid, multag, mulargs, "arm64_sqdmull_scalar"); @@ -10137,7 +10120,7 @@ process_bb (EmitContext *ctx, MonoBasicBlock *bb) scalar_acc_result = TRUE; } if (scalar_acc_result) - result = vector_from_scalar_ty (ctx, ret_t, result); + result = vector_from_scalar (ctx, ret_t, result); else result = keep_lowest_element (ctx, ret_t, result); values [ins->dreg] = result; @@ -10147,7 +10130,7 @@ process_bb (EmitContext *ctx, MonoBasicBlock *bb) LLVMValueRef mul2 = LLVMBuildExtractElement (builder, rhs, arg3, ""); LLVMValueRef mul1 = scalar_from_vector (ctx, lhs); LLVMValueRef result = LLVMBuildFMul (builder, mul1, mul2, "arm64_fmul_sel"); - result = vector_from_scalar (ctx, lhs, result); + result = vector_from_scalar (ctx, LLVMTypeOf (lhs), result); values [ins->dreg] = result; break; } @@ -10266,7 +10249,7 @@ process_bb (EmitContext *ctx, MonoBasicBlock *bb) else result = LLVMBuildNeg (builder, result, "arm64_xneg"); if (scalar) - result = vector_from_scalar (ctx, lhs, result); + result = vector_from_scalar (ctx, LLVMTypeOf (lhs), result); values [ins->dreg] = result; break; } @@ -10422,7 +10405,7 @@ process_bb (EmitContext *ctx, MonoBasicBlock *bb) if (scalar) { unsigned int elems = LLVMGetVectorSize (shift_arg_t); LLVMValueRef lo = scalar_from_vector (ctx, shift_arg); - shift_arg = vector_from_scalar_ty (ctx, LLVMVectorType (shift_arg_elem_t, elems * 2), lo); + shift_arg = vector_from_scalar (ctx, LLVMVectorType (shift_arg_elem_t, elems * 2), lo); } int max_index = range_max - range_min + 1; ImmediateUnrollCtx ictx = immediate_unroll_begin (ctx, bb, max_index, shift_amount, intrin_result_t, "arm64_xnshift"); @@ -10751,7 +10734,7 @@ process_bb (EmitContext *ctx, MonoBasicBlock *bb) // @llvm.aarch64.neon.saddv.i32.v8i16 ought to return an i16, but doesn't in LLVM 9. result = LLVMBuildTrunc (builder, result, elem_t, ""); } - result = vector_from_scalar_ty (ctx, ret_t, result); + result = vector_from_scalar (ctx, ret_t, result); values [ins->dreg] = result; break; } @@ -10768,7 +10751,7 @@ process_bb (EmitContext *ctx, MonoBasicBlock *bb) // @llvm.aarch64.neon.saddlv.i32.v16i8 ought to return an i16, but doesn't in LLVM 9. result = LLVMBuildTrunc (builder, result, i2_t, ""); } - result = vector_from_scalar_ty (ctx, ret_t, result); + result = vector_from_scalar (ctx, ret_t, result); values [ins->dreg] = result; break; } @@ -10887,7 +10870,7 @@ process_bb (EmitContext *ctx, MonoBasicBlock *bb) LLVMValueRef result = call_overloaded_intrins (ctx, iid, ovr_tag, args, ""); result = LLVMBuildBitCast (builder, result, result_t, ""); if (scalar) - result = vector_from_scalar_ty (ctx, ret_t, result); + result = vector_from_scalar (ctx, ret_t, result); values [ins->dreg] = result; break; } From 24a89e1478b73704aafb73f9b8d41a846673e2b5 Mon Sep 17 00:00:00 2001 From: Imran Hameed Date: Wed, 10 Mar 2021 12:45:30 -0800 Subject: [PATCH 57/58] Fix the vector concatenation overloads of Vector128/256 --- src/mono/mono/mini/mini-llvm.c | 4 ++++ src/mono/mono/mini/mini-ops.h | 2 ++ src/mono/mono/mini/simd-intrinsics.c | 36 +++++++++++++++++++++++++++- 3 files changed, 41 insertions(+), 1 deletion(-) diff --git a/src/mono/mono/mini/mini-llvm.c b/src/mono/mono/mini/mini-llvm.c index 5d27e263997aa5..5c97d4e316217e 100644 --- a/src/mono/mono/mini/mini-llvm.c +++ b/src/mono/mono/mini/mini-llvm.c @@ -7550,6 +7550,10 @@ process_bb (EmitContext *ctx, MonoBasicBlock *bb) values [ins->dreg] = LLVMBuildBitCast (builder, lhs, t, ""); break; } + case OP_XCONCAT: { + values [ins->dreg] = concatenate_vectors (ctx, lhs, rhs); + break; + } #endif // defined(TARGET_X86) || defined(TARGET_AMD64) || defined(TARGET_ARM64) || defined(TARGET_WASM) #if defined(TARGET_X86) || defined(TARGET_AMD64) || defined(TARGET_WASM) diff --git a/src/mono/mono/mini/mini-ops.h b/src/mono/mono/mini/mini-ops.h index 4bd23c5e803b99..f11da0f5ebf390 100644 --- a/src/mono/mono/mini/mini-ops.h +++ b/src/mono/mono/mini/mini-ops.h @@ -1565,6 +1565,8 @@ MINI_OP(OP_XOP_OVR_SCALAR_X_X_X, "xop_ovr_scalar_x_x_x", XREG, XREG, XREG) MINI_OP3(OP_XOP_OVR_SCALAR_X_X_X_X, "xop_ovr_scalar_x_x_x_x", XREG, XREG, XREG, XREG) MINI_OP(OP_XOP_OVR_BYSCALAR_X_X_X, "xop_ovr_byscalar_x_x_x", XREG, XREG, XREG) +MINI_OP(OP_XCONCAT, "xconcat", XREG, XREG, XREG) + MINI_OP(OP_XCAST, "xcast", XREG, XREG, NONE) /* Extract element of vector */ /* The index is assumed to be in range */ diff --git a/src/mono/mono/mini/simd-intrinsics.c b/src/mono/mono/mini/simd-intrinsics.c index a7da1df3d41c6a..04b9d903a20c43 100644 --- a/src/mono/mono/mini/simd-intrinsics.c +++ b/src/mono/mono/mini/simd-intrinsics.c @@ -257,6 +257,15 @@ emit_xcompare (MonoCompile *cfg, MonoClass *klass, MonoTypeEnum etype, MonoInst return ins; } +static gboolean +is_intrinsics_vector_type (MonoType *vector_type) +{ + if (vector_type->type != MONO_TYPE_GENERICINST) return FALSE; + MonoClass *klass = mono_class_from_mono_type_internal (vector_type); + const char *name = m_class_get_name (klass); + return !strcmp (name, "Vector64`1") || !strcmp (name, "Vector128`1") || !strcmp (name, "Vector256`1"); +} + static MonoType* get_vector_t_elem_type (MonoType *vector_type) { @@ -487,6 +496,28 @@ static guint16 sri_vector_methods [] = { SN_CreateScalarUnsafe, }; +static gboolean +is_elementwise_create_overload (MonoMethodSignature *fsig, MonoType *ret_type) +{ + uint16_t param_count = fsig->param_count; + if (param_count < 1) return FALSE; + MonoType *type = fsig->params [0]; + gboolean is_vector_primitive = MONO_TYPE_IS_PRIMITIVE (type) && (type->type >= MONO_TYPE_I1 && type->type <= MONO_TYPE_R8); + if (!is_vector_primitive) return FALSE; + if (!mono_metadata_type_equal (ret_type, type)) return FALSE; + for (uint16_t i = 1; i < param_count; ++i) + if (!mono_metadata_type_equal (type, fsig->params [i])) return FALSE; + return TRUE; +} + +static gboolean +is_create_from_half_vectors_overload (MonoMethodSignature *fsig) +{ + if (fsig->param_count != 2) return FALSE; + if (!is_intrinsics_vector_type (fsig->params [0])) return FALSE; + return mono_metadata_type_equal (fsig->params [0], fsig->params [1]); +} + static MonoInst* emit_sri_vector (MonoCompile *cfg, MonoMethod *cmethod, MonoMethodSignature *fsig, MonoInst **args) { @@ -519,8 +550,11 @@ emit_sri_vector (MonoCompile *cfg, MonoMethod *cmethod, MonoMethodSignature *fsi MonoType *etype = get_vector_t_elem_type (fsig->ret); if (fsig->param_count == 1 && mono_metadata_type_equal (fsig->params [0], etype)) return emit_simd_ins (cfg, klass, type_to_expand_op (etype), args [0]->dreg, -1); - else + else if (is_create_from_half_vectors_overload (fsig)) + return emit_simd_ins (cfg, klass, OP_XCONCAT, args [0]->dreg, args [1]->dreg); + else if (is_elementwise_create_overload (fsig, etype)) return emit_vector_create_elementwise (cfg, fsig, fsig->ret, etype, args); + break; } case SN_CreateScalarUnsafe: return emit_simd_ins_for_sig (cfg, klass, OP_CREATE_SCALAR_UNSAFE, -1, arg0_type, fsig, args); From 14602df828f72d1bb22e8b08313179d1393e618b Mon Sep 17 00:00:00 2001 From: Imran Hameed Date: Thu, 11 Mar 2021 11:21:29 -0800 Subject: [PATCH 58/58] Sha1.FixedRotate is a scalar-in-vector op. TODO: refactor to use XOP_SCALAR_X_X --- src/mono/mono/mini/mini-llvm.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/mono/mono/mini/mini-llvm.c b/src/mono/mono/mini/mini-llvm.c index 5c97d4e316217e..2ea44c5dcccdce 100644 --- a/src/mono/mono/mini/mini-llvm.c +++ b/src/mono/mono/mini/mini-llvm.c @@ -9616,6 +9616,7 @@ process_bb (EmitContext *ctx, MonoBasicBlock *bb) } case OP_XOP_X_X: { IntrinsicId id = (IntrinsicId)0; + LLVMTypeRef ret_t = simd_class_to_llvm_type (ctx, ins->klass); gboolean getLowerElement = FALSE; switch (ins->opcode) { default: @@ -9630,6 +9631,8 @@ process_bb (EmitContext *ctx, MonoBasicBlock *bb) if (getLowerElement) arg0 = LLVMBuildExtractElement (ctx->builder, arg0, const_int32 (0), ""); LLVMValueRef result = call_intrins (ctx, id, &arg0, ""); + if (getLowerElement) + result = vector_from_scalar (ctx, ret_t, result); values [ins->dreg] = result; break; }