diff --git a/src/CodeGen_X86.cpp b/src/CodeGen_X86.cpp index 7529c31688f4..31325b43239d 100644 --- a/src/CodeGen_X86.cpp +++ b/src/CodeGen_X86.cpp @@ -185,9 +185,15 @@ const x86Intrinsic intrinsic_defs[] = { // LLVM does not provide an unmasked 128bit cvtneps2bf16 intrinsic, so provide a wrapper around the masked version. {"vcvtneps2bf16x4", BFloat(16, 4), "f32_to_bf16", {Float(32, 4)}, Target::AVX512_SapphireRapids}, - // 2-way dot products - {"llvm.x86.avx2.pmadd.ub.sw", Int(16, 16), "saturating_dot_product", {UInt(8, 32), Int(8, 32)}, Target::AVX2}, - {"llvm.x86.ssse3.pmadd.ub.sw.128", Int(16, 8), "saturating_dot_product", {UInt(8, 16), Int(8, 16)}, Target::SSE41}, + // Horizontal adds that use (v)phadd(w | d). + {"phaddw_sse3", UInt(16, 8), "horizontal_add", {UInt(16, 16)}, Target::SSE41}, + {"phaddw_sse3", Int(16, 8), "horizontal_add", {Int(16, 16)}, Target::SSE41}, + {"phaddw_avx2", UInt(16, 16), "horizontal_add", {UInt(16, 32)}, Target::AVX2}, + {"phaddw_avx2", Int(16, 16), "horizontal_add", {Int(16, 32)}, Target::AVX2}, + {"phaddd_sse3", UInt(32, 4), "horizontal_add", {UInt(32, 8)}, Target::SSE41}, + {"phaddd_sse3", Int(32, 4), "horizontal_add", {Int(32, 8)}, Target::SSE41}, + {"phaddd_avx2", UInt(32, 8), "horizontal_add", {UInt(32, 16)}, Target::AVX2}, + {"phaddd_avx2", Int(32, 8), "horizontal_add", {Int(32, 16)}, Target::AVX2}, // Horizontal widening adds using 2-way dot products. {"hadd_pmadd_u8_sse3", UInt(16, 8), "horizontal_widening_add", {UInt(8, 16)}, Target::SSE41}, @@ -196,6 +202,12 @@ const x86Intrinsic intrinsic_defs[] = { {"hadd_pmadd_u8_avx2", UInt(16, 16), "horizontal_widening_add", {UInt(8, 32)}, Target::AVX2}, {"hadd_pmadd_u8_avx2", Int(16, 16), "horizontal_widening_add", {UInt(8, 32)}, Target::AVX2}, {"hadd_pmadd_i8_avx2", Int(16, 16), "horizontal_widening_add", {Int(8, 32)}, Target::AVX2}, + {"hadd_pmadd_i16_sse2", Int(32, 4), "horizontal_widening_add", {Int(16, 8)}}, + {"hadd_pmadd_i16_avx2", Int(32, 8), "horizontal_widening_add", {Int(16, 16)}, Target::AVX2}, + + // 2-way dot products + {"llvm.x86.avx2.pmadd.ub.sw", Int(16, 16), "saturating_dot_product", {UInt(8, 32), Int(8, 32)}, Target::AVX2}, + {"llvm.x86.ssse3.pmadd.ub.sw.128", Int(16, 8), "saturating_dot_product", {UInt(8, 16), Int(8, 16)}, Target::SSE41}, {"llvm.x86.avx512.pmaddw.d.512", Int(32, 16), "dot_product", {Int(16, 32), Int(16, 32)}, Target::AVX512_Skylake}, {"llvm.x86.avx512.pmaddw.d.512", Int(32, 16), "dot_product", {Int(16, 32), Int(16, 32)}, Target::AVX512_Cannonlake}, @@ -639,6 +651,11 @@ void CodeGen_X86::codegen_vector_reduce(const VectorReduce *op, const Expr &init {VectorReduce::Add, 2, u16(wild_u8x_), "horizontal_widening_add", {}, Pattern::SingleArg}, {VectorReduce::Add, 2, i16(wild_u8x_), "horizontal_widening_add", {}, Pattern::SingleArg}, {VectorReduce::Add, 2, i16(wild_i8x_), "horizontal_widening_add", {}, Pattern::SingleArg}, + {VectorReduce::Add, 2, i32(wild_i16x_), "horizontal_widening_add", {}, Pattern::SingleArg}, + {VectorReduce::Add, 2, wild_u16x_, "horizontal_add", {}, Pattern::SingleArg}, + {VectorReduce::Add, 2, wild_i16x_, "horizontal_add", {}, Pattern::SingleArg}, + {VectorReduce::Add, 2, wild_u32x_, "horizontal_add", {}, Pattern::SingleArg}, + {VectorReduce::Add, 2, wild_i32x_, "horizontal_add", {}, Pattern::SingleArg}, }; // clang-format on diff --git a/src/runtime/x86.ll b/src/runtime/x86.ll index 6e33e9f9f51f..8e40b14e6e22 100644 --- a/src/runtime/x86.ll +++ b/src/runtime/x86.ll @@ -51,17 +51,6 @@ define weak_odr <8 x i16> @packssdwx8(<8 x i32> %arg) nounwind alwaysinline { ret <8 x i16> %3 } -define weak_odr <8 x i32> @wmul_pmaddwd_avx2(<8 x i16> %a, <8 x i16> %b) nounwind alwaysinline { - %1 = zext <8 x i16> %a to <8 x i32> - %2 = zext <8 x i16> %b to <8 x i32> - %3 = bitcast <8 x i32> %1 to <16 x i16> - %4 = bitcast <8 x i32> %2 to <16 x i16> - %res = call <8 x i32> @llvm.x86.avx2.pmadd.wd(<16 x i16> %3, <16 x i16> %4) - ret <8 x i32> %res -} - -declare <8 x i32> @llvm.x86.avx2.pmadd.wd(<16 x i16>, <16 x i16>) nounwind readnone - define weak_odr <4 x i32> @wmul_pmaddwd_sse2(<4 x i16> %a, <4 x i16> %b) nounwind alwaysinline { %1 = zext <4 x i16> %a to <4 x i32> %2 = zext <4 x i16> %b to <4 x i32> @@ -71,6 +60,10 @@ define weak_odr <4 x i32> @wmul_pmaddwd_sse2(<4 x i16> %a, <4 x i16> %b) nounwin ret <4 x i32> %res } +define weak_odr <4 x i32> @hadd_pmadd_i16_sse2(<8 x i16> %a) nounwind alwaysinline { + %1 = tail call <4 x i32> @llvm.x86.sse2.pmadd.wd(<8 x i16> %a, <8 x i16> ) + ret <4 x i32> %1 +} declare <4 x i32> @llvm.x86.sse2.pmadd.wd(<8 x i16>, <8 x i16>) nounwind readnone define weak_odr <4 x float> @sqrt_f32x4(<4 x float> %x) nounwind uwtable readnone alwaysinline { diff --git a/src/runtime/x86_avx2.ll b/src/runtime/x86_avx2.ll index 1a80f5b583d3..2b9223dd640d 100644 --- a/src/runtime/x86_avx2.ll +++ b/src/runtime/x86_avx2.ll @@ -72,3 +72,34 @@ define weak_odr <16 x i16> @hadd_pmadd_i8_avx2(<32 x i8> %a) nounwind alwaysinli ret <16 x i16> %1 } declare <16 x i16> @llvm.x86.avx2.pmadd.ub.sw(<32 x i8>, <32 x i8>) nounwind readnone + +define weak_odr <16 x i16> @phaddw_avx2(<32 x i16> %a) nounwind alwaysinline { + %1 = shufflevector <32 x i16> %a, <32 x i16> undef, <16 x i32> + %2 = shufflevector <32 x i16> %a, <32 x i16> undef, <16 x i32> + %3 = tail call <16 x i16> @llvm.x86.avx2.phadd.w(<16 x i16> %1, <16 x i16> %2) + ret <16 x i16> %3 +} +declare <16 x i16> @llvm.x86.avx2.phadd.w(<16 x i16>, <16 x i16>) nounwind readnone + +define weak_odr <8 x i32> @phaddd_avx2(<16 x i32> %a) nounwind alwaysinline { + %1 = shufflevector <16 x i32> %a, <16 x i32> undef, <8 x i32> + %2 = shufflevector <16 x i32> %a, <16 x i32> undef, <8 x i32> + %3 = tail call <8 x i32> @llvm.x86.avx2.phadd.d(<8 x i32> %1, <8 x i32> %2) + ret <8 x i32> %3 +} +declare <8 x i32> @llvm.x86.avx2.phadd.d(<8 x i32>, <8 x i32>) nounwind readnone + +define weak_odr <8 x i32> @hadd_pmadd_i16_avx2(<16 x i16> %a) nounwind alwaysinline { + %1 = tail call <8 x i32> @llvm.x86.avx2.pmadd.wd(<16 x i16> %a, <16 x i16> ) + ret <8 x i32> %1 +} + +define weak_odr <8 x i32> @wmul_pmaddwd_avx2(<8 x i16> %a, <8 x i16> %b) nounwind alwaysinline { + %1 = zext <8 x i16> %a to <8 x i32> + %2 = zext <8 x i16> %b to <8 x i32> + %3 = bitcast <8 x i32> %1 to <16 x i16> + %4 = bitcast <8 x i32> %2 to <16 x i16> + %res = call <8 x i32> @llvm.x86.avx2.pmadd.wd(<16 x i16> %3, <16 x i16> %4) + ret <8 x i32> %res +} +declare <8 x i32> @llvm.x86.avx2.pmadd.wd(<16 x i16>, <16 x i16>) nounwind readnone diff --git a/src/runtime/x86_sse41.ll b/src/runtime/x86_sse41.ll index f109ee37ec23..87c729218bcc 100644 --- a/src/runtime/x86_sse41.ll +++ b/src/runtime/x86_sse41.ll @@ -92,3 +92,19 @@ define weak_odr <8 x i16> @hadd_pmadd_i8_sse3(<16 x i8> %a) nounwind alwaysinlin ret <8 x i16> %1 } declare <8 x i16> @llvm.x86.ssse3.pmadd.ub.sw.128(<16 x i8>, <16 x i8>) nounwind readnone + +define weak_odr <8 x i16> @phaddw_sse3(<16 x i16> %a) nounwind alwaysinline { + %1 = shufflevector <16 x i16> %a, <16 x i16> undef, <8 x i32> + %2 = shufflevector <16 x i16> %a, <16 x i16> undef, <8 x i32> + %3 = tail call <8 x i16> @llvm.x86.ssse3.phadd.w.128(<8 x i16> %1, <8 x i16> %2) + ret <8 x i16> %3 +} +declare <8 x i16> @llvm.x86.ssse3.phadd.w.128(<8 x i16>, <8 x i16>) nounwind readnone + +define weak_odr <4 x i32> @phaddd_sse3(<8 x i32> %a) nounwind alwaysinline { + %1 = shufflevector <8 x i32> %a, <8 x i32> undef, <4 x i32> + %2 = shufflevector <8 x i32> %a, <8 x i32> undef, <4 x i32> + %3 = tail call <4 x i32> @llvm.x86.ssse3.phadd.d.128(<4 x i32> %1, <4 x i32> %2) + ret <4 x i32> %3 +} +declare <4 x i32> @llvm.x86.ssse3.phadd.d.128(<4 x i32>, <4 x i32>) nounwind readnone diff --git a/test/correctness/simd_op_check.cpp b/test/correctness/simd_op_check.cpp index 5f2258b65f55..6fff40e5d27b 100644 --- a/test/correctness/simd_op_check.cpp +++ b/test/correctness/simd_op_check.cpp @@ -277,10 +277,19 @@ class SimdOpCheck : public SimdOpCheckTest { check("movshdup", 1, sum(in_f32(RDom(0, 4) + 4 * x))); check("movshdup", 1, sum(in_f32(RDom(0, 16) + 16 * x))); - // The integer horizontal add operations are pretty - // terrible on all x86 variants, and LLVM does its best to - // avoid generating those too, so we won't test that here - // either. + // Integer horizontal add operations are cheaper in reductions + // than the permute + padd alternatives. + for (int w = 1; w <= 8; w++) { + const char *check_phaddw = + (use_avx2 && w >= 4) ? "vphaddw" : "phaddw"; + const char *check_phaddd = + (use_avx2 && w >= 4) ? "vphaddd" : "phaddd"; + + const int factor = 2 * w; + RDom r2(0, factor); + check(check_phaddw, 4 * w, sum(in_i16(r2 + (factor * x)))); + check(check_phaddd, 4 * w, sum(in_i32(r2 + (factor * x)))); + } // Min reductions should use phminposuw when // possible. This only exists for u16. X86 is weird. @@ -331,6 +340,9 @@ class SimdOpCheck : public SimdOpCheckTest { // Also generate for widening_mul check(check_pmaddwd, 2 * w, i32(i16_1) * i32(i16_2)); + + // Also generated for horizontal widening adds + check(check_pmaddwd, 2 * w, sum(i32(in_i16(RDom(0, 2) + (2 * x))))); } // llvm doesn't distinguish between signed and unsigned multiplies