Skip to content

[x86] Codegen phaddw, phaddd, and pmaddwd #6878

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
rootjalex wants to merge 4 commits into from
Closed

[x86] Codegen phaddw, phaddd, and pmaddwd #6878

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
625c18b
x86 should use phaddw and phaddd
Jul 21, 2022
fba66fd
remove accidental duplicat code
Jul 21, 2022
23e3d13
add simd_op_check tests
Jul 21, 2022
0acae70
fix avx2 phadd shuffles
Jul 22, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
23 changes: 20 additions & 3 deletions src/CodeGen_X86.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -185,9 +185,15 @@ const x86Intrinsic intrinsic_defs[] = {
// LLVM does not provide an unmasked 128bit cvtneps2bf16 intrinsic, so provide a wrapper around the masked version.
{"vcvtneps2bf16x4", BFloat(16, 4), "f32_to_bf16", {Float(32, 4)}, Target::AVX512_SapphireRapids},

// 2-way dot products
{"llvm.x86.avx2.pmadd.ub.sw", Int(16, 16), "saturating_dot_product", {UInt(8, 32), Int(8, 32)}, Target::AVX2},
{"llvm.x86.ssse3.pmadd.ub.sw.128", Int(16, 8), "saturating_dot_product", {UInt(8, 16), Int(8, 16)}, Target::SSE41},
// Horizontal adds that use (v)phadd(w | d).
{"phaddw_sse3", UInt(16, 8), "horizontal_add", {UInt(16, 16)}, Target::SSE41},
{"phaddw_sse3", Int(16, 8), "horizontal_add", {Int(16, 16)}, Target::SSE41},
{"phaddw_avx2", UInt(16, 16), "horizontal_add", {UInt(16, 32)}, Target::AVX2},
{"phaddw_avx2", Int(16, 16), "horizontal_add", {Int(16, 32)}, Target::AVX2},
{"phaddd_sse3", UInt(32, 4), "horizontal_add", {UInt(32, 8)}, Target::SSE41},
{"phaddd_sse3", Int(32, 4), "horizontal_add", {Int(32, 8)}, Target::SSE41},
{"phaddd_avx2", UInt(32, 8), "horizontal_add", {UInt(32, 16)}, Target::AVX2},
{"phaddd_avx2", Int(32, 8), "horizontal_add", {Int(32, 16)}, Target::AVX2},

// Horizontal widening adds using 2-way dot products.
{"hadd_pmadd_u8_sse3", UInt(16, 8), "horizontal_widening_add", {UInt(8, 16)}, Target::SSE41},
Expand All @@ -196,6 +202,12 @@ const x86Intrinsic intrinsic_defs[] = {
{"hadd_pmadd_u8_avx2", UInt(16, 16), "horizontal_widening_add", {UInt(8, 32)}, Target::AVX2},
{"hadd_pmadd_u8_avx2", Int(16, 16), "horizontal_widening_add", {UInt(8, 32)}, Target::AVX2},
{"hadd_pmadd_i8_avx2", Int(16, 16), "horizontal_widening_add", {Int(8, 32)}, Target::AVX2},
{"hadd_pmadd_i16_sse2", Int(32, 4), "horizontal_widening_add", {Int(16, 8)}},
{"hadd_pmadd_i16_avx2", Int(32, 8), "horizontal_widening_add", {Int(16, 16)}, Target::AVX2},

// 2-way dot products
{"llvm.x86.avx2.pmadd.ub.sw", Int(16, 16), "saturating_dot_product", {UInt(8, 32), Int(8, 32)}, Target::AVX2},
{"llvm.x86.ssse3.pmadd.ub.sw.128", Int(16, 8), "saturating_dot_product", {UInt(8, 16), Int(8, 16)}, Target::SSE41},

{"llvm.x86.avx512.pmaddw.d.512", Int(32, 16), "dot_product", {Int(16, 32), Int(16, 32)}, Target::AVX512_Skylake},
{"llvm.x86.avx512.pmaddw.d.512", Int(32, 16), "dot_product", {Int(16, 32), Int(16, 32)}, Target::AVX512_Cannonlake},
Expand Down Expand Up @@ -639,6 +651,11 @@ void CodeGen_X86::codegen_vector_reduce(const VectorReduce *op, const Expr &init
{VectorReduce::Add, 2, u16(wild_u8x_), "horizontal_widening_add", {}, Pattern::SingleArg},
{VectorReduce::Add, 2, i16(wild_u8x_), "horizontal_widening_add", {}, Pattern::SingleArg},
{VectorReduce::Add, 2, i16(wild_i8x_), "horizontal_widening_add", {}, Pattern::SingleArg},
{VectorReduce::Add, 2, i32(wild_i16x_), "horizontal_widening_add", {}, Pattern::SingleArg},
{VectorReduce::Add, 2, wild_u16x_, "horizontal_add", {}, Pattern::SingleArg},
{VectorReduce::Add, 2, wild_i16x_, "horizontal_add", {}, Pattern::SingleArg},
{VectorReduce::Add, 2, wild_u32x_, "horizontal_add", {}, Pattern::SingleArg},
{VectorReduce::Add, 2, wild_i32x_, "horizontal_add", {}, Pattern::SingleArg},
};
// clang-format on

Expand Down
15 changes: 4 additions & 11 deletions src/runtime/x86.ll
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -51,17 +51,6 @@ define weak_odr <8 x i16> @packssdwx8(<8 x i32> %arg) nounwind alwaysinline {
ret <8 x i16> %3
}

define weak_odr <8 x i32> @wmul_pmaddwd_avx2(<8 x i16> %a, <8 x i16> %b) nounwind alwaysinline {
%1 = zext <8 x i16> %a to <8 x i32>
%2 = zext <8 x i16> %b to <8 x i32>
%3 = bitcast <8 x i32> %1 to <16 x i16>
%4 = bitcast <8 x i32> %2 to <16 x i16>
%res = call <8 x i32> @llvm.x86.avx2.pmadd.wd(<16 x i16> %3, <16 x i16> %4)
ret <8 x i32> %res
}

declare <8 x i32> @llvm.x86.avx2.pmadd.wd(<16 x i16>, <16 x i16>) nounwind readnone

define weak_odr <4 x i32> @wmul_pmaddwd_sse2(<4 x i16> %a, <4 x i16> %b) nounwind alwaysinline {
%1 = zext <4 x i16> %a to <4 x i32>
%2 = zext <4 x i16> %b to <4 x i32>
Expand All @@ -71,6 +60,10 @@ define weak_odr <4 x i32> @wmul_pmaddwd_sse2(<4 x i16> %a, <4 x i16> %b) nounwin
ret <4 x i32> %res
}

define weak_odr <4 x i32> @hadd_pmadd_i16_sse2(<8 x i16> %a) nounwind alwaysinline {
%1 = tail call <4 x i32> @llvm.x86.sse2.pmadd.wd(<8 x i16> %a, <8 x i16> <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1>)
ret <4 x i32> %1
}
declare <4 x i32> @llvm.x86.sse2.pmadd.wd(<8 x i16>, <8 x i16>) nounwind readnone

define weak_odr <4 x float> @sqrt_f32x4(<4 x float> %x) nounwind uwtable readnone alwaysinline {
Expand Down
31 changes: 31 additions & 0 deletions src/runtime/x86_avx2.ll
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -72,3 +72,34 @@ define weak_odr <16 x i16> @hadd_pmadd_i8_avx2(<32 x i8> %a) nounwind alwaysinli
ret <16 x i16> %1
}
declare <16 x i16> @llvm.x86.avx2.pmadd.ub.sw(<32 x i8>, <32 x i8>) nounwind readnone

define weak_odr <16 x i16> @phaddw_avx2(<32 x i16> %a) nounwind alwaysinline {
%1 = shufflevector <32 x i16> %a, <32 x i16> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23>
%2 = shufflevector <32 x i16> %a, <32 x i16> undef, <16 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
%3 = tail call <16 x i16> @llvm.x86.avx2.phadd.w(<16 x i16> %1, <16 x i16> %2)
ret <16 x i16> %3
}
declare <16 x i16> @llvm.x86.avx2.phadd.w(<16 x i16>, <16 x i16>) nounwind readnone

define weak_odr <8 x i32> @phaddd_avx2(<16 x i32> %a) nounwind alwaysinline {
%1 = shufflevector <16 x i32> %a, <16 x i32> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>
%2 = shufflevector <16 x i32> %a, <16 x i32> undef, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 12, i32 13, i32 14, i32 15>
%3 = tail call <8 x i32> @llvm.x86.avx2.phadd.d(<8 x i32> %1, <8 x i32> %2)
ret <8 x i32> %3
}
declare <8 x i32> @llvm.x86.avx2.phadd.d(<8 x i32>, <8 x i32>) nounwind readnone

define weak_odr <8 x i32> @hadd_pmadd_i16_avx2(<16 x i16> %a) nounwind alwaysinline {
%1 = tail call <8 x i32> @llvm.x86.avx2.pmadd.wd(<16 x i16> %a, <16 x i16> <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1>)
ret <8 x i32> %1
}

define weak_odr <8 x i32> @wmul_pmaddwd_avx2(<8 x i16> %a, <8 x i16> %b) nounwind alwaysinline {
%1 = zext <8 x i16> %a to <8 x i32>
%2 = zext <8 x i16> %b to <8 x i32>
%3 = bitcast <8 x i32> %1 to <16 x i16>
%4 = bitcast <8 x i32> %2 to <16 x i16>
%res = call <8 x i32> @llvm.x86.avx2.pmadd.wd(<16 x i16> %3, <16 x i16> %4)
ret <8 x i32> %res
}
declare <8 x i32> @llvm.x86.avx2.pmadd.wd(<16 x i16>, <16 x i16>) nounwind readnone
16 changes: 16 additions & 0 deletions src/runtime/x86_sse41.ll
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -92,3 +92,19 @@ define weak_odr <8 x i16> @hadd_pmadd_i8_sse3(<16 x i8> %a) nounwind alwaysinlin
ret <8 x i16> %1
}
declare <8 x i16> @llvm.x86.ssse3.pmadd.ub.sw.128(<16 x i8>, <16 x i8>) nounwind readnone

define weak_odr <8 x i16> @phaddw_sse3(<16 x i16> %a) nounwind alwaysinline {
%1 = shufflevector <16 x i16> %a, <16 x i16> undef, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
%2 = shufflevector <16 x i16> %a, <16 x i16> undef, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
%3 = tail call <8 x i16> @llvm.x86.ssse3.phadd.w.128(<8 x i16> %1, <8 x i16> %2)
ret <8 x i16> %3
}
declare <8 x i16> @llvm.x86.ssse3.phadd.w.128(<8 x i16>, <8 x i16>) nounwind readnone

define weak_odr <4 x i32> @phaddd_sse3(<8 x i32> %a) nounwind alwaysinline {
%1 = shufflevector <8 x i32> %a, <8 x i32> undef, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
%2 = shufflevector <8 x i32> %a, <8 x i32> undef, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
%3 = tail call <4 x i32> @llvm.x86.ssse3.phadd.d.128(<4 x i32> %1, <4 x i32> %2)
ret <4 x i32> %3
}
declare <4 x i32> @llvm.x86.ssse3.phadd.d.128(<4 x i32>, <4 x i32>) nounwind readnone
20 changes: 16 additions & 4 deletions test/correctness/simd_op_check.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -277,10 +277,19 @@ class SimdOpCheck : public SimdOpCheckTest {
check("movshdup", 1, sum(in_f32(RDom(0, 4) + 4 * x)));
check("movshdup", 1, sum(in_f32(RDom(0, 16) + 16 * x)));

// The integer horizontal add operations are pretty
// terrible on all x86 variants, and LLVM does its best to
// avoid generating those too, so we won't test that here
// either.
// Integer horizontal add operations are cheaper in reductions
// than the permute + padd alternatives.
for (int w = 1; w <= 8; w++) {
const char *check_phaddw =
(use_avx2 && w >= 4) ? "vphaddw" : "phaddw";
const char *check_phaddd =
(use_avx2 && w >= 4) ? "vphaddd" : "phaddd";

const int factor = 2 * w;
RDom r2(0, factor);
check(check_phaddw, 4 * w, sum(in_i16(r2 + (factor * x))));
check(check_phaddd, 4 * w, sum(in_i32(r2 + (factor * x))));
}

// Min reductions should use phminposuw when
// possible. This only exists for u16. X86 is weird.
Expand Down Expand Up @@ -331,6 +340,9 @@ class SimdOpCheck : public SimdOpCheckTest {

// Also generate for widening_mul
check(check_pmaddwd, 2 * w, i32(i16_1) * i32(i16_2));

// Also generated for horizontal widening adds
check(check_pmaddwd, 2 * w, sum(i32(in_i16(RDom(0, 2) + (2 * x)))));
}

// llvm doesn't distinguish between signed and unsigned multiplies
Expand Down