From a584a6f521423926edee60d3caa55704d026d5e1 Mon Sep 17 00:00:00 2001 From: James Burton Date: Wed, 1 Jul 2026 13:27:20 +0100 Subject: [PATCH 1/6] Add AvxIfma + AvxIfma.V512 hardware intrinsics (structural) Adds the managed AvxIfma class (V128/V256 VEX form) and nested AvxIfma.V512 (EVEX form) for VPMADD52LUQ / VPMADD52HUQ. Structural changes only (no tests yet, no build validation yet): - hwintrinsiclistxarch.h: new AVXIFMA range + AVX512v2 IFMA entries - hwintrinsicxarch.cpp: V512VersionOfIsa AVXIFMA/AVX512v2 case, lookupInstructionSet Ifma dispatch - hwintrinsiccodegenxarch.cpp: dispatch cases for MultiplyAdd52Low/High - lsraxarch.cpp: RMW cases - lowerxarch.cpp: default-branch assert extension - CorInfoInstructionSet.cs: nested type lookup + LookupPlatformIntrinsicTypes - InstructionSetDesc.txt: nested type alias AvxIfma_V512 - InstructionSetHelpers.cs: optimistic support - projitems: include new files - AvxIfma.cs + AvxIfma.PlatformNotSupported.cs: managed surface - System.Runtime.Intrinsics.cs: ref assembly Closes #98833, closes #96476. --- src/coreclr/jit/hwintrinsiccodegenxarch.cpp | 4 + src/coreclr/jit/hwintrinsiclistxarch.h | 14 +++- src/coreclr/jit/hwintrinsicxarch.cpp | 17 +++- src/coreclr/jit/lowerxarch.cpp | 3 + src/coreclr/jit/lsraxarch.cpp | 4 + .../tools/Common/InstructionSetHelpers.cs | 1 + .../JitInterface/CorInfoInstructionSet.cs | 52 ++++++++++++ .../ThunkGenerator/InstructionSetDesc.txt | 2 +- .../System.Private.CoreLib.Shared.projitems | 2 + .../X86/AvxIfma.PlatformNotSupported.cs | 77 +++++++++++++++++ .../System/Runtime/Intrinsics/X86/AvxIfma.cs | 82 +++++++++++++++++++ .../ref/System.Runtime.Intrinsics.cs | 22 +++++ 12 files changed, 277 insertions(+), 3 deletions(-) create mode 100644 src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/X86/AvxIfma.PlatformNotSupported.cs create mode 100644 src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/X86/AvxIfma.cs diff --git a/src/coreclr/jit/hwintrinsiccodegenxarch.cpp b/src/coreclr/jit/hwintrinsiccodegenxarch.cpp index 25dbdff2b9150b..e77a10a8e68348 100644 --- a/src/coreclr/jit/hwintrinsiccodegenxarch.cpp +++ b/src/coreclr/jit/hwintrinsiccodegenxarch.cpp @@ -904,6 +904,10 @@ void CodeGen::genHWIntrinsic(GenTreeHWIntrinsic* node) case NI_AVXVNNI_MultiplyWideningAndAddSaturate: case NI_AVX512v3_MultiplyWideningAndAdd: case NI_AVX512v3_MultiplyWideningAndAddSaturate: + case NI_AVXIFMA_MultiplyAdd52Low: + case NI_AVXIFMA_MultiplyAdd52High: + case NI_AVX512v2_MultiplyAdd52Low: + case NI_AVX512v2_MultiplyAdd52High: case NI_AVX512BMM_BitMultiplyMatrix16x16WithOrReduction: case NI_AVX512BMM_BitMultiplyMatrix16x16WithXorReduction: { diff --git a/src/coreclr/jit/hwintrinsiclistxarch.h b/src/coreclr/jit/hwintrinsiclistxarch.h index 93424d9966f1bc..d50997833dfac2 100644 --- a/src/coreclr/jit/hwintrinsiclistxarch.h +++ b/src/coreclr/jit/hwintrinsiclistxarch.h @@ -1096,7 +1096,9 @@ HARDWARE_INTRINSIC(AVX512v2, PermuteVar32x8, HARDWARE_INTRINSIC(AVX512v2, PermuteVar32x8x2, 32, 3, {INS_vpermt2b, INS_vpermt2b, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, 4, -1, HW_Category_SimpleSIMD, HW_Flag_SpecialCodeGen|HW_Flag_PermuteVar2x|HW_Flag_RmwIntrinsic) HARDWARE_INTRINSIC(AVX512v2, PermuteVar64x8, 64, 2, {INS_vpermb, INS_vpermb, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, 3, -1, HW_Category_SimpleSIMD, HW_Flag_SpecialImport) HARDWARE_INTRINSIC(AVX512v2, PermuteVar64x8x2, 64, 3, {INS_vpermt2b, INS_vpermt2b, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, 4, -1, HW_Category_SimpleSIMD, HW_Flag_SpecialCodeGen|HW_Flag_PermuteVar2x|HW_Flag_RmwIntrinsic) -#define LAST_NI_AVX512v2 NI_AVX512v2_PermuteVar64x8x2 +HARDWARE_INTRINSIC(AVX512v2, MultiplyAdd52Low, 64, 3, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vpmadd52luq, INS_invalid, INS_invalid}, 5, -1, HW_Category_SimpleSIMD, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AVX512v2, MultiplyAdd52High, 64, 3, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vpmadd52huq, INS_invalid, INS_invalid}, 5, -1, HW_Category_SimpleSIMD, HW_Flag_NoFlag) +#define LAST_NI_AVX512v2 NI_AVX512v2_MultiplyAdd52High // *************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************** // ISA Function name SIMD size NumArg Instructions IntCost FltCost Category Flags @@ -1161,6 +1163,16 @@ HARDWARE_INTRINSIC(AVXVNNI, MultiplyWideningAndAdd, HARDWARE_INTRINSIC(AVXVNNI, MultiplyWideningAndAddSaturate, -1, 3, {INS_invalid, INS_vpdpbusds, INS_vpdpwssds, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, 5, -1, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromSecondArg|HW_Flag_NoEvexSemantics) #define LAST_NI_AVXVNNI NI_AVXVNNI_MultiplyWideningAndAddSaturate +// *************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************** +// ISA Function name SIMD size NumArg Instructions IntCost FltCost Category Flags +// {TYP_BYTE, TYP_UBYTE, TYP_SHORT, TYP_USHORT, TYP_INT, TYP_UINT, TYP_LONG, TYP_ULONG, TYP_FLOAT, TYP_DOUBLE} +// *************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************** +// Intrinsics for AVX-IFMA (VEX-encoded VPMADD52*UQ on V128 / V256) +#define FIRST_NI_AVXIFMA NI_AVXIFMA_MultiplyAdd52Low +HARDWARE_INTRINSIC(AVXIFMA, MultiplyAdd52Low, -1, 3, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vpmadd52luq, INS_invalid, INS_invalid}, 5, -1, HW_Category_SimpleSIMD, HW_Flag_NoEvexSemantics) +HARDWARE_INTRINSIC(AVXIFMA, MultiplyAdd52High, -1, 3, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vpmadd52huq, INS_invalid, INS_invalid}, 5, -1, HW_Category_SimpleSIMD, HW_Flag_NoEvexSemantics) +#define LAST_NI_AVXIFMA NI_AVXIFMA_MultiplyAdd52High + // *************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************** // ISA Function name SIMD size NumArg Instructions IntCost FltCost Category Flags // {TYP_BYTE, TYP_UBYTE, TYP_SHORT, TYP_USHORT, TYP_INT, TYP_UINT, TYP_LONG, TYP_ULONG, TYP_FLOAT, TYP_DOUBLE} diff --git a/src/coreclr/jit/hwintrinsicxarch.cpp b/src/coreclr/jit/hwintrinsicxarch.cpp index 17ed42ed753151..7fec81cdf03c0d 100644 --- a/src/coreclr/jit/hwintrinsicxarch.cpp +++ b/src/coreclr/jit/hwintrinsicxarch.cpp @@ -166,6 +166,14 @@ static CORINFO_InstructionSet V512VersionOfIsa(CORINFO_InstructionSet isa) return InstructionSet_AVX512v3; } + case InstructionSet_AVXIFMA: + case InstructionSet_AVX512v2: + { + // AvxIfma.V512 lifts under AVX512v2, which carries the EVEX-encoded + // VPMADD52*UQ on ZMM. Same rationale as AvxVnni.V512 above. + return InstructionSet_AVX512v2; + } + default: { return InstructionSet_NONE; @@ -276,7 +284,14 @@ CORINFO_InstructionSet Compiler::lookupInstructionSet(const char* className) } else if (strcmp(className + 3, "Ifma") == 0) { - return InstructionSet_AVXIFMA; + if (compSupportsHWIntrinsic(InstructionSet_AVXIFMA)) + { + return InstructionSet_AVXIFMA; + } + else + { + return InstructionSet_AVX512v2; + } } else if (strncmp(className + 3, "Vnni", 4) == 0) { diff --git a/src/coreclr/jit/lowerxarch.cpp b/src/coreclr/jit/lowerxarch.cpp index eeea2751b537b0..a6648deaaabec8 100644 --- a/src/coreclr/jit/lowerxarch.cpp +++ b/src/coreclr/jit/lowerxarch.cpp @@ -10249,6 +10249,9 @@ void Lowering::ContainCheckHWIntrinsic(GenTreeHWIntrinsic* node) assert((intrinsicId == NI_X86Base_DivRem) || (intrinsicId == NI_X86Base_X64_DivRem) || (intrinsicId == NI_AVX512v3_MultiplyWideningAndAdd) || (intrinsicId == NI_AVX512v3_MultiplyWideningAndAddSaturate) || + (intrinsicId == NI_AVX512v2_MultiplyAdd52Low) || + (intrinsicId == NI_AVX512v2_MultiplyAdd52High) || + (intrinsicId >= FIRST_NI_AVXIFMA && intrinsicId <= LAST_NI_AVXIFMA) || (intrinsicId >= FIRST_NI_AVXVNNI && intrinsicId <= LAST_NI_AVXVNNIINT_V512)); TryMakeSrcContainedOrRegOptional(node, op3); break; diff --git a/src/coreclr/jit/lsraxarch.cpp b/src/coreclr/jit/lsraxarch.cpp index 05781578c7c608..9c4e0a234122b1 100644 --- a/src/coreclr/jit/lsraxarch.cpp +++ b/src/coreclr/jit/lsraxarch.cpp @@ -2750,6 +2750,10 @@ int LinearScan::BuildHWIntrinsic(GenTreeHWIntrinsic* intrinsicTree, int* pDstCou case NI_AVXVNNI_MultiplyWideningAndAddSaturate: case NI_AVX512v3_MultiplyWideningAndAdd: case NI_AVX512v3_MultiplyWideningAndAddSaturate: + case NI_AVXIFMA_MultiplyAdd52Low: + case NI_AVXIFMA_MultiplyAdd52High: + case NI_AVX512v2_MultiplyAdd52Low: + case NI_AVX512v2_MultiplyAdd52High: case NI_AVXVNNIINT_MultiplyWideningAndAdd: case NI_AVXVNNIINT_MultiplyWideningAndAddSaturate: case NI_AVXVNNIINT_V512_MultiplyWideningAndAdd: diff --git a/src/coreclr/tools/Common/InstructionSetHelpers.cs b/src/coreclr/tools/Common/InstructionSetHelpers.cs index f167a4328e94d2..df753d97da6b9b 100644 --- a/src/coreclr/tools/Common/InstructionSetHelpers.cs +++ b/src/coreclr/tools/Common/InstructionSetHelpers.cs @@ -291,6 +291,7 @@ public static InstructionSetSupport ConfigureInstructionSetSupport(string instru optimisticInstructionSetSupportBuilder.AddSupportedInstructionSet("avx10v2"); optimisticInstructionSetSupportBuilder.AddSupportedInstructionSet("avxvnniint_v512"); optimisticInstructionSetSupportBuilder.AddSupportedInstructionSet("avxvnni_v512"); + optimisticInstructionSetSupportBuilder.AddSupportedInstructionSet("avxifma_v512"); optimisticInstructionSetSupportBuilder.AddSupportedInstructionSet("avx512vp2intersect"); optimisticInstructionSetSupportBuilder.AddSupportedInstructionSet("aes_v512"); optimisticInstructionSetSupportBuilder.AddSupportedInstructionSet("gfni_v512"); diff --git a/src/coreclr/tools/Common/JitInterface/CorInfoInstructionSet.cs b/src/coreclr/tools/Common/JitInterface/CorInfoInstructionSet.cs index 264d2cd0bdbcd6..a5f3f330b8453f 100644 --- a/src/coreclr/tools/Common/JitInterface/CorInfoInstructionSet.cs +++ b/src/coreclr/tools/Common/JitInterface/CorInfoInstructionSet.cs @@ -1193,6 +1193,7 @@ public static IEnumerable ArchitectureToValidInstructionSets yield return new InstructionSetInfo("avx512v2", "", InstructionSet.X64_AVX512v2, true); yield return new InstructionSetInfo("avx512v2", "Avx512Vbmi", InstructionSet.X64_AVX512v2, true); yield return new InstructionSetInfo("avx512v2", "Avx512Vbmi_VL", InstructionSet.X64_AVX512v2, true); + yield return new InstructionSetInfo("avx512v2", "AvxIfma_V512", InstructionSet.X64_AVX512v2, true); yield return new InstructionSetInfo("avx512v3", "", InstructionSet.X64_AVX512v3, true); yield return new InstructionSetInfo("avx512v3", "", InstructionSet.X64_AVX512v3, true); yield return new InstructionSetInfo("avx512v3", "Avx512Vbmi2", InstructionSet.X64_AVX512v3, true); @@ -1267,6 +1268,7 @@ public static IEnumerable ArchitectureToValidInstructionSets yield return new InstructionSetInfo("avx512v2", "", InstructionSet.X86_AVX512v2, true); yield return new InstructionSetInfo("avx512v2", "Avx512Vbmi", InstructionSet.X86_AVX512v2, true); yield return new InstructionSetInfo("avx512v2", "Avx512Vbmi_VL", InstructionSet.X86_AVX512v2, true); + yield return new InstructionSetInfo("avx512v2", "AvxIfma_V512", InstructionSet.X86_AVX512v2, true); yield return new InstructionSetInfo("avx512v3", "", InstructionSet.X86_AVX512v3, true); yield return new InstructionSetInfo("avx512v3", "", InstructionSet.X86_AVX512v3, true); yield return new InstructionSetInfo("avx512v3", "Avx512Vbmi2", InstructionSet.X86_AVX512v3, true); @@ -1844,6 +1846,18 @@ public static InstructionSet LookupPlatformIntrinsicInstructionSet(TargetArchite else return InstructionSet.X64_AVXVNNI; + case "AvxIfma": + if (nestedTypeName == "X64") + return InstructionSet.X64_AVXIFMA_X64; + else + if (nestedTypeName == "V512_X64") + return InstructionSet.X64_AVX512v2_X64; + else + if (nestedTypeName == "V512") + return InstructionSet.X64_AVX512v2; + else + return InstructionSet.X64_AVXIFMA; + case "Avx512Bmm": return InstructionSet.X64_AVX512BMM; @@ -1991,6 +2005,12 @@ public static InstructionSet LookupPlatformIntrinsicInstructionSet(TargetArchite else return InstructionSet.X86_AVXVNNI; + case "AvxIfma": + if (nestedTypeName == "V512") + return InstructionSet.X86_AVX512v2; + else + return InstructionSet.X86_AVXIFMA; + case "Avx512Bmm": return InstructionSet.X86_AVX512BMM; @@ -2737,6 +2757,26 @@ public static IEnumerable LookupPlatformIntrinsicTypes(TypeSystemC } } } + { + var parentType = context.SystemModule.GetType("System.Runtime.Intrinsics.X86"u8, "AvxIfma"u8, false); + if (parentType != null) + { + yield return parentType; + var nestedType = parentType.GetNestedType("V512"u8); + if (nestedType != null) + { + yield return nestedType; + if (instructionSet == InstructionSet.X64_AVX512v2_X64) + { + var nestedType64 = parentType.GetNestedType("V512_X64"u8); + if (nestedType64 != null) + { + yield return nestedType64; + } + } + } + } + } break; case (InstructionSet.X64_AVX512v3, TargetArchitecture.X64): @@ -3287,6 +3327,18 @@ public static IEnumerable LookupPlatformIntrinsicTypes(TypeSystemC } } } + { + var parentType = context.SystemModule.GetType("System.Runtime.Intrinsics.X86"u8, "AvxIfma"u8, false); + if (parentType != null) + { + yield return parentType; + var nestedType = parentType.GetNestedType("V512"u8); + if (nestedType != null) + { + yield return nestedType; + } + } + } break; case (InstructionSet.X86_AVX512v3, TargetArchitecture.X86): diff --git a/src/coreclr/tools/Common/JitInterface/ThunkGenerator/InstructionSetDesc.txt b/src/coreclr/tools/Common/JitInterface/ThunkGenerator/InstructionSetDesc.txt index b961d2accdb9e3..1019b14afba917 100644 --- a/src/coreclr/tools/Common/JitInterface/ThunkGenerator/InstructionSetDesc.txt +++ b/src/coreclr/tools/Common/JitInterface/ThunkGenerator/InstructionSetDesc.txt @@ -61,7 +61,7 @@ instructionset ,X86 ,Avx512CD_VL , ,34 ,AVX512 instructionset ,X86 ,Avx512DQ , ,35 ,AVX512 ,avx512 instructionset ,X86 ,Avx512DQ_VL , ,36 ,AVX512 ,avx512 -instructionset ,X86 , ,Avx512Ifma ,76 ,AVX512v2 ,avx512v2 +instructionset ,X86 ,AvxIfma_V512 ,Avx512Ifma ,76 ,AVX512v2 ,avx512v2 instructionset ,X86 ,Avx512Vbmi , ,37 ,AVX512v2 ,avx512v2 instructionset ,X86 ,Avx512Vbmi_VL , ,38 ,AVX512v2 ,avx512v2 diff --git a/src/libraries/System.Private.CoreLib/src/System.Private.CoreLib.Shared.projitems b/src/libraries/System.Private.CoreLib/src/System.Private.CoreLib.Shared.projitems index fba53e000e31bd..70a9e46f1a33f6 100644 --- a/src/libraries/System.Private.CoreLib/src/System.Private.CoreLib.Shared.projitems +++ b/src/libraries/System.Private.CoreLib/src/System.Private.CoreLib.Shared.projitems @@ -2814,6 +2814,7 @@ + @@ -2847,6 +2848,7 @@ + diff --git a/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/X86/AvxIfma.PlatformNotSupported.cs b/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/X86/AvxIfma.PlatformNotSupported.cs new file mode 100644 index 00000000000000..8699f2a3275936 --- /dev/null +++ b/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/X86/AvxIfma.PlatformNotSupported.cs @@ -0,0 +1,77 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Runtime.CompilerServices; + +namespace System.Runtime.Intrinsics.X86 +{ + /// Provides access to the x86 AVX-IFMA hardware instructions via intrinsics. + [CLSCompliant(false)] + public abstract class AvxIfma : Avx2 + { + internal AvxIfma() { } + + /// Gets a value that indicates whether the APIs in this class are supported. + /// if the APIs are supported; otherwise, . + /// A value of indicates that the APIs will throw . + public static new bool IsSupported { [Intrinsic] get { return false; } } + + /// Provides access to the x86 AVX-IFMA hardware instructions, that are only available to 64-bit processes, via intrinsics. + public new abstract class X64 : Avx2.X64 + { + internal X64() { } + + /// Gets a value that indicates whether the APIs in this class are supported. + /// if the APIs are supported; otherwise, . + /// A value of indicates that the APIs will throw . + public static new bool IsSupported { [Intrinsic] get { return false; } } + } + + /// + /// __m128i _mm_madd52lo_epu64 (__m128i a, __m128i b, __m128i c) + /// VPMADD52LUQ xmm, xmm, xmm/m128 + /// + public static Vector128 MultiplyAdd52Low(Vector128 addend, Vector128 left, Vector128 right) { throw new PlatformNotSupportedException(); } + + /// + /// __m256i _mm256_madd52lo_epu64 (__m256i a, __m256i b, __m256i c) + /// VPMADD52LUQ ymm, ymm, ymm/m256 + /// + public static Vector256 MultiplyAdd52Low(Vector256 addend, Vector256 left, Vector256 right) { throw new PlatformNotSupportedException(); } + + /// + /// __m128i _mm_madd52hi_epu64 (__m128i a, __m128i b, __m128i c) + /// VPMADD52HUQ xmm, xmm, xmm/m128 + /// + public static Vector128 MultiplyAdd52High(Vector128 addend, Vector128 left, Vector128 right) { throw new PlatformNotSupportedException(); } + + /// + /// __m256i _mm256_madd52hi_epu64 (__m256i a, __m256i b, __m256i c) + /// VPMADD52HUQ ymm, ymm, ymm/m256 + /// + public static Vector256 MultiplyAdd52High(Vector256 addend, Vector256 left, Vector256 right) { throw new PlatformNotSupportedException(); } + + /// Provides access to the x86 AVX-512-IFMA hardware instructions, that operate on 512-bit vectors, via intrinsics. + public abstract class V512 + { + internal V512() { } + + /// Gets a value that indicates whether the APIs in this class are supported. + /// if the APIs are supported; otherwise, . + /// A value of indicates that the APIs will throw . + public static bool IsSupported { [Intrinsic] get { return false; } } + + /// + /// __m512i _mm512_madd52lo_epu64 (__m512i a, __m512i b, __m512i c) + /// VPMADD52LUQ zmm, zmm, zmm/m512 + /// + public static Vector512 MultiplyAdd52Low(Vector512 addend, Vector512 left, Vector512 right) { throw new PlatformNotSupportedException(); } + + /// + /// __m512i _mm512_madd52hi_epu64 (__m512i a, __m512i b, __m512i c) + /// VPMADD52HUQ zmm, zmm, zmm/m512 + /// + public static Vector512 MultiplyAdd52High(Vector512 addend, Vector512 left, Vector512 right) { throw new PlatformNotSupportedException(); } + } + } +} diff --git a/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/X86/AvxIfma.cs b/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/X86/AvxIfma.cs new file mode 100644 index 00000000000000..a027fc90da635e --- /dev/null +++ b/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/X86/AvxIfma.cs @@ -0,0 +1,82 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Runtime.CompilerServices; + +namespace System.Runtime.Intrinsics.X86 +{ + /// Provides access to the x86 AVX-IFMA hardware instructions via intrinsics. + [Intrinsic] + [CLSCompliant(false)] + public abstract class AvxIfma : Avx2 + { + internal AvxIfma() { } + + /// Gets a value that indicates whether the APIs in this class are supported. + /// if the APIs are supported; otherwise, . + /// A value of indicates that the APIs will throw . + public static new bool IsSupported { get => IsSupported; } + + /// Provides access to the x86 AVX-IFMA hardware instructions, that are only available to 64-bit processes, via intrinsics. + [Intrinsic] + public new abstract class X64 : Avx2.X64 + { + internal X64() { } + + /// Gets a value that indicates whether the APIs in this class are supported. + /// if the APIs are supported; otherwise, . + /// A value of indicates that the APIs will throw . + public static new bool IsSupported { get => IsSupported; } + } + + /// + /// __m128i _mm_madd52lo_epu64 (__m128i a, __m128i b, __m128i c) + /// VPMADD52LUQ xmm, xmm, xmm/m128 + /// + public static Vector128 MultiplyAdd52Low(Vector128 addend, Vector128 left, Vector128 right) => MultiplyAdd52Low(addend, left, right); + + /// + /// __m256i _mm256_madd52lo_epu64 (__m256i a, __m256i b, __m256i c) + /// VPMADD52LUQ ymm, ymm, ymm/m256 + /// + public static Vector256 MultiplyAdd52Low(Vector256 addend, Vector256 left, Vector256 right) => MultiplyAdd52Low(addend, left, right); + + /// + /// __m128i _mm_madd52hi_epu64 (__m128i a, __m128i b, __m128i c) + /// VPMADD52HUQ xmm, xmm, xmm/m128 + /// + public static Vector128 MultiplyAdd52High(Vector128 addend, Vector128 left, Vector128 right) => MultiplyAdd52High(addend, left, right); + + /// + /// __m256i _mm256_madd52hi_epu64 (__m256i a, __m256i b, __m256i c) + /// VPMADD52HUQ ymm, ymm, ymm/m256 + /// + public static Vector256 MultiplyAdd52High(Vector256 addend, Vector256 left, Vector256 right) => MultiplyAdd52High(addend, left, right); + + /// Provides access to the x86 AVX-512-IFMA hardware instructions, that operate on 512-bit vectors, via intrinsics. + [Intrinsic] + public abstract class V512 + { + internal V512() { } + + /// Gets a value that indicates whether the APIs in this class are supported. + /// if the APIs are supported; otherwise, . + /// A value of indicates that the APIs will throw . + public static bool IsSupported { [Intrinsic] get => IsSupported; } + + /// + /// __m512i _mm512_madd52lo_epu64 (__m512i a, __m512i b, __m512i c) + /// VPMADD52LUQ zmm, zmm, zmm/m512 + /// + [Intrinsic] + public static Vector512 MultiplyAdd52Low(Vector512 addend, Vector512 left, Vector512 right) => MultiplyAdd52Low(addend, left, right); + + /// + /// __m512i _mm512_madd52hi_epu64 (__m512i a, __m512i b, __m512i c) + /// VPMADD52HUQ zmm, zmm, zmm/m512 + /// + [Intrinsic] + public static Vector512 MultiplyAdd52High(Vector512 addend, Vector512 left, Vector512 right) => MultiplyAdd52High(addend, left, right); + } + } +} diff --git a/src/libraries/System.Runtime.Intrinsics/ref/System.Runtime.Intrinsics.cs b/src/libraries/System.Runtime.Intrinsics/ref/System.Runtime.Intrinsics.cs index fc60e6690163a1..493f9812abca33 100644 --- a/src/libraries/System.Runtime.Intrinsics/ref/System.Runtime.Intrinsics.cs +++ b/src/libraries/System.Runtime.Intrinsics/ref/System.Runtime.Intrinsics.cs @@ -10570,6 +10570,28 @@ internal V512() { } } } [System.CLSCompliantAttribute(false)] + public abstract class AvxIfma : System.Runtime.Intrinsics.X86.Avx2 + { + internal AvxIfma() { } + public static new bool IsSupported { get { throw null; } } + public static System.Runtime.Intrinsics.Vector128 MultiplyAdd52Low(System.Runtime.Intrinsics.Vector128 addend, System.Runtime.Intrinsics.Vector128 left, System.Runtime.Intrinsics.Vector128 right) { throw null; } + public static System.Runtime.Intrinsics.Vector128 MultiplyAdd52High(System.Runtime.Intrinsics.Vector128 addend, System.Runtime.Intrinsics.Vector128 left, System.Runtime.Intrinsics.Vector128 right) { throw null; } + public static System.Runtime.Intrinsics.Vector256 MultiplyAdd52Low(System.Runtime.Intrinsics.Vector256 addend, System.Runtime.Intrinsics.Vector256 left, System.Runtime.Intrinsics.Vector256 right) { throw null; } + public static System.Runtime.Intrinsics.Vector256 MultiplyAdd52High(System.Runtime.Intrinsics.Vector256 addend, System.Runtime.Intrinsics.Vector256 left, System.Runtime.Intrinsics.Vector256 right) { throw null; } + public new abstract partial class X64 : System.Runtime.Intrinsics.X86.Avx2.X64 + { + internal X64() { } + public static new bool IsSupported { get { throw null; } } + } + public abstract partial class V512 + { + internal V512() { } + public static bool IsSupported { get { throw null; } } + public static System.Runtime.Intrinsics.Vector512 MultiplyAdd52Low(System.Runtime.Intrinsics.Vector512 addend, System.Runtime.Intrinsics.Vector512 left, System.Runtime.Intrinsics.Vector512 right) { throw null; } + public static System.Runtime.Intrinsics.Vector512 MultiplyAdd52High(System.Runtime.Intrinsics.Vector512 addend, System.Runtime.Intrinsics.Vector512 left, System.Runtime.Intrinsics.Vector512 right) { throw null; } + } + } + [System.CLSCompliantAttribute(false)] public abstract partial class Bmi1 : System.Runtime.Intrinsics.X86.X86Base { internal Bmi1() { } From e99f0646c03bbd11a33c55c3a2097caa7671c765 Mon Sep 17 00:00:00 2001 From: James Burton Date: Wed, 1 Jul 2026 13:44:16 +0100 Subject: [PATCH 2/6] Add AvxIfma/AvxIfma.V512 tests + NativeAOT smoke wiring MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds sample tests mirroring the AvxVnni.V512 test shape: - X86_Avx/AvxIfma/ — V128/V256 VEX form - X86_Avx/AvxIfma_V512/ — V512 EVEX form NativeAOT smoke test: uncomments AvxIfma / AvxIfma.X64 checks (were pre-approved but not yet wired) and adds AvxIfma.V512 check. AvxIfma.V512 folds under AVX512v2 (Avx512Vbmi is a representative sibling). --- .../X86_Avx/AvxIfma/AvxIfmaSampleTest.cs | 114 ++++++++++++++++++ .../AvxIfma/AvxIfma_handwritten_r.csproj | 12 ++ .../AvxIfma/AvxIfma_handwritten_ro.csproj | 13 ++ .../X86_Avx/AvxIfma/AvxIfma_r.csproj | 14 +++ .../X86_Avx/AvxIfma/AvxIfma_ro.csproj | 15 +++ .../X86_Avx/AvxIfma/Program.AvxIfma.cs | 9 ++ .../AvxIfma_V512/AvxIfma_V512SampleTest.cs | 107 ++++++++++++++++ .../AvxIfma_V512_handwritten_r.csproj | 12 ++ .../AvxIfma_V512_handwritten_ro.csproj | 13 ++ .../AvxIfma_V512/AvxIfma_V512_r.csproj | 14 +++ .../AvxIfma_V512/AvxIfma_V512_ro.csproj | 15 +++ .../AvxIfma_V512/Program.AvxIfma_V512.cs | 9 ++ .../SmokeTests/HardwareIntrinsics/Program.cs | 15 ++- 13 files changed, 358 insertions(+), 4 deletions(-) create mode 100644 src/tests/JIT/HardwareIntrinsics/X86_Avx/AvxIfma/AvxIfmaSampleTest.cs create mode 100644 src/tests/JIT/HardwareIntrinsics/X86_Avx/AvxIfma/AvxIfma_handwritten_r.csproj create mode 100644 src/tests/JIT/HardwareIntrinsics/X86_Avx/AvxIfma/AvxIfma_handwritten_ro.csproj create mode 100644 src/tests/JIT/HardwareIntrinsics/X86_Avx/AvxIfma/AvxIfma_r.csproj create mode 100644 src/tests/JIT/HardwareIntrinsics/X86_Avx/AvxIfma/AvxIfma_ro.csproj create mode 100644 src/tests/JIT/HardwareIntrinsics/X86_Avx/AvxIfma/Program.AvxIfma.cs create mode 100644 src/tests/JIT/HardwareIntrinsics/X86_Avx/AvxIfma_V512/AvxIfma_V512SampleTest.cs create mode 100644 src/tests/JIT/HardwareIntrinsics/X86_Avx/AvxIfma_V512/AvxIfma_V512_handwritten_r.csproj create mode 100644 src/tests/JIT/HardwareIntrinsics/X86_Avx/AvxIfma_V512/AvxIfma_V512_handwritten_ro.csproj create mode 100644 src/tests/JIT/HardwareIntrinsics/X86_Avx/AvxIfma_V512/AvxIfma_V512_r.csproj create mode 100644 src/tests/JIT/HardwareIntrinsics/X86_Avx/AvxIfma_V512/AvxIfma_V512_ro.csproj create mode 100644 src/tests/JIT/HardwareIntrinsics/X86_Avx/AvxIfma_V512/Program.AvxIfma_V512.cs diff --git a/src/tests/JIT/HardwareIntrinsics/X86_Avx/AvxIfma/AvxIfmaSampleTest.cs b/src/tests/JIT/HardwareIntrinsics/X86_Avx/AvxIfma/AvxIfmaSampleTest.cs new file mode 100644 index 00000000000000..b9e52f8835517d --- /dev/null +++ b/src/tests/JIT/HardwareIntrinsics/X86_Avx/AvxIfma/AvxIfmaSampleTest.cs @@ -0,0 +1,114 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// + +using System; +using System.Runtime.Intrinsics; +using System.Runtime.Intrinsics.X86; +using Xunit; + +namespace IntelHardwareIntrinsicTest._AvxIfma +{ + public partial class Program + { + [Fact] + public static void AvxIfmaSampleTest() + { + Vector128 zero128 = Vector128.Zero; + Vector128 one128 = Vector128.Create((ulong)1); + Vector128 forty_two128 = Vector128.Create((ulong)42); + Vector128 big128 = Vector128.Create((ulong)(1UL << 26)); + + Vector256 zero256 = Vector256.Zero; + Vector256 one256 = Vector256.Create((ulong)1); + Vector256 forty_two256 = Vector256.Create((ulong)42); + Vector256 big256 = Vector256.Create((ulong)(1UL << 26)); + + if (!AvxIfma.IsSupported) + { + Assert.Throws( + () => AvxIfma.MultiplyAdd52Low(zero128, one128, forty_two128)); + Assert.Throws( + () => AvxIfma.MultiplyAdd52High(zero128, big128, big128)); + Assert.Throws( + () => AvxIfma.MultiplyAdd52Low(zero256, one256, forty_two256)); + Assert.Throws( + () => AvxIfma.MultiplyAdd52High(zero256, big256, big256)); + return; + } + + // V128: VPMADD52LUQ semantics + // dst[i] = addend[i] + ((left[i] & 0xF_FFFF_FFFF_FFFF) * (right[i] & 0xF_FFFF_FFFF_FFFF))[51:0] + AssertAllLanesEqual128(AvxIfma.MultiplyAdd52Low(zero128, one128, forty_two128), 42UL); + AssertAllLanesEqual128(AvxIfma.MultiplyAdd52High(zero128, big128, big128), 1UL); + AssertAllLanesEqual128(AvxIfma.MultiplyAdd52High(zero128, one128, forty_two128), 0UL); + + // V256: same shape + AssertAllLanesEqual256(AvxIfma.MultiplyAdd52Low(zero256, one256, forty_two256), 42UL); + AssertAllLanesEqual256(AvxIfma.MultiplyAdd52High(zero256, big256, big256), 1UL); + AssertAllLanesEqual256(AvxIfma.MultiplyAdd52High(zero256, one256, forty_two256), 0UL); + + // Addend accumulates + Vector128 preloaded128 = Vector128.Create((ulong)1000); + AssertAllLanesEqual128(AvxIfma.MultiplyAdd52Low(preloaded128, one128, forty_two128), 1042UL); + + Vector256 preloaded256 = Vector256.Create((ulong)1000); + AssertAllLanesEqual256(AvxIfma.MultiplyAdd52Low(preloaded256, one256, forty_two256), 1042UL); + + // Only low 52 bits of source operands participate + ulong topMask = 0xFFF0_0000_0000_0000UL; + Vector128 leftMasked128 = Vector128.Create((ulong)1 | topMask); + Vector128 rightMasked128 = Vector128.Create((ulong)42 | topMask); + AssertAllLanesEqual128(AvxIfma.MultiplyAdd52Low(zero128, leftMasked128, rightMasked128), 42UL); + + Vector256 leftMasked256 = Vector256.Create((ulong)1 | topMask); + Vector256 rightMasked256 = Vector256.Create((ulong)42 | topMask); + AssertAllLanesEqual256(AvxIfma.MultiplyAdd52Low(zero256, leftMasked256, rightMasked256), 42UL); + + // Non-trivial lane-varying pattern on V256 + const ulong Mask52 = 0x000F_FFFF_FFFF_FFFFUL; + ulong[] leftValues = new ulong[Vector256.Count]; + ulong[] rightValues = new ulong[Vector256.Count]; + ulong[] addendValues = new ulong[Vector256.Count]; + ulong[] expectedLow = new ulong[Vector256.Count]; + for (int index = 0; index < leftValues.Length; index++) + { + leftValues[index] = ((ulong)index + 1) * 1_000_003UL; + rightValues[index] = ((ulong)index + 5) * 999_983UL; + addendValues[index] = (ulong)index * 7UL; + ulong product = (leftValues[index] & Mask52) * (rightValues[index] & Mask52); + expectedLow[index] = addendValues[index] + (product & Mask52); + } + AssertLanesEqual256( + AvxIfma.MultiplyAdd52Low( + Vector256.Create(addendValues), + Vector256.Create(leftValues), + Vector256.Create(rightValues)), + expectedLow); + } + + private static void AssertAllLanesEqual128(Vector128 value, ulong expected) + { + for (int index = 0; index < Vector128.Count; index++) + { + Assert.Equal(expected, value.GetElement(index)); + } + } + + private static void AssertAllLanesEqual256(Vector256 value, ulong expected) + { + for (int index = 0; index < Vector256.Count; index++) + { + Assert.Equal(expected, value.GetElement(index)); + } + } + + private static void AssertLanesEqual256(Vector256 value, ulong[] expected) + { + for (int index = 0; index < Vector256.Count; index++) + { + Assert.Equal(expected[index], value.GetElement(index)); + } + } + } +} diff --git a/src/tests/JIT/HardwareIntrinsics/X86_Avx/AvxIfma/AvxIfma_handwritten_r.csproj b/src/tests/JIT/HardwareIntrinsics/X86_Avx/AvxIfma/AvxIfma_handwritten_r.csproj new file mode 100644 index 00000000000000..fb658581f6de17 --- /dev/null +++ b/src/tests/JIT/HardwareIntrinsics/X86_Avx/AvxIfma/AvxIfma_handwritten_r.csproj @@ -0,0 +1,12 @@ + + + X86_AvxIfma_handwritten_r + true + + + Embedded + + + + + diff --git a/src/tests/JIT/HardwareIntrinsics/X86_Avx/AvxIfma/AvxIfma_handwritten_ro.csproj b/src/tests/JIT/HardwareIntrinsics/X86_Avx/AvxIfma/AvxIfma_handwritten_ro.csproj new file mode 100644 index 00000000000000..e9269fdb1ce42d --- /dev/null +++ b/src/tests/JIT/HardwareIntrinsics/X86_Avx/AvxIfma/AvxIfma_handwritten_ro.csproj @@ -0,0 +1,13 @@ + + + X86_AvxIfma_handwritten_ro + true + + + Embedded + True + + + + + diff --git a/src/tests/JIT/HardwareIntrinsics/X86_Avx/AvxIfma/AvxIfma_r.csproj b/src/tests/JIT/HardwareIntrinsics/X86_Avx/AvxIfma/AvxIfma_r.csproj new file mode 100644 index 00000000000000..12686cea58e553 --- /dev/null +++ b/src/tests/JIT/HardwareIntrinsics/X86_Avx/AvxIfma/AvxIfma_r.csproj @@ -0,0 +1,14 @@ + + + X86_AvxIfma_r + true + + + Embedded + + + + + + + diff --git a/src/tests/JIT/HardwareIntrinsics/X86_Avx/AvxIfma/AvxIfma_ro.csproj b/src/tests/JIT/HardwareIntrinsics/X86_Avx/AvxIfma/AvxIfma_ro.csproj new file mode 100644 index 00000000000000..0e60d7f912c908 --- /dev/null +++ b/src/tests/JIT/HardwareIntrinsics/X86_Avx/AvxIfma/AvxIfma_ro.csproj @@ -0,0 +1,15 @@ + + + X86_AvxIfma_ro + true + + + Embedded + True + + + + + + + diff --git a/src/tests/JIT/HardwareIntrinsics/X86_Avx/AvxIfma/Program.AvxIfma.cs b/src/tests/JIT/HardwareIntrinsics/X86_Avx/AvxIfma/Program.AvxIfma.cs new file mode 100644 index 00000000000000..770c442c676b98 --- /dev/null +++ b/src/tests/JIT/HardwareIntrinsics/X86_Avx/AvxIfma/Program.AvxIfma.cs @@ -0,0 +1,9 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +namespace IntelHardwareIntrinsicTest._AvxIfma +{ + public partial class Program + { + } +} diff --git a/src/tests/JIT/HardwareIntrinsics/X86_Avx/AvxIfma_V512/AvxIfma_V512SampleTest.cs b/src/tests/JIT/HardwareIntrinsics/X86_Avx/AvxIfma_V512/AvxIfma_V512SampleTest.cs new file mode 100644 index 00000000000000..155c3e577c6fe4 --- /dev/null +++ b/src/tests/JIT/HardwareIntrinsics/X86_Avx/AvxIfma_V512/AvxIfma_V512SampleTest.cs @@ -0,0 +1,107 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// + +using System; +using System.Runtime.Intrinsics; +using System.Runtime.Intrinsics.X86; +using Xunit; + +namespace IntelHardwareIntrinsicTest._AvxIfma_V512 +{ + public partial class Program + { + [Fact] + public static void AvxIfma_V512SampleTest() + { + Vector512 zero = Vector512.Zero; + Vector512 one = Vector512.Create((ulong)1); + Vector512 forty_two = Vector512.Create((ulong)42); + Vector512 big = Vector512.Create((ulong)(1UL << 26)); + + if (!AvxIfma.V512.IsSupported) + { + Assert.Throws( + () => AvxIfma.V512.MultiplyAdd52Low(zero, one, forty_two)); + Assert.Throws( + () => AvxIfma.V512.MultiplyAdd52High(zero, big, big)); + return; + } + + // VPMADD52LUQ semantics per Intel SDM: + // dst[i] = addend[i] + ((left[i] & 0x000F_FFFF_FFFF_FFFF) + // * (right[i] & 0x000F_FFFF_FFFF_FFFF))[51:0] + // + // With addend=0, left=1, right=42: result = (1 * 42) & 0xF_FFFF_FFFF_FFFF = 42 + Vector512 lowResult = AvxIfma.V512.MultiplyAdd52Low(zero, one, forty_two); + AssertAllLanesEqual(lowResult, 42UL); + + // Addend accumulates: pre-loaded addend + product low + Vector512 preloaded = Vector512.Create((ulong)1000); + Vector512 accumulated = AvxIfma.V512.MultiplyAdd52Low(preloaded, one, forty_two); + AssertAllLanesEqual(accumulated, 1042UL); + + // VPMADD52HUQ semantics: + // dst[i] = addend[i] + ((left[i] & 0x000F_FFFF_FFFF_FFFF) + // * (right[i] & 0x000F_FFFF_FFFF_FFFF))[103:52] + // + // With addend=0, left = 1<<26, right = 1<<26: + // product = 1<<52; high 52 bits of product = 1 + Vector512 highResult = AvxIfma.V512.MultiplyAdd52High(zero, big, big); + AssertAllLanesEqual(highResult, 1UL); + + // High 52 bits of a smaller product are zero: 1 * 42 = 42 fits in low 52 bits, + // so the "high" 52 bits are 0. + Vector512 highZero = AvxIfma.V512.MultiplyAdd52High(zero, one, forty_two); + AssertAllLanesEqual(highZero, 0UL); + + // Only the low 52 bits of the source operands participate — bits above 52 + // should be ignored. Set the top bits and confirm no change: + ulong topMask = 0xFFF0_0000_0000_0000UL; + Vector512 leftMasked = Vector512.Create((ulong)1 | topMask); + Vector512 rightMasked = Vector512.Create((ulong)42 | topMask); + Vector512 lowResultMasked = AvxIfma.V512.MultiplyAdd52Low(zero, leftMasked, rightMasked); + AssertAllLanesEqual(lowResultMasked, 42UL); + + // Non-trivial lane-varying pattern: + ulong[] leftValues = new ulong[Vector512.Count]; + ulong[] rightValues = new ulong[Vector512.Count]; + ulong[] addendValues = new ulong[Vector512.Count]; + ulong[] expectedLow = new ulong[Vector512.Count]; + + const ulong Mask52 = 0x000F_FFFF_FFFF_FFFFUL; + for (int index = 0; index < leftValues.Length; index++) + { + leftValues[index] = ((ulong)index + 1) * 1_000_003UL; + rightValues[index] = ((ulong)index + 5) * 999_983UL; + addendValues[index] = (ulong)index * 7UL; + + ulong product = (leftValues[index] & Mask52) * (rightValues[index] & Mask52); + expectedLow[index] = addendValues[index] + (product & Mask52); + } + + Vector512 addendVec = Vector512.Create(addendValues); + Vector512 leftVec = Vector512.Create(leftValues); + Vector512 rightVec = Vector512.Create(rightValues); + + Vector512 patternedLow = AvxIfma.V512.MultiplyAdd52Low(addendVec, leftVec, rightVec); + AssertLanesEqual(patternedLow, expectedLow); + } + + private static void AssertAllLanesEqual(Vector512 value, ulong expected) + { + for (int index = 0; index < Vector512.Count; index++) + { + Assert.Equal(expected, value.GetElement(index)); + } + } + + private static void AssertLanesEqual(Vector512 value, ulong[] expected) + { + for (int index = 0; index < Vector512.Count; index++) + { + Assert.Equal(expected[index], value.GetElement(index)); + } + } + } +} diff --git a/src/tests/JIT/HardwareIntrinsics/X86_Avx/AvxIfma_V512/AvxIfma_V512_handwritten_r.csproj b/src/tests/JIT/HardwareIntrinsics/X86_Avx/AvxIfma_V512/AvxIfma_V512_handwritten_r.csproj new file mode 100644 index 00000000000000..f72bac155536cf --- /dev/null +++ b/src/tests/JIT/HardwareIntrinsics/X86_Avx/AvxIfma_V512/AvxIfma_V512_handwritten_r.csproj @@ -0,0 +1,12 @@ + + + X86_AvxIfma_V512_handwritten_r + true + + + Embedded + + + + + diff --git a/src/tests/JIT/HardwareIntrinsics/X86_Avx/AvxIfma_V512/AvxIfma_V512_handwritten_ro.csproj b/src/tests/JIT/HardwareIntrinsics/X86_Avx/AvxIfma_V512/AvxIfma_V512_handwritten_ro.csproj new file mode 100644 index 00000000000000..57a7d38c9c1099 --- /dev/null +++ b/src/tests/JIT/HardwareIntrinsics/X86_Avx/AvxIfma_V512/AvxIfma_V512_handwritten_ro.csproj @@ -0,0 +1,13 @@ + + + X86_AvxIfma_V512_handwritten_ro + true + + + Embedded + True + + + + + diff --git a/src/tests/JIT/HardwareIntrinsics/X86_Avx/AvxIfma_V512/AvxIfma_V512_r.csproj b/src/tests/JIT/HardwareIntrinsics/X86_Avx/AvxIfma_V512/AvxIfma_V512_r.csproj new file mode 100644 index 00000000000000..e2aaaaf21ba860 --- /dev/null +++ b/src/tests/JIT/HardwareIntrinsics/X86_Avx/AvxIfma_V512/AvxIfma_V512_r.csproj @@ -0,0 +1,14 @@ + + + X86_AvxIfma_V512_r + true + + + Embedded + + + + + + + diff --git a/src/tests/JIT/HardwareIntrinsics/X86_Avx/AvxIfma_V512/AvxIfma_V512_ro.csproj b/src/tests/JIT/HardwareIntrinsics/X86_Avx/AvxIfma_V512/AvxIfma_V512_ro.csproj new file mode 100644 index 00000000000000..fe6d8604fc33af --- /dev/null +++ b/src/tests/JIT/HardwareIntrinsics/X86_Avx/AvxIfma_V512/AvxIfma_V512_ro.csproj @@ -0,0 +1,15 @@ + + + X86_AvxIfma_V512_ro + true + + + Embedded + True + + + + + + + diff --git a/src/tests/JIT/HardwareIntrinsics/X86_Avx/AvxIfma_V512/Program.AvxIfma_V512.cs b/src/tests/JIT/HardwareIntrinsics/X86_Avx/AvxIfma_V512/Program.AvxIfma_V512.cs new file mode 100644 index 00000000000000..1b48c78154b304 --- /dev/null +++ b/src/tests/JIT/HardwareIntrinsics/X86_Avx/AvxIfma_V512/Program.AvxIfma_V512.cs @@ -0,0 +1,9 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +namespace IntelHardwareIntrinsicTest._AvxIfma_V512 +{ + public partial class Program + { + } +} diff --git a/src/tests/nativeaot/SmokeTests/HardwareIntrinsics/Program.cs b/src/tests/nativeaot/SmokeTests/HardwareIntrinsics/Program.cs index b1d340da18e895..6860db42c6af44 100644 --- a/src/tests/nativeaot/SmokeTests/HardwareIntrinsics/Program.cs +++ b/src/tests/nativeaot/SmokeTests/HardwareIntrinsics/Program.cs @@ -451,8 +451,8 @@ static int Main() // Check("Avx512Vp2intersect.VL", ExpectedAvx512Vp2intersect, &Avx512Vp2intersectVLIsSupported, Avx512Vp2intersect.VL.IsSupported, null); // Check("Avx512Vp2intersect.X64", ExpectedAvx512Vp2intersect, &Avx512Vp2intersectX64IsSupported, Avx512Vp2intersect.X64.IsSupported, null); - // Check("AvxIfma", ExpectedAvxIfma, &AvxIfmaIsSupported, AvxIfma.IsSupported, null); - // Check("AvxIfma.X64", ExpectedAvxIfma, &AvxIfmaX64IsSupported, AvxIfma.X64.IsSupported, null); + Check("AvxIfma", ExpectedAvxIfma, &AvxIfmaIsSupported, AvxIfma.IsSupported, () => AvxIfma.MultiplyAdd52Low(Vector128.Zero, Vector128.Zero, Vector128.Zero).Equals(Vector128.Zero)); + Check("AvxIfma.X64", ExpectedAvxIfma, &AvxIfmaX64IsSupported, AvxIfma.X64.IsSupported, null); Check("AvxVnni", ExpectedAvxVnni, &AvxVnniIsSupported, AvxVnni.IsSupported, () => AvxVnni.MultiplyWideningAndAdd(Vector128.Zero, Vector128.Zero, Vector128.Zero).Equals(Vector128.Zero)); Check("AvxVnni.X64", ExpectedAvxVnni, &AvxVnniX64IsSupported, AvxVnni.X64.IsSupported, null); @@ -463,6 +463,12 @@ static int Main() // must still report true. Check("AvxVnni.V512", ExpectedAvx512Vbmi2, &AvxVnniV512IsSupported, AvxVnni.V512.IsSupported, () => AvxVnni.V512.MultiplyWideningAndAdd(Vector512.Zero, Vector512.Zero, Vector512.Zero).Equals(Vector512.Zero)); + // AvxIfma.V512 is folded under AVX512v2 (Avx512Vbmi is a representative + // sibling). On a machine that has AVX-512-IFMA but lacks the dedicated + // VEX AvxIfma CPUID bit, AvxIfma.V512.IsSupported (and AvxIfma.IsSupported) + // must still report true. + Check("AvxIfma.V512", ExpectedAvx512Vbmi, &AvxIfmaV512IsSupported, AvxIfma.V512.IsSupported, () => AvxIfma.V512.MultiplyAdd52Low(Vector512.Zero, Vector512.Zero, Vector512.Zero).Equals(Vector512.Zero)); + Check("Gfni", ExpectedGfni, &GfniIsSupported, Gfni.IsSupported, () => Gfni.GaloisFieldMultiply(Vector128.Zero, Vector128.Zero).Equals(Vector128.Zero)); Check("Gfni.V256", ExpectedGfniV256, &GfniV256IsSupported, Gfni.V256.IsSupported, () => Gfni.V256.GaloisFieldMultiply(Vector256.Zero, Vector256.Zero).Equals(Vector256.Zero)); Check("Gfni.V512", ExpectedGfniV512, &GfniV512IsSupported, Gfni.V512.IsSupported, () => Gfni.V512.GaloisFieldMultiply(Vector512.Zero, Vector512.Zero).Equals(Vector512.Zero)); @@ -591,8 +597,9 @@ static int Main() // static bool Avx512Vp2intersectVLIsSupported() => Avx512Vp2intersect.VL.IsSupported; // static bool Avx512Vp2intersectX64IsSupported() => Avx512Vp2intersect.X64.IsSupported; - // static bool AvxIfmaIsSupported() => AvxIfma.IsSupported; - // static bool AvxIfmaX64IsSupported() => AvxIfma.X64.IsSupported; + static bool AvxIfmaIsSupported() => AvxIfma.IsSupported; + static bool AvxIfmaX64IsSupported() => AvxIfma.X64.IsSupported; + static bool AvxIfmaV512IsSupported() => AvxIfma.V512.IsSupported; static bool AvxVnniIsSupported() => AvxVnni.IsSupported; static bool AvxVnniX64IsSupported() => AvxVnni.X64.IsSupported; From add8927e21d358e93839959ab8309a25d2f728d2 Mon Sep 17 00:00:00 2001 From: James Burton Date: Wed, 1 Jul 2026 15:27:33 +0100 Subject: [PATCH 3/6] =?UTF-8?q?Fix=20hwintrinsiclistxarch.h=20ordering=20?= =?UTF-8?q?=E2=80=94=20binarySearchId=20requires=20alphabetical?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit binarySearchId uses strcmp (case-sensitive ASCII order) to find intrinsics within an ISA range. My initial ordering put MultiplyAdd52Low before MultiplyAdd52High, but 'H' (0x48) < 'L' (0x4C), so the binary search would fail on High. Same issue for both AVXIFMA and AVX512v2 ranges. Also reorders MultiplyAdd52*/PermuteVar* so Multiply* precedes PermuteVar* (M < P). This surfaced as ILC 'Code generation failed' on the AvxIfma_handwritten_r NativeAOT tests. --- src/coreclr/jit/hwintrinsiclistxarch.h | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/coreclr/jit/hwintrinsiclistxarch.h b/src/coreclr/jit/hwintrinsiclistxarch.h index d50997833dfac2..d750a3ed6e51ef 100644 --- a/src/coreclr/jit/hwintrinsiclistxarch.h +++ b/src/coreclr/jit/hwintrinsiclistxarch.h @@ -1090,15 +1090,15 @@ HARDWARE_INTRINSIC(AVX512_X64, ConvertToUInt64WithTruncation, // Intrinsics for AVX512-IFMA, AVX512-VBMI #define FIRST_NI_AVX512v2 NI_AVX512v2_MultiShift HARDWARE_INTRINSIC(AVX512v2, MultiShift, -1, 2, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vpmultishiftqb, INS_vpmultishiftqb, INS_invalid, INS_invalid}, 3, -1, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromSecondArg) +HARDWARE_INTRINSIC(AVX512v2, MultiplyAdd52High, 64, 3, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vpmadd52huq, INS_invalid, INS_invalid}, 5, -1, HW_Category_SimpleSIMD, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AVX512v2, MultiplyAdd52Low, 64, 3, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vpmadd52luq, INS_invalid, INS_invalid}, 5, -1, HW_Category_SimpleSIMD, HW_Flag_NoFlag) HARDWARE_INTRINSIC(AVX512v2, PermuteVar16x8, 16, 2, {INS_vpermb, INS_vpermb, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, 3, -1, HW_Category_SimpleSIMD, HW_Flag_SpecialImport) HARDWARE_INTRINSIC(AVX512v2, PermuteVar16x8x2, 16, 3, {INS_vpermt2b, INS_vpermt2b, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, 4, -1, HW_Category_SimpleSIMD, HW_Flag_SpecialCodeGen|HW_Flag_PermuteVar2x|HW_Flag_RmwIntrinsic) HARDWARE_INTRINSIC(AVX512v2, PermuteVar32x8, 32, 2, {INS_vpermb, INS_vpermb, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, 3, -1, HW_Category_SimpleSIMD, HW_Flag_SpecialImport) HARDWARE_INTRINSIC(AVX512v2, PermuteVar32x8x2, 32, 3, {INS_vpermt2b, INS_vpermt2b, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, 4, -1, HW_Category_SimpleSIMD, HW_Flag_SpecialCodeGen|HW_Flag_PermuteVar2x|HW_Flag_RmwIntrinsic) HARDWARE_INTRINSIC(AVX512v2, PermuteVar64x8, 64, 2, {INS_vpermb, INS_vpermb, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, 3, -1, HW_Category_SimpleSIMD, HW_Flag_SpecialImport) HARDWARE_INTRINSIC(AVX512v2, PermuteVar64x8x2, 64, 3, {INS_vpermt2b, INS_vpermt2b, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, 4, -1, HW_Category_SimpleSIMD, HW_Flag_SpecialCodeGen|HW_Flag_PermuteVar2x|HW_Flag_RmwIntrinsic) -HARDWARE_INTRINSIC(AVX512v2, MultiplyAdd52Low, 64, 3, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vpmadd52luq, INS_invalid, INS_invalid}, 5, -1, HW_Category_SimpleSIMD, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(AVX512v2, MultiplyAdd52High, 64, 3, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vpmadd52huq, INS_invalid, INS_invalid}, 5, -1, HW_Category_SimpleSIMD, HW_Flag_NoFlag) -#define LAST_NI_AVX512v2 NI_AVX512v2_MultiplyAdd52High +#define LAST_NI_AVX512v2 NI_AVX512v2_PermuteVar64x8x2 // *************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************** // ISA Function name SIMD size NumArg Instructions IntCost FltCost Category Flags @@ -1168,10 +1168,10 @@ HARDWARE_INTRINSIC(AVXVNNI, MultiplyWideningAndAddSaturate, // {TYP_BYTE, TYP_UBYTE, TYP_SHORT, TYP_USHORT, TYP_INT, TYP_UINT, TYP_LONG, TYP_ULONG, TYP_FLOAT, TYP_DOUBLE} // *************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************** // Intrinsics for AVX-IFMA (VEX-encoded VPMADD52*UQ on V128 / V256) -#define FIRST_NI_AVXIFMA NI_AVXIFMA_MultiplyAdd52Low -HARDWARE_INTRINSIC(AVXIFMA, MultiplyAdd52Low, -1, 3, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vpmadd52luq, INS_invalid, INS_invalid}, 5, -1, HW_Category_SimpleSIMD, HW_Flag_NoEvexSemantics) +#define FIRST_NI_AVXIFMA NI_AVXIFMA_MultiplyAdd52High HARDWARE_INTRINSIC(AVXIFMA, MultiplyAdd52High, -1, 3, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vpmadd52huq, INS_invalid, INS_invalid}, 5, -1, HW_Category_SimpleSIMD, HW_Flag_NoEvexSemantics) -#define LAST_NI_AVXIFMA NI_AVXIFMA_MultiplyAdd52High +HARDWARE_INTRINSIC(AVXIFMA, MultiplyAdd52Low, -1, 3, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vpmadd52luq, INS_invalid, INS_invalid}, 5, -1, HW_Category_SimpleSIMD, HW_Flag_NoEvexSemantics) +#define LAST_NI_AVXIFMA NI_AVXIFMA_MultiplyAdd52Low // *************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************** // ISA Function name SIMD size NumArg Instructions IntCost FltCost Category Flags From fc7299fd915279f4885306a29cfb4f339b0de2a8 Mon Sep 17 00:00:00 2001 From: James Burton Date: Wed, 1 Jul 2026 16:38:20 +0100 Subject: [PATCH 4/6] Add AvxIfma type lookups for X64_AVXIFMA / X86_AVXIFMA cases The LookupPlatformIntrinsicTypes switch was missing cases for InstructionSet.X64_AVXIFMA/AVXIFMA_X64 and X86_AVXIFMA. When the AOT compiler tried to resolve AvxIfma against an AVXIFMA-only target (e.g. NativeAOT smoke tests X64Baseline / X64Avx2), the type wasn't found and ILC failed with 'Code generation failed for AvxIfma.get_IsSupported()'. Mirrors the existing X64_AVXVNNI/X86_AVXVNNI cases. --- .../JitInterface/CorInfoInstructionSet.cs | 29 +++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/src/coreclr/tools/Common/JitInterface/CorInfoInstructionSet.cs b/src/coreclr/tools/Common/JitInterface/CorInfoInstructionSet.cs index a5f3f330b8453f..c653b171485827 100644 --- a/src/coreclr/tools/Common/JitInterface/CorInfoInstructionSet.cs +++ b/src/coreclr/tools/Common/JitInterface/CorInfoInstructionSet.cs @@ -2999,6 +2999,25 @@ public static IEnumerable LookupPlatformIntrinsicTypes(TypeSystemC } break; + case (InstructionSet.X64_AVXIFMA, TargetArchitecture.X64): + case (InstructionSet.X64_AVXIFMA_X64, TargetArchitecture.X64): + { + var type = context.SystemModule.GetType("System.Runtime.Intrinsics.X86"u8, "AvxIfma"u8, false); + if (type != null) + { + yield return type; + if (instructionSet == InstructionSet.X64_AVXIFMA_X64) + { + var nestedType = type.GetNestedType("X64"u8); + if (nestedType != null) + { + yield return nestedType; + } + } + } + } + break; + case (InstructionSet.X64_AVX512BMM, TargetArchitecture.X64): { var type = context.SystemModule.GetType("System.Runtime.Intrinsics.X86"u8, "Avx512Bmm"u8, false); @@ -3476,6 +3495,16 @@ public static IEnumerable LookupPlatformIntrinsicTypes(TypeSystemC } break; + case (InstructionSet.X86_AVXIFMA, TargetArchitecture.X86): + { + var type = context.SystemModule.GetType("System.Runtime.Intrinsics.X86"u8, "AvxIfma"u8, false); + if (type != null) + { + yield return type; + } + } + break; + case (InstructionSet.X86_AVX512BMM, TargetArchitecture.X86): { var type = context.SystemModule.GetType("System.Runtime.Intrinsics.X86"u8, "Avx512Bmm"u8, false); From ff40b908575e122cfe95f71f7302d4acbc541bf9 Mon Sep 17 00:00:00 2001 From: James Burton Date: Wed, 1 Jul 2026 19:44:30 +0100 Subject: [PATCH 5/6] Set 'AvxIfma' as the ManagedName so ILC can look it up InstructionSetDesc.txt had AvxIfma in the R2R-name column (col 4) instead of the managed-name column (col 3), so the generated ArchitectureToValidInstructionSets emitted 'avxifma' with an empty ManagedName. HardwareIntrinsicILProvider builds its class-name -> InstructionSet dictionary from ArchitectureToValidInstructionSets, skipping entries with empty ManagedName, so lookups for 'AvxIfma' threw KeyNotFoundException at ILC time. Fixed the .txt source-of-truth (so any regeneration is consistent) and the already-generated CorInfoInstructionSet.cs to set ManagedName='AvxIfma'. --- .../tools/Common/JitInterface/CorInfoInstructionSet.cs | 4 ++-- .../Common/JitInterface/ThunkGenerator/InstructionSetDesc.txt | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/coreclr/tools/Common/JitInterface/CorInfoInstructionSet.cs b/src/coreclr/tools/Common/JitInterface/CorInfoInstructionSet.cs index c653b171485827..07d6d731a2a519 100644 --- a/src/coreclr/tools/Common/JitInterface/CorInfoInstructionSet.cs +++ b/src/coreclr/tools/Common/JitInterface/CorInfoInstructionSet.cs @@ -1218,7 +1218,7 @@ public static IEnumerable ArchitectureToValidInstructionSets yield return new InstructionSetInfo("aes_v512", "Pclmulqdq_V512", InstructionSet.X64_AES_V512, true); yield return new InstructionSetInfo("avx512vp2intersect", "", InstructionSet.X64_AVX512VP2INTERSECT, true); yield return new InstructionSetInfo("avx512vp2intersect", "", InstructionSet.X64_AVX512VP2INTERSECT, true); - yield return new InstructionSetInfo("avxifma", "", InstructionSet.X64_AVXIFMA, true); + yield return new InstructionSetInfo("avxifma", "AvxIfma", InstructionSet.X64_AVXIFMA, true); yield return new InstructionSetInfo("avxvnni", "AvxVnni", InstructionSet.X64_AVXVNNI, true); yield return new InstructionSetInfo("avx512bmm", "Avx512Bmm", InstructionSet.X64_AVX512BMM, true); yield return new InstructionSetInfo("gfni", "Gfni", InstructionSet.X64_GFNI, true); @@ -1293,7 +1293,7 @@ public static IEnumerable ArchitectureToValidInstructionSets yield return new InstructionSetInfo("aes_v512", "Pclmulqdq_V512", InstructionSet.X86_AES_V512, true); yield return new InstructionSetInfo("avx512vp2intersect", "", InstructionSet.X86_AVX512VP2INTERSECT, true); yield return new InstructionSetInfo("avx512vp2intersect", "", InstructionSet.X86_AVX512VP2INTERSECT, true); - yield return new InstructionSetInfo("avxifma", "", InstructionSet.X86_AVXIFMA, true); + yield return new InstructionSetInfo("avxifma", "AvxIfma", InstructionSet.X86_AVXIFMA, true); yield return new InstructionSetInfo("avxvnni", "AvxVnni", InstructionSet.X86_AVXVNNI, true); yield return new InstructionSetInfo("avx512bmm", "Avx512Bmm", InstructionSet.X86_AVX512BMM, true); yield return new InstructionSetInfo("gfni", "Gfni", InstructionSet.X86_GFNI, true); diff --git a/src/coreclr/tools/Common/JitInterface/ThunkGenerator/InstructionSetDesc.txt b/src/coreclr/tools/Common/JitInterface/ThunkGenerator/InstructionSetDesc.txt index 1019b14afba917..92484692db9dcc 100644 --- a/src/coreclr/tools/Common/JitInterface/ThunkGenerator/InstructionSetDesc.txt +++ b/src/coreclr/tools/Common/JitInterface/ThunkGenerator/InstructionSetDesc.txt @@ -96,7 +96,7 @@ instructionset ,X86 ,Pclmulqdq_V512 , ,50 ,AES_V51 instructionset ,X86 , ,Avx512Vp2intersect ,80 ,AVX512VP2INTERSECT ,avx512vp2intersect instructionset ,X86 , ,Avx512Vp2intersect_VL ,81 ,AVX512VP2INTERSECT ,avx512vp2intersect -instructionset ,X86 , ,AvxIfma ,66 ,AVXIFMA ,avxifma +instructionset ,X86 ,AvxIfma , ,66 ,AVXIFMA ,avxifma instructionset ,X86 ,AvxVnni , ,25 ,AVXVNNI ,avxvnni instructionset ,X86 ,Avx512Bmm , ,85 ,AVX512BMM ,avx512bmm From 52cbd74cbe1bb8640c2abdbcf35fe0cad48ba2de Mon Sep 17 00:00:00 2001 From: James Burton Date: Sat, 4 Jul 2026 13:01:25 +0100 Subject: [PATCH 6/6] Fix simdSize for AVX512v2 MultiplyAdd52High/Low fallback entries Per Tanner's review feedback: these entries are reachable via the AvxIfma class-name fallback (when AVXIFMA isn't supported but AVX512v2 is), so a fixed simdSize=64 causes the importer's lookupSimdSize short-circuit to hardcode the node's SIMD size to 64 even for genuine Vector128/256 call sites. Setting simdSize=-1 matches the AVXIFMA entries and lets the importer derive the actual width from the call site signature instead. --- src/coreclr/jit/hwintrinsiclistxarch.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/coreclr/jit/hwintrinsiclistxarch.h b/src/coreclr/jit/hwintrinsiclistxarch.h index 6b1209b815a585..448c943f529e40 100644 --- a/src/coreclr/jit/hwintrinsiclistxarch.h +++ b/src/coreclr/jit/hwintrinsiclistxarch.h @@ -634,8 +634,8 @@ HARDWARE_INTRINSIC(AVX512_X64, ConvertToUInt64WithTruncation, // Intrinsics for AVX512-IFMA, AVX512-VBMI #define FIRST_NI_AVX512v2 NI_AVX512v2_MultiShift HARDWARE_INTRINSIC(AVX512v2, MultiShift, -1, 2, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vpmultishiftqb, INS_vpmultishiftqb, INS_invalid, INS_invalid, 3, -1, HW_Category_SimpleSIMD, HW_Flag_BaseTypeFromSecondArg) -HARDWARE_INTRINSIC(AVX512v2, MultiplyAdd52High, 64, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vpmadd52huq, INS_invalid, INS_invalid, 5, -1, HW_Category_SimpleSIMD, HW_Flag_NoFlag) -HARDWARE_INTRINSIC(AVX512v2, MultiplyAdd52Low, 64, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vpmadd52luq, INS_invalid, INS_invalid, 5, -1, HW_Category_SimpleSIMD, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AVX512v2, MultiplyAdd52High, -1, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vpmadd52huq, INS_invalid, INS_invalid, 5, -1, HW_Category_SimpleSIMD, HW_Flag_NoFlag) +HARDWARE_INTRINSIC(AVX512v2, MultiplyAdd52Low, -1, 3, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_vpmadd52luq, INS_invalid, INS_invalid, 5, -1, HW_Category_SimpleSIMD, HW_Flag_NoFlag) HARDWARE_INTRINSIC(AVX512v2, PermuteVar16x8, 16, 2, INS_vpermb, INS_vpermb, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 3, -1, HW_Category_SimpleSIMD, HW_Flag_SpecialImport) HARDWARE_INTRINSIC(AVX512v2, PermuteVar16x8x2, 16, 3, INS_vpermt2b, INS_vpermt2b, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 4, -1, HW_Category_SimpleSIMD, HW_Flag_SpecialCodeGen|HW_Flag_PermuteVar2x|HW_Flag_RmwIntrinsic) HARDWARE_INTRINSIC(AVX512v2, PermuteVar32x8, 32, 2, INS_vpermb, INS_vpermb, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, 3, -1, HW_Category_SimpleSIMD, HW_Flag_SpecialImport)