From 6d43eb27c8fda9e1307928bb2ffc1018faa52365 Mon Sep 17 00:00:00 2001 From: Sebastian Nickolls Date: Fri, 5 Sep 2025 09:57:23 +0000 Subject: [PATCH 1/3] Use unpredicated movprfx with SVE2 fminnmp and fmaxnmp Codegen is currently emitting a predicated `movprfx` with these instructions in combination with `ConditionalSelect`, which results in unpredictable behaviour. Use unpredicated `movprfx` instead and apply the mask to the result of the instruction. --- src/coreclr/jit/hwintrinsiccodegenarm64.cpp | 12 ++++++++++++ src/coreclr/jit/lowerarmarch.cpp | 16 ++++++++++++++-- 2 files changed, 26 insertions(+), 2 deletions(-) diff --git a/src/coreclr/jit/hwintrinsiccodegenarm64.cpp b/src/coreclr/jit/hwintrinsiccodegenarm64.cpp index 13c7163c4fcbb1..173006f034f574 100644 --- a/src/coreclr/jit/hwintrinsiccodegenarm64.cpp +++ b/src/coreclr/jit/hwintrinsiccodegenarm64.cpp @@ -783,6 +783,18 @@ void CodeGen::genHWIntrinsic(GenTreeHWIntrinsic* node) emitInsHelper(targetReg, maskReg, embMaskOp2Reg); break; + case NI_Sve2_MaxNumberPairwise: + case NI_Sve2_MinNumberPairwise: + // These instructions have unpredictable behaviour when using predicated movprfx, + // so the unpredicated variant must be used here. + assert(!intrin.op3->isContained() && falseReg != REG_NA); + GetEmitter()->emitIns_R_R(INS_sve_movprfx, EA_SCALABLE, targetReg, embMaskOp1Reg); + GetEmitter()->emitIns_R_R_R(insEmbMask, emitSize, targetReg, maskReg, embMaskOp2Reg, + embOpt, sopt); + GetEmitter()->emitIns_R_R_R_R(INS_sve_sel, emitSize, targetReg, maskReg, targetReg, + falseReg, opt); + break; + default: assert(targetReg != embMaskOp2Reg); diff --git a/src/coreclr/jit/lowerarmarch.cpp b/src/coreclr/jit/lowerarmarch.cpp index f1d450661bdbcd..c1bb7a1fc5cdc2 100644 --- a/src/coreclr/jit/lowerarmarch.cpp +++ b/src/coreclr/jit/lowerarmarch.cpp @@ -4061,8 +4061,20 @@ void Lowering::ContainCheckHWIntrinsic(GenTreeHWIntrinsic* node) // When we are merging with zero, we can specialize // and avoid instantiating the vector constant. // Do this only if op1 was AllTrueMask - MakeSrcContained(node, op3); - LABELEDDISPTREERANGE("Contained false mask op3 in ConditionalSelect", BlockRange(), op3); + switch (op2->AsHWIntrinsic()->GetHWIntrinsicId()) + { + case NI_Sve2_MinNumberPairwise: + case NI_Sve2_MaxNumberPairwise: + // This is an edge case where these instructions have unpredictable behaviour when + // using predicated movprfx, so the unpredicated variant must be used here. This + // prevents us from performing this optimization as we will need the constant vector + // for masking the result. + break; + + default: + MakeSrcContained(node, op3); + LABELEDDISPTREERANGE("Contained false mask op3 in ConditionalSelect", BlockRange(), op3); + } } break; From ffeede6db77b9417bea2aa3e96a1110e6782e148 Mon Sep 17 00:00:00 2001 From: Sebastian Nickolls Date: Thu, 11 Sep 2025 08:42:19 +0000 Subject: [PATCH 2/3] Add predicated movprfx checker to emitter, add more exception cases to movprfx optimization --- src/coreclr/jit/emitarm64sve.cpp | 1011 +++++++++++++++++++ src/coreclr/jit/hwintrinsiccodegenarm64.cpp | 3 + src/coreclr/jit/lowerarmarch.cpp | 5 +- 3 files changed, 1018 insertions(+), 1 deletion(-) diff --git a/src/coreclr/jit/emitarm64sve.cpp b/src/coreclr/jit/emitarm64sve.cpp index 83e367392ac9c7..98c45ef2f13759 100644 --- a/src/coreclr/jit/emitarm64sve.cpp +++ b/src/coreclr/jit/emitarm64sve.cpp @@ -18690,6 +18690,1017 @@ void emitter::emitInsPairSanityCheck(instrDesc* firstId, instrDesc* secondId) // "predicated using the same governing predicate register and source element size as this instruction." assert(firstId->idInsOpt() == secondId->idInsOpt()); } + + // The following instructions cannot use predicated movprfx, else the behaviour will be unpredictable. + switch (secondId->idInsFmt()) + { + case IF_SVE_AA_3A: + { + switch (secondId->idIns()) + { + case INS_sve_addp: + case INS_sve_smaxp: + case INS_sve_sminp: + case INS_sve_umaxp: + case INS_sve_uminp: + assert(!movprefxIsPredicated); + break; + default: + break; + } + break; + } + case IF_SVE_AT_3A: + { + switch (secondId->idIns()) + { + case INS_sve_eorbt: + case INS_sve_eortb: + case INS_sve_fclamp: + case INS_sve_sclamp: + case INS_sve_uclamp: + assert(!movprefxIsPredicated); + break; + default: + break; + } + break; + } + case IF_SVE_AV_3A: + { + switch (secondId->idIns()) + { + case INS_sve_bcax: + case INS_sve_bsl: + case INS_sve_bsl1n: + case INS_sve_bsl2n: + case INS_sve_eor3: + case INS_sve_nbsl: + assert(!movprefxIsPredicated); + break; + default: + break; + } + break; + } + case IF_SVE_AW_2A: + { + switch (secondId->idIns()) + { + case INS_sve_xar: + assert(!movprefxIsPredicated); + break; + default: + break; + } + break; + } + case IF_SVE_BN_1A: + { + switch (secondId->idIns()) + { + case INS_sve_decd: + case INS_sve_dech: + case INS_sve_decw: + case INS_sve_incd: + case INS_sve_inch: + case INS_sve_incw: + assert(!movprefxIsPredicated); + break; + default: + break; + } + break; + } + case IF_SVE_BP_1A: + { + switch (secondId->idIns()) + { + case INS_sve_sqdecd: + case INS_sve_sqdech: + case INS_sve_sqdecw: + case INS_sve_sqincd: + case INS_sve_sqinch: + case INS_sve_sqincw: + case INS_sve_uqdecd: + case INS_sve_uqdech: + case INS_sve_uqdecw: + case INS_sve_uqincd: + case INS_sve_uqinch: + case INS_sve_uqincw: + assert(!movprefxIsPredicated); + break; + default: + break; + } + break; + } + case IF_SVE_BS_1A: + { + switch (secondId->idIns()) + { + case INS_sve_and: + case INS_sve_bic: + case INS_sve_eon: + case INS_sve_eor: + case INS_sve_orn: + case INS_sve_orr: + assert(!movprefxIsPredicated); + break; + default: + break; + } + break; + } + case IF_SVE_BY_2A: + { + switch (secondId->idIns()) + { + case INS_sve_extq: + assert(!movprefxIsPredicated); + break; + default: + break; + } + break; + } + case IF_SVE_CC_2A: + { + switch (secondId->idIns()) + { + case INS_sve_insr: + assert(!movprefxIsPredicated); + break; + default: + break; + } + break; + } + case IF_SVE_CD_2A: + { + switch (secondId->idIns()) + { + case INS_sve_insr: + assert(!movprefxIsPredicated); + break; + default: + break; + } + break; + } + case IF_SVE_CM_3A: + { + switch (secondId->idIns()) + { + case INS_sve_clasta: + case INS_sve_clastb: + assert(!movprefxIsPredicated); + break; + default: + break; + } + break; + } + case IF_SVE_CT_3A: + { + switch (secondId->idIns()) + { + case INS_sve_revd: + assert(!movprefxIsPredicated); + break; + default: + break; + } + break; + } + case IF_SVE_DN_2A: + { + switch (secondId->idIns()) + { + case INS_sve_decp: + case INS_sve_incp: + assert(!movprefxIsPredicated); + break; + default: + break; + } + break; + } + case IF_SVE_DP_2A: + { + switch (secondId->idIns()) + { + case INS_sve_sqdecp: + case INS_sve_sqincp: + case INS_sve_uqdecp: + case INS_sve_uqincp: + assert(!movprefxIsPredicated); + break; + default: + break; + } + break; + } + case IF_SVE_EC_1A: + { + switch (secondId->idIns()) + { + case INS_sve_add: + case INS_sve_sqadd: + case INS_sve_sqsub: + case INS_sve_sub: + case INS_sve_subr: + case INS_sve_uqadd: + case INS_sve_uqsub: + assert(!movprefxIsPredicated); + break; + default: + break; + } + break; + } + case IF_SVE_ED_1A: + { + switch (secondId->idIns()) + { + case INS_sve_smax: + case INS_sve_smin: + case INS_sve_umax: + case INS_sve_umin: + assert(!movprefxIsPredicated); + break; + default: + break; + } + break; + } + case IF_SVE_EE_1A: + { + switch (secondId->idIns()) + { + case INS_sve_mul: + assert(!movprefxIsPredicated); + break; + default: + break; + } + break; + } + case IF_SVE_EF_3A: + { + switch (secondId->idIns()) + { + case INS_sve_sdot: + case INS_sve_udot: + assert(!movprefxIsPredicated); + break; + default: + break; + } + break; + } + case IF_SVE_EG_3A: + { + switch (secondId->idIns()) + { + case INS_sve_sdot: + case INS_sve_udot: + assert(!movprefxIsPredicated); + break; + default: + break; + } + break; + } + case IF_SVE_EH_3A: + { + switch (secondId->idIns()) + { + case INS_sve_sdot: + case INS_sve_udot: + assert(!movprefxIsPredicated); + break; + default: + break; + } + break; + } + case IF_SVE_EI_3A: + { + switch (secondId->idIns()) + { + case INS_sve_usdot: + assert(!movprefxIsPredicated); + break; + default: + break; + } + break; + } + case IF_SVE_EJ_3A: + { + switch (secondId->idIns()) + { + case INS_sve_cdot: + assert(!movprefxIsPredicated); + break; + default: + break; + } + break; + } + case IF_SVE_EK_3A: + { + switch (secondId->idIns()) + { + case INS_sve_cmla: + case INS_sve_sqrdcmlah: + assert(!movprefxIsPredicated); + break; + default: + break; + } + break; + } + case IF_SVE_EL_3A: + { + switch (secondId->idIns()) + { + case INS_sve_sabalb: + case INS_sve_sabalt: + case INS_sve_smlalb: + case INS_sve_smlalt: + case INS_sve_smlslb: + case INS_sve_smlslt: + case INS_sve_sqdmlalb: + case INS_sve_sqdmlalbt: + case INS_sve_sqdmlalt: + case INS_sve_sqdmlslb: + case INS_sve_sqdmlslbt: + case INS_sve_sqdmlslt: + case INS_sve_uabalb: + case INS_sve_uabalt: + case INS_sve_umlalb: + case INS_sve_umlalt: + case INS_sve_umlslb: + case INS_sve_umlslt: + assert(!movprefxIsPredicated); + break; + default: + break; + } + break; + } + case IF_SVE_EM_3A: + { + switch (secondId->idIns()) + { + case INS_sve_sqrdmlah: + case INS_sve_sqrdmlsh: + assert(!movprefxIsPredicated); + break; + default: + break; + } + break; + } + case IF_SVE_EW_3A: + { + switch (secondId->idIns()) + { + case INS_sve_mlapt: + assert(!movprefxIsPredicated); + break; + default: + break; + } + break; + } + case IF_SVE_EW_3B: + { + switch (secondId->idIns()) + { + case INS_sve_madpt: + assert(!movprefxIsPredicated); + break; + default: + break; + } + break; + } + case IF_SVE_EY_3A: + { + switch (secondId->idIns()) + { + case INS_sve_sdot: + case INS_sve_udot: + assert(!movprefxIsPredicated); + break; + default: + break; + } + break; + } + case IF_SVE_EY_3B: + { + switch (secondId->idIns()) + { + case INS_sve_sdot: + case INS_sve_udot: + assert(!movprefxIsPredicated); + break; + default: + break; + } + break; + } + case IF_SVE_EZ_3A: + { + switch (secondId->idIns()) + { + case INS_sve_sudot: + case INS_sve_usdot: + assert(!movprefxIsPredicated); + break; + default: + break; + } + break; + } + case IF_SVE_FA_3A: + { + switch (secondId->idIns()) + { + case INS_sve_cdot: + assert(!movprefxIsPredicated); + break; + default: + break; + } + break; + } + case IF_SVE_FA_3B: + { + switch (secondId->idIns()) + { + case INS_sve_cdot: + assert(!movprefxIsPredicated); + break; + default: + break; + } + break; + } + case IF_SVE_FB_3A: + { + switch (secondId->idIns()) + { + case INS_sve_cmla: + assert(!movprefxIsPredicated); + break; + default: + break; + } + break; + } + case IF_SVE_FB_3B: + { + switch (secondId->idIns()) + { + case INS_sve_cmla: + assert(!movprefxIsPredicated); + break; + default: + break; + } + break; + } + case IF_SVE_FC_3A: + { + switch (secondId->idIns()) + { + case INS_sve_sqrdcmlah: + assert(!movprefxIsPredicated); + break; + default: + break; + } + break; + } + case IF_SVE_FC_3B: + { + switch (secondId->idIns()) + { + case INS_sve_sqrdcmlah: + assert(!movprefxIsPredicated); + break; + default: + break; + } + break; + } + case IF_SVE_FF_3A: + { + switch (secondId->idIns()) + { + case INS_sve_mla: + case INS_sve_mls: + assert(!movprefxIsPredicated); + break; + default: + break; + } + break; + } + case IF_SVE_FF_3B: + { + switch (secondId->idIns()) + { + case INS_sve_mla: + case INS_sve_mls: + assert(!movprefxIsPredicated); + break; + default: + break; + } + break; + } + case IF_SVE_FF_3C: + { + switch (secondId->idIns()) + { + case INS_sve_mla: + case INS_sve_mls: + assert(!movprefxIsPredicated); + break; + default: + break; + } + break; + } + case IF_SVE_FG_3A: + { + switch (secondId->idIns()) + { + case INS_sve_smlalb: + case INS_sve_smlalt: + case INS_sve_smlslb: + case INS_sve_smlslt: + case INS_sve_umlalb: + case INS_sve_umlalt: + case INS_sve_umlslb: + case INS_sve_umlslt: + assert(!movprefxIsPredicated); + break; + default: + break; + } + break; + } + case IF_SVE_FG_3B: + { + switch (secondId->idIns()) + { + case INS_sve_smlalb: + case INS_sve_smlalt: + case INS_sve_smlslb: + case INS_sve_smlslt: + case INS_sve_umlalb: + case INS_sve_umlalt: + case INS_sve_umlslb: + case INS_sve_umlslt: + assert(!movprefxIsPredicated); + break; + default: + break; + } + break; + } + case IF_SVE_FJ_3A: + { + switch (secondId->idIns()) + { + case INS_sve_sqdmlalb: + case INS_sve_sqdmlalt: + case INS_sve_sqdmlslb: + case INS_sve_sqdmlslt: + assert(!movprefxIsPredicated); + break; + default: + break; + } + break; + } + case IF_SVE_FJ_3B: + { + switch (secondId->idIns()) + { + case INS_sve_sqdmlalb: + case INS_sve_sqdmlalt: + case INS_sve_sqdmlslb: + case INS_sve_sqdmlslt: + assert(!movprefxIsPredicated); + break; + default: + break; + } + break; + } + case IF_SVE_FK_3A: + { + switch (secondId->idIns()) + { + case INS_sve_sqrdmlah: + case INS_sve_sqrdmlsh: + assert(!movprefxIsPredicated); + break; + default: + break; + } + break; + } + case IF_SVE_FK_3B: + { + switch (secondId->idIns()) + { + case INS_sve_sqrdmlah: + case INS_sve_sqrdmlsh: + assert(!movprefxIsPredicated); + break; + default: + break; + } + break; + } + case IF_SVE_FK_3C: + { + switch (secondId->idIns()) + { + case INS_sve_sqrdmlah: + case INS_sve_sqrdmlsh: + assert(!movprefxIsPredicated); + break; + default: + break; + } + break; + } + case IF_SVE_FO_3A: + { + switch (secondId->idIns()) + { + case INS_sve_smmla: + case INS_sve_ummla: + case INS_sve_usmmla: + assert(!movprefxIsPredicated); + break; + default: + break; + } + break; + } + case IF_SVE_FU_2A: + { + switch (secondId->idIns()) + { + case INS_sve_srsra: + case INS_sve_ssra: + case INS_sve_ursra: + case INS_sve_usra: + assert(!movprefxIsPredicated); + break; + default: + break; + } + break; + } + case IF_SVE_FV_2A: + { + switch (secondId->idIns()) + { + case INS_sve_cadd: + case INS_sve_sqcadd: + assert(!movprefxIsPredicated); + break; + default: + break; + } + break; + } + case IF_SVE_FW_3A: + { + switch (secondId->idIns()) + { + case INS_sve_saba: + case INS_sve_uaba: + assert(!movprefxIsPredicated); + break; + default: + break; + } + break; + } + case IF_SVE_FY_3A: + { + switch (secondId->idIns()) + { + case INS_sve_adclb: + case INS_sve_adclt: + case INS_sve_sbclb: + case INS_sve_sbclt: + assert(!movprefxIsPredicated); + break; + default: + break; + } + break; + } + case IF_SVE_GM_3A: + { + switch (secondId->idIns()) + { + case INS_sve_fmlalb: + case INS_sve_fmlalt: + assert(!movprefxIsPredicated); + break; + default: + break; + } + break; + } + case IF_SVE_GN_3A: + { + switch (secondId->idIns()) + { + case INS_sve_fmlalb: + case INS_sve_fmlalt: + assert(!movprefxIsPredicated); + break; + default: + break; + } + break; + } + case IF_SVE_GO_3A: + { + switch (secondId->idIns()) + { + case INS_sve_fmlallbb: + case INS_sve_fmlallbt: + case INS_sve_fmlalltb: + case INS_sve_fmlalltt: + assert(!movprefxIsPredicated); + break; + default: + break; + } + break; + } + case IF_SVE_GR_3A: + { + switch (secondId->idIns()) + { + case INS_sve_faddp: + case INS_sve_fmaxnmp: + case INS_sve_fmaxp: + case INS_sve_fminnmp: + case INS_sve_fminp: + assert(!movprefxIsPredicated); + break; + default: + break; + } + break; + } + case IF_SVE_GU_3A: + { + switch (secondId->idIns()) + { + case INS_sve_fmla: + case INS_sve_fmls: + assert(!movprefxIsPredicated); + break; + default: + break; + } + break; + } + case IF_SVE_GU_3B: + { + switch (secondId->idIns()) + { + case INS_sve_fmla: + case INS_sve_fmls: + assert(!movprefxIsPredicated); + break; + default: + break; + } + break; + } + case IF_SVE_GU_3C: + { + switch (secondId->idIns()) + { + case INS_sve_bfmla: + case INS_sve_bfmls: + assert(!movprefxIsPredicated); + break; + default: + break; + } + break; + } + case IF_SVE_GV_3A: + { + switch (secondId->idIns()) + { + case INS_sve_fcmla: + assert(!movprefxIsPredicated); + break; + default: + break; + } + break; + } + case IF_SVE_GW_3B: + { + switch (secondId->idIns()) + { + case INS_sve_bfclamp: + assert(!movprefxIsPredicated); + break; + default: + break; + } + break; + } + case IF_SVE_GY_3A: + { + switch (secondId->idIns()) + { + case INS_sve_fdot: + assert(!movprefxIsPredicated); + break; + default: + break; + } + break; + } + case IF_SVE_GY_3B: + { + switch (secondId->idIns()) + { + case INS_sve_bfdot: + case INS_sve_fdot: + assert(!movprefxIsPredicated); + break; + default: + break; + } + break; + } + case IF_SVE_GY_3B_D: + { + switch (secondId->idIns()) + { + case INS_sve_fdot: + assert(!movprefxIsPredicated); + break; + default: + break; + } + break; + } + case IF_SVE_GZ_3A: + { + switch (secondId->idIns()) + { + case INS_sve_bfmlalb: + case INS_sve_bfmlalt: + case INS_sve_bfmlslb: + case INS_sve_bfmlslt: + case INS_sve_fmlalb: + case INS_sve_fmlalt: + case INS_sve_fmlslb: + case INS_sve_fmlslt: + assert(!movprefxIsPredicated); + break; + default: + break; + } + break; + } + case IF_SVE_HA_3A: + { + switch (secondId->idIns()) + { + case INS_sve_bfdot: + case INS_sve_fdot: + assert(!movprefxIsPredicated); + break; + default: + break; + } + break; + } + case IF_SVE_HA_3A_E: + { + switch (secondId->idIns()) + { + case INS_sve_fdot: + assert(!movprefxIsPredicated); + break; + default: + break; + } + break; + } + case IF_SVE_HA_3A_F: + { + switch (secondId->idIns()) + { + case INS_sve_fdot: + assert(!movprefxIsPredicated); + break; + default: + break; + } + break; + } + case IF_SVE_HB_3A: + { + switch (secondId->idIns()) + { + case INS_sve_bfmlalb: + case INS_sve_bfmlalt: + case INS_sve_bfmlslb: + case INS_sve_bfmlslt: + case INS_sve_fmlalb: + case INS_sve_fmlalt: + case INS_sve_fmlslb: + case INS_sve_fmlslt: + assert(!movprefxIsPredicated); + break; + default: + break; + } + break; + } + case IF_SVE_HC_3A: + { + switch (secondId->idIns()) + { + case INS_sve_fmlallbb: + case INS_sve_fmlallbt: + case INS_sve_fmlalltb: + case INS_sve_fmlalltt: + assert(!movprefxIsPredicated); + break; + default: + break; + } + break; + } + case IF_SVE_HD_3A: + { + switch (secondId->idIns()) + { + case INS_sve_bfmmla: + assert(!movprefxIsPredicated); + break; + default: + break; + } + break; + } + case IF_SVE_HD_3A_A: + { + switch (secondId->idIns()) + { + case INS_sve_fmmla: + assert(!movprefxIsPredicated); + break; + default: + break; + } + break; + } + case IF_SVE_HN_2A: + { + switch (secondId->idIns()) + { + case INS_sve_ftmad: + assert(!movprefxIsPredicated); + break; + default: + break; + } + break; + } + default: + break; + } } #endif // DEBUG diff --git a/src/coreclr/jit/hwintrinsiccodegenarm64.cpp b/src/coreclr/jit/hwintrinsiccodegenarm64.cpp index 173006f034f574..81ecbc07fa1bd1 100644 --- a/src/coreclr/jit/hwintrinsiccodegenarm64.cpp +++ b/src/coreclr/jit/hwintrinsiccodegenarm64.cpp @@ -783,8 +783,11 @@ void CodeGen::genHWIntrinsic(GenTreeHWIntrinsic* node) emitInsHelper(targetReg, maskReg, embMaskOp2Reg); break; + case NI_Sve2_AddPairwise: case NI_Sve2_MaxNumberPairwise: + case NI_Sve2_MaxPairwise: case NI_Sve2_MinNumberPairwise: + case NI_Sve2_MinPairwise: // These instructions have unpredictable behaviour when using predicated movprfx, // so the unpredicated variant must be used here. assert(!intrin.op3->isContained() && falseReg != REG_NA); diff --git a/src/coreclr/jit/lowerarmarch.cpp b/src/coreclr/jit/lowerarmarch.cpp index c1bb7a1fc5cdc2..a75814b38d3cc4 100644 --- a/src/coreclr/jit/lowerarmarch.cpp +++ b/src/coreclr/jit/lowerarmarch.cpp @@ -4063,8 +4063,11 @@ void Lowering::ContainCheckHWIntrinsic(GenTreeHWIntrinsic* node) // Do this only if op1 was AllTrueMask switch (op2->AsHWIntrinsic()->GetHWIntrinsicId()) { - case NI_Sve2_MinNumberPairwise: + case NI_Sve2_AddPairwise: case NI_Sve2_MaxNumberPairwise: + case NI_Sve2_MaxPairwise: + case NI_Sve2_MinNumberPairwise: + case NI_Sve2_MinPairwise: // This is an edge case where these instructions have unpredictable behaviour when // using predicated movprfx, so the unpredicated variant must be used here. This // prevents us from performing this optimization as we will need the constant vector From 947b0918fc1cd5e927e709cb6493acbc91f7ac32 Mon Sep 17 00:00:00 2001 From: Sebastian Nickolls Date: Tue, 16 Sep 2025 12:01:11 +0000 Subject: [PATCH 3/3] Invert emitter validation check --- src/coreclr/jit/emitarm64sve.cpp | 850 ++++++++++--------------------- 1 file changed, 267 insertions(+), 583 deletions(-) diff --git a/src/coreclr/jit/emitarm64sve.cpp b/src/coreclr/jit/emitarm64sve.cpp index 98c45ef2f13759..cc21be120234a6 100644 --- a/src/coreclr/jit/emitarm64sve.cpp +++ b/src/coreclr/jit/emitarm64sve.cpp @@ -18692,383 +18692,24 @@ void emitter::emitInsPairSanityCheck(instrDesc* firstId, instrDesc* secondId) } // The following instructions cannot use predicated movprfx, else the behaviour will be unpredictable. - switch (secondId->idInsFmt()) + switch (secondId->idIns()) { - case IF_SVE_AA_3A: - { - switch (secondId->idIns()) - { - case INS_sve_addp: - case INS_sve_smaxp: - case INS_sve_sminp: - case INS_sve_umaxp: - case INS_sve_uminp: - assert(!movprefxIsPredicated); - break; - default: - break; - } - break; - } - case IF_SVE_AT_3A: - { - switch (secondId->idIns()) - { - case INS_sve_eorbt: - case INS_sve_eortb: - case INS_sve_fclamp: - case INS_sve_sclamp: - case INS_sve_uclamp: - assert(!movprefxIsPredicated); - break; - default: - break; - } - break; - } - case IF_SVE_AV_3A: - { - switch (secondId->idIns()) - { - case INS_sve_bcax: - case INS_sve_bsl: - case INS_sve_bsl1n: - case INS_sve_bsl2n: - case INS_sve_eor3: - case INS_sve_nbsl: - assert(!movprefxIsPredicated); - break; - default: - break; - } - break; - } - case IF_SVE_AW_2A: - { - switch (secondId->idIns()) - { - case INS_sve_xar: - assert(!movprefxIsPredicated); - break; - default: - break; - } - break; - } - case IF_SVE_BN_1A: - { - switch (secondId->idIns()) - { - case INS_sve_decd: - case INS_sve_dech: - case INS_sve_decw: - case INS_sve_incd: - case INS_sve_inch: - case INS_sve_incw: - assert(!movprefxIsPredicated); - break; - default: - break; - } - break; - } - case IF_SVE_BP_1A: - { - switch (secondId->idIns()) - { - case INS_sve_sqdecd: - case INS_sve_sqdech: - case INS_sve_sqdecw: - case INS_sve_sqincd: - case INS_sve_sqinch: - case INS_sve_sqincw: - case INS_sve_uqdecd: - case INS_sve_uqdech: - case INS_sve_uqdecw: - case INS_sve_uqincd: - case INS_sve_uqinch: - case INS_sve_uqincw: - assert(!movprefxIsPredicated); - break; - default: - break; - } - break; - } - case IF_SVE_BS_1A: - { - switch (secondId->idIns()) - { - case INS_sve_and: - case INS_sve_bic: - case INS_sve_eon: - case INS_sve_eor: - case INS_sve_orn: - case INS_sve_orr: - assert(!movprefxIsPredicated); - break; - default: - break; - } - break; - } - case IF_SVE_BY_2A: - { - switch (secondId->idIns()) - { - case INS_sve_extq: - assert(!movprefxIsPredicated); - break; - default: - break; - } - break; - } - case IF_SVE_CC_2A: - { - switch (secondId->idIns()) - { - case INS_sve_insr: - assert(!movprefxIsPredicated); - break; - default: - break; - } - break; - } - case IF_SVE_CD_2A: - { - switch (secondId->idIns()) - { - case INS_sve_insr: - assert(!movprefxIsPredicated); - break; - default: - break; - } - break; - } - case IF_SVE_CM_3A: - { - switch (secondId->idIns()) - { - case INS_sve_clasta: - case INS_sve_clastb: - assert(!movprefxIsPredicated); - break; - default: - break; - } - break; - } - case IF_SVE_CT_3A: - { - switch (secondId->idIns()) - { - case INS_sve_revd: - assert(!movprefxIsPredicated); - break; - default: - break; - } - break; - } - case IF_SVE_DN_2A: - { - switch (secondId->idIns()) - { - case INS_sve_decp: - case INS_sve_incp: - assert(!movprefxIsPredicated); - break; - default: - break; - } - break; - } - case IF_SVE_DP_2A: - { - switch (secondId->idIns()) - { - case INS_sve_sqdecp: - case INS_sve_sqincp: - case INS_sve_uqdecp: - case INS_sve_uqincp: - assert(!movprefxIsPredicated); - break; - default: - break; - } - break; - } - case IF_SVE_EC_1A: - { - switch (secondId->idIns()) - { - case INS_sve_add: - case INS_sve_sqadd: - case INS_sve_sqsub: - case INS_sve_sub: - case INS_sve_subr: - case INS_sve_uqadd: - case INS_sve_uqsub: - assert(!movprefxIsPredicated); - break; - default: - break; - } - break; - } - case IF_SVE_ED_1A: - { - switch (secondId->idIns()) - { - case INS_sve_smax: - case INS_sve_smin: - case INS_sve_umax: - case INS_sve_umin: - assert(!movprefxIsPredicated); - break; - default: - break; - } - break; - } - case IF_SVE_EE_1A: - { - switch (secondId->idIns()) - { - case INS_sve_mul: - assert(!movprefxIsPredicated); - break; - default: - break; - } - break; - } - case IF_SVE_EF_3A: - { - switch (secondId->idIns()) - { - case INS_sve_sdot: - case INS_sve_udot: - assert(!movprefxIsPredicated); - break; - default: - break; - } - break; - } - case IF_SVE_EG_3A: - { - switch (secondId->idIns()) - { - case INS_sve_sdot: - case INS_sve_udot: - assert(!movprefxIsPredicated); - break; - default: - break; - } - break; - } - case IF_SVE_EH_3A: - { - switch (secondId->idIns()) - { - case INS_sve_sdot: - case INS_sve_udot: - assert(!movprefxIsPredicated); - break; - default: - break; - } - break; - } - case IF_SVE_EI_3A: - { - switch (secondId->idIns()) - { - case INS_sve_usdot: - assert(!movprefxIsPredicated); - break; - default: - break; - } - break; - } - case IF_SVE_EJ_3A: - { - switch (secondId->idIns()) - { - case INS_sve_cdot: - assert(!movprefxIsPredicated); - break; - default: - break; - } - break; - } - case IF_SVE_EK_3A: - { - switch (secondId->idIns()) - { - case INS_sve_cmla: - case INS_sve_sqrdcmlah: - assert(!movprefxIsPredicated); - break; - default: - break; - } - break; - } - case IF_SVE_EL_3A: - { - switch (secondId->idIns()) - { - case INS_sve_sabalb: - case INS_sve_sabalt: - case INS_sve_smlalb: - case INS_sve_smlalt: - case INS_sve_smlslb: - case INS_sve_smlslt: - case INS_sve_sqdmlalb: - case INS_sve_sqdmlalbt: - case INS_sve_sqdmlalt: - case INS_sve_sqdmlslb: - case INS_sve_sqdmlslbt: - case INS_sve_sqdmlslt: - case INS_sve_uabalb: - case INS_sve_uabalt: - case INS_sve_umlalb: - case INS_sve_umlalt: - case INS_sve_umlslb: - case INS_sve_umlslt: - assert(!movprefxIsPredicated); - break; - default: - break; - } - break; - } - case IF_SVE_EM_3A: - { - switch (secondId->idIns()) - { - case INS_sve_sqrdmlah: - case INS_sve_sqrdmlsh: - assert(!movprefxIsPredicated); - break; - default: - break; - } - break; - } - case IF_SVE_EW_3A: + case INS_sve_sqdecd: + case INS_sve_sqdech: + case INS_sve_sqdecw: + case INS_sve_sqincd: + case INS_sve_sqinch: + case INS_sve_sqincw: + case INS_sve_uqdecd: + case INS_sve_uqdech: + case INS_sve_uqdecw: + case INS_sve_uqincd: + case INS_sve_uqinch: + case INS_sve_uqincw: { - switch (secondId->idIns()) + switch (secondId->idInsFmt()) { - case INS_sve_mlapt: + case IF_SVE_BP_1A: assert(!movprefxIsPredicated); break; default: @@ -19076,11 +18717,20 @@ void emitter::emitInsPairSanityCheck(instrDesc* firstId, instrDesc* secondId) } break; } - case IF_SVE_EW_3B: + case INS_sve_smlalb: + case INS_sve_smlalt: + case INS_sve_smlslb: + case INS_sve_smlslt: + case INS_sve_umlalb: + case INS_sve_umlalt: + case INS_sve_umlslb: + case INS_sve_umlslt: { - switch (secondId->idIns()) + switch (secondId->idInsFmt()) { - case INS_sve_madpt: + case IF_SVE_EL_3A: + case IF_SVE_FG_3A: + case IF_SVE_FG_3B: assert(!movprefxIsPredicated); break; default: @@ -19088,12 +18738,17 @@ void emitter::emitInsPairSanityCheck(instrDesc* firstId, instrDesc* secondId) } break; } - case IF_SVE_EY_3A: + case INS_sve_add: + case INS_sve_sqadd: + case INS_sve_sqsub: + case INS_sve_sub: + case INS_sve_subr: + case INS_sve_uqadd: + case INS_sve_uqsub: { - switch (secondId->idIns()) + switch (secondId->idInsFmt()) { - case INS_sve_sdot: - case INS_sve_udot: + case IF_SVE_EC_1A: assert(!movprefxIsPredicated); break; default: @@ -19101,12 +18756,16 @@ void emitter::emitInsPairSanityCheck(instrDesc* firstId, instrDesc* secondId) } break; } - case IF_SVE_EY_3B: + case INS_sve_and: + case INS_sve_bic: + case INS_sve_eon: + case INS_sve_eor: + case INS_sve_orn: + case INS_sve_orr: { - switch (secondId->idIns()) + switch (secondId->idInsFmt()) { - case INS_sve_sdot: - case INS_sve_udot: + case IF_SVE_BS_1A: assert(!movprefxIsPredicated); break; default: @@ -19114,12 +18773,16 @@ void emitter::emitInsPairSanityCheck(instrDesc* firstId, instrDesc* secondId) } break; } - case IF_SVE_EZ_3A: + case INS_sve_bcax: + case INS_sve_bsl: + case INS_sve_bsl1n: + case INS_sve_bsl2n: + case INS_sve_eor3: + case INS_sve_nbsl: { - switch (secondId->idIns()) + switch (secondId->idInsFmt()) { - case INS_sve_sudot: - case INS_sve_usdot: + case IF_SVE_AV_3A: assert(!movprefxIsPredicated); break; default: @@ -19127,11 +18790,17 @@ void emitter::emitInsPairSanityCheck(instrDesc* firstId, instrDesc* secondId) } break; } - case IF_SVE_FA_3A: + case INS_sve_bfmlalb: + case INS_sve_bfmlalt: + case INS_sve_bfmlslb: + case INS_sve_bfmlslt: + case INS_sve_fmlslb: + case INS_sve_fmlslt: { - switch (secondId->idIns()) + switch (secondId->idInsFmt()) { - case INS_sve_cdot: + case IF_SVE_GZ_3A: + case IF_SVE_HB_3A: assert(!movprefxIsPredicated); break; default: @@ -19139,11 +18808,16 @@ void emitter::emitInsPairSanityCheck(instrDesc* firstId, instrDesc* secondId) } break; } - case IF_SVE_FA_3B: + case INS_sve_decd: + case INS_sve_dech: + case INS_sve_decw: + case INS_sve_incd: + case INS_sve_inch: + case INS_sve_incw: { - switch (secondId->idIns()) + switch (secondId->idInsFmt()) { - case INS_sve_cdot: + case IF_SVE_BN_1A: assert(!movprefxIsPredicated); break; default: @@ -19151,11 +18825,16 @@ void emitter::emitInsPairSanityCheck(instrDesc* firstId, instrDesc* secondId) } break; } - case IF_SVE_FB_3A: + case INS_sve_sabalb: + case INS_sve_sabalt: + case INS_sve_sqdmlalbt: + case INS_sve_sqdmlslbt: + case INS_sve_uabalb: + case INS_sve_uabalt: { - switch (secondId->idIns()) + switch (secondId->idInsFmt()) { - case INS_sve_cmla: + case IF_SVE_EL_3A: assert(!movprefxIsPredicated); break; default: @@ -19163,11 +18842,15 @@ void emitter::emitInsPairSanityCheck(instrDesc* firstId, instrDesc* secondId) } break; } - case IF_SVE_FB_3B: + case INS_sve_addp: + case INS_sve_smaxp: + case INS_sve_sminp: + case INS_sve_umaxp: + case INS_sve_uminp: { - switch (secondId->idIns()) + switch (secondId->idInsFmt()) { - case INS_sve_cmla: + case IF_SVE_AA_3A: assert(!movprefxIsPredicated); break; default: @@ -19175,11 +18858,15 @@ void emitter::emitInsPairSanityCheck(instrDesc* firstId, instrDesc* secondId) } break; } - case IF_SVE_FC_3A: + case INS_sve_eorbt: + case INS_sve_eortb: + case INS_sve_fclamp: + case INS_sve_sclamp: + case INS_sve_uclamp: { - switch (secondId->idIns()) + switch (secondId->idInsFmt()) { - case INS_sve_sqrdcmlah: + case IF_SVE_AT_3A: assert(!movprefxIsPredicated); break; default: @@ -19187,11 +18874,15 @@ void emitter::emitInsPairSanityCheck(instrDesc* firstId, instrDesc* secondId) } break; } - case IF_SVE_FC_3B: + case INS_sve_faddp: + case INS_sve_fmaxnmp: + case INS_sve_fmaxp: + case INS_sve_fminnmp: + case INS_sve_fminp: { - switch (secondId->idIns()) + switch (secondId->idInsFmt()) { - case INS_sve_sqrdcmlah: + case IF_SVE_GR_3A: assert(!movprefxIsPredicated); break; default: @@ -19199,12 +18890,14 @@ void emitter::emitInsPairSanityCheck(instrDesc* firstId, instrDesc* secondId) } break; } - case IF_SVE_FF_3A: + case INS_sve_adclb: + case INS_sve_adclt: + case INS_sve_sbclb: + case INS_sve_sbclt: { - switch (secondId->idIns()) + switch (secondId->idInsFmt()) { - case INS_sve_mla: - case INS_sve_mls: + case IF_SVE_FY_3A: assert(!movprefxIsPredicated); break; default: @@ -19212,12 +18905,15 @@ void emitter::emitInsPairSanityCheck(instrDesc* firstId, instrDesc* secondId) } break; } - case IF_SVE_FF_3B: + case INS_sve_fmlallbb: + case INS_sve_fmlallbt: + case INS_sve_fmlalltb: + case INS_sve_fmlalltt: { - switch (secondId->idIns()) + switch (secondId->idInsFmt()) { - case INS_sve_mla: - case INS_sve_mls: + case IF_SVE_GO_3A: + case IF_SVE_HC_3A: assert(!movprefxIsPredicated); break; default: @@ -19225,12 +18921,14 @@ void emitter::emitInsPairSanityCheck(instrDesc* firstId, instrDesc* secondId) } break; } - case IF_SVE_FF_3C: + case INS_sve_smax: + case INS_sve_smin: + case INS_sve_umax: + case INS_sve_umin: { - switch (secondId->idIns()) + switch (secondId->idInsFmt()) { - case INS_sve_mla: - case INS_sve_mls: + case IF_SVE_ED_1A: assert(!movprefxIsPredicated); break; default: @@ -19238,18 +18936,14 @@ void emitter::emitInsPairSanityCheck(instrDesc* firstId, instrDesc* secondId) } break; } - case IF_SVE_FG_3A: + case INS_sve_sqdecp: + case INS_sve_sqincp: + case INS_sve_uqdecp: + case INS_sve_uqincp: { - switch (secondId->idIns()) + switch (secondId->idInsFmt()) { - case INS_sve_smlalb: - case INS_sve_smlalt: - case INS_sve_smlslb: - case INS_sve_smlslt: - case INS_sve_umlalb: - case INS_sve_umlalt: - case INS_sve_umlslb: - case INS_sve_umlslt: + case IF_SVE_DP_2A: assert(!movprefxIsPredicated); break; default: @@ -19257,18 +18951,16 @@ void emitter::emitInsPairSanityCheck(instrDesc* firstId, instrDesc* secondId) } break; } - case IF_SVE_FG_3B: + case INS_sve_sqdmlalb: + case INS_sve_sqdmlalt: + case INS_sve_sqdmlslb: + case INS_sve_sqdmlslt: { - switch (secondId->idIns()) + switch (secondId->idInsFmt()) { - case INS_sve_smlalb: - case INS_sve_smlalt: - case INS_sve_smlslb: - case INS_sve_smlslt: - case INS_sve_umlalb: - case INS_sve_umlalt: - case INS_sve_umlslb: - case INS_sve_umlslt: + case IF_SVE_EL_3A: + case IF_SVE_FJ_3A: + case IF_SVE_FJ_3B: assert(!movprefxIsPredicated); break; default: @@ -19276,14 +18968,14 @@ void emitter::emitInsPairSanityCheck(instrDesc* firstId, instrDesc* secondId) } break; } - case IF_SVE_FJ_3A: + case INS_sve_srsra: + case INS_sve_ssra: + case INS_sve_ursra: + case INS_sve_usra: { - switch (secondId->idIns()) + switch (secondId->idInsFmt()) { - case INS_sve_sqdmlalb: - case INS_sve_sqdmlalt: - case INS_sve_sqdmlslb: - case INS_sve_sqdmlslt: + case IF_SVE_FU_2A: assert(!movprefxIsPredicated); break; default: @@ -19291,14 +18983,13 @@ void emitter::emitInsPairSanityCheck(instrDesc* firstId, instrDesc* secondId) } break; } - case IF_SVE_FJ_3B: + case INS_sve_smmla: + case INS_sve_ummla: + case INS_sve_usmmla: { - switch (secondId->idIns()) + switch (secondId->idInsFmt()) { - case INS_sve_sqdmlalb: - case INS_sve_sqdmlalt: - case INS_sve_sqdmlslb: - case INS_sve_sqdmlslt: + case IF_SVE_FO_3A: assert(!movprefxIsPredicated); break; default: @@ -19306,12 +18997,12 @@ void emitter::emitInsPairSanityCheck(instrDesc* firstId, instrDesc* secondId) } break; } - case IF_SVE_FK_3A: + case INS_sve_bfmla: + case INS_sve_bfmls: { - switch (secondId->idIns()) + switch (secondId->idInsFmt()) { - case INS_sve_sqrdmlah: - case INS_sve_sqrdmlsh: + case IF_SVE_GU_3C: assert(!movprefxIsPredicated); break; default: @@ -19319,12 +19010,12 @@ void emitter::emitInsPairSanityCheck(instrDesc* firstId, instrDesc* secondId) } break; } - case IF_SVE_FK_3B: + case INS_sve_cadd: + case INS_sve_sqcadd: { - switch (secondId->idIns()) + switch (secondId->idInsFmt()) { - case INS_sve_sqrdmlah: - case INS_sve_sqrdmlsh: + case IF_SVE_FV_2A: assert(!movprefxIsPredicated); break; default: @@ -19332,12 +19023,12 @@ void emitter::emitInsPairSanityCheck(instrDesc* firstId, instrDesc* secondId) } break; } - case IF_SVE_FK_3C: + case INS_sve_clasta: + case INS_sve_clastb: { - switch (secondId->idIns()) + switch (secondId->idInsFmt()) { - case INS_sve_sqrdmlah: - case INS_sve_sqrdmlsh: + case IF_SVE_CM_3A: assert(!movprefxIsPredicated); break; default: @@ -19345,13 +19036,12 @@ void emitter::emitInsPairSanityCheck(instrDesc* firstId, instrDesc* secondId) } break; } - case IF_SVE_FO_3A: + case INS_sve_decp: + case INS_sve_incp: { - switch (secondId->idIns()) + switch (secondId->idInsFmt()) { - case INS_sve_smmla: - case INS_sve_ummla: - case INS_sve_usmmla: + case IF_SVE_DN_2A: assert(!movprefxIsPredicated); break; default: @@ -19359,14 +19049,13 @@ void emitter::emitInsPairSanityCheck(instrDesc* firstId, instrDesc* secondId) } break; } - case IF_SVE_FU_2A: + case INS_sve_fmla: + case INS_sve_fmls: { - switch (secondId->idIns()) + switch (secondId->idInsFmt()) { - case INS_sve_srsra: - case INS_sve_ssra: - case INS_sve_ursra: - case INS_sve_usra: + case IF_SVE_GU_3A: + case IF_SVE_GU_3B: assert(!movprefxIsPredicated); break; default: @@ -19374,12 +19063,15 @@ void emitter::emitInsPairSanityCheck(instrDesc* firstId, instrDesc* secondId) } break; } - case IF_SVE_FV_2A: + case INS_sve_fmlalb: + case INS_sve_fmlalt: { - switch (secondId->idIns()) + switch (secondId->idInsFmt()) { - case INS_sve_cadd: - case INS_sve_sqcadd: + case IF_SVE_GM_3A: + case IF_SVE_GN_3A: + case IF_SVE_GZ_3A: + case IF_SVE_HB_3A: assert(!movprefxIsPredicated); break; default: @@ -19387,12 +19079,14 @@ void emitter::emitInsPairSanityCheck(instrDesc* firstId, instrDesc* secondId) } break; } - case IF_SVE_FW_3A: + case INS_sve_mla: + case INS_sve_mls: { - switch (secondId->idIns()) + switch (secondId->idInsFmt()) { - case INS_sve_saba: - case INS_sve_uaba: + case IF_SVE_FF_3A: + case IF_SVE_FF_3B: + case IF_SVE_FF_3C: assert(!movprefxIsPredicated); break; default: @@ -19400,14 +19094,12 @@ void emitter::emitInsPairSanityCheck(instrDesc* firstId, instrDesc* secondId) } break; } - case IF_SVE_FY_3A: + case INS_sve_saba: + case INS_sve_uaba: { - switch (secondId->idIns()) + switch (secondId->idInsFmt()) { - case INS_sve_adclb: - case INS_sve_adclt: - case INS_sve_sbclb: - case INS_sve_sbclt: + case IF_SVE_FW_3A: assert(!movprefxIsPredicated); break; default: @@ -19415,12 +19107,16 @@ void emitter::emitInsPairSanityCheck(instrDesc* firstId, instrDesc* secondId) } break; } - case IF_SVE_GM_3A: + case INS_sve_sdot: + case INS_sve_udot: { - switch (secondId->idIns()) + switch (secondId->idInsFmt()) { - case INS_sve_fmlalb: - case INS_sve_fmlalt: + case IF_SVE_EF_3A: + case IF_SVE_EG_3A: + case IF_SVE_EH_3A: + case IF_SVE_EY_3A: + case IF_SVE_EY_3B: assert(!movprefxIsPredicated); break; default: @@ -19428,12 +19124,15 @@ void emitter::emitInsPairSanityCheck(instrDesc* firstId, instrDesc* secondId) } break; } - case IF_SVE_GN_3A: + case INS_sve_sqrdmlah: + case INS_sve_sqrdmlsh: { - switch (secondId->idIns()) + switch (secondId->idInsFmt()) { - case INS_sve_fmlalb: - case INS_sve_fmlalt: + case IF_SVE_EM_3A: + case IF_SVE_FK_3A: + case IF_SVE_FK_3B: + case IF_SVE_FK_3C: assert(!movprefxIsPredicated); break; default: @@ -19441,14 +19140,11 @@ void emitter::emitInsPairSanityCheck(instrDesc* firstId, instrDesc* secondId) } break; } - case IF_SVE_GO_3A: + case INS_sve_bfclamp: { - switch (secondId->idIns()) + switch (secondId->idInsFmt()) { - case INS_sve_fmlallbb: - case INS_sve_fmlallbt: - case INS_sve_fmlalltb: - case INS_sve_fmlalltt: + case IF_SVE_GW_3B: assert(!movprefxIsPredicated); break; default: @@ -19456,15 +19152,12 @@ void emitter::emitInsPairSanityCheck(instrDesc* firstId, instrDesc* secondId) } break; } - case IF_SVE_GR_3A: + case INS_sve_bfdot: { - switch (secondId->idIns()) + switch (secondId->idInsFmt()) { - case INS_sve_faddp: - case INS_sve_fmaxnmp: - case INS_sve_fmaxp: - case INS_sve_fminnmp: - case INS_sve_fminp: + case IF_SVE_GY_3B: + case IF_SVE_HA_3A: assert(!movprefxIsPredicated); break; default: @@ -19472,12 +19165,11 @@ void emitter::emitInsPairSanityCheck(instrDesc* firstId, instrDesc* secondId) } break; } - case IF_SVE_GU_3A: + case INS_sve_bfmmla: { - switch (secondId->idIns()) + switch (secondId->idInsFmt()) { - case INS_sve_fmla: - case INS_sve_fmls: + case IF_SVE_HD_3A: assert(!movprefxIsPredicated); break; default: @@ -19485,12 +19177,13 @@ void emitter::emitInsPairSanityCheck(instrDesc* firstId, instrDesc* secondId) } break; } - case IF_SVE_GU_3B: + case INS_sve_cdot: { - switch (secondId->idIns()) + switch (secondId->idInsFmt()) { - case INS_sve_fmla: - case INS_sve_fmls: + case IF_SVE_EJ_3A: + case IF_SVE_FA_3A: + case IF_SVE_FA_3B: assert(!movprefxIsPredicated); break; default: @@ -19498,12 +19191,13 @@ void emitter::emitInsPairSanityCheck(instrDesc* firstId, instrDesc* secondId) } break; } - case IF_SVE_GU_3C: + case INS_sve_cmla: { - switch (secondId->idIns()) + switch (secondId->idInsFmt()) { - case INS_sve_bfmla: - case INS_sve_bfmls: + case IF_SVE_EK_3A: + case IF_SVE_FB_3A: + case IF_SVE_FB_3B: assert(!movprefxIsPredicated); break; default: @@ -19511,11 +19205,11 @@ void emitter::emitInsPairSanityCheck(instrDesc* firstId, instrDesc* secondId) } break; } - case IF_SVE_GV_3A: + case INS_sve_extq: { - switch (secondId->idIns()) + switch (secondId->idInsFmt()) { - case INS_sve_fcmla: + case IF_SVE_BY_2A: assert(!movprefxIsPredicated); break; default: @@ -19523,11 +19217,11 @@ void emitter::emitInsPairSanityCheck(instrDesc* firstId, instrDesc* secondId) } break; } - case IF_SVE_GW_3B: + case INS_sve_fcmla: { - switch (secondId->idIns()) + switch (secondId->idInsFmt()) { - case INS_sve_bfclamp: + case IF_SVE_GV_3A: assert(!movprefxIsPredicated); break; default: @@ -19535,11 +19229,16 @@ void emitter::emitInsPairSanityCheck(instrDesc* firstId, instrDesc* secondId) } break; } - case IF_SVE_GY_3A: + case INS_sve_fdot: { - switch (secondId->idIns()) - { - case INS_sve_fdot: + switch (secondId->idInsFmt()) + { + case IF_SVE_GY_3A: + case IF_SVE_GY_3B: + case IF_SVE_GY_3B_D: + case IF_SVE_HA_3A: + case IF_SVE_HA_3A_E: + case IF_SVE_HA_3A_F: assert(!movprefxIsPredicated); break; default: @@ -19547,12 +19246,11 @@ void emitter::emitInsPairSanityCheck(instrDesc* firstId, instrDesc* secondId) } break; } - case IF_SVE_GY_3B: + case INS_sve_fmmla: { - switch (secondId->idIns()) + switch (secondId->idInsFmt()) { - case INS_sve_bfdot: - case INS_sve_fdot: + case IF_SVE_HD_3A_A: assert(!movprefxIsPredicated); break; default: @@ -19560,11 +19258,11 @@ void emitter::emitInsPairSanityCheck(instrDesc* firstId, instrDesc* secondId) } break; } - case IF_SVE_GY_3B_D: + case INS_sve_ftmad: { - switch (secondId->idIns()) + switch (secondId->idInsFmt()) { - case INS_sve_fdot: + case IF_SVE_HN_2A: assert(!movprefxIsPredicated); break; default: @@ -19572,18 +19270,12 @@ void emitter::emitInsPairSanityCheck(instrDesc* firstId, instrDesc* secondId) } break; } - case IF_SVE_GZ_3A: + case INS_sve_insr: { - switch (secondId->idIns()) + switch (secondId->idInsFmt()) { - case INS_sve_bfmlalb: - case INS_sve_bfmlalt: - case INS_sve_bfmlslb: - case INS_sve_bfmlslt: - case INS_sve_fmlalb: - case INS_sve_fmlalt: - case INS_sve_fmlslb: - case INS_sve_fmlslt: + case IF_SVE_CC_2A: + case IF_SVE_CD_2A: assert(!movprefxIsPredicated); break; default: @@ -19591,12 +19283,11 @@ void emitter::emitInsPairSanityCheck(instrDesc* firstId, instrDesc* secondId) } break; } - case IF_SVE_HA_3A: + case INS_sve_madpt: { - switch (secondId->idIns()) + switch (secondId->idInsFmt()) { - case INS_sve_bfdot: - case INS_sve_fdot: + case IF_SVE_EW_3B: assert(!movprefxIsPredicated); break; default: @@ -19604,11 +19295,11 @@ void emitter::emitInsPairSanityCheck(instrDesc* firstId, instrDesc* secondId) } break; } - case IF_SVE_HA_3A_E: + case INS_sve_mlapt: { - switch (secondId->idIns()) + switch (secondId->idInsFmt()) { - case INS_sve_fdot: + case IF_SVE_EW_3A: assert(!movprefxIsPredicated); break; default: @@ -19616,11 +19307,11 @@ void emitter::emitInsPairSanityCheck(instrDesc* firstId, instrDesc* secondId) } break; } - case IF_SVE_HA_3A_F: + case INS_sve_mul: { - switch (secondId->idIns()) + switch (secondId->idInsFmt()) { - case INS_sve_fdot: + case IF_SVE_EE_1A: assert(!movprefxIsPredicated); break; default: @@ -19628,18 +19319,11 @@ void emitter::emitInsPairSanityCheck(instrDesc* firstId, instrDesc* secondId) } break; } - case IF_SVE_HB_3A: + case INS_sve_revd: { - switch (secondId->idIns()) + switch (secondId->idInsFmt()) { - case INS_sve_bfmlalb: - case INS_sve_bfmlalt: - case INS_sve_bfmlslb: - case INS_sve_bfmlslt: - case INS_sve_fmlalb: - case INS_sve_fmlalt: - case INS_sve_fmlslb: - case INS_sve_fmlslt: + case IF_SVE_CT_3A: assert(!movprefxIsPredicated); break; default: @@ -19647,14 +19331,13 @@ void emitter::emitInsPairSanityCheck(instrDesc* firstId, instrDesc* secondId) } break; } - case IF_SVE_HC_3A: + case INS_sve_sqrdcmlah: { - switch (secondId->idIns()) + switch (secondId->idInsFmt()) { - case INS_sve_fmlallbb: - case INS_sve_fmlallbt: - case INS_sve_fmlalltb: - case INS_sve_fmlalltt: + case IF_SVE_EK_3A: + case IF_SVE_FC_3A: + case IF_SVE_FC_3B: assert(!movprefxIsPredicated); break; default: @@ -19662,11 +19345,11 @@ void emitter::emitInsPairSanityCheck(instrDesc* firstId, instrDesc* secondId) } break; } - case IF_SVE_HD_3A: + case INS_sve_sudot: { - switch (secondId->idIns()) + switch (secondId->idInsFmt()) { - case INS_sve_bfmmla: + case IF_SVE_EZ_3A: assert(!movprefxIsPredicated); break; default: @@ -19674,11 +19357,12 @@ void emitter::emitInsPairSanityCheck(instrDesc* firstId, instrDesc* secondId) } break; } - case IF_SVE_HD_3A_A: + case INS_sve_usdot: { - switch (secondId->idIns()) + switch (secondId->idInsFmt()) { - case INS_sve_fmmla: + case IF_SVE_EI_3A: + case IF_SVE_EZ_3A: assert(!movprefxIsPredicated); break; default: @@ -19686,11 +19370,11 @@ void emitter::emitInsPairSanityCheck(instrDesc* firstId, instrDesc* secondId) } break; } - case IF_SVE_HN_2A: + case INS_sve_xar: { - switch (secondId->idIns()) + switch (secondId->idInsFmt()) { - case INS_sve_ftmad: + case IF_SVE_AW_2A: assert(!movprefxIsPredicated); break; default: