diff --git a/src/coreclr/jit/codegen.h b/src/coreclr/jit/codegen.h index f8f9eddf0c6322..4d82b6210ba71c 100644 --- a/src/coreclr/jit/codegen.h +++ b/src/coreclr/jit/codegen.h @@ -1019,6 +1019,12 @@ class CodeGen final : public CodeGenInterface #endif // defined(TARGET_XARCH) #ifdef TARGET_ARM64 + void genEmbeddedMaskedHWIntrinsic(GenTreeHWIntrinsic* cndSelNode, regNumber targetReg); + +#ifdef DEBUG + void checkRMWRegisters(const HWIntrinsic intrin, regNumber targetReg); +#endif + class HWIntrinsicImmOpHelper final { public: diff --git a/src/coreclr/jit/emitarm64sve.cpp b/src/coreclr/jit/emitarm64sve.cpp index 4bae37de2082e4..a2eab684ee8000 100644 --- a/src/coreclr/jit/emitarm64sve.cpp +++ b/src/coreclr/jit/emitarm64sve.cpp @@ -19318,7 +19318,10 @@ void emitter::emitInsPairSanityCheck(instrDesc* firstId, instrDesc* secondId) assert(firstId->idReg2() == secondId->idReg2()); // "predicated using the same governing predicate register and source element size as this instruction." - assert(firstId->idInsOpt() == secondId->idInsOpt()); + emitAttr dstSize1 = optGetSveElemsize(firstId->idInsOpt()); + emitAttr dstSize2 = insOptsScalableStandard(secondId->idInsOpt()) ? optGetSveElemsize(secondId->idInsOpt()) + : optGetDstsize(secondId->idInsOpt()); + assert(dstSize1 == dstSize2); } // The following instructions cannot use predicated movprfx, else the behaviour will be unpredictable. diff --git a/src/coreclr/jit/hwintrinsiccodegenarm64.cpp b/src/coreclr/jit/hwintrinsiccodegenarm64.cpp index 6255e64de4173d..eb0e670046c147 100644 --- a/src/coreclr/jit/hwintrinsiccodegenarm64.cpp +++ b/src/coreclr/jit/hwintrinsiccodegenarm64.cpp @@ -288,6 +288,605 @@ static void genEmitCreateWhileMask(emitter* emit, emit->emitIns_R_R_R(ins, emitSize, targetReg, op1Reg, op2Reg, opt); } +//------------------------------------------------------------------------ +// genEmbeddedMaskedHWIntrinsic: Generates the code for an embedded masked hardware intrinsic. +// +// Arguments: +// cndSelNode -- the conditional select HWIntrinsic node. +// targetReg -- the target register of the HWIntrinsic node. +// +void CodeGen::genEmbeddedMaskedHWIntrinsic(GenTreeHWIntrinsic* cndSelNode, regNumber targetReg) +{ + const HWIntrinsic intrinCndSel(cndSelNode); + assert(intrinCndSel.id == NI_Sve_ConditionalSelect); + + GenTree* maskOp = intrinCndSel.op1; + GenTree* embMaskOp = intrinCndSel.op2; + GenTree* falseOp = intrinCndSel.op3; + + assert(embMaskOp->OperIsHWIntrinsic()); + assert(embMaskOp->isContained()); + assert(embMaskOp->IsEmbMaskOp()); + + const HWIntrinsic intrinEmbMask(embMaskOp->AsHWIntrinsic()); + instruction insEmbMask = HWIntrinsicInfo::lookupIns(intrinEmbMask.id, intrinEmbMask.baseType, m_compiler); + + const bool isRMW = embMaskOp->isRMWHWIntrinsic(m_compiler); + bool isOptionalEmbMask = HWIntrinsicInfo::IsOptionalEmbeddedMaskedOperation(intrinEmbMask.id); + + regNumber maskReg = maskOp->GetRegNum(); + regNumber embMaskOp1Reg = REG_NA; + regNumber embMaskOp2Reg = REG_NA; + regNumber embMaskOp3Reg = REG_NA; + regNumber embMaskOp4Reg = REG_NA; + regNumber falseReg = falseOp->GetRegNum(); + regNumber tempReg = REG_NA; + + switch (intrinEmbMask.numOperands) + { + case 4: + assert(intrinEmbMask.op4 != nullptr); + embMaskOp4Reg = intrinEmbMask.op4->GetRegNum(); + FALLTHROUGH; + + case 3: + assert(intrinEmbMask.op3 != nullptr); + embMaskOp3Reg = intrinEmbMask.op3->GetRegNum(); + FALLTHROUGH; + + case 2: + assert(intrinEmbMask.op2 != nullptr); + embMaskOp2Reg = intrinEmbMask.op2->GetRegNum(); + FALLTHROUGH; + + case 1: + assert(intrinEmbMask.op1 != nullptr); + embMaskOp1Reg = intrinEmbMask.op1->GetRegNum(); + break; + + default: + unreached(); + } + + if (intrinEmbMask.id == NI_Sve_MultiplyAddRotateComplex) + { + assert(intrinEmbMask.numOperands == 4); + tempReg = internalRegisters.GetSingle(cndSelNode, RBM_ALLFLOAT); + } + + emitAttr emitSize = EA_SCALABLE; + insOpts opt = emitter::optGetSveInsOpt(emitTypeSize(intrinCndSel.baseType)); + insOpts embOpt = opt; + insScalableOpts sopt = INS_SCALABLE_OPTS_NONE; + +#ifdef DEBUG + checkRMWRegisters(intrinEmbMask, targetReg); +#endif + + // Setup instruction options and handle special cases. + if (intrinEmbMask.numOperands == 1) + { + assert(!isRMW); + + if (HWIntrinsicInfo::IsReduceOperation(intrinEmbMask.id)) + { + // For reduce operations, targetReg will always be overwritten by the scalar result. + // So falseReg can be ignored and just perform the operation. + GetEmitter()->emitInsSve_R_R_R(insEmbMask, emitSize, targetReg, maskReg, embMaskOp1Reg, embOpt); + return; + } + + switch (intrinEmbMask.id) + { + case NI_Sve2_ConvertToDoubleOdd: + // This instruction does not support movprfx, so use conditional select instead. + embOpt = emitTypeSize(intrinEmbMask.baseType) == EA_4BYTE ? INS_OPTS_S_TO_D : INS_OPTS_SCALABLE_D; + if (!maskOp->IsTrueMask(intrinCndSel.baseType) && (targetReg != falseReg)) + { + // Move falseReg to the inactive lanes of targetReg + // if mask is not all-true and falseReg is not the same as targetReg. + assert(!falseOp->isContained()); + GetEmitter()->emitIns_R_R_R_R(INS_sve_sel, emitSize, targetReg, maskReg, targetReg, falseReg, opt); + } + GetEmitter()->emitInsSve_R_R_R(insEmbMask, emitSize, targetReg, maskReg, embMaskOp1Reg, embOpt, sopt); + return; + + case NI_Sve_ConvertToInt32: + case NI_Sve_ConvertToUInt32: + case NI_Sve_ConvertToSingle: + case NI_Sve2_ConvertToSingleEvenRoundToOdd: + embOpt = emitTypeSize(intrinEmbMask.baseType) == EA_8BYTE ? INS_OPTS_D_TO_S : INS_OPTS_SCALABLE_S; + break; + + case NI_Sve_ConvertToInt64: + case NI_Sve_ConvertToUInt64: + case NI_Sve_ConvertToDouble: + embOpt = emitTypeSize(intrinEmbMask.baseType) == EA_4BYTE ? INS_OPTS_S_TO_D : INS_OPTS_SCALABLE_D; + break; + + default: + break; + } + + if (targetReg == falseReg) + { + // targetReg == falseReg: Just perform the masked operation. + GetEmitter()->emitIns_R_R_R(insEmbMask, emitSize, targetReg, maskReg, embMaskOp1Reg, embOpt); + } + else + { + // targetReg != falseReg: Move falseReg into targetReg. + if (falseOp->isContained()) + { + assert(falseOp->IsVectorZero()); + if (maskOp->IsTrueMask(intrinCndSel.baseType)) + { + // If maskOp is all-true, no need to move falseReg to targetReg + // because the predicated instruction will eventually set it. + GetEmitter()->emitIns_R_R_R(insEmbMask, emitSize, targetReg, maskReg, embMaskOp1Reg, embOpt); + } + else + { + // If falseValue is zero, just zero out those lanes of targetReg using zeroing movprfx. + GetEmitter()->emitInsSve_R_R_R_R(insEmbMask, emitSize, targetReg, maskReg, targetReg, embMaskOp1Reg, + embOpt, sopt, INS_SVE_MOV_OPTS_ZEROING); + } + } + else if (emitter::isVectorRegister(embMaskOp1Reg) && (targetReg == embMaskOp1Reg)) + { + // We cannot use use `movprfx` here to move falseReg to targetReg because that will + // overwrite the value of embMaskOp1Reg which is present in targetReg. + GetEmitter()->emitIns_R_R_R(insEmbMask, emitSize, targetReg, maskReg, embMaskOp1Reg, embOpt); + GetEmitter()->emitIns_R_R_R_R(INS_sve_sel, emitSize, targetReg, maskReg, targetReg, falseReg, opt); + } + else + { + // targetReg != embMaskOp1Reg != falseReg: Move falseReg unpredicated into targetReg. + GetEmitter()->emitInsSve_R_R_R_R(insEmbMask, emitSize, targetReg, maskReg, falseReg, embMaskOp1Reg, + embOpt, sopt, INS_SVE_MOV_OPTS_UNPRED); + } + } + return; + } + else if (intrinEmbMask.numOperands == 2) + { + switch (intrinEmbMask.id) + { + case NI_Sve_CreateBreakPropagateMask: + { + embOpt = INS_OPTS_SCALABLE_B; + // This instruction is zeroing predicated, just use unpredicated mov. + assert(falseOp->IsVectorZero()); + GetEmitter()->emitInsSve_R_R_R_R(insEmbMask, emitSize, targetReg, maskReg, embMaskOp1Reg, embMaskOp2Reg, + embOpt, sopt); + return; + } + + case NI_Sve_AddSequentialAcross: + { + // Predicate functionality is currently not exposed for this API, + // but the FADDA instruction only has a predicated variant. + // Thus, we expect the JIT to wrap this with CndSel. + assert(falseOp->IsVectorZero()); + GetEmitter()->emitInsSve_R_R_R_R(insEmbMask, emitSize, targetReg, maskReg, embMaskOp1Reg, embMaskOp2Reg, + embOpt, sopt); + return; + } + + case NI_Sve2_ConvertToSingleOdd: + case NI_Sve2_ConvertToSingleOddRoundToOdd: + { + // These instructions do not support movprfx. + embOpt = INS_OPTS_D_TO_S; + if (falseOp->IsVectorZero() && !maskOp->IsTrueMask(intrinCndSel.baseType) && (targetReg == falseReg)) + { + GetEmitter()->emitIns_R_R_R(INS_sve_mov, emitSize, targetReg, maskReg, embMaskOp1Reg, opt, + INS_SCALABLE_OPTS_PREDICATE_MERGE); + GetEmitter()->emitInsSve_R_R_R(insEmbMask, emitSize, targetReg, maskReg, embMaskOp2Reg, embOpt, + sopt); + return; + } + FALLTHROUGH; + } + + case NI_Sve2_AddPairwise: + case NI_Sve2_MaxNumberPairwise: + case NI_Sve2_MaxPairwise: + case NI_Sve2_MinNumberPairwise: + case NI_Sve2_MinPairwise: + { + // These instructions have unpredictable behaviour when using predicated movprfx. + // Move embMaskOp1Reg to targetReg unpredicated. + GetEmitter()->emitInsSve_R_R_R_R(insEmbMask, emitSize, targetReg, maskReg, embMaskOp1Reg, embMaskOp2Reg, + embOpt, sopt); + if (!maskOp->IsTrueMask(intrinCndSel.baseType) && (targetReg != falseReg)) + { + // Use conditional select to move falseReg to the inactive lanes of targetReg if necessary. + assert(!falseOp->isContained()); + GetEmitter()->emitInsSve_R_R_R_R(INS_sve_sel, emitSize, targetReg, maskReg, targetReg, falseReg, + opt); + } + return; + } + + case NI_Sve2_AddSaturate: + { + var_types baseType = embMaskOp->AsHWIntrinsic()->GetSimdBaseType(); + var_types auxType = embMaskOp->AsHWIntrinsic()->GetAuxiliaryType(); + if (baseType != auxType) + { + insEmbMask = (varTypeIsUnsigned(baseType)) ? INS_sve_usqadd : INS_sve_suqadd; + // SUQADD and USQADD must be predicated. + isOptionalEmbMask = false; + } + else + { + // SQADD and UQADD can be unpredicated. + isOptionalEmbMask = true; + } + break; + } + + case NI_Sve_ShiftLeftLogical: + case NI_Sve_ShiftRightArithmetic: + case NI_Sve_ShiftRightLogical: + { + const emitAttr op2Size = emitTypeSize(embMaskOp->AsHWIntrinsic()->GetAuxiliaryType()); + if (op2Size != emitTypeSize(intrinEmbMask.baseType)) + { + assert(emitter::optGetSveInsOpt(op2Size) == INS_OPTS_SCALABLE_D); + sopt = INS_SCALABLE_OPTS_WIDE; + } + break; + } + + default: + break; + } + + if (!isRMW) + { + // Perform the actual "predicated" operation so that `embMaskOp1Reg` is the first operand.. + switch (intrinEmbMask.id) + { + case NI_Sve_And_Predicates: + case NI_Sve_BitwiseClear_Predicates: + case NI_Sve_Or_Predicates: + case NI_Sve_Xor_Predicates: + embOpt = INS_OPTS_SCALABLE_B; + break; + + default: + break; + } + + GetEmitter()->emitIns_R_R_R_R(insEmbMask, emitSize, targetReg, maskReg, embMaskOp1Reg, embMaskOp2Reg, + embOpt); + return; + } + else if (isOptionalEmbMask) + { + if (maskOp->IsTrueMask(intrinEmbMask.baseType) || + (!falseOp->IsVectorZero() && (targetReg != falseReg) && (falseReg != embMaskOp1Reg))) + { + // If the embedded instruction supports optional mask operation, and when movprfx is not needed, + // use the "unpredicated" version of the instruction. + if (HWIntrinsicInfo::HasImmediateOperand(intrinEmbMask.id)) + { + HWIntrinsicImmOpHelper helper(this, intrinEmbMask.op2, embMaskOp->AsHWIntrinsic()); + for (helper.EmitBegin(); !helper.Done(); helper.EmitCaseEnd()) + { + GetEmitter()->emitInsSve_R_R_I(insEmbMask, emitSize, targetReg, embMaskOp1Reg, + helper.ImmValue(), embOpt, sopt); + } + } + else + { + GetEmitter()->emitIns_R_R_R(insEmbMask, emitSize, targetReg, embMaskOp1Reg, embMaskOp2Reg, embOpt, + sopt); + } + + if (!maskOp->IsTrueMask(intrinCndSel.baseType)) + { + // Use "sel" to select the active lanes if mask is not all-true. + GetEmitter()->emitIns_R_R_R_R(INS_sve_sel, emitSize, targetReg, maskReg, targetReg, falseReg, opt); + } + return; + } + } + } + else if (HWIntrinsicInfo::IsFmaIntrinsic(intrinEmbMask.id) && (intrinEmbMask.numOperands == 3)) + { + // For FMA, the operation we are trying to perform is: + // result = op1 + (op2 * op3) + // + // There are two instructions that can be used depending on which operand's register, + // optionally, will store the final result. + // + // 1. If the result is stored in the operand that was used as an "addend" in the operation, + // then we use `FMLA` format: + // reg1 = reg1 + (reg2 * reg3) + // + // 2. If the result is stored in the operand that was used as a "multiplicand" in the operation, + // then we use `FMAD` format: + // reg1 = (reg1 * reg2) + reg3 + // + // Check if the result's register is same as that of one of the operand's register and + // accordingly pick the appropriate format. Suppose `targetReg` holds the result, then we have + // following cases: + // + // Case# 1: Result is stored in the operand that held the "addend" + // targetReg == reg1 + // + // We generate the FMLA instruction format and no further changes are needed. + // + // Case# 2: Result is stored in the operand `op2` that held the "multiplicand" + // targetReg == reg2 + // + // So we basically have an operation: + // reg2 = reg1 + (reg2 * reg3) + // + // Since, the result will be stored in the "multiplicand", we pick format `FMAD`. + // Then, we rearrange the operands to ensure that the operation is done correctly. + // reg2 = reg1 + (reg2 * reg3) // to start with + // reg2 = reg3 + (reg2 * reg1) // swap reg1 <--> reg3 + // reg1 = reg3 + (reg1 * reg2) // swap reg1 <--> reg2 + // reg1 = (reg1 * reg2) + reg3 // rearrange to get FMAD format + // + // Case# 3: Result is stored in the operand `op3` that held the "multiplier" + // targetReg == reg3 + // + // So we basically have an operation: + // reg3 = reg1 + (reg2 * reg3) + // Since, the result will be stored in the "multiplier", we again pick format `FMAD`. + // Then, we rearrange the operands to ensure that the operation is done correctly. + // reg3 = reg1 + (reg2 * reg3) // to start with + // reg1 = reg3 + (reg2 * reg1) // swap reg1 <--> reg3 + // reg1 = (reg1 * reg2) + reg3 // rearrange to get FMAD format + bool useAddend = true; + if (targetReg == embMaskOp2Reg) + { + // Case# 2 + useAddend = false; + std::swap(embMaskOp1Reg, embMaskOp3Reg); + std::swap(embMaskOp1Reg, embMaskOp2Reg); + } + else if (targetReg == embMaskOp3Reg) + { + // Case# 3 + useAddend = false; + std::swap(embMaskOp1Reg, embMaskOp3Reg); + } + else + { + // Case# 1 + } + switch (intrinEmbMask.id) + { + case NI_Sve_FusedMultiplyAdd: + insEmbMask = useAddend ? INS_sve_fmla : INS_sve_fmad; + break; + case NI_Sve_FusedMultiplyAddNegated: + insEmbMask = useAddend ? INS_sve_fnmla : INS_sve_fnmad; + break; + case NI_Sve_FusedMultiplySubtract: + insEmbMask = useAddend ? INS_sve_fmls : INS_sve_fmsb; + break; + case NI_Sve_FusedMultiplySubtractNegated: + insEmbMask = useAddend ? INS_sve_fnmls : INS_sve_fnmsb; + break; + case NI_Sve_MultiplyAdd: + insEmbMask = useAddend ? INS_sve_mla : INS_sve_mad; + break; + case NI_Sve_MultiplySubtract: + insEmbMask = useAddend ? INS_sve_mls : INS_sve_msb; + break; + default: + unreached(); + } + } + + // Determine the move option, based on the register usage. + insSveMovOpts mopt = INS_SVE_MOV_OPTS_UNPRED; + if (falseOp->IsVectorZero()) + { + // If `falseReg` is zero, then move the first operand of `intrinEmbMask` in the + // destination using /Z. + mopt = INS_SVE_MOV_OPTS_ZEROING; + } + else if (targetReg != falseReg) + { + // If `targetReg` and `falseReg` are not same, then we need to move it to `targetReg` first + // so the `insEmbMask` operation can be merged on top of it. + if (falseReg != embMaskOp1Reg) + { + // targetReg != embMaskOp1Reg != falseReg: Use conditional select. + // Move embMaskOp1Reg to active lanes and falseReg to inactive lanes of targetReg. + assert(HWIntrinsicInfo::IsEmbeddedMaskedOperation(intrinEmbMask.id)); + assert(!HWIntrinsicInfo::IsZeroingMaskedOperation(intrinEmbMask.id)); + GetEmitter()->emitIns_R_R_R_R(INS_sve_sel, emitSize, targetReg, maskReg, embMaskOp1Reg, falseReg, opt); + // embMaskOp1Reg becomes targetReg, then use unpredicated movprfx. + embMaskOp1Reg = targetReg; + mopt = INS_SVE_MOV_OPTS_UNPRED; + } + else + { + // targetReg != falseReg == embMaskOp1Reg: Use unpredicated movprfx. + mopt = INS_SVE_MOV_OPTS_UNPRED; + } + } + else if (falseReg != embMaskOp1Reg) + { + // targetReg == falseReg != embMaskOp1Reg: Use merging movprfx. + mopt = INS_SVE_MOV_OPTS_MERGING; + } + + if (maskOp->IsTrueMask(intrinCndSel.baseType)) + { + // Prefer using unpredicated movprfx when possible. + mopt = INS_SVE_MOV_OPTS_UNPRED; + } + + // Emit the embedded masked intrinsics + if (HWIntrinsicInfo::HasImmediateOperand(intrinEmbMask.id)) + { + // The immediate operand is the last operand. + GenTree* immOp = embMaskOp->AsHWIntrinsic()->Op(intrinEmbMask.numOperands); + assert(immOp->isContained() == (immOp->GetRegNum() == REG_NA)); + + if ((intrinEmbMask.id == NI_Sve_MultiplyAddRotateComplex) && (targetReg != embMaskOp1Reg)) + { + if (targetReg == embMaskOp2Reg) + { + GetEmitter()->emitInsSve_Mov(INS_sve_mov, EA_SCALABLE, tempReg, embMaskOp2Reg, /* canSkip */ true, opt); + embMaskOp2Reg = tempReg; + if (embMaskOp3Reg == targetReg) + { + embMaskOp3Reg = tempReg; + } + } + else if (targetReg == embMaskOp3Reg) + { + GetEmitter()->emitInsSve_Mov(INS_sve_mov, EA_SCALABLE, tempReg, embMaskOp3Reg, /* canSkip */ true, opt); + embMaskOp3Reg = tempReg; + } + } + + int numInstrs = ((mopt != INS_SVE_MOV_OPTS_UNPRED) || (targetReg != embMaskOp1Reg)) ? 2 : 1; + HWIntrinsicImmOpHelper helper(this, immOp, embMaskOp->AsHWIntrinsic(), numInstrs); + for (helper.EmitBegin(); !helper.Done(); helper.EmitCaseEnd()) + { + ssize_t imm = helper.ImmValue(); + switch (intrinEmbMask.numOperands) + { + case 2: + GetEmitter()->emitInsSve_R_R_R_I(insEmbMask, emitSize, targetReg, maskReg, embMaskOp1Reg, imm, + embOpt, sopt, mopt); + break; + case 3: + GetEmitter()->emitInsSve_R_R_R_R_I(insEmbMask, emitSize, targetReg, maskReg, embMaskOp1Reg, + embMaskOp2Reg, imm, embOpt, sopt, mopt); + break; + case 4: + GetEmitter()->emitInsSve_R_R_R_R_R_I(insEmbMask, emitSize, targetReg, maskReg, embMaskOp1Reg, + embMaskOp2Reg, embMaskOp3Reg, imm, embOpt, sopt, mopt); + break; + default: + unreached(); + } + } + } + else + { + switch (intrinEmbMask.numOperands) + { + case 2: + GetEmitter()->emitInsSve_R_R_R_R(insEmbMask, emitSize, targetReg, maskReg, embMaskOp1Reg, embMaskOp2Reg, + embOpt, sopt, mopt); + break; + case 3: + GetEmitter()->emitInsSve_R_R_R_R_R(insEmbMask, emitSize, targetReg, maskReg, embMaskOp1Reg, + embMaskOp2Reg, embMaskOp3Reg, embOpt, sopt, mopt); + break; + default: + unreached(); + } + } +} + +#ifdef DEBUG +void CodeGen::checkRMWRegisters(const HWIntrinsic intrin, regNumber targetReg) +{ + const bool canRepairTargetOverlap = (intrin.id == NI_Sve_MultiplyAddRotateComplex); + + GenTree* rmwOp; + if (HWIntrinsicInfo::IsFmaIntrinsic(intrin.id) && (intrin.numOperands == 3)) + { + // SVE FMA intrinsics can use either the addend or a multiplicand as the destructive operand. Codegen + // selects the matching instruction form and rearranges the operands based on the allocated target. + if (targetReg == intrin.op2->GetRegNum()) + { + rmwOp = intrin.op2; + } + else if (targetReg == intrin.op3->GetRegNum()) + { + rmwOp = intrin.op3; + } + else + { + rmwOp = intrin.op1; + } + } + else + { + switch (intrin.id) + { + case NI_Sve2_AddCarryWideningEven: + case NI_Sve2_AddCarryWideningOdd: + // RMW operates on op3 + rmwOp = intrin.op3; + break; + case NI_Sve_CreateBreakPropagateMask: + case NI_Sve2_BitwiseSelect: + case NI_Sve2_BitwiseSelectLeftInverted: + case NI_Sve2_BitwiseSelectRightInverted: + // RMW operates on op2 + rmwOp = intrin.op2; + break; + default: + if (HWIntrinsicInfo::IsExplicitMaskedOperation(intrin.id)) + { + rmwOp = intrin.op2; + } + else + { + rmwOp = intrin.op1; + } + break; + } + } + + regNumber rmwReg = rmwOp->GetRegNum(); + if (targetReg != rmwReg) + { + switch (intrin.numOperands) + { + case 5: + assert((targetReg != intrin.op5->GetRegNum()) || genIsSameLocalVar(rmwOp, intrin.op5)); + FALLTHROUGH; + + case 4: + assert((targetReg != intrin.op4->GetRegNum()) || genIsSameLocalVar(rmwOp, intrin.op4)); + FALLTHROUGH; + + case 3: + if (rmwReg != intrin.op3->GetRegNum()) + { + assert(canRepairTargetOverlap || (targetReg != intrin.op3->GetRegNum()) || + genIsSameLocalVar(rmwOp, intrin.op3)); + } + FALLTHROUGH; + + case 2: + if (rmwReg != intrin.op2->GetRegNum()) + { + assert(canRepairTargetOverlap || (targetReg != intrin.op2->GetRegNum()) || + genIsSameLocalVar(rmwOp, intrin.op2)); + } + if (rmwReg != intrin.op1->GetRegNum()) + { + assert((targetReg != intrin.op1->GetRegNum()) || genIsSameLocalVar(rmwOp, intrin.op1)); + } + break; + + default: + break; + } + } +} +#endif // DEBUG + +//------------------------------------------------------------------------ // genHWIntrinsic: Generates the code for a given hardware intrinsic node. // // Arguments: @@ -384,68 +983,7 @@ void CodeGen::genHWIntrinsic(GenTreeHWIntrinsic* node) // If we see an optional embedded masked operation here, it is not embedded (and not RMW). if (isRMW && !HWIntrinsicInfo::IsOptionalEmbeddedMaskedOperation(intrin.id)) { - GenTree* rmwOp; - switch (intrin.id) - { - case NI_Sve2_AddCarryWideningEven: - case NI_Sve2_AddCarryWideningOdd: - // RMW operates on op3 - rmwOp = intrin.op3; - break; - case NI_Sve_CreateBreakPropagateMask: - case NI_Sve2_BitwiseSelect: - case NI_Sve2_BitwiseSelectLeftInverted: - case NI_Sve2_BitwiseSelectRightInverted: - // RMW operates on op2 - rmwOp = intrin.op2; - break; - default: - if (HWIntrinsicInfo::IsExplicitMaskedOperation(intrin.id)) - { - rmwOp = intrin.op2; - } - else - { - rmwOp = intrin.op1; - } - break; - } - - regNumber rmwReg = rmwOp->GetRegNum(); - if (targetReg != rmwReg) - { - switch (intrin.numOperands) - { - case 5: - assert((targetReg != op5Reg) || genIsSameLocalVar(rmwOp, intrin.op5)); - FALLTHROUGH; - - case 4: - assert((targetReg != op4Reg) || genIsSameLocalVar(rmwOp, intrin.op4)); - FALLTHROUGH; - - case 3: - if (rmwReg != op3Reg) - { - assert((targetReg != op3Reg) || genIsSameLocalVar(rmwOp, intrin.op3)); - } - FALLTHROUGH; - - case 2: - if (rmwReg != op2Reg) - { - assert((targetReg != op2Reg) || genIsSameLocalVar(rmwOp, intrin.op2)); - } - if (rmwReg != op1Reg) - { - assert((targetReg != op1Reg) || genIsSameLocalVar(rmwOp, intrin.op1)); - } - break; - - default: - break; - } - } + checkRMWRegisters(intrin, targetReg); } #endif // DEBUG @@ -548,642 +1086,10 @@ void CodeGen::genHWIntrinsic(GenTreeHWIntrinsic* node) } } } - else if (intrin.numOperands >= 2 && intrin.op2->IsEmbMaskOp()) + else if (intrin.id == NI_Sve_ConditionalSelect && intrin.op2->IsEmbMaskOp()) { // Handle case where op2 is operation that needs embedded mask - GenTree* op2 = intrin.op2; - assert(intrin.id == NI_Sve_ConditionalSelect); - assert(op2->OperIsHWIntrinsic()); - assert(op2->isContained()); - - // Get the registers and intrinsics that needs embedded mask - const HWIntrinsic intrinEmbMask(op2->AsHWIntrinsic()); - instruction insEmbMask = HWIntrinsicInfo::lookupIns(intrinEmbMask.id, intrinEmbMask.baseType, m_compiler); - const bool instrIsRMW = op2->isRMWHWIntrinsic(m_compiler); - - regNumber maskReg = op1Reg; - regNumber embMaskOp1Reg = REG_NA; - regNumber embMaskOp2Reg = REG_NA; - regNumber embMaskOp3Reg = REG_NA; - regNumber embMaskOp4Reg = REG_NA; - regNumber falseReg = op3Reg; - - switch (intrinEmbMask.numOperands) - { - case 4: - assert(intrinEmbMask.op4 != nullptr); - embMaskOp4Reg = intrinEmbMask.op4->GetRegNum(); - FALLTHROUGH; - - case 3: - assert(intrinEmbMask.op3 != nullptr); - embMaskOp3Reg = intrinEmbMask.op3->GetRegNum(); - FALLTHROUGH; - - case 2: - assert(intrinEmbMask.op2 != nullptr); - embMaskOp2Reg = intrinEmbMask.op2->GetRegNum(); - FALLTHROUGH; - - case 1: - assert(intrinEmbMask.op1 != nullptr); - embMaskOp1Reg = intrinEmbMask.op1->GetRegNum(); - break; - - default: - unreached(); - } - - // Shared code for setting up embedded mask arg for intrinsics with 3+ operands - - auto emitEmbeddedMaskSetupInstrs = [&] { - if (intrin.op3->IsVectorZero() || (targetReg != falseReg) || (targetReg != embMaskOp1Reg)) - { - return 1; - } - return 0; - }; - - auto emitEmbeddedMaskSetup = [&] { - if (intrin.op3->IsVectorZero()) - { - // If `falseReg` is zero, then move the first operand of `intrinEmbMask` in the - // destination using /Z. - - assert((targetReg != embMaskOp2Reg) || (embMaskOp1Reg == embMaskOp2Reg) || - genIsSameLocalVar(intrinEmbMask.op1, intrinEmbMask.op2)); - assert(intrin.op3->isContained() || !intrin.op1->IsTrueMask(node->GetSimdBaseType())); - return INS_SVE_MOV_OPTS_ZEROING; - } - else - { - // Below are the considerations we need to handle: - // - // targetReg == falseReg && targetReg == embMaskOp1Reg - // fmla Zd, P/m, Zn, Zm - // - // targetReg == falseReg && targetReg != embMaskOp1Reg - // movprfx target, P/m, embMaskOp1Reg - // fmla target, P/m, embMaskOp2Reg, embMaskOp3Reg - // - // targetReg != falseReg && targetReg == embMaskOp1Reg - // sel target, P/m, embMaskOp1Reg, falseReg - // fmla target, P/m, embMaskOp2Reg, embMaskOp3Reg - // - // targetReg != falseReg && targetReg != embMaskOp1Reg - // sel target, P/m, embMaskOp1Reg, falseReg - // fmla target, P/m, embMaskOp2Reg, embMaskOp3Reg - // - // Note that, we just check if the targetReg/falseReg or targetReg/embMaskOp1Reg - // coincides or not. - - if (targetReg != falseReg) - { - if (falseReg == embMaskOp1Reg) - { - // If falseReg value and embMaskOp1Reg value are same, then just mov the value - // to the target unpredicated. - - return INS_SVE_MOV_OPTS_UNPRED; - } - else - { - // If falseReg value is not present in targetReg yet, move the inactive lanes - // into the targetReg using `sel`. Since this is RMW, the active lanes should - // have the value from embMaskOp1Reg - - GetEmitter()->emitInsSve_R_R_R_R(INS_sve_sel, emitSize, targetReg, maskReg, embMaskOp1Reg, - falseReg, opt); - // embMaskOp1Reg becomes targetReg, then use unpredicated movprfx. - embMaskOp1Reg = targetReg; - return INS_SVE_MOV_OPTS_UNPRED; - } - } - else if (targetReg != embMaskOp1Reg) - { - // If target already contains the values of `falseReg`, just merge the lanes from - // `embMaskOp1Reg`, again because this is RMW semantics. - - return INS_SVE_MOV_OPTS_MERGING; - } - } - return INS_SVE_MOV_OPTS_UNPRED; - }; - - switch (intrinEmbMask.numOperands) - { - case 1: - { - assert(!instrIsRMW); - - insOpts embOpt = emitter::optGetSveInsOpt(emitTypeSize(intrinEmbMask.baseType)); - // Special handling for ConvertTo* APIs - // Just need to change the opt here. - switch (intrinEmbMask.id) - { - case NI_Sve_ConvertToInt32: - case NI_Sve_ConvertToUInt32: - case NI_Sve_ConvertToSingle: - case NI_Sve2_ConvertToSingleEvenRoundToOdd: - { - embOpt = emitTypeSize(intrinEmbMask.baseType) == EA_8BYTE ? INS_OPTS_D_TO_S - : INS_OPTS_SCALABLE_S; - break; - } - - case NI_Sve_ConvertToInt64: - case NI_Sve_ConvertToUInt64: - case NI_Sve_ConvertToDouble: - case NI_Sve2_ConvertToDoubleOdd: - { - embOpt = emitTypeSize(intrinEmbMask.baseType) == EA_4BYTE ? INS_OPTS_S_TO_D - : INS_OPTS_SCALABLE_D; - break; - } - - default: - break; - } - - if (targetReg != falseReg) - { - // If targetReg is not the same as `falseReg` then need to move - // the `falseReg` to `targetReg`. - - if (intrin.op3->isContained()) - { - assert(intrin.op3->IsVectorZero()); - - if (intrin.op1->isContained() || intrin.op1->IsTrueMask(node->GetSimdBaseType())) - { - // We already skip importing ConditionalSelect if op1 == trueAll, however - // if we still see it here, it is because we wrapped the predicated instruction - // inside ConditionalSelect. - // As such, no need to move the `falseReg` to `targetReg` - // because the predicated instruction will eventually set it. - GetEmitter()->emitIns_R_R_R(insEmbMask, emitSize, targetReg, maskReg, embMaskOp1Reg, - embOpt); - } - else - { - assert(!HWIntrinsicInfo::IsZeroingMaskedOperation(intrinEmbMask.id)); - - // If falseValue is zero, just zero out those lanes of targetReg using `movprfx` - // and /Z - GetEmitter()->emitInsSve_R_R_R_R(insEmbMask, emitSize, targetReg, maskReg, targetReg, - embMaskOp1Reg, embOpt, INS_SCALABLE_OPTS_NONE, - INS_SVE_MOV_OPTS_ZEROING); - } - } - else if (emitter::isVectorRegister(embMaskOp1Reg) && (targetReg == embMaskOp1Reg)) - { - // target != falseValue, but we do not want to overwrite target with `embMaskOp1Reg`. - // We will first do the predicate operation and then do conditionalSelect inactive - // elements from falseValue - - // We cannot use use `movprfx` here to move falseReg to targetReg because that will - // overwrite the value of embMaskOp1Reg which is present in targetReg. - GetEmitter()->emitIns_R_R_R(insEmbMask, emitSize, targetReg, maskReg, embMaskOp1Reg, - embOpt); - - GetEmitter()->emitIns_R_R_R_R(INS_sve_sel, emitSize, targetReg, maskReg, targetReg, - falseReg, opt); - } - else - { - // At this point, target != embMaskOp1Reg != falseReg, so just go ahead - // and move the falseReg unpredicated into targetReg. - GetEmitter()->emitIns_R_R_R_R(insEmbMask, emitSize, targetReg, maskReg, falseReg, - embMaskOp1Reg, embOpt); - } - } - else - { - GetEmitter()->emitIns_R_R_R(insEmbMask, emitSize, targetReg, maskReg, embMaskOp1Reg, embOpt); - } - break; - } - - case 2: - { - if (!instrIsRMW) - { - // Perform the actual "predicated" operation so that `embMaskOp1Reg` is the first operand.. - switch (intrinEmbMask.id) - { - case NI_Sve_And_Predicates: - case NI_Sve_BitwiseClear_Predicates: - case NI_Sve_Or_Predicates: - case NI_Sve_Xor_Predicates: - GetEmitter()->emitIns_R_R_R_R(insEmbMask, emitSize, targetReg, maskReg, embMaskOp1Reg, - embMaskOp2Reg, INS_OPTS_SCALABLE_B); - break; - - default: - { - GetEmitter()->emitIns_R_R_R_R(insEmbMask, emitSize, targetReg, maskReg, embMaskOp1Reg, - embMaskOp2Reg, opt); - } - } - break; - } - - insScalableOpts sopt = INS_SCALABLE_OPTS_NONE; - bool hasImmShift = (intrinEmbMask.category == HW_Category_ShiftLeftByImmediate || - intrinEmbMask.category == HW_Category_ShiftRightByImmediate) && - HWIntrinsicInfo::HasImmediateOperand(intrinEmbMask.id); - bool hasOptionalEmbMask = HWIntrinsicInfo::IsOptionalEmbeddedMaskedOperation(intrinEmbMask.id); - - insOpts embOpt = opt; - switch (intrinEmbMask.id) - { - case NI_Sve_ShiftLeftLogical: - case NI_Sve_ShiftRightArithmetic: - case NI_Sve_ShiftRightLogical: - { - const emitAttr op2Size = emitTypeSize(op2->AsHWIntrinsic()->GetAuxiliaryType()); - if (op2Size != emitTypeSize(intrinEmbMask.baseType)) - { - assert(emitter::optGetSveInsOpt(op2Size) == INS_OPTS_SCALABLE_D); - sopt = INS_SCALABLE_OPTS_WIDE; - } - break; - } - - case NI_Sve_CreateBreakPropagateMask: - embOpt = INS_OPTS_SCALABLE_B; - break; - - case NI_Sve_AddSequentialAcross: - // Predicate functionality is currently not exposed for this API, - // but the FADDA instruction only has a predicated variant. - // Thus, we expect the JIT to wrap this with CndSel. - assert(intrin.op3->IsVectorZero()); - break; - - case NI_Sve2_AddSaturate: - { - var_types baseType = op2->AsHWIntrinsic()->GetSimdBaseType(); - var_types auxType = op2->AsHWIntrinsic()->GetAuxiliaryType(); - if (baseType != auxType) - { - insEmbMask = (varTypeIsUnsigned(baseType)) ? INS_sve_usqadd : INS_sve_suqadd; - // SUQADD and USQADD must be predicated. - hasOptionalEmbMask = false; - } - else - { - // SQADD and UQADD can be unpredicated. - hasOptionalEmbMask = true; - } - break; - } - - case NI_Sve2_ConvertToSingleOdd: - case NI_Sve2_ConvertToSingleOddRoundToOdd: - embOpt = INS_OPTS_D_TO_S; - break; - - default: - break; - } - - auto emitInsHelper = [&](regNumber reg1, regNumber reg2, regNumber reg3) { - if (hasImmShift) - { - HWIntrinsicImmOpHelper helper(this, intrinEmbMask.op2, op2->AsHWIntrinsic()); - for (helper.EmitBegin(); !helper.Done(); helper.EmitCaseEnd()) - { - GetEmitter()->emitInsSve_R_R_I(insEmbMask, emitSize, reg1, reg2, helper.ImmValue(), - embOpt, sopt); - } - } - else - { - GetEmitter()->emitIns_R_R_R(insEmbMask, emitSize, reg1, reg2, reg3, embOpt, sopt); - } - }; - - auto emitInsMovPrfxHelper = [&](regNumber reg1, regNumber reg2, regNumber reg3, regNumber reg4) { - if (hasImmShift) - { - int numInstrs = (reg1 != reg3) ? 2 : 1; - HWIntrinsicImmOpHelper helper(this, intrinEmbMask.op2, op2->AsHWIntrinsic(), numInstrs); - for (helper.EmitBegin(); !helper.Done(); helper.EmitCaseEnd()) - { - GetEmitter()->emitInsSve_R_R_R_I(insEmbMask, emitSize, reg1, reg2, reg3, - helper.ImmValue(), embOpt, sopt); - } - } - else - { - GetEmitter()->emitIns_R_R_R_R(insEmbMask, emitSize, reg1, reg2, reg3, reg4, embOpt, sopt); - } - }; - - if (intrin.op3->IsVectorZero()) - { - // If `falseReg` is zero, then move the first operand of `intrinEmbMask` in the - // destination using /Z. - - switch (intrinEmbMask.id) - { - case NI_Sve_CreateBreakPropagateMask: - assert((targetReg == embMaskOp2Reg) || (targetReg != embMaskOp1Reg) || - genIsSameLocalVar(intrinEmbMask.op1, intrinEmbMask.op2)); - GetEmitter()->emitIns_R_R_R_R(insEmbMask, emitSize, targetReg, maskReg, embMaskOp1Reg, - embMaskOp2Reg, embOpt, sopt); - break; - - case NI_Sve_AddSequentialAcross: - assert((targetReg == op1Reg) || (targetReg != embMaskOp2Reg) || - genIsSameLocalVar(intrinEmbMask.op1, intrinEmbMask.op2)); - GetEmitter()->emitIns_R_R_R_R(insEmbMask, emitSize, targetReg, maskReg, embMaskOp1Reg, - embMaskOp2Reg, embOpt, sopt); - break; - - case NI_Sve2_AddSaturate: - assert((targetReg == op1Reg) || (targetReg != embMaskOp2Reg) || - genIsSameLocalVar(intrinEmbMask.op1, intrinEmbMask.op2)); - - if (hasOptionalEmbMask && intrin.op1->IsTrueMask(intrinEmbMask.baseType)) - { - // Use unpredicated SQADD/UQADD if the mask is all-true. - GetEmitter()->emitIns_R_R_R(insEmbMask, emitSize, targetReg, embMaskOp1Reg, - embMaskOp2Reg, embOpt, sopt); - } - else - { - GetEmitter()->emitIns_R_R_R(INS_sve_movprfx, EA_SCALABLE, targetReg, maskReg, - embMaskOp1Reg, opt); - emitInsHelper(targetReg, maskReg, embMaskOp2Reg); - } - break; - - case NI_Sve2_AddPairwise: - case NI_Sve2_MaxNumberPairwise: - case NI_Sve2_MaxPairwise: - case NI_Sve2_MinNumberPairwise: - case NI_Sve2_MinPairwise: - // These instructions have unpredictable behaviour when using predicated movprfx, - // so the unpredicated variant must be used here. - assert(!intrin.op3->isContained() && falseReg != REG_NA); - GetEmitter()->emitIns_R_R_R_R(insEmbMask, emitSize, targetReg, maskReg, embMaskOp1Reg, - embMaskOp2Reg, embOpt, sopt); - GetEmitter()->emitIns_R_R_R_R(INS_sve_sel, emitSize, targetReg, maskReg, targetReg, - falseReg, opt); - break; - - case NI_Sve2_ConvertToSingleOdd: - case NI_Sve2_ConvertToSingleOddRoundToOdd: - // TODO-SVE: Optimise away the explicit copying of `embMaskOp1Reg` to `targetReg`. - // For these intrinsics we cannot use movprfx instruction to populate `targetReg` with - // `embMaskOp1Reg`. Thus, we need to perform move before the operation. - GetEmitter()->emitIns_R_R_R_R(insEmbMask, emitSize, targetReg, maskReg, embMaskOp1Reg, - embMaskOp2Reg, embOpt, sopt); - break; - - default: - assert(targetReg != embMaskOp2Reg); - - // Finally, perform the actual "predicated" operation so that `targetReg` is the first - // operand and `embMaskOp2Reg` is the second operand. - - if (hasImmShift) - { - HWIntrinsicImmOpHelper helper(this, intrinEmbMask.op2, op2->AsHWIntrinsic(), 2); - for (helper.EmitBegin(); !helper.Done(); helper.EmitCaseEnd()) - { - GetEmitter()->emitInsSve_R_R_R_I(insEmbMask, emitSize, targetReg, maskReg, - embMaskOp1Reg, helper.ImmValue(), embOpt, sopt, - INS_SVE_MOV_OPTS_ZEROING); - } - } - else - { - GetEmitter()->emitInsSve_R_R_R_R(insEmbMask, emitSize, targetReg, maskReg, - embMaskOp1Reg, embMaskOp2Reg, embOpt, sopt, - INS_SVE_MOV_OPTS_ZEROING); - } - break; - } - } - // If `targetReg` and `falseReg` are not same, then we need to move it to `targetReg` first - // so the `insEmbMask` operation can be merged on top of it. - else if (targetReg != falseReg) - { - - if ((intrinEmbMask.id == NI_Sve2_ConvertToSingleOdd) || - (intrinEmbMask.id == NI_Sve2_ConvertToSingleOddRoundToOdd)) - { - // TODO-SVE: Optimise away the explicit copying of `embMaskOp1Reg` to `targetReg`. - // For these intrinsics we cannot use movprfx instruction to populate `targetReg` with - // `embMaskOp1Reg`. Thus, we need to perform move before the operation, and then "sel" to - // select the active lanes. - GetEmitter()->emitIns_R_R_R_R(insEmbMask, emitSize, targetReg, maskReg, embMaskOp1Reg, - embMaskOp2Reg, embOpt, sopt); - GetEmitter()->emitIns_R_R_R_R(INS_sve_sel, emitSize, targetReg, maskReg, targetReg, - falseReg, opt); - } - else if (falseReg != embMaskOp1Reg) - { - // At the point, targetReg != embMaskOp1Reg != falseReg - if (hasOptionalEmbMask) - { - // If the embedded instruction supports optional mask operation, use the "unpredicated" - // version of the instruction, followed by "sel" to select the active lanes. - emitInsHelper(targetReg, embMaskOp1Reg, embMaskOp2Reg); - } - else - { - // If the instruction just has "predicated" version, then move the "embMaskOp1Reg" - // into targetReg. Next, do the predicated operation on the targetReg and last, - // use "sel" to select the active lanes based on mask, and set inactive lanes - // to falseReg. - assert(targetReg != embMaskOp2Reg); - assert(HWIntrinsicInfo::IsEmbeddedMaskedOperation(intrinEmbMask.id)); - - emitInsMovPrfxHelper(targetReg, maskReg, embMaskOp1Reg, embMaskOp2Reg); - } - - GetEmitter()->emitIns_R_R_R_R(INS_sve_sel, emitSize, targetReg, maskReg, targetReg, - falseReg, opt); - } - else if (targetReg != embMaskOp1Reg) - { - // embMaskOp1Reg is same as `falseReg`, but not same as `targetReg`. Move the - // `embMaskOp1Reg` i.e. `falseReg` in `targetReg`, using "unpredicated movprfx", so the - // subsequent `insEmbMask` operation can be merged on top of it. - - emitInsMovPrfxHelper(targetReg, maskReg, falseReg, embMaskOp2Reg); - } - else - { - // Finally, perform the actual "predicated" operation so that `targetReg` is the first - // operand and `embMaskOp2Reg` is the second operand. - emitInsHelper(targetReg, maskReg, embMaskOp2Reg); - } - } - else - { - // Just perform the actual "predicated" operation so that `targetReg` is the first operand - // and `embMaskOp2Reg` is the second operand. - emitInsHelper(targetReg, maskReg, embMaskOp2Reg); - } - - break; - } - - case 3: - { - assert(instrIsRMW); - - if (HWIntrinsicInfo::IsFmaIntrinsic(intrinEmbMask.id)) - { - // For FMA, the operation we are trying to perform is: - // result = op1 + (op2 * op3) - // - // There are two instructions that can be used depending on which operand's register, - // optionally, will store the final result. - // - // 1. If the result is stored in the operand that was used as an "addend" in the operation, - // then we use `FMLA` format: - // reg1 = reg1 + (reg2 * reg3) - // - // 2. If the result is stored in the operand that was used as a "multiplicand" in the operation, - // then we use `FMAD` format: - // reg1 = (reg1 * reg2) + reg3 - // - // Check if the result's register is same as that of one of the operand's register and - // accordingly pick the appropriate format. Suppose `targetReg` holds the result, then we have - // following cases: - // - // Case# 1: Result is stored in the operand that held the "addend" - // targetReg == reg1 - // - // We generate the FMLA instruction format and no further changes are needed. - // - // Case# 2: Result is stored in the operand `op2` that held the "multiplicand" - // targetReg == reg2 - // - // So we basically have an operation: - // reg2 = reg1 + (reg2 * reg3) - // - // Since, the result will be stored in the "multiplicand", we pick format `FMAD`. - // Then, we rearrange the operands to ensure that the operation is done correctly. - // reg2 = reg1 + (reg2 * reg3) // to start with - // reg2 = reg3 + (reg2 * reg1) // swap reg1 <--> reg3 - // reg1 = reg3 + (reg1 * reg2) // swap reg1 <--> reg2 - // reg1 = (reg1 * reg2) + reg3 // rearrange to get FMAD format - // - // Case# 3: Result is stored in the operand `op3` that held the "multiplier" - // targetReg == reg3 - // - // So we basically have an operation: - // reg3 = reg1 + (reg2 * reg3) - // Since, the result will be stored in the "multiplier", we again pick format `FMAD`. - // Then, we rearrange the operands to ensure that the operation is done correctly. - // reg3 = reg1 + (reg2 * reg3) // to start with - // reg1 = reg3 + (reg2 * reg1) // swap reg1 <--> reg3 - // reg1 = (reg1 * reg2) + reg3 // rearrange to get FMAD format - - bool useAddend = true; - if (targetReg == embMaskOp2Reg) - { - // Case# 2 - useAddend = false; - std::swap(embMaskOp1Reg, embMaskOp3Reg); - std::swap(embMaskOp1Reg, embMaskOp2Reg); - } - else if (targetReg == embMaskOp3Reg) - { - // Case# 3 - useAddend = false; - std::swap(embMaskOp1Reg, embMaskOp3Reg); - } - else - { - // Case# 1 - } - - switch (intrinEmbMask.id) - { - case NI_Sve_FusedMultiplyAdd: - insEmbMask = useAddend ? INS_sve_fmla : INS_sve_fmad; - break; - - case NI_Sve_FusedMultiplyAddNegated: - insEmbMask = useAddend ? INS_sve_fnmla : INS_sve_fnmad; - break; - - case NI_Sve_FusedMultiplySubtract: - insEmbMask = useAddend ? INS_sve_fmls : INS_sve_fmsb; - break; - - case NI_Sve_FusedMultiplySubtractNegated: - insEmbMask = useAddend ? INS_sve_fnmls : INS_sve_fnmsb; - break; - - case NI_Sve_MultiplyAdd: - insEmbMask = useAddend ? INS_sve_mla : INS_sve_mad; - break; - - case NI_Sve_MultiplySubtract: - insEmbMask = useAddend ? INS_sve_mls : INS_sve_msb; - break; - - default: - unreached(); - } - } - - // Finally, perform the desired operation. - if (HWIntrinsicInfo::HasImmediateOperand(intrinEmbMask.id)) - { - HWIntrinsicImmOpHelper helper(this, intrinEmbMask.op3, op2->AsHWIntrinsic(), - emitEmbeddedMaskSetupInstrs() + 1); - for (helper.EmitBegin(); !helper.Done(); helper.EmitCaseEnd()) - { - insSveMovOpts mopt = emitEmbeddedMaskSetup(); - GetEmitter()->emitInsSve_R_R_R_R_I(insEmbMask, emitSize, targetReg, maskReg, embMaskOp1Reg, - embMaskOp2Reg, helper.ImmValue(), opt, - INS_SCALABLE_OPTS_NONE, mopt); - } - } - else - { - assert(HWIntrinsicInfo::IsFmaIntrinsic(intrinEmbMask.id)); - insSveMovOpts mopt = emitEmbeddedMaskSetup(); - GetEmitter()->emitInsSve_R_R_R_R_R(insEmbMask, emitSize, targetReg, maskReg, embMaskOp1Reg, - embMaskOp2Reg, embMaskOp3Reg, opt, INS_SCALABLE_OPTS_NONE, - mopt); - } - - break; - } - - case 4: - { - assert(instrIsRMW); - assert(intrinEmbMask.op4->isContained() == (embMaskOp4Reg == REG_NA)); - assert(HWIntrinsicInfo::HasImmediateOperand(intrinEmbMask.id)); - - HWIntrinsicImmOpHelper helper(this, intrinEmbMask.op4, op2->AsHWIntrinsic(), - emitEmbeddedMaskSetupInstrs() + 1); - for (helper.EmitBegin(); !helper.Done(); helper.EmitCaseEnd()) - { - insSveMovOpts mopt = emitEmbeddedMaskSetup(); - GetEmitter()->emitInsSve_R_R_R_R_R_I(insEmbMask, emitSize, targetReg, maskReg, embMaskOp1Reg, - embMaskOp2Reg, embMaskOp3Reg, helper.ImmValue(), opt, - INS_SCALABLE_OPTS_NONE, mopt); - } - - break; - } - - default: - unreached(); - } + genEmbeddedMaskedHWIntrinsic(node, targetReg); } else { diff --git a/src/coreclr/jit/lowerarmarch.cpp b/src/coreclr/jit/lowerarmarch.cpp index d0ef0b68627d5c..a8515b337ec241 100644 --- a/src/coreclr/jit/lowerarmarch.cpp +++ b/src/coreclr/jit/lowerarmarch.cpp @@ -3965,11 +3965,9 @@ void Lowering::ContainCheckHWIntrinsic(GenTreeHWIntrinsic* node) } // Handle op3 - if (op3->IsVectorZero() && op1->IsTrueMask(node->GetSimdBaseType()) && op2->IsEmbMaskOp()) + if (op3->IsVectorZero() && op2->IsEmbMaskOp()) { - // When we are merging with zero, we can specialize - // and avoid instantiating the vector constant. - // Do this only if op1 was AllTrueMask + // When we are merging with zero, we can specialize and avoid instantiating the vector constant. switch (op2->AsHWIntrinsic()->GetHWIntrinsicId()) { case NI_Sve2_AddPairwise: @@ -3977,11 +3975,16 @@ void Lowering::ContainCheckHWIntrinsic(GenTreeHWIntrinsic* node) case NI_Sve2_MaxPairwise: case NI_Sve2_MinNumberPairwise: case NI_Sve2_MinPairwise: - // This is an edge case where these instructions have unpredictable behaviour when - // using predicated movprfx, so the unpredicated variant must be used here. This - // prevents us from performing this optimization as we will need the constant vector - // for masking the result. - break; + case NI_Sve2_ConvertToDoubleOdd: + case NI_Sve2_ConvertToSingleOdd: + case NI_Sve2_ConvertToSingleOddRoundToOdd: + // This is an edge case where these instructions do not support predicated or any movprfx. + if (!op1->IsTrueMask(node->GetSimdBaseType())) + { + // When op1 is not all-true, we will need the constant vector for masking the result. + break; + } + FALLTHROUGH; default: MakeSrcContained(node, op3); diff --git a/src/coreclr/jit/lsra.h b/src/coreclr/jit/lsra.h index 22493f4a479018..ba30cb3f742c59 100644 --- a/src/coreclr/jit/lsra.h +++ b/src/coreclr/jit/lsra.h @@ -1436,7 +1436,7 @@ class LinearScan : public RegAllocInterface FORCEINLINE RefPosition* getNextConsecutiveRefPosition(RefPosition* refPosition); FORCEINLINE regNumber getNextFPRegWraparound(regNumber reg); SingleTypeRegSet getOperandCandidates(GenTreeHWIntrinsic* intrinsicTree, HWIntrinsic intrin, size_t opNum); - GenTree* getDelayFreeOperand(GenTreeHWIntrinsic* intrinsicTree, bool embedded = false); + GenTree* getDelayFreeOperand(GenTreeHWIntrinsic* intrinsicTree, GenTreeHWIntrinsic* user = nullptr); GenTree* getVectorAddrOperand(GenTreeHWIntrinsic* intrinsicTree); GenTree* getConsecutiveRegistersOperand(const HWIntrinsic intrin, bool* destIsConsecutive); GenTreeHWIntrinsic* getEmbeddedMaskOperand(const HWIntrinsic intrin); @@ -2078,6 +2078,9 @@ class LinearScan : public RegAllocInterface int BuildConsecutiveRegistersForUse(GenTree* treeNode, GenTree* rmwNode = nullptr); void BuildConsecutiveRegistersForDef(GenTree* treeNode, int fieldCount); void BuildHWIntrinsicImmediate(GenTreeHWIntrinsic* intrinsicTree, const HWIntrinsic intrin); + void BuildHWIntrinsicTempRegs(GenTreeHWIntrinsic* intrinsicTree, + const HWIntrinsic intrin, + GenTreeHWIntrinsic* embeddedOp); int BuildEmbeddedOperandUses(GenTreeHWIntrinsic* embeddedOpNode, GenTree* embeddedDelayFreeOp); int BuildContainedCselUses(GenTreeHWIntrinsic* containedCselOpNode, GenTree* delayFreeOp, diff --git a/src/coreclr/jit/lsraarm64.cpp b/src/coreclr/jit/lsraarm64.cpp index b49a65a61841ff..898ddfb4d9c66b 100644 --- a/src/coreclr/jit/lsraarm64.cpp +++ b/src/coreclr/jit/lsraarm64.cpp @@ -1430,32 +1430,14 @@ int LinearScan::BuildHWIntrinsic(GenTreeHWIntrinsic* intrinsicTree, int* pDstCou if (embeddedOp != nullptr) { assert(delayFreeOp == nullptr); - delayFreeOp = getDelayFreeOperand(embeddedOp, /* embedded */ true); + delayFreeOp = getDelayFreeOperand(embeddedOp, intrinsicTree); } // Build any immediates BuildHWIntrinsicImmediate(intrinsicTree, intrin); - // Build any additional special cases - switch (intrin.id) - { - case NI_Sve2_GatherVectorInt16SignExtendNonTemporal: - case NI_Sve2_GatherVectorInt32SignExtendNonTemporal: - case NI_Sve2_GatherVectorNonTemporal: - case NI_Sve2_GatherVectorUInt16ZeroExtendNonTemporal: - case NI_Sve2_GatherVectorUInt32ZeroExtendNonTemporal: - case NI_Sve2_Scatter16BitNarrowingNonTemporal: - case NI_Sve2_Scatter32BitNarrowingNonTemporal: - case NI_Sve2_ScatterNonTemporal: - if (!varTypeIsSIMD(intrin.op2->gtType)) - { - buildInternalFloatRegisterDefForNode(intrinsicTree, internalFloatRegCandidates()); - } - break; - - default: - break; - } + // Build any internal temporary registers + BuildHWIntrinsicTempRegs(intrinsicTree, intrin, embeddedOp); // Build all Operands for (size_t opNum = 1; opNum <= intrin.numOperands; opNum++) @@ -1809,6 +1791,47 @@ void LinearScan::BuildHWIntrinsicImmediate(GenTreeHWIntrinsic* intrinsicTree, co } } +//------------------------------------------------------------------------ +// BuildHWIntrinsicTempRegs: Build temporary registers used by HWIntrinsic codegen. +// +// Arguments: +// intrinsicTree - Intrinsic tree node for which need to build RefPositions for +// intrin - Underlying intrinsic +// embeddedOp - Embedded mask operand of intrinsicTree, if any +// +void LinearScan::BuildHWIntrinsicTempRegs(GenTreeHWIntrinsic* intrinsicTree, + const HWIntrinsic intrin, + GenTreeHWIntrinsic* embeddedOp) +{ + switch (intrin.id) + { + case NI_Sve2_GatherVectorInt16SignExtendNonTemporal: + case NI_Sve2_GatherVectorInt32SignExtendNonTemporal: + case NI_Sve2_GatherVectorNonTemporal: + case NI_Sve2_GatherVectorUInt16ZeroExtendNonTemporal: + case NI_Sve2_GatherVectorUInt32ZeroExtendNonTemporal: + case NI_Sve2_Scatter16BitNarrowingNonTemporal: + case NI_Sve2_Scatter32BitNarrowingNonTemporal: + case NI_Sve2_ScatterNonTemporal: + if (!varTypeIsSIMD(intrin.op2->gtType)) + { + buildInternalFloatRegisterDefForNode(intrinsicTree, internalFloatRegCandidates()); + } + break; + + case NI_Sve_ConditionalSelect: + if ((embeddedOp != nullptr) && embeddedOp->OperIsHWIntrinsic(NI_Sve_MultiplyAddRotateComplex)) + { + buildInternalFloatRegisterDefForNode(intrinsicTree, internalFloatRegCandidates()); + setInternalRegsDelayFree = true; + } + break; + + default: + break; + } +} + //------------------------------------------------------------------------ // BuildEmbeddedOperandUses: Build the uses for an Embedded Mask operand // @@ -2315,17 +2338,18 @@ SingleTypeRegSet LinearScan::getOperandCandidates(GenTreeHWIntrinsic* intrinsicT // // Arguments: // intrinsicTree - Tree to check -// embedded - If this is an embedded operand +// user - The user of intrinsicTree if intrinsicTree is an embedded operand // // Return Value: // The operand that needs to be delay freed // -GenTree* LinearScan::getDelayFreeOperand(GenTreeHWIntrinsic* intrinsicTree, bool embedded) +GenTree* LinearScan::getDelayFreeOperand(GenTreeHWIntrinsic* intrinsicTree, GenTreeHWIntrinsic* user) { bool isRMW = intrinsicTree->isRMWHWIntrinsic(m_compiler); const NamedIntrinsic intrinsicId = intrinsicTree->GetHWIntrinsicId(); GenTree* delayFreeOp = nullptr; + const bool embedded = user != nullptr; switch (intrinsicId) { @@ -2381,6 +2405,23 @@ GenTree* LinearScan::getDelayFreeOperand(GenTreeHWIntrinsic* intrinsicTree, bool assert(delayFreeOp != nullptr); break; + case NI_Sve2_ConvertToSingleOdd: + case NI_Sve2_ConvertToSingleOddRoundToOdd: + assert(isRMW); + if (embedded && user->OperIsHWIntrinsic(NI_Sve_ConditionalSelect) && user->Op(3)->IsVectorZero() && + !user->Op(1)->IsTrueMask(user->GetSimdBaseType())) + { + // These instructions cannot use movprfx. Prefer the zero falseOp as the target so codegen can + // preserve inactive lanes by first copying active lanes from op1. + delayFreeOp = user->Op(3); + } + else + { + delayFreeOp = intrinsicTree->Op(1); + } + assert(delayFreeOp != nullptr); + break; + default: if (isRMW) { @@ -2406,6 +2447,25 @@ GenTree* LinearScan::getDelayFreeOperand(GenTreeHWIntrinsic* intrinsicTree, bool assert(delayFreeOp != nullptr); } } + else if ((intrinsicTree->GetOperandCount() == 1) && embedded && + !HWIntrinsicInfo::IsReduceOperation(intrinsicId)) + { + // Unary embedded masked operations are non-RMW, but also support movprfx. + assert(user->OperIsHWIntrinsic(NI_Sve_ConditionalSelect)); + + if (user->Op(1)->IsTrueMask(user->GetSimdBaseType()) && user->Op(3)->IsVectorZero()) + { + // When all lanes are active, falseOp is irrelevant and the embedded operation can write back to + // op1. + delayFreeOp = intrinsicTree->Op(1); + } + else + { + // Mark the falseOp of the conditional select as delay free. + delayFreeOp = user->Op(3); + } + assert(delayFreeOp != nullptr); + } break; } diff --git a/src/tests/JIT/HardwareIntrinsics/Arm/Shared/SveGatherVector.template b/src/tests/JIT/HardwareIntrinsics/Arm/Shared/SveGatherVector.template index 1fc8fb6398241b..e2c62064c16ce4 100644 --- a/src/tests/JIT/HardwareIntrinsics/Arm/Shared/SveGatherVector.template +++ b/src/tests/JIT/HardwareIntrinsics/Arm/Shared/SveGatherVector.template @@ -418,7 +418,9 @@ namespace {Namespace} [method: MethodImpl(MethodImplOptions.AggressiveInlining)] private void ConditionalSelectScenario_TrueValue({RetVectorType}<{RetBaseType}> mask, {Op1VectorType}<{Op1BaseType}> op1, {Op2BaseType}* op2, {Op3VectorType}<{Op3BaseType}> op3, {Op1VectorType}<{Op1BaseType}> falseOp) { - var result = Sve.ConditionalSelect(mask, {Isa}.{Method}(op1, op2, op3), falseOp); + var result = (falseOp == {Op1VectorType}<{Op1BaseType}>.Zero) ? + Sve.ConditionalSelect(mask, {Isa}.{Method}(op1, op2, op3), {Op1VectorType}<{Op1BaseType}>.Zero) : + Sve.ConditionalSelect(mask, {Isa}.{Method}(op1, op2, op3), falseOp); Unsafe.Write(_dataTable.outArrayPtr, result); ValidateConditionalSelectResult_TrueValue(mask, op1, op2, op3, falseOp, _dataTable.outArrayPtr); diff --git a/src/tests/JIT/HardwareIntrinsics/Arm/Shared/SveGatherVectorByteOffsetFirstFaulting.template b/src/tests/JIT/HardwareIntrinsics/Arm/Shared/SveGatherVectorByteOffsetFirstFaulting.template index dc5d26283d8d08..5667205cf96ac1 100644 --- a/src/tests/JIT/HardwareIntrinsics/Arm/Shared/SveGatherVectorByteOffsetFirstFaulting.template +++ b/src/tests/JIT/HardwareIntrinsics/Arm/Shared/SveGatherVectorByteOffsetFirstFaulting.template @@ -470,7 +470,9 @@ namespace {Namespace} [method: MethodImpl(MethodImplOptions.AggressiveInlining)] private void ConditionalSelectScenario_TrueValue({RetVectorType}<{RetBaseType}> mask, {Op1VectorType}<{Op1BaseType}> op1, {Op2BaseType}* op2, {Op3VectorType}<{Op3BaseType}> op3, {Op1VectorType}<{Op1BaseType}> falseOp) { - var result = Sve.ConditionalSelect(mask, {Isa}.{Method}(op1, op2, op3), falseOp); + var result = (falseOp == {Op1VectorType}<{Op1BaseType}>.Zero) ? + Sve.ConditionalSelect(mask, {Isa}.{Method}(op1, op2, op3), {Op1VectorType}<{Op1BaseType}>.Zero) : + Sve.ConditionalSelect(mask, {Isa}.{Method}(op1, op2, op3), falseOp); Unsafe.Write(_dataTable.outArrayPtr, result); ValidateConditionalSelectResult_TrueValue(mask, op1, op2, op3, falseOp, _dataTable.outArrayPtr); diff --git a/src/tests/JIT/HardwareIntrinsics/Arm/Shared/SveGatherVectorByteOffsets.template b/src/tests/JIT/HardwareIntrinsics/Arm/Shared/SveGatherVectorByteOffsets.template index a5b16f2b781edf..bb8dd48fed47e7 100644 --- a/src/tests/JIT/HardwareIntrinsics/Arm/Shared/SveGatherVectorByteOffsets.template +++ b/src/tests/JIT/HardwareIntrinsics/Arm/Shared/SveGatherVectorByteOffsets.template @@ -396,7 +396,9 @@ namespace {Namespace} [method: MethodImpl(MethodImplOptions.AggressiveInlining)] private void ConditionalSelectScenario_TrueValue({RetVectorType}<{RetBaseType}> mask, {Op1VectorType}<{Op1BaseType}> op1, {Op2BaseType}* op2, {Op3VectorType}<{Op3BaseType}> op3, {Op1VectorType}<{Op1BaseType}> falseOp) { - var result = Sve.ConditionalSelect(mask, {Isa}.{Method}(op1, op2, op3), falseOp); + var result = (falseOp == {Op1VectorType}<{Op1BaseType}>.Zero) ? + Sve.ConditionalSelect(mask, {Isa}.{Method}(op1, op2, op3), {Op1VectorType}<{Op1BaseType}>.Zero) : + Sve.ConditionalSelect(mask, {Isa}.{Method}(op1, op2, op3), falseOp); Unsafe.Write(_dataTable.outArrayPtr, result); ValidateConditionalSelectResult_TrueValue(mask, op1, op2, op3, falseOp, _dataTable.outArrayPtr); diff --git a/src/tests/JIT/HardwareIntrinsics/Arm/Shared/SveGatherVectorFirstFaultingVectorBases.template b/src/tests/JIT/HardwareIntrinsics/Arm/Shared/SveGatherVectorFirstFaultingVectorBases.template index fa40f4a85f9f84..dc8c424185420b 100644 --- a/src/tests/JIT/HardwareIntrinsics/Arm/Shared/SveGatherVectorFirstFaultingVectorBases.template +++ b/src/tests/JIT/HardwareIntrinsics/Arm/Shared/SveGatherVectorFirstFaultingVectorBases.template @@ -470,7 +470,9 @@ namespace {Namespace} [method: MethodImpl(MethodImplOptions.AggressiveInlining)] private void ConditionalSelectScenario_TrueValue({RetVectorType}<{RetBaseType}> mask, {Op1VectorType}<{Op1BaseType}> op1, {Op2VectorType}<{Op2BaseType}> op2, {Op1VectorType}<{Op1BaseType}> falseOp) { - var result = Sve.ConditionalSelect(mask, {Isa}.{Method}(op1, op2), falseOp); + var result = (falseOp == {Op1VectorType}<{Op1BaseType}>.Zero) ? + Sve.ConditionalSelect(mask, {Isa}.{Method}(op1, op2), {Op1VectorType}<{Op1BaseType}>.Zero) : + Sve.ConditionalSelect(mask, {Isa}.{Method}(op1, op2), falseOp); Unsafe.Write(_dataTable.outArrayPtr, result); ValidateConditionalSelectResult_TrueValue(mask, op1, op2, falseOp, _dataTable.outArrayPtr); diff --git a/src/tests/JIT/HardwareIntrinsics/Arm/Shared/SveGatherVectorVectorBases.template b/src/tests/JIT/HardwareIntrinsics/Arm/Shared/SveGatherVectorVectorBases.template index 3cba82d4453b7d..507c417f58013f 100644 --- a/src/tests/JIT/HardwareIntrinsics/Arm/Shared/SveGatherVectorVectorBases.template +++ b/src/tests/JIT/HardwareIntrinsics/Arm/Shared/SveGatherVectorVectorBases.template @@ -412,7 +412,9 @@ namespace {Namespace} [method: MethodImpl(MethodImplOptions.AggressiveInlining)] private void ConditionalSelectScenario_TrueValue({RetVectorType}<{RetBaseType}> mask, {Op1VectorType}<{Op1BaseType}> op1, {Op2VectorType}<{Op2BaseType}> op2, {Op1VectorType}<{Op1BaseType}> falseOp) { - var result = Sve.ConditionalSelect(mask, {Isa}.{Method}(op1, op2), falseOp); + var result = (falseOp == {Op1VectorType}<{Op1BaseType}>.Zero) ? + Sve.ConditionalSelect(mask, {Isa}.{Method}(op1, op2), {Op1VectorType}<{Op1BaseType}>.Zero) : + Sve.ConditionalSelect(mask, {Isa}.{Method}(op1, op2), falseOp); Unsafe.Write(_dataTable.outArrayPtr, result); ValidateConditionalSelectResult_TrueValue(mask, op1, op2, falseOp, _dataTable.outArrayPtr); diff --git a/src/tests/JIT/HardwareIntrinsics/Arm/Shared/_SveBinaryMaskOpTestTemplate.template b/src/tests/JIT/HardwareIntrinsics/Arm/Shared/_SveBinaryMaskOpTestTemplate.template index 023fc61e917062..9406e9933641b5 100644 --- a/src/tests/JIT/HardwareIntrinsics/Arm/Shared/_SveBinaryMaskOpTestTemplate.template +++ b/src/tests/JIT/HardwareIntrinsics/Arm/Shared/_SveBinaryMaskOpTestTemplate.template @@ -345,7 +345,9 @@ namespace {Namespace} [method: MethodImpl(MethodImplOptions.AggressiveInlining)] private void ConditionalSelectScenario_TrueValue({Op1VectorType}<{Op1BaseType}> mask, {Op1VectorType}<{Op1BaseType}> op1, {Op1VectorType}<{Op2BaseType}> op2, {Op1VectorType}<{Op1BaseType}> falseOp) { - var result = Sve.ConditionalSelect(mask, {Isa}.{Method}(op1, op2), falseOp); + var result = (falseOp == {Op1VectorType}<{Op1BaseType}>.Zero) ? + Sve.ConditionalSelect(mask, {Isa}.{Method}(op1, op2), {Op1VectorType}<{Op1BaseType}>.Zero) : + Sve.ConditionalSelect(mask, {Isa}.{Method}(op1, op2), falseOp); Unsafe.Write(_dataTable.outArrayPtr, result); ValidateConditionalSelectResult_TrueValue(mask, op1, op2, falseOp, _dataTable.outArrayPtr); diff --git a/src/tests/JIT/HardwareIntrinsics/Arm/Shared/_SveBinaryOpDifferentRetTypeTestTemplate.template b/src/tests/JIT/HardwareIntrinsics/Arm/Shared/_SveBinaryOpDifferentRetTypeTestTemplate.template index 411a59d414d6df..b32f3b62a089c6 100644 --- a/src/tests/JIT/HardwareIntrinsics/Arm/Shared/_SveBinaryOpDifferentRetTypeTestTemplate.template +++ b/src/tests/JIT/HardwareIntrinsics/Arm/Shared/_SveBinaryOpDifferentRetTypeTestTemplate.template @@ -403,7 +403,9 @@ namespace {Namespace} [method: MethodImpl(MethodImplOptions.AggressiveInlining)] private void ConditionalSelectScenario_TrueValue({RetVectorType}<{RetBaseType}> mask, {Op1VectorType}<{Op1BaseType}> op1, {Op2VectorType}<{Op2BaseType}> op2, {RetVectorType}<{RetBaseType}> falseOp) { - var result = Sve.ConditionalSelect(mask, {Isa}.{Method}(op1, op2), falseOp); + var result = (falseOp == {RetVectorType}<{RetBaseType}>.Zero) ? + Sve.ConditionalSelect(mask, {Isa}.{Method}(op1, op2), {RetVectorType}<{RetBaseType}>.Zero) : + Sve.ConditionalSelect(mask, {Isa}.{Method}(op1, op2), falseOp); Unsafe.Write(_dataTable.outArrayPtr, result); ValidateConditionalSelectResult_TrueValue(mask, op1, op2, falseOp, _dataTable.outArrayPtr); diff --git a/src/tests/JIT/HardwareIntrinsics/Arm/Shared/_SveBinaryOpDifferentTypesTestTemplate.template b/src/tests/JIT/HardwareIntrinsics/Arm/Shared/_SveBinaryOpDifferentTypesTestTemplate.template index a8f3b92347663b..5d46b27c0dc554 100644 --- a/src/tests/JIT/HardwareIntrinsics/Arm/Shared/_SveBinaryOpDifferentTypesTestTemplate.template +++ b/src/tests/JIT/HardwareIntrinsics/Arm/Shared/_SveBinaryOpDifferentTypesTestTemplate.template @@ -322,7 +322,9 @@ namespace {Namespace} [method: MethodImpl(MethodImplOptions.AggressiveInlining)] private void ConditionalSelectScenario_TrueValue({Op1VectorType}<{Op1BaseType}> mask, {Op1VectorType}<{Op1BaseType}> op1, {Op2VectorType}<{Op2BaseType}> op2, {Op1VectorType}<{Op1BaseType}> falseOp) { - var result = Sve.ConditionalSelect(mask, {Isa}.{Method}(op1, op2), falseOp); + var result = (falseOp == {Op1VectorType}<{Op1BaseType}>.Zero) ? + Sve.ConditionalSelect(mask, {Isa}.{Method}(op1, op2), {Op1VectorType}<{Op1BaseType}>.Zero) : + Sve.ConditionalSelect(mask, {Isa}.{Method}(op1, op2), falseOp); Unsafe.Write(_dataTable.outArrayPtr, result); ValidateConditionalSelectResult_TrueValue(mask, op1, op2, falseOp, _dataTable.outArrayPtr); diff --git a/src/tests/JIT/HardwareIntrinsics/Arm/Shared/_SveBinaryOpTestTemplate.template b/src/tests/JIT/HardwareIntrinsics/Arm/Shared/_SveBinaryOpTestTemplate.template index 7f13e6ee46405f..0bfbc7b0cf82db 100644 --- a/src/tests/JIT/HardwareIntrinsics/Arm/Shared/_SveBinaryOpTestTemplate.template +++ b/src/tests/JIT/HardwareIntrinsics/Arm/Shared/_SveBinaryOpTestTemplate.template @@ -313,7 +313,9 @@ namespace {Namespace} [method: MethodImpl(MethodImplOptions.AggressiveInlining)] private void ConditionalSelectScenario_TrueValue(Vector<{RetBaseType}> mask, Vector<{Op1BaseType}> op1, Vector<{Op2BaseType}> op2, Vector<{RetBaseType}> falseOp) { - var result = Sve.ConditionalSelect(mask, {Isa}.{Method}(op1, op2), falseOp); + var result = (falseOp == Vector<{RetBaseType}>.Zero) ? + Sve.ConditionalSelect(mask, {Isa}.{Method}(op1, op2), Vector<{RetBaseType}>.Zero) : + Sve.ConditionalSelect(mask, {Isa}.{Method}(op1, op2), falseOp); ValidateConditionalSelectResult(mask, op1, op2, falseOp, result); } diff --git a/src/tests/JIT/HardwareIntrinsics/Arm/Shared/_SveBinaryRetMaskOpTestTemplate.template b/src/tests/JIT/HardwareIntrinsics/Arm/Shared/_SveBinaryRetMaskOpTestTemplate.template index 7540f53e15f234..cc5bce2e5f0817 100644 --- a/src/tests/JIT/HardwareIntrinsics/Arm/Shared/_SveBinaryRetMaskOpTestTemplate.template +++ b/src/tests/JIT/HardwareIntrinsics/Arm/Shared/_SveBinaryRetMaskOpTestTemplate.template @@ -383,7 +383,9 @@ namespace {Namespace} [method: MethodImpl(MethodImplOptions.AggressiveInlining)] private void ConditionalSelectScenario_TrueValue({Op1VectorType}<{Op1BaseType}> mask, {Op1VectorType}<{Op1BaseType}> op1, {Op1VectorType}<{Op1BaseType}> op2, {Op1VectorType}<{Op1BaseType}> falseOp) { - var result = Sve.ConditionalSelect(mask, {Isa}.{Method}(op1, op2), falseOp); + var result = (falseOp == {Op1VectorType}<{Op1BaseType}>.Zero) ? + Sve.ConditionalSelect(mask, {Isa}.{Method}(op1, op2), {Op1VectorType}<{Op1BaseType}>.Zero) : + Sve.ConditionalSelect(mask, {Isa}.{Method}(op1, op2), falseOp); Unsafe.Write(_dataTable.outArrayPtr, result); ValidateConditionalSelectResult_TrueValue(mask, op1, op2, falseOp, _dataTable.outArrayPtr); diff --git a/src/tests/JIT/HardwareIntrinsics/Arm/Shared/_SveImm2TernOpTestTemplate.template b/src/tests/JIT/HardwareIntrinsics/Arm/Shared/_SveImm2TernOpTestTemplate.template index 416cadb3ee4c12..484f7f3b863d42 100644 --- a/src/tests/JIT/HardwareIntrinsics/Arm/Shared/_SveImm2TernOpTestTemplate.template +++ b/src/tests/JIT/HardwareIntrinsics/Arm/Shared/_SveImm2TernOpTestTemplate.template @@ -485,7 +485,9 @@ namespace {Namespace} [method: MethodImpl(MethodImplOptions.AggressiveInlining)] private void ConditionalSelectScenario_TrueValue({RetVectorType}<{RetBaseType}> mask, {Op1VectorType}<{Op1BaseType}> op1, {Op2VectorType}<{Op2BaseType}> op2, {Op3VectorType}<{Op3BaseType}> op3, {RetVectorType}<{RetBaseType}> falseOp) { - var result = Sve.ConditionalSelect(mask, {Isa}.{Method}(op1, op2, op3, {Imm1}, {Imm2}), falseOp); + var result = (falseOp == {RetVectorType}<{RetBaseType}>.Zero) ? + Sve.ConditionalSelect(mask, {Isa}.{Method}(op1, op2, op3, {Imm1}, {Imm2}), {RetVectorType}<{RetBaseType}>.Zero) : + Sve.ConditionalSelect(mask, {Isa}.{Method}(op1, op2, op3, {Imm1}, {Imm2}), falseOp); Unsafe.Write(_dataTable.outArrayPtr, result); ValidateConditionalSelectResult_TrueValue(mask, op1, op2, op3, falseOp, _dataTable.outArrayPtr); } diff --git a/src/tests/JIT/HardwareIntrinsics/Arm/Shared/_SveImmBinaryOpDifferentRetTypeTestTemplate.template b/src/tests/JIT/HardwareIntrinsics/Arm/Shared/_SveImmBinaryOpDifferentRetTypeTestTemplate.template index 70da8d237ef284..38bb9da70d01e5 100644 --- a/src/tests/JIT/HardwareIntrinsics/Arm/Shared/_SveImmBinaryOpDifferentRetTypeTestTemplate.template +++ b/src/tests/JIT/HardwareIntrinsics/Arm/Shared/_SveImmBinaryOpDifferentRetTypeTestTemplate.template @@ -407,7 +407,9 @@ namespace {Namespace} [method: MethodImpl(MethodImplOptions.AggressiveInlining)] private void ConditionalSelectScenario_TrueValue({RetVectorType}<{RetBaseType}> mask, {Op1VectorType}<{Op1BaseType}> op1, {Op2VectorType}<{Op2BaseType}> op2, {RetVectorType}<{RetBaseType}> falseOp) { - var result = Sve.ConditionalSelect(mask, {Isa}.{Method}(op1, op2, Imm), falseOp); + var result = (falseOp == {RetVectorType}<{RetBaseType}>.Zero) ? + Sve.ConditionalSelect(mask, {Isa}.{Method}(op1, op2, Imm), {RetVectorType}<{RetBaseType}>.Zero) : + Sve.ConditionalSelect(mask, {Isa}.{Method}(op1, op2, Imm), falseOp); Unsafe.Write(_dataTable.outArrayPtr, result); ValidateConditionalSelectResult_TrueValue(mask, op1, op2, falseOp, _dataTable.outArrayPtr); diff --git a/src/tests/JIT/HardwareIntrinsics/Arm/Shared/_SveImmBinaryOpTestTemplate.template b/src/tests/JIT/HardwareIntrinsics/Arm/Shared/_SveImmBinaryOpTestTemplate.template index bfe50ea72026ae..d72f1d4b1dc01b 100644 --- a/src/tests/JIT/HardwareIntrinsics/Arm/Shared/_SveImmBinaryOpTestTemplate.template +++ b/src/tests/JIT/HardwareIntrinsics/Arm/Shared/_SveImmBinaryOpTestTemplate.template @@ -403,7 +403,9 @@ namespace {Namespace} [method: MethodImpl(MethodImplOptions.AggressiveInlining)] private void ConditionalSelectScenario_TrueValue({RetVectorType}<{RetBaseType}> mask, {Op1VectorType}<{Op1BaseType}> op1, {Op2VectorType}<{Op2BaseType}> op2, {RetVectorType}<{RetBaseType}> falseOp) { - var result = Sve.ConditionalSelect(mask, {Isa}.{Method}(op1, op2, {Imm}), falseOp); + var result = (falseOp == {RetVectorType}<{RetBaseType}>.Zero) ? + Sve.ConditionalSelect(mask, {Isa}.{Method}(op1, op2, {Imm}), {RetVectorType}<{RetBaseType}>.Zero) : + Sve.ConditionalSelect(mask, {Isa}.{Method}(op1, op2, {Imm}), falseOp); Unsafe.Write(_dataTable.outArrayPtr, result); ValidateConditionalSelectResult_TrueValue(mask, op1, op2, falseOp, _dataTable.outArrayPtr); } diff --git a/src/tests/JIT/HardwareIntrinsics/Arm/Shared/_SveImmTernOpFirstArgTestTemplate.template b/src/tests/JIT/HardwareIntrinsics/Arm/Shared/_SveImmTernOpFirstArgTestTemplate.template index a31717a22880f3..4f1413ca8981c3 100644 --- a/src/tests/JIT/HardwareIntrinsics/Arm/Shared/_SveImmTernOpFirstArgTestTemplate.template +++ b/src/tests/JIT/HardwareIntrinsics/Arm/Shared/_SveImmTernOpFirstArgTestTemplate.template @@ -371,7 +371,9 @@ namespace {Namespace} [method: MethodImpl(MethodImplOptions.AggressiveInlining)] private void ConditionalSelectScenario_TrueValue({Op1VectorType}<{Op1BaseType}> mask, {Op1VectorType}<{Op1BaseType}> op1, {Op1VectorType}<{Op2BaseType}> op2, {Op1VectorType}<{Op3BaseType}> op3, {Op1VectorType}<{Op1BaseType}> falseOp) { - var result = Sve.ConditionalSelect(mask, {Isa}.{Method}(op1, op2, op3, Imm), falseOp); + var result = (falseOp == {Op1VectorType}<{Op1BaseType}>.Zero) ? + Sve.ConditionalSelect(mask, {Isa}.{Method}(op1, op2, op3, Imm), {Op1VectorType}<{Op1BaseType}>.Zero) : + Sve.ConditionalSelect(mask, {Isa}.{Method}(op1, op2, op3, Imm), falseOp); Unsafe.Write(_dataTable.outArrayPtr, result); ValidateConditionalSelectResult_TrueValue(mask, op1, op2, op3, falseOp, _dataTable.outArrayPtr); } diff --git a/src/tests/JIT/HardwareIntrinsics/Arm/Shared/_SveImmTernOpTestTemplate.template b/src/tests/JIT/HardwareIntrinsics/Arm/Shared/_SveImmTernOpTestTemplate.template index 7412ef208727c0..6c3c81f5bdd050 100644 --- a/src/tests/JIT/HardwareIntrinsics/Arm/Shared/_SveImmTernOpTestTemplate.template +++ b/src/tests/JIT/HardwareIntrinsics/Arm/Shared/_SveImmTernOpTestTemplate.template @@ -451,7 +451,9 @@ namespace {Namespace} [method: MethodImpl(MethodImplOptions.AggressiveInlining)] private void ConditionalSelectScenario_TrueValue({Op1VectorType}<{RetBaseType}> mask, {Op1VectorType}<{Op1BaseType}> op1, {Op1VectorType}<{Op2BaseType}> op2, {Op1VectorType}<{Op3BaseType}> op3, {Op1VectorType}<{RetBaseType}> falseOp) { - var result = Sve.ConditionalSelect(mask, {Isa}.{Method}(op1, op2, op3, Imm), falseOp); + var result = (falseOp == {Op1VectorType}<{RetBaseType}>.Zero) ? + Sve.ConditionalSelect(mask, {Isa}.{Method}(op1, op2, op3, Imm), {Op1VectorType}<{RetBaseType}>.Zero) : + Sve.ConditionalSelect(mask, {Isa}.{Method}(op1, op2, op3, Imm), falseOp); Unsafe.Write(_dataTable.outArrayPtr, result); ValidateConditionalSelectResult_TrueValue(mask, op1, op2, op3, falseOp, _dataTable.outArrayPtr); } diff --git a/src/tests/JIT/HardwareIntrinsics/Arm/Shared/_SveImmUnaryOpTestTemplate.template b/src/tests/JIT/HardwareIntrinsics/Arm/Shared/_SveImmUnaryOpTestTemplate.template index 16ec9a0cacb964..1b8209b85e0b53 100644 --- a/src/tests/JIT/HardwareIntrinsics/Arm/Shared/_SveImmUnaryOpTestTemplate.template +++ b/src/tests/JIT/HardwareIntrinsics/Arm/Shared/_SveImmUnaryOpTestTemplate.template @@ -352,7 +352,9 @@ namespace {Namespace} [method: MethodImpl(MethodImplOptions.AggressiveInlining)] private void ConditionalSelectScenario_TrueValue({RetVectorType}<{RetBaseType}> mask, {Op1VectorType}<{Op1BaseType}> op, {RetVectorType}<{RetBaseType}> falseOp) { - var result = Sve.ConditionalSelect(mask, {Isa}.{Method}(op, {Imm}), falseOp); + var result = (falseOp == {RetVectorType}<{RetBaseType}>.Zero) ? + Sve.ConditionalSelect(mask, {Isa}.{Method}(op, {Imm}), {RetVectorType}<{RetBaseType}>.Zero) : + Sve.ConditionalSelect(mask, {Isa}.{Method}(op, {Imm}), falseOp); Unsafe.Write(_dataTable.outArrayPtr, result); ValidateConditionalSelectResult_TrueValue(mask, op, falseOp, Imm, _dataTable.outArrayPtr); } diff --git a/src/tests/JIT/HardwareIntrinsics/Arm/Shared/_SveMinimalUnaryOpTestTemplate.template b/src/tests/JIT/HardwareIntrinsics/Arm/Shared/_SveMinimalUnaryOpTestTemplate.template index 365569f1b55bac..c39d6cb4895c8e 100644 --- a/src/tests/JIT/HardwareIntrinsics/Arm/Shared/_SveMinimalUnaryOpTestTemplate.template +++ b/src/tests/JIT/HardwareIntrinsics/Arm/Shared/_SveMinimalUnaryOpTestTemplate.template @@ -323,7 +323,9 @@ namespace {Namespace} [method: MethodImpl(MethodImplOptions.AggressiveInlining)] private void ConditionalSelectScenario_TrueValue({RetVectorType}<{RetBaseType}> mask, {Op1VectorType}<{Op1BaseType}> op1, {RetVectorType}<{RetBaseType}> falseOp) { - var result = Sve.ConditionalSelect(mask, {Isa}.{Method}(op1), falseOp); + var result = (falseOp == {RetVectorType}<{RetBaseType}>.Zero) ? + Sve.ConditionalSelect(mask, {Isa}.{Method}(op1), {RetVectorType}<{RetBaseType}>.Zero) : + Sve.ConditionalSelect(mask, {Isa}.{Method}(op1), falseOp); Unsafe.Write(_dataTable.outArrayPtr, result); ValidateConditionalSelectResult_TrueValue(mask, op1, falseOp, _dataTable.outArrayPtr); diff --git a/src/tests/JIT/HardwareIntrinsics/Arm/Shared/_SveTernOpFirstArgTestTemplate.template b/src/tests/JIT/HardwareIntrinsics/Arm/Shared/_SveTernOpFirstArgTestTemplate.template index 4efab35b8fa3de..4979fec95cab3f 100644 --- a/src/tests/JIT/HardwareIntrinsics/Arm/Shared/_SveTernOpFirstArgTestTemplate.template +++ b/src/tests/JIT/HardwareIntrinsics/Arm/Shared/_SveTernOpFirstArgTestTemplate.template @@ -365,7 +365,9 @@ namespace {Namespace} [method: MethodImpl(MethodImplOptions.AggressiveInlining)] private void ConditionalSelectScenario_TrueValue({Op1VectorType}<{Op1BaseType}> mask, {Op1VectorType}<{Op1BaseType}> op1, {Op1VectorType}<{Op2BaseType}> op2, {Op1VectorType}<{Op3BaseType}> op3, {Op1VectorType}<{Op1BaseType}> falseOp) { - var result = Sve.ConditionalSelect(mask, {Isa}.{Method}(op1, op2, op3), falseOp); + var result = (falseOp == {Op1VectorType}<{Op1BaseType}>.Zero) ? + Sve.ConditionalSelect(mask, {Isa}.{Method}(op1, op2, op3), {Op1VectorType}<{Op1BaseType}>.Zero) : + Sve.ConditionalSelect(mask, {Isa}.{Method}(op1, op2, op3), falseOp); Unsafe.Write(_dataTable.outArrayPtr, result); ValidateConditionalSelectResult_TrueValue(mask, op1, op2, op3, falseOp, _dataTable.outArrayPtr); diff --git a/src/tests/JIT/HardwareIntrinsics/Arm/Shared/_SveTernOpMaskedOpTestTemplate.template b/src/tests/JIT/HardwareIntrinsics/Arm/Shared/_SveTernOpMaskedOpTestTemplate.template index b10d3c4fd8d821..f0afaf8109550b 100644 --- a/src/tests/JIT/HardwareIntrinsics/Arm/Shared/_SveTernOpMaskedOpTestTemplate.template +++ b/src/tests/JIT/HardwareIntrinsics/Arm/Shared/_SveTernOpMaskedOpTestTemplate.template @@ -245,7 +245,9 @@ namespace {Namespace}._{Isa} [method: MethodImpl(MethodImplOptions.AggressiveInlining)] private void ConditionalSelectScenario_TrueValue(Vector<{Op1BaseType}> mask, Vector<{Op1BaseType}> op1, Vector<{Op2BaseType}> op2, Vector<{RetBaseType}> falseOp) { - var result = Sve.ConditionalSelect((Vector<{RetBaseType}>)mask, {Isa}.{Method}(mask, op1, op2), falseOp); + var result = (falseOp == Vector<{RetBaseType}>.Zero) ? + Sve.ConditionalSelect((Vector<{RetBaseType}>)mask, {Isa}.{Method}(mask, op1, op2), Vector<{RetBaseType}>.Zero) : + Sve.ConditionalSelect((Vector<{RetBaseType}>)mask, {Isa}.{Method}(mask, op1, op2), falseOp); Unsafe.Write(_dataTable.outArrayPtr, result); ValidateConditionalSelectResult_TrueValue(mask, op1, op2, falseOp, _dataTable.outArrayPtr); diff --git a/src/tests/JIT/HardwareIntrinsics/Arm/Shared/_SveTernOpTestTemplate.template b/src/tests/JIT/HardwareIntrinsics/Arm/Shared/_SveTernOpTestTemplate.template index 79a1f69f892682..e7a6ea1230989e 100644 --- a/src/tests/JIT/HardwareIntrinsics/Arm/Shared/_SveTernOpTestTemplate.template +++ b/src/tests/JIT/HardwareIntrinsics/Arm/Shared/_SveTernOpTestTemplate.template @@ -423,7 +423,9 @@ namespace {Namespace} [method: MethodImpl(MethodImplOptions.AggressiveInlining)] private void ConditionalSelectScenario_TrueValue({Op1VectorType}<{RetBaseType}> mask, {Op1VectorType}<{Op1BaseType}> op1, {Op1VectorType}<{Op2BaseType}> op2, {Op1VectorType}<{Op3BaseType}> op3, {Op1VectorType}<{RetBaseType}> falseOp) { - var result = Sve.ConditionalSelect(mask, {Isa}.{Method}(op1, op2, op3), falseOp); + var result = (falseOp == {Op1VectorType}<{RetBaseType}>.Zero) ? + Sve.ConditionalSelect(mask, {Isa}.{Method}(op1, op2, op3), {Op1VectorType}<{RetBaseType}>.Zero) : + Sve.ConditionalSelect(mask, {Isa}.{Method}(op1, op2, op3), falseOp); Unsafe.Write(_dataTable.outArrayPtr, result); ValidateConditionalSelectResult_TrueValue(mask, op1, op2, op3, falseOp, _dataTable.outArrayPtr); diff --git a/src/tests/JIT/HardwareIntrinsics/Arm/Shared/_SveUnaryOpDifferentRetTypeTestTemplate.template b/src/tests/JIT/HardwareIntrinsics/Arm/Shared/_SveUnaryOpDifferentRetTypeTestTemplate.template index e616becbbef0bf..3bd00354ddd4a8 100644 --- a/src/tests/JIT/HardwareIntrinsics/Arm/Shared/_SveUnaryOpDifferentRetTypeTestTemplate.template +++ b/src/tests/JIT/HardwareIntrinsics/Arm/Shared/_SveUnaryOpDifferentRetTypeTestTemplate.template @@ -301,7 +301,9 @@ namespace {Namespace} [method: MethodImpl(MethodImplOptions.AggressiveInlining)] private void ConditionalSelectScenario_TrueValue({Op1VectorType}<{RetBaseType}> mask, {Op1VectorType}<{Op1BaseType}> op1, {Op1VectorType}<{RetBaseType}> falseOp) { - var result = Sve.ConditionalSelect(mask, {Isa}.{Method}(op1), falseOp); + var result = (falseOp == {Op1VectorType}<{RetBaseType}>.Zero) ? + Sve.ConditionalSelect(mask, {Isa}.{Method}(op1), {Op1VectorType}<{RetBaseType}>.Zero) : + Sve.ConditionalSelect(mask, {Isa}.{Method}(op1), falseOp); Unsafe.Write(_dataTable.outArrayPtr, result); ValidateConditionalSelectResult_TrueValue(mask, op1, falseOp, _dataTable.outArrayPtr); diff --git a/src/tests/JIT/HardwareIntrinsics/Arm/Shared/_SveUnaryOpTestTemplate.template b/src/tests/JIT/HardwareIntrinsics/Arm/Shared/_SveUnaryOpTestTemplate.template index 38c53561fce72c..aa0f3122b5235d 100644 --- a/src/tests/JIT/HardwareIntrinsics/Arm/Shared/_SveUnaryOpTestTemplate.template +++ b/src/tests/JIT/HardwareIntrinsics/Arm/Shared/_SveUnaryOpTestTemplate.template @@ -251,7 +251,9 @@ namespace {Namespace} [method: MethodImpl(MethodImplOptions.AggressiveInlining)] private void ConditionalSelectScenario_TrueValue(Vector<{Op1BaseType}> mask, Vector<{Op1BaseType}> op1, Vector<{Op1BaseType}> falseOp) { - var result = Sve.ConditionalSelect(mask, {Isa}.{Method}(op1), falseOp); + var result = (falseOp == Vector<{Op1BaseType}>.Zero) ? + Sve.ConditionalSelect(mask, {Isa}.{Method}(op1), Vector<{Op1BaseType}>.Zero) : + Sve.ConditionalSelect(mask, {Isa}.{Method}(op1), falseOp); ValidateConditionalSelectResult(mask, op1, falseOp, result); } diff --git a/src/tests/JIT/opt/SVE/ConstantMasks.cs b/src/tests/JIT/opt/SVE/ConstantMasks.cs index 078e60e9b55411..c6c52fc1f7e491 100644 --- a/src/tests/JIT/opt/SVE/ConstantMasks.cs +++ b/src/tests/JIT/opt/SVE/ConstantMasks.cs @@ -82,7 +82,6 @@ static void CndSelectEmbeddedZero(Vector op1, Vector op2) { [MethodImpl(MethodImplOptions.NoInlining)] static void CndSelectEmbeddedTrueMask(Vector op1, Vector op2) { //ARM64-FULL-LINE: ptrue {{p[0-9]+}}.s - //ARM64-FULL-LINE-NEXT: movprfx {{z[0-9]+}}.s, {{p[0-9]+}}/z, {{z[0-9]+}}.s //ARM64-FULL-LINE-NEXT: sabd {{z[0-9]+}}.s, {{p[0-9]+}}/m, {{z[0-9]+}}.s, {{z[0-9]+}}.s //ARM64-FULL-LINE-NEXT: movz {{.*}} Vector result = Sve.ConditionalSelect(Sve.CreateTrueMaskInt32(), Sve.AbsoluteDifference(op1, op2), op1); @@ -92,7 +91,6 @@ static void CndSelectEmbeddedTrueMask(Vector op1, Vector op2) { [MethodImpl(MethodImplOptions.NoInlining)] static void CndSelectEmbeddedAllBits(Vector op1, Vector op2) { //ARM64-FULL-LINE: ptrue {{p[0-9]+}}.s - //ARM64-FULL-LINE-NEXT: movprfx {{z[0-9]+}}.s, {{p[0-9]+}}/z, {{z[0-9]+}}.s //ARM64-FULL-LINE-NEXT: sabd {{z[0-9]+}}.s, {{p[0-9]+}}/m, {{z[0-9]+}}.s, {{z[0-9]+}}.s //ARM64-FULL-LINE-NEXT: movz {{.*}} Vector result = Sve.ConditionalSelect(Vector.AllBitsSet, Sve.AbsoluteDifference(op1, op2), op1); diff --git a/src/tests/JIT/opt/SVE/ConstantMasksOp2Fixed.cs b/src/tests/JIT/opt/SVE/ConstantMasksOp2Fixed.cs index ba23ebe08f07c9..ba4df0e245d095 100644 --- a/src/tests/JIT/opt/SVE/ConstantMasksOp2Fixed.cs +++ b/src/tests/JIT/opt/SVE/ConstantMasksOp2Fixed.cs @@ -119,7 +119,6 @@ static void CndSelectEmbeddedTrueMaskF(Vector op1, Vector op2) { [MethodImpl(MethodImplOptions.NoInlining)] static void CndSelectEmbeddedTrueMaskZ(Vector op1, Vector op2) { //ARM6-FULL-LINE: ptrue {{p[0-9]+}}.s - //ARM6-FULL-LINE-NEXT: movprfx {{z[0-9]+}}.s, {{p[0-9]+}}/m, {{z[0-9]+}}.s //ARM6-FULL-LINE-NEXT: sabd {{z[0-9]+}}.s, {{p[0-9]+}}/m, {{z[0-9]+}}.s, {{z[0-9]+}}.s var result = Sve.ConditionalSelect(Sve.CreateTrueMaskInt32(), Sve.AbsoluteDifference(op1, op2), Vector.Zero); Consume(result);