diff --git a/src/coreclr/jit/compiler.cpp b/src/coreclr/jit/compiler.cpp index aa1daa1ccfca19..aefbe6653a51a3 100644 --- a/src/coreclr/jit/compiler.cpp +++ b/src/coreclr/jit/compiler.cpp @@ -3734,6 +3734,29 @@ bool Compiler::compPromoteFewerStructs(unsigned lclNum) return rejectThisPromo; } +//------------------------------------------------------------------------ +// dumpRegMask: display a register mask. For well-known sets of registers, display a well-known token instead of +// a potentially large number of registers. +// +// Arguments: +// regs - The set of registers to display +// type - The type of `regs` +// +void Compiler::dumpRegMask(SingleTypeRegSet regs, var_types type) const +{ +#ifdef FEATURE_MASKED_HW_INTRINSICS + if (varTypeIsMask(type)) + { + dumpRegMask(regMaskTP(RBM_NONE, regs)); + } + else +#endif + { + assert(varTypeUsesIntReg(type) || varTypeUsesFloatReg(type)); + dumpRegMask(regMaskTP(regs, RBM_NONE)); + } +} + //------------------------------------------------------------------------ // dumpRegMask: display a register mask. For well-known sets of registers, display a well-known token instead of // a potentially large number of registers. diff --git a/src/coreclr/jit/compiler.h b/src/coreclr/jit/compiler.h index 7e105eb8a00ea1..4504de02a4adfd 100644 --- a/src/coreclr/jit/compiler.h +++ b/src/coreclr/jit/compiler.h @@ -11112,6 +11112,7 @@ class Compiler bool compJitHaltMethod(); void dumpRegMask(regMaskTP regs) const; + void dumpRegMask(SingleTypeRegSet regs, var_types type) const; #endif diff --git a/src/coreclr/jit/compiler.hpp b/src/coreclr/jit/compiler.hpp index 4c5910d53dacab..e9714586bb10c8 100644 --- a/src/coreclr/jit/compiler.hpp +++ b/src/coreclr/jit/compiler.hpp @@ -1031,11 +1031,12 @@ inline regNumber genFirstRegNumFromMask(const regMaskTP& mask) // // Arguments: // mask - the register mask +// type - type of the register mask // // Return Value: // The number of the first register contained in the mask. // -inline regNumber genFirstRegNumFromMask(SingleTypeRegSet mask) +inline regNumber genFirstRegNumFromMask(SingleTypeRegSet mask, var_types type) { assert(mask != RBM_NONE); // Must have one bit set, so can't have a mask of zero @@ -1043,6 +1044,13 @@ inline regNumber genFirstRegNumFromMask(SingleTypeRegSet mask) regNumber regNum = (regNumber)BitOperations::BitScanForward(mask); +#ifdef HAS_MORE_THAN_64_REGISTERS + if (varTypeIsMask(type)) + { + regNum = (regNumber)(64 + regNum); + } +#endif + return regNum; } @@ -1075,13 +1083,14 @@ inline regNumber genFirstRegNumFromMaskAndToggle(regMaskTP& mask) // register number and also toggle the bit in the `mask`. // Arguments: // mask - the register mask +// type - type of the register mask // // Return Value: // The number of the first register contained in the mask and updates the `mask` to toggle // the bit. // -inline regNumber genFirstRegNumFromMaskAndToggle(SingleTypeRegSet& mask) +inline regNumber genFirstRegNumFromMaskAndToggle(SingleTypeRegSet& mask, var_types type) { assert(mask != RBM_NONE); // Must have one bit set, so can't have a mask of zero @@ -1091,6 +1100,13 @@ inline regNumber genFirstRegNumFromMaskAndToggle(SingleTypeRegSet& mask) mask ^= genSingleTypeRegMask(regNum); +#ifdef HAS_MORE_THAN_64_REGISTERS + if (varTypeIsMask(type)) + { + regNum = (regNumber)(64 + regNum); + } +#endif + return regNum; } diff --git a/src/coreclr/jit/lsra.cpp b/src/coreclr/jit/lsra.cpp index 96d75ba3d1ceac..eb9817b560e970 100644 --- a/src/coreclr/jit/lsra.cpp +++ b/src/coreclr/jit/lsra.cpp @@ -178,12 +178,22 @@ void lsraAssignRegToTree(GenTree* tree, regNumber reg, unsigned regIdx) // // Returns: // Weight of ref position. -weight_t LinearScan::getWeight(RefPosition* refPos) +weight_t LinearScan::getWeight(RefPosition* refPos DEBUG_ARG(bool forDump)) { weight_t weight; GenTree* treeNode = refPos->treeNode; - if (treeNode != nullptr) +#ifdef HAS_MORE_THAN_64_REGISTERS + // If refType is `RefTypeKill`, we are using the killRegisterAssignment + // and treeNode field is garbage. + assert(forDump || refPos->refType != RefTypeKill); +#endif + + if (treeNode != nullptr +#ifdef DEBUG + && (refPos->refType != RefTypeKill) +#endif + ) { if (isCandidateLocalRef(treeNode)) { @@ -231,6 +241,21 @@ weight_t LinearScan::getWeight(RefPosition* refPos) return weight; } +#ifdef DEBUG +//------------------------------------------------------------- +// getWeightForDump: Returns the weight of the RefPosition, for dump +// +// Arguments: +// refPos - ref position +// +// Returns: +// Weight of ref position. +weight_t LinearScan::getWeightForDump(RefPosition* refPos) +{ + return getWeight(refPos, true); +} +#endif + // allRegs represents a set of registers that can // be used to allocate the specified type in any point // in time (more of a 'bank' of registers). @@ -312,7 +337,7 @@ SingleTypeRegSet LinearScan::getMatchingConstants(SingleTypeRegSet mask, SingleTypeRegSet result = RBM_NONE; while (candidates != RBM_NONE) { - regNumber regNum = genFirstRegNumFromMask(candidates); + regNumber regNum = genFirstRegNumFromMask(candidates, currentInterval->registerType); SingleTypeRegSet candidateBit = genSingleTypeRegMask(regNum); candidates ^= candidateBit; @@ -3080,7 +3105,7 @@ regNumber LinearScan::allocateReg(Interval* currentInterval, return REG_NA; } - regNumber foundReg = genRegNumFromMask(foundRegBit); + regNumber foundReg = genRegNumFromMask(foundRegBit, currentInterval->registerType); RegRecord* availablePhysRegRecord = getRegisterRecord(foundReg); Interval* assignedInterval = availablePhysRegRecord->assignedInterval; if ((assignedInterval != currentInterval) && @@ -4087,7 +4112,7 @@ void LinearScan::spillGCRefs(RefPosition* killRefPosition) INDEBUG(bool killedRegs = false); while (candidateRegs != RBM_NONE) { - regNumber nextReg = genFirstRegNumFromMaskAndToggle(candidateRegs); + regNumber nextReg = genFirstRegNumFromMaskAndToggle(candidateRegs, IntRegisterType); RegRecord* regRecord = getRegisterRecord(nextReg); Interval* assignedInterval = regRecord->assignedInterval; @@ -4182,7 +4207,7 @@ regNumber LinearScan::rotateBlockStartLocation(Interval* interval, regNumber tar regNumber newReg = REG_NA; while (candidateRegs != RBM_NONE) { - regNumber nextReg = genFirstRegNumFromMaskAndToggle(candidateRegs); + regNumber nextReg = genFirstRegNumFromMaskAndToggle(candidateRegs, interval->registerType); if (nextReg > targetReg) { newReg = nextReg; @@ -9772,8 +9797,7 @@ void LinearScan::resolveEdge(BasicBlock* fromBlock, regMaskTP targetCandidates = targetRegsToDo; while (targetCandidates.IsNonEmpty()) { - regNumber targetReg = genFirstRegNumFromMask(targetCandidates); - targetCandidates.RemoveRegNumFromMask(targetReg); + regNumber targetReg = genFirstRegNumFromMaskAndToggle(targetCandidates); if (location[targetReg] == REG_NA) { #ifdef TARGET_ARM @@ -9802,9 +9826,8 @@ void LinearScan::resolveEdge(BasicBlock* fromBlock, { while (targetRegsReady.IsNonEmpty()) { - regNumber targetReg = genFirstRegNumFromMask(targetRegsReady); + regNumber targetReg = genFirstRegNumFromMaskAndToggle(targetRegsReady); targetRegsToDo.RemoveRegNumFromMask(targetReg); - targetRegsReady.RemoveRegNumFromMask(targetReg); assert(location[targetReg] != targetReg); assert(targetReg < REG_COUNT); regNumber sourceReg = (regNumber)source[targetReg]; @@ -10397,7 +10420,7 @@ void RefPosition::dump(LinearScan* linearScan) { this->getInterval()->tinyDump(); } - if (this->treeNode) + if ((refType != RefTypeKill) && this->treeNode) { printf("%s", treeNode->OpName(treeNode->OperGet())); if (this->treeNode->IsMultiRegNode()) @@ -10409,7 +10432,26 @@ void RefPosition::dump(LinearScan* linearScan) printf(FMT_BB " ", this->bbNum); printf("regmask="); - linearScan->compiler->dumpRegMask(registerAssignment); +#ifdef HAS_MORE_THAN_64_REGISTERS + if (refType == RefTypeKill) + { + linearScan->compiler->dumpRegMask(getKillRegisterAssignment()); + } + else +#endif // HAS_MORE_THAN_64_REGISTERS + { + var_types type = TYP_UNKNOWN; + if ((refType == RefTypeBB) || (refType == RefTypeKillGCRefs)) + { + // These refTypes do not have intervals + type = TYP_INT; + } + else + { + type = getRegisterType(); + } + linearScan->compiler->dumpRegMask(registerAssignment, type); + } printf(" minReg=%d", minRegCandidateCount); @@ -10463,7 +10505,7 @@ void RefPosition::dump(LinearScan* linearScan) printf(" regOptional"); } - printf(" wt=%.2f", linearScan->getWeight(this)); + printf(" wt=%.2f", linearScan->getWeightForDump(this)); printf(">\n"); } @@ -10545,10 +10587,10 @@ void Interval::dump(Compiler* compiler) printf(" physReg:%s", getRegName(physReg)); printf(" Preferences="); - compiler->dumpRegMask(this->registerPreferences); + compiler->dumpRegMask(this->registerPreferences, this->registerType); printf(" Aversions="); - compiler->dumpRegMask(this->registerAversion); + compiler->dumpRegMask(this->registerAversion, this->registerType); if (relatedInterval) { printf(" RelatedInterval "); @@ -11066,7 +11108,12 @@ void LinearScan::TupleStyleDump(LsraTupleDumpMode mode) printf("\n Kill: "); killPrinted = true; } - compiler->dumpRegMask(currentRefPosition->registerAssignment); +#ifdef HAS_MORE_THAN_64_REGISTERS + compiler->dumpRegMask(currentRefPosition->getKillRegisterAssignment()); +#else + compiler->dumpRegMask(currentRefPosition->registerAssignment, + currentRefPosition->getRegisterType()); +#endif printf(" "); break; case RefTypeFixedReg: @@ -12228,7 +12275,7 @@ void LinearScan::verifyFinalAllocation() SingleTypeRegSet candidateRegs = currentRefPosition.registerAssignment; while (candidateRegs != RBM_NONE) { - regNumber nextReg = genFirstRegNumFromMaskAndToggle(candidateRegs); + regNumber nextReg = genFirstRegNumFromMaskAndToggle(candidateRegs, IntRegisterType); RegRecord* regRecord = getRegisterRecord(nextReg); Interval* assignedInterval = regRecord->assignedInterval; @@ -12872,7 +12919,7 @@ void LinearScan::RegisterSelection::try_BEST_FIT() LsraLocation bestFitLocation = earliestIsBest ? MaxLocation : MinLocation; for (SingleTypeRegSet bestFitCandidates = candidates; bestFitCandidates != RBM_NONE;) { - regNumber bestFitCandidateRegNum = genFirstRegNumFromMask(bestFitCandidates); + regNumber bestFitCandidateRegNum = genFirstRegNumFromMask(bestFitCandidates, regType); SingleTypeRegSet bestFitCandidateBit = genSingleTypeRegMask(bestFitCandidateRegNum); bestFitCandidates ^= bestFitCandidateBit; @@ -12971,7 +13018,7 @@ void LinearScan::RegisterSelection::try_REG_ORDER() SingleTypeRegSet lowestRegOrderBit = RBM_NONE; for (SingleTypeRegSet regOrderCandidates = candidates; regOrderCandidates != RBM_NONE;) { - regNumber regOrderCandidateRegNum = genFirstRegNumFromMask(regOrderCandidates); + regNumber regOrderCandidateRegNum = genFirstRegNumFromMask(regOrderCandidates, regType); SingleTypeRegSet regOrderCandidateBit = genSingleTypeRegMask(regOrderCandidateRegNum); regOrderCandidates ^= regOrderCandidateBit; @@ -13007,7 +13054,7 @@ void LinearScan::RegisterSelection::try_SPILL_COST() for (SingleTypeRegSet spillCandidates = candidates; spillCandidates != RBM_NONE;) { - regNumber spillCandidateRegNum = genFirstRegNumFromMask(spillCandidates); + regNumber spillCandidateRegNum = genFirstRegNumFromMask(spillCandidates, regType); SingleTypeRegSet spillCandidateBit = genSingleTypeRegMask(spillCandidateRegNum); spillCandidates ^= spillCandidateBit; @@ -13132,7 +13179,7 @@ void LinearScan::RegisterSelection::try_FAR_NEXT_REF() SingleTypeRegSet farthestSet = RBM_NONE; for (SingleTypeRegSet farthestCandidates = candidates; farthestCandidates != RBM_NONE;) { - regNumber farthestCandidateRegNum = genFirstRegNumFromMask(farthestCandidates); + regNumber farthestCandidateRegNum = genFirstRegNumFromMask(farthestCandidates, regType); SingleTypeRegSet farthestCandidateBit = genSingleTypeRegMask(farthestCandidateRegNum); farthestCandidates ^= farthestCandidateBit; @@ -13165,7 +13212,7 @@ void LinearScan::RegisterSelection::try_PREV_REG_OPT() SingleTypeRegSet prevRegOptSet = RBM_NONE; for (SingleTypeRegSet prevRegOptCandidates = candidates; prevRegOptCandidates != RBM_NONE;) { - regNumber prevRegOptCandidateRegNum = genFirstRegNumFromMask(prevRegOptCandidates); + regNumber prevRegOptCandidateRegNum = genFirstRegNumFromMask(prevRegOptCandidates, regType); SingleTypeRegSet prevRegOptCandidateBit = genSingleTypeRegMask(prevRegOptCandidateRegNum); prevRegOptCandidates ^= prevRegOptCandidateBit; Interval* assignedInterval = linearScan->physRegs[prevRegOptCandidateRegNum].assignedInterval; @@ -13268,7 +13315,7 @@ void LinearScan::RegisterSelection::calculateUnassignedSets() // TODO: Seperate SingleTypeRegSet coversCandidates = candidates; for (; coversCandidates != RBM_NONE;) { - regNumber coversCandidateRegNum = genFirstRegNumFromMask(coversCandidates); + regNumber coversCandidateRegNum = genFirstRegNumFromMask(coversCandidates, regType); SingleTypeRegSet coversCandidateBit = genSingleTypeRegMask(coversCandidateRegNum); coversCandidates ^= coversCandidateBit; @@ -13296,7 +13343,7 @@ void LinearScan::RegisterSelection::calculateCoversSets() SingleTypeRegSet coversCandidates = (preferenceSet == RBM_NONE) ? candidates : preferenceSet; for (; coversCandidates != RBM_NONE;) { - regNumber coversCandidateRegNum = genFirstRegNumFromMask(coversCandidates); + regNumber coversCandidateRegNum = genFirstRegNumFromMask(coversCandidates, regType); SingleTypeRegSet coversCandidateBit = genSingleTypeRegMask(coversCandidateRegNum); coversCandidates ^= coversCandidateBit; @@ -13607,7 +13654,7 @@ SingleTypeRegSet LinearScan::RegisterSelection::select(Interval* SingleTypeRegSet checkConflictMask = candidates & linearScan->fixedRegs.GetRegSetForType(regType); while (checkConflictMask != RBM_NONE) { - regNumber checkConflictReg = genFirstRegNumFromMask(checkConflictMask); + regNumber checkConflictReg = genFirstRegNumFromMask(checkConflictMask, regType); SingleTypeRegSet checkConflictBit = genSingleTypeRegMask(checkConflictReg); checkConflictMask ^= checkConflictBit; @@ -13926,7 +13973,7 @@ SingleTypeRegSet LinearScan::RegisterSelection::selectMinimal( SingleTypeRegSet checkConflictMask = candidates & linearScan->fixedRegs.GetRegSetForType(regType); while (checkConflictMask != RBM_NONE) { - regNumber checkConflictReg = genFirstRegNumFromMask(checkConflictMask); + regNumber checkConflictReg = genFirstRegNumFromMask(checkConflictMask, regType); SingleTypeRegSet checkConflictBit = genSingleTypeRegMask(checkConflictReg); checkConflictMask ^= checkConflictBit; diff --git a/src/coreclr/jit/lsra.h b/src/coreclr/jit/lsra.h index d0a34f6ddcb53b..5d49673ffc4bee 100644 --- a/src/coreclr/jit/lsra.h +++ b/src/coreclr/jit/lsra.h @@ -1185,7 +1185,10 @@ class LinearScan : public LinearScanInterface void associateRefPosWithInterval(RefPosition* rp); - weight_t getWeight(RefPosition* refPos); + weight_t getWeight(RefPosition* refPos DEBUG_ARG(bool forDump = false)); +#ifdef DEBUG + weight_t getWeightForDump(RefPosition* refPos); +#endif // DEBUG /***************************************************************************** * Register management diff --git a/src/coreclr/jit/lsrabuild.cpp b/src/coreclr/jit/lsrabuild.cpp index d711922718d81a..33f9b804bc2cc5 100644 --- a/src/coreclr/jit/lsrabuild.cpp +++ b/src/coreclr/jit/lsrabuild.cpp @@ -647,7 +647,18 @@ RefPosition* LinearScan::newRefPosition(Interval* theInterval, assert(theInterval != nullptr); theInterval->isSingleDef = theInterval->firstRefPosition == newRP; } - +#ifdef DEBUG +#ifdef HAS_MORE_THAN_64_REGISTERS + // Need to do this here do the dump can print the mask correctly. + // Doing in DEBUG so we do not incur of cost of this check for + // every RefPosition. We will set this anyway in addKillForRegs() + // in RELEASE. + if (theRefType == RefTypeKill) + { + newRP->killRegisterAssignment = mask; + } +#endif // HAS_MORE_THAN_64_REGISTERS +#endif DBEXEC(VERBOSE, newRP->dump(this)); return newRP; } diff --git a/src/coreclr/jit/utils.cpp b/src/coreclr/jit/utils.cpp index badc448928ed7b..05abfbf774ccd0 100644 --- a/src/coreclr/jit/utils.cpp +++ b/src/coreclr/jit/utils.cpp @@ -284,14 +284,14 @@ const char* getRegNameFloat(regNumber reg, var_types type) */ const char* dspRegRange(regMaskTP regMask, size_t& minSiz, const char* sep, regNumber regFirst, regNumber regLast) { -#ifdef TARGET_XARCH +#ifdef FEATURE_MASKED_HW_INTRINSICS assert(((regFirst == REG_INT_FIRST) && (regLast == REG_INT_LAST)) || ((regFirst == REG_FP_FIRST) && (regLast == REG_FP_LAST)) || ((regFirst == REG_MASK_FIRST) && (regLast == REG_MASK_LAST))); #else assert(((regFirst == REG_INT_FIRST) && (regLast == REG_INT_LAST)) || ((regFirst == REG_FP_FIRST) && (regLast == REG_FP_LAST))); -#endif +#endif // FEATURE_MASKED_HW_INTRINSICS if (strlen(sep) > 0) { @@ -436,10 +436,9 @@ void dspRegMask(regMaskTP regMask, size_t minSiz) sep = dspRegRange(regMask, minSiz, sep, REG_INT_FIRST, REG_INT_LAST); sep = dspRegRange(regMask, minSiz, sep, REG_FP_FIRST, REG_FP_LAST); - -#ifdef TARGET_XARCH +#ifdef FEATURE_MASKED_HW_INTRINSICS sep = dspRegRange(regMask, minSiz, sep, REG_MASK_FIRST, REG_MASK_LAST); -#endif // TARGET_XARCH +#endif // FEATURE_MASKED_HW_INTRINSICS printf("]");