diff --git a/src/coreclr/inc/corinfo.h b/src/coreclr/inc/corinfo.h index 9427339ee8d88d..4dfa9d39bc10c7 100644 --- a/src/coreclr/inc/corinfo.h +++ b/src/coreclr/inc/corinfo.h @@ -858,6 +858,7 @@ enum class CorInfoReloc ARM64_BRANCH26, // Arm64: B, BL ARM64_PAGEBASE_REL21, // ADRP ARM64_PAGEOFFSET_12A, // ADD/ADDS (immediate) with zero shift, for page offset + ARM64_PAGEOFFSET_12L, // LDR (indexed, unsigned immediate), for page offset // Linux arm64 ARM64_LIN_TLSDESC_ADR_PAGE21, ARM64_LIN_TLSDESC_LD64_LO12, diff --git a/src/coreclr/inc/jiteeversionguid.h b/src/coreclr/inc/jiteeversionguid.h index 12347eb227a951..e5725f397fa132 100644 --- a/src/coreclr/inc/jiteeversionguid.h +++ b/src/coreclr/inc/jiteeversionguid.h @@ -37,11 +37,11 @@ #include -constexpr GUID JITEEVersionIdentifier = { /* dcace1ba-8b1e-45ad-890f-055c4e190ddb */ - 0xdcace1ba, - 0x8b1e, - 0x45ad, - {0x89, 0x0f, 0x05, 0x5c, 0x4e, 0x19, 0x0d, 0xdb} +constexpr GUID JITEEVersionIdentifier = { /* e5612019-5e77-477b-9148-506eb4179bac */ + 0xe5612019, + 0x5e77, + 0x477b, + {0x91, 0x48, 0x50, 0x6e, 0xb4, 0x17, 0x9b, 0xac} }; #endif // JIT_EE_VERSIONING_GUID_H diff --git a/src/coreclr/inc/utilcode.h b/src/coreclr/inc/utilcode.h index 76e15fc8da444e..d91c68186f2cc9 100644 --- a/src/coreclr/inc/utilcode.h +++ b/src/coreclr/inc/utilcode.h @@ -3011,6 +3011,11 @@ INT32 GetArm64Rel21(UINT32 * pCode); //***************************************************************************** INT32 GetArm64Rel12(UINT32 * pCode); +//***************************************************************************** +// Extract the page offset from an ldr (unsigned immediate) instruction +//***************************************************************************** +INT32 GetArm64Rel12Ldr(UINT32 * pCode); + //***************************************************************************** // Deposit the PC-Relative offset 'imm28' into a b or bl instruction //***************************************************************************** @@ -3026,6 +3031,11 @@ void PutArm64Rel21(UINT32 * pCode, INT32 imm21); //***************************************************************************** void PutArm64Rel12(UINT32 * pCode, INT32 imm12); +//***************************************************************************** +// Deposit the page offset 'imm12' into an ldr (unsigned immediate) instruction +//***************************************************************************** +void PutArm64Rel12Ldr(UINT32 * pCode, INT32 imm12); + //***************************************************************************** // Extract the PC-Relative page address and page offset from pcalau12i+add/ld //***************************************************************************** diff --git a/src/coreclr/jit/emitarm64.cpp b/src/coreclr/jit/emitarm64.cpp index 71b03f23c7d7c3..34b98a4c6b84de 100644 --- a/src/coreclr/jit/emitarm64.cpp +++ b/src/coreclr/jit/emitarm64.cpp @@ -237,7 +237,7 @@ void emitter::emitInsSanityCheck(instrDesc* id) assert(isIntegerRegister(id->idReg1()) || // ZR isVectorRegister(id->idReg1())); assert(isIntegerRegister(id->idReg2())); // SP - assert((emitGetInsSC(id) == 0) || (id->idIsTlsGD())); + assert((emitGetInsSC(id) == 0) || id->idIsTlsGD() || id->idIsReloc()); assert(insOptsNone(id->idInsOpt())); break; @@ -5906,6 +5906,13 @@ void emitter::emitIns_R_R_I(instruction ins, } } + // Try to fold a preceding relocatable "adrp/add" page-offset into this load, turning + // "adrp Rd,sym; add Rd,Rd,#:lo12:sym; ldr Rd,[Rd]" into "adrp Rd,sym; ldr Rd,[Rd,#:lo12:sym]". + if ((fmt == IF_LS_2A) && m_compiler->opts.compReloc && TryFoldPageOffsetIntoLdr(ins, attr, reg1, reg2)) + { + return; + } + // Try to optimize a load/store with an alternative instruction. if (isLdrStr && m_compiler->opts.OptimizationEnabled() && OptimizeLdrStr(ins, attr, reg1, reg2, imm, size, fmt, false, -1, -1 DEBUG_ARG(false))) @@ -11423,6 +11430,11 @@ size_t emitter::emitOutputInstr(insGroup* ig, instrDesc* id, BYTE** dp) { emitRecordRelocation(odst, (void*)emitGetInsSC(id), CorInfoReloc::ARM64_LIN_TLSDESC_LD64_LO12); } + else if (id->idIsReloc()) + { + // "ldr Rt,[Rn,#:lo12:sym]" with the page offset folded in from a preceding adrp/add pair. + emitRecordRelocation(odst, (void*)emitGetInsSC(id), CorInfoReloc::ARM64_PAGEOFFSET_12L); + } break; case IF_LS_2B: // LS_2B .X.......Xiiiiii iiiiiinnnnnttttt Rt Rn imm(0-4095) @@ -13856,9 +13868,21 @@ void emitter::emitDispInsHelp( case IF_LS_2A: // LS_2A .X.......X...... ......nnnnnttttt Rt Rn assert(insOptsNone(id->idInsOpt())); - assert((emitGetInsSC(id) == 0) || id->idIsTlsGD()); + assert((emitGetInsSC(id) == 0) || id->idIsTlsGD() || id->idIsReloc()); emitDispReg(id->idReg1(), emitInsTargetRegSize(id), true); - emitDispAddrRI(id->idReg2(), id->idInsOpt(), 0); + if (id->idIsReloc() && !id->idIsTlsGD()) + { + // "ldr Rt,[Rn,#:lo12:sym]" with the page offset folded in from a preceding adrp/add pair. + printf("["); + emitDispReg(id->idReg2(), EA_PTRSIZE, true); + printf("[LOW RELOC "); + emitDispImm((ssize_t)emitGetInsSC(id), false); + printf("]]"); + } + else + { + emitDispAddrRI(id->idReg2(), id->idInsOpt(), 0); + } break; case IF_LS_2B: // LS_2B .X.......Xiiiiii iiiiiinnnnnttttt Rt Rn imm(0-4095) @@ -17891,7 +17915,7 @@ bool emitter::OptimizePostIndexed(instruction ins, regNumber reg, ssize_t imm, e return false; } - if ((emitLastIns->idInsFmt() != IF_LS_2A) || emitLastIns->idIsTlsGD()) + if ((emitLastIns->idInsFmt() != IF_LS_2A) || emitLastIns->idIsTlsGD() || emitLastIns->idIsReloc()) { return false; } @@ -17996,6 +18020,80 @@ bool emitter::OptimizePostIndexed(instruction ins, regNumber reg, ssize_t imm, e return true; } +//----------------------------------------------------------------------------------- +// TryFoldPageOffsetIntoLdr: Fold the page offset of an immediately preceding relocatable +// "add Rd, Rd, #:lo12:sym" (PAGEOFFSET_12A) into a "ldr Rd, [Rd]" so that the pair +// "adrp Rd, sym; add Rd, Rd, #:lo12:sym; ldr Rd, [Rd]" becomes the shorter +// "adrp Rd, sym; ldr Rd, [Rd, #:lo12:sym]" using a PAGEOFFSET_12L relocation. +// +// Arguments: +// ins - The load instruction being emitted (must be INS_ldr to fold). +// attr - The operand attributes of the load. +// reg1 - The destination register of the load. +// reg2 - The base (address) register of the load. +// +// Returns: +// True if the fold was performed and the (relocatable) load was emitted; false otherwise. +// +bool emitter::TryFoldPageOffsetIntoLdr(instruction ins, emitAttr attr, regNumber reg1, regNumber reg2) +{ + if (ins != INS_ldr) + { + return false; + } + + // The load must overwrite its own base so the full cell address is provably dead. + if (reg1 != reg2) + { + return false; + } + + // PAGEOFFSET_12L patches a 64-bit LDR (offset scaled by 8); restrict to 64-bit general-register + // loads (this includes GCREF/BYREF, which are EA_8BYTE in a general register). + if ((EA_SIZE(attr) != EA_8BYTE) || !isGeneralRegister(reg1)) + { + return false; + } + + if (!emitCanPeepholeLastIns()) + { + return false; + } + + if (m_compiler->compGeneratingUnwindProlog || m_compiler->compGeneratingUnwindEpilog) + { + // Don't remove instructions while generating the "unwind" part of prologs or epilogs. + return false; + } + + // The previous instruction must be the "add Rd, Rd, #:lo12:sym" half of an adrp/add reloc pair. + instrDesc* prevId = emitLastIns; + if ((prevId->idIns() != INS_add) || (prevId->idInsFmt() != IF_DI_2A) || !prevId->idIsReloc() || + prevId->idIsTlsGD() || (prevId->idReg1() != reg1) || (prevId->idReg2() != reg1)) + { + return false; + } + + void* sym = prevId->idAddr()->iiaAddr; + + // Drop the "add"; the preceding "adrp" already put the page base of 'sym' into reg1. + emitRemoveLastInstruction(); + + // Emit "ldr reg1, [reg1]" carrying a PAGEOFFSET_12L relocation against 'sym'. The instruction + // encodes a zero offset; the relocation deposits the scaled :lo12: page offset of 'sym'. + instrDesc* id = emitNewInstrSC(attr, (ssize_t)sym); + id->idIns(INS_ldr); + id->idInsFmt(IF_LS_2A); + id->idInsOpt(INS_OPTS_NONE); + id->idReg1(reg1); + id->idReg2(reg1); + id->idSetIsDspReloc(); + + dispIns(id); + appendToCurIG(id); + return true; +} + #if defined(FEATURE_SIMD) //----------------------------------------------------------------------------------- // emitStoreSimd12ToLclOffset: store SIMD12 value from dataReg to varNum+offset. diff --git a/src/coreclr/jit/emitarm64.h b/src/coreclr/jit/emitarm64.h index 3fa360c2c06a87..f2a4c9e7f0b652 100644 --- a/src/coreclr/jit/emitarm64.h +++ b/src/coreclr/jit/emitarm64.h @@ -197,6 +197,10 @@ FORCEINLINE bool OptimizeLdrStr(instruction ins, bool OptimizePostIndexed(instruction ins, regNumber reg, ssize_t imm, emitAttr regAttr); +// Try to fold the page offset of a preceding relocatable "add Rd,Rd,#:lo12:sym" (PAGEOFFSET_12A) +// into a "ldr Rd,[Rd]" to form "ldr Rd,[Rd,#:lo12:sym]" (PAGEOFFSET_12L), removing the add. +bool TryFoldPageOffsetIntoLdr(instruction ins, emitAttr attr, regNumber reg1, regNumber reg2); + emitLclVarAddr* emitGetLclVarPairLclVar2(instrDesc* id) { assert(id->idIsLclVarPair()); diff --git a/src/coreclr/tools/Common/JitInterface/CorInfoImpl.cs b/src/coreclr/tools/Common/JitInterface/CorInfoImpl.cs index e3dea4bbdc585a..442be2390dc0ba 100644 --- a/src/coreclr/tools/Common/JitInterface/CorInfoImpl.cs +++ b/src/coreclr/tools/Common/JitInterface/CorInfoImpl.cs @@ -4296,6 +4296,7 @@ private RelocType GetRelocType(CorInfoReloc reloc) CorInfoReloc.ARM64_BRANCH26 => RelocType.IMAGE_REL_BASED_ARM64_BRANCH26, CorInfoReloc.ARM64_PAGEBASE_REL21 => RelocType.IMAGE_REL_BASED_ARM64_PAGEBASE_REL21, CorInfoReloc.ARM64_PAGEOFFSET_12A => RelocType.IMAGE_REL_BASED_ARM64_PAGEOFFSET_12A, + CorInfoReloc.ARM64_PAGEOFFSET_12L => RelocType.IMAGE_REL_BASED_ARM64_PAGEOFFSET_12L, CorInfoReloc.ARM64_LIN_TLSDESC_ADR_PAGE21 => RelocType.IMAGE_REL_AARCH64_TLSDESC_ADR_PAGE21, CorInfoReloc.ARM64_LIN_TLSDESC_LD64_LO12 => RelocType.IMAGE_REL_AARCH64_TLSDESC_LD64_LO12, CorInfoReloc.ARM64_LIN_TLSDESC_ADD_LO12 => RelocType.IMAGE_REL_AARCH64_TLSDESC_ADD_LO12, diff --git a/src/coreclr/tools/Common/JitInterface/CorInfoTypes.cs b/src/coreclr/tools/Common/JitInterface/CorInfoTypes.cs index d21a2018b7ba44..0c843d13aa1a7f 100644 --- a/src/coreclr/tools/Common/JitInterface/CorInfoTypes.cs +++ b/src/coreclr/tools/Common/JitInterface/CorInfoTypes.cs @@ -479,6 +479,7 @@ public enum CorInfoReloc ARM64_BRANCH26, // Arm64: B, BL ARM64_PAGEBASE_REL21, // ADRP ARM64_PAGEOFFSET_12A, // ADD/ADDS (immediate) with zero shift, for page offset + ARM64_PAGEOFFSET_12L, // LDR (indexed, unsigned immediate), for page offset // Linux arm64 ARM64_LIN_TLSDESC_ADR_PAGE21, ARM64_LIN_TLSDESC_LD64_LO12, diff --git a/src/coreclr/tools/superpmi/superpmi-shared/compileresult.cpp b/src/coreclr/tools/superpmi/superpmi-shared/compileresult.cpp index 0683828dce8479..1dcbbf1bca20e4 100644 --- a/src/coreclr/tools/superpmi/superpmi-shared/compileresult.cpp +++ b/src/coreclr/tools/superpmi/superpmi-shared/compileresult.cpp @@ -681,6 +681,7 @@ const char* relocationTypeToString(CorInfoReloc fRelocType) ADD_CASE(ARM64_BRANCH26); ADD_CASE(ARM64_PAGEBASE_REL21); ADD_CASE(ARM64_PAGEOFFSET_12A); + ADD_CASE(ARM64_PAGEOFFSET_12L); ADD_CASE(ARM64_LIN_TLSDESC_ADR_PAGE21); ADD_CASE(ARM64_LIN_TLSDESC_LD64_LO12); ADD_CASE(ARM64_LIN_TLSDESC_ADD_LO12); @@ -846,7 +847,7 @@ void CompileResult::applyRelocs(RelocContext* rc, unsigned char* block1, ULONG b DWORDLONG target = tmp.target + (int32_t)tmp.addlDelta; if (relocType == CorInfoReloc::ARM64_PAGEBASE_REL21 || relocType == CorInfoReloc::ARM64_LIN_TLSDESC_ADR_PAGE21 || - relocType == CorInfoReloc::ARM64_PAGEOFFSET_12A) + relocType == CorInfoReloc::ARM64_PAGEOFFSET_12A || relocType == CorInfoReloc::ARM64_PAGEOFFSET_12L) { if ((rc->originalRoDataAddress2 <= (size_t)target) && ((size_t)target < rc->originalRoDataAddress2 + rc->roDataSize2)) @@ -913,6 +914,17 @@ void CompileResult::applyRelocs(RelocContext* rc, unsigned char* block1, ULONG b } break; + case CorInfoReloc::ARM64_PAGEOFFSET_12L: // LDR 12 bit page offset + { + if ((section_begin <= address) && (address < section_end)) // A reloc for our section? + { + INT32 imm12 = (INT32)(SIZE_T)target & 0xFFFLL; + PutArm64Rel12Ldr((UINT32*)address, imm12); + } + wasRelocHandled = true; + } + break; + case CorInfoReloc::ARM64_WIN_TLS_SECREL_HIGH12A: // TLSDESC ADD for High-12 Add case CorInfoReloc::ARM64_WIN_TLS_SECREL_LOW12A: // TLSDESC ADD for Low-12 Add case CorInfoReloc::ARM64_LIN_TLSDESC_LD64_LO12: diff --git a/src/coreclr/tools/superpmi/superpmi-shared/spmiutil.cpp b/src/coreclr/tools/superpmi/superpmi-shared/spmiutil.cpp index a614d51a74de2e..1044df01ba7581 100644 --- a/src/coreclr/tools/superpmi/superpmi-shared/spmiutil.cpp +++ b/src/coreclr/tools/superpmi/superpmi-shared/spmiutil.cpp @@ -295,6 +295,16 @@ void PutArm64Rel12(UINT32* pCode, INT32 imm12) *pCode = addInstr; } +void PutArm64Rel12Ldr(UINT32* pCode, INT32 imm12) +{ + // For a 64-bit LDR the encoded immediate is scaled by the access size (8). + INT32 scaledImm12 = imm12 >> 3; + UINT32 ldrInstr = *pCode; + ldrInstr &= 0xFFC003FF; + ldrInstr |= (scaledImm12 << 10); + *pCode = ldrInstr; +} + void PutThumb2Imm16(UINT16* p, UINT16 imm16) { USHORT Opcode0 = p[0]; diff --git a/src/coreclr/tools/superpmi/superpmi-shared/spmiutil.h b/src/coreclr/tools/superpmi/superpmi-shared/spmiutil.h index 26a85de9c170f9..232eaa8a04861b 100644 --- a/src/coreclr/tools/superpmi/superpmi-shared/spmiutil.h +++ b/src/coreclr/tools/superpmi/superpmi-shared/spmiutil.h @@ -93,6 +93,7 @@ inline size_t SpmiTargetPointerSize() void PutArm64Rel28(UINT32* pCode, INT32 imm28); void PutArm64Rel21(UINT32* pCode, INT32 imm21); void PutArm64Rel12(UINT32* pCode, INT32 imm12); +void PutArm64Rel12Ldr(UINT32* pCode, INT32 imm12); void PutThumb2Mov32(UINT16* p, UINT32 imm32); void PutThumb2BlRel24(UINT16* p, INT32 imm24); diff --git a/src/coreclr/utilcode/util.cpp b/src/coreclr/utilcode/util.cpp index a60973d6fe18e6..1ebb2c02009ea6 100644 --- a/src/coreclr/utilcode/util.cpp +++ b/src/coreclr/utilcode/util.cpp @@ -2109,6 +2109,49 @@ void PutArm64Rel12(UINT32 * pCode, INT32 imm12) _ASSERTE(GetArm64Rel12(pCode) == imm12); } +//***************************************************************************** +// Extract the 12-bit page offset from an LDR instruction (unsigned immediate). +// For a 64-bit LDR the encoded immediate is scaled by 8 bytes. +//***************************************************************************** +INT32 GetArm64Rel12Ldr(UINT32 * pCode) +{ + LIMITED_METHOD_CONTRACT; + + UINT32 ldrInstr = *pCode; + + // 21-10 contains the scaled immediate. Mask 12 bits and shift by 10 bits. + INT32 scaledImm12 = (INT32)(ldrInstr & 0x003FFC00) >> 10; + + // Scale back to a byte offset (multiply by 8). + return scaledImm12 << 3; +} + +//***************************************************************************** +// Deposit the PC-Relative page offset 'imm12' into an LDR instruction (unsigned +// immediate). For a 64-bit LDR the immediate represents offset/8 (scaled by 8). +//***************************************************************************** +void PutArm64Rel12Ldr(UINT32 * pCode, INT32 imm12) +{ + LIMITED_METHOD_CONTRACT; + + // Verify that we got a valid offset that is aligned to 8 bytes. + _ASSERTE(FitsInRel12(imm12)); + _ASSERTE((imm12 & 7) == 0); + + UINT32 ldrInstr = *pCode; + // Check ldr opcode: 1111 1001 0100 .... (LDR 64-bit, unsigned immediate) + _ASSERTE((ldrInstr & 0xFFC00000) == 0xF9400000); + + INT32 scaledImm12 = imm12 >> 3; // scale the offset by the access size (8) + + ldrInstr &= 0xFFC003FF; // keep bits 31-22, 9-0 + ldrInstr |= (scaledImm12 << 10); // Occupy 21-10. + + *pCode = ldrInstr; // write the assembled instruction + + _ASSERTE(GetArm64Rel12Ldr(pCode) == imm12); +} + //***************************************************************************** // Extract the PC-Relative page address and page offset from pcalau12i+add/ld //***************************************************************************** diff --git a/src/coreclr/vm/jitinterface.cpp b/src/coreclr/vm/jitinterface.cpp index 1cb80773f3148a..c75a158e4df49e 100644 --- a/src/coreclr/vm/jitinterface.cpp +++ b/src/coreclr/vm/jitinterface.cpp @@ -12254,6 +12254,16 @@ void CEEJitInfo::recordRelocation(void * location, break; } + case CorInfoReloc::ARM64_PAGEOFFSET_12L: + { + _ASSERTE(addlDelta == 0); + + // Write the 12 bits page offset into the ldr instruction. + INT32 imm12 = (INT32)(SIZE_T)target & 0xFFFLL; + PutArm64Rel12Ldr((UINT32 *)locationRW, imm12); + break; + } + #endif // TARGET_ARM64 #ifdef TARGET_LOONGARCH64