Arm64 R2R: fold adrp+add+ldr into adrp+ldr[#:lo12:]

src/coreclr/inc/corinfo.h

@@ -858,6 +858,7 @@ enum class CorInfoReloc
     ARM64_BRANCH26,                        // Arm64: B, BL
     ARM64_PAGEBASE_REL21,                  // ADRP
     ARM64_PAGEOFFSET_12A,                  // ADD/ADDS (immediate) with zero shift, for page offset
+    ARM64_PAGEOFFSET_12L,                  // LDR (indexed, unsigned immediate), for page offset
     // Linux arm64
     ARM64_LIN_TLSDESC_ADR_PAGE21,
     ARM64_LIN_TLSDESC_LD64_LO12,

src/coreclr/inc/jiteeversionguid.h

@@ -37,11 +37,11 @@
 
 #include <minipal/guid.h>
 
-constexpr GUID JITEEVersionIdentifier = { /* dcace1ba-8b1e-45ad-890f-055c4e190ddb */
-    0xdcace1ba,
-    0x8b1e,
-    0x45ad,
-    {0x89, 0x0f, 0x05, 0x5c, 0x4e, 0x19, 0x0d, 0xdb}
+constexpr GUID JITEEVersionIdentifier = { /* e5612019-5e77-477b-9148-506eb4179bac */
+    0xe5612019,
+    0x5e77,
+    0x477b,
+    {0x91, 0x48, 0x50, 0x6e, 0xb4, 0x17, 0x9b, 0xac}
   };
 
 #endif // JIT_EE_VERSIONING_GUID_H

src/coreclr/inc/utilcode.h

@@ -3011,6 +3011,11 @@ INT32 GetArm64Rel21(UINT32 * pCode);
 //*****************************************************************************
 INT32 GetArm64Rel12(UINT32 * pCode);
 
+//*****************************************************************************
+//  Extract the page offset from an ldr (unsigned immediate) instruction
+//*****************************************************************************
+INT32 GetArm64Rel12Ldr(UINT32 * pCode);
+
 //*****************************************************************************
 //  Deposit the PC-Relative offset 'imm28' into a b or bl instruction
 //*****************************************************************************
@@ -3026,6 +3031,11 @@ void PutArm64Rel21(UINT32 * pCode, INT32 imm21);
 //*****************************************************************************
 void PutArm64Rel12(UINT32 * pCode, INT32 imm12);
 
+//*****************************************************************************
+//  Deposit the page offset 'imm12' into an ldr (unsigned immediate) instruction
+//*****************************************************************************
+void PutArm64Rel12Ldr(UINT32 * pCode, INT32 imm12);
+
 //*****************************************************************************
 //  Extract the PC-Relative page address and page offset from pcalau12i+add/ld
 //*****************************************************************************

src/coreclr/jit/emitarm64.cpp

@@ -237,7 +237,7 @@ void emitter::emitInsSanityCheck(instrDesc* id)
             assert(isIntegerRegister(id->idReg1()) || // ZR
                    isVectorRegister(id->idReg1()));
             assert(isIntegerRegister(id->idReg2())); // SP
-            assert((emitGetInsSC(id) == 0) || (id->idIsTlsGD()));
+            assert((emitGetInsSC(id) == 0) || id->idIsTlsGD() || id->idIsReloc());
             assert(insOptsNone(id->idInsOpt()));
             break;
 
@@ -5906,6 +5906,13 @@ void emitter::emitIns_R_R_I(instruction     ins,
             }
         }
 
+        // Try to fold a preceding relocatable "adrp/add" page-offset into this load, turning
+        // "adrp Rd,sym; add Rd,Rd,#:lo12:sym; ldr Rd,[Rd]" into "adrp Rd,sym; ldr Rd,[Rd,#:lo12:sym]".
+        if ((fmt == IF_LS_2A) && m_compiler->opts.compReloc && TryFoldPageOffsetIntoLdr(ins, attr, reg1, reg2))
+        {
+            return;
+        }
+
         // Try to optimize a load/store with an alternative instruction.
         if (isLdrStr && m_compiler->opts.OptimizationEnabled() &&
             OptimizeLdrStr(ins, attr, reg1, reg2, imm, size, fmt, false, -1, -1 DEBUG_ARG(false)))
@@ -11423,6 +11430,11 @@ size_t emitter::emitOutputInstr(insGroup* ig, instrDesc* id, BYTE** dp)
             {
                 emitRecordRelocation(odst, (void*)emitGetInsSC(id), CorInfoReloc::ARM64_LIN_TLSDESC_LD64_LO12);
             }
+            else if (id->idIsReloc())
+            {
+                // "ldr Rt,[Rn,#:lo12:sym]" with the page offset folded in from a preceding adrp/add pair.
+                emitRecordRelocation(odst, (void*)emitGetInsSC(id), CorInfoReloc::ARM64_PAGEOFFSET_12L);
+            }
             break;
 
         case IF_LS_2B: // LS_2B   .X.......Xiiiiii iiiiiinnnnnttttt      Rt Rn    imm(0-4095)
@@ -13856,9 +13868,21 @@ void emitter::emitDispInsHelp(
 
         case IF_LS_2A: // LS_2A   .X.......X...... ......nnnnnttttt      Rt Rn
             assert(insOptsNone(id->idInsOpt()));
-            assert((emitGetInsSC(id) == 0) || id->idIsTlsGD());
+            assert((emitGetInsSC(id) == 0) || id->idIsTlsGD() || id->idIsReloc());
             emitDispReg(id->idReg1(), emitInsTargetRegSize(id), true);
-            emitDispAddrRI(id->idReg2(), id->idInsOpt(), 0);
+            if (id->idIsReloc() && !id->idIsTlsGD())
+            {
+                // "ldr Rt,[Rn,#:lo12:sym]" with the page offset folded in from a preceding adrp/add pair.
+                printf("[");
+                emitDispReg(id->idReg2(), EA_PTRSIZE, true);
+                printf("[LOW RELOC ");
+                emitDispImm((ssize_t)emitGetInsSC(id), false);
+                printf("]]");
+            }
+            else
+            {
+                emitDispAddrRI(id->idReg2(), id->idInsOpt(), 0);
+            }
             break;
 
         case IF_LS_2B: // LS_2B   .X.......Xiiiiii iiiiiinnnnnttttt      Rt Rn    imm(0-4095)
@@ -17891,7 +17915,7 @@ bool emitter::OptimizePostIndexed(instruction ins, regNumber reg, ssize_t imm, e
         return false;
     }
 
-    if ((emitLastIns->idInsFmt() != IF_LS_2A) || emitLastIns->idIsTlsGD())
+    if ((emitLastIns->idInsFmt() != IF_LS_2A) || emitLastIns->idIsTlsGD() || emitLastIns->idIsReloc())
     {
         return false;
     }
@@ -17996,6 +18020,80 @@ bool emitter::OptimizePostIndexed(instruction ins, regNumber reg, ssize_t imm, e
     return true;
 }
 
+//-----------------------------------------------------------------------------------
+// TryFoldPageOffsetIntoLdr: Fold the page offset of an immediately preceding relocatable
+//   "add Rd, Rd, #:lo12:sym" (PAGEOFFSET_12A) into a "ldr Rd, [Rd]" so that the pair
+//   "adrp Rd, sym; add Rd, Rd, #:lo12:sym; ldr Rd, [Rd]" becomes the shorter
+//   "adrp Rd, sym; ldr Rd, [Rd, #:lo12:sym]" using a PAGEOFFSET_12L relocation.
+//
+// Arguments:
+//   ins  - The load instruction being emitted (must be INS_ldr to fold).
+//   attr - The operand attributes of the load.
+//   reg1 - The destination register of the load.
+//   reg2 - The base (address) register of the load.
+//
+// Returns:
+//   True if the fold was performed and the (relocatable) load was emitted; false otherwise.
+//
+bool emitter::TryFoldPageOffsetIntoLdr(instruction ins, emitAttr attr, regNumber reg1, regNumber reg2)
+{
+    if (ins != INS_ldr)
+    {
+        return false;
+    }
+
+    // The load must overwrite its own base so the full cell address is provably dead.
+    if (reg1 != reg2)
+    {
+        return false;
+    }
+
+    // PAGEOFFSET_12L patches a 64-bit LDR (offset scaled by 8); restrict to 64-bit general-register
+    // loads (this includes GCREF/BYREF, which are EA_8BYTE in a general register).
+    if ((EA_SIZE(attr) != EA_8BYTE) || !isGeneralRegister(reg1))
+    {
+        return false;
+    }
+
+    if (!emitCanPeepholeLastIns())
+    {
+        return false;
+    }
+
+    if (m_compiler->compGeneratingUnwindProlog || m_compiler->compGeneratingUnwindEpilog)
+    {
+        // Don't remove instructions while generating the "unwind" part of prologs or epilogs.
+        return false;
+    }
+
+    // The previous instruction must be the "add Rd, Rd, #:lo12:sym" half of an adrp/add reloc pair.
+    instrDesc* prevId = emitLastIns;
+    if ((prevId->idIns() != INS_add) || (prevId->idInsFmt() != IF_DI_2A) || !prevId->idIsReloc() ||
+        prevId->idIsTlsGD() || (prevId->idReg1() != reg1) || (prevId->idReg2() != reg1))
+    {
+        return false;
+    }
+
+    void* sym = prevId->idAddr()->iiaAddr;
+
+    // Drop the "add"; the preceding "adrp" already put the page base of 'sym' into reg1.
+    emitRemoveLastInstruction();
+
+    // Emit "ldr reg1, [reg1]" carrying a PAGEOFFSET_12L relocation against 'sym'. The instruction
+    // encodes a zero offset; the relocation deposits the scaled :lo12: page offset of 'sym'.
+    instrDesc* id = emitNewInstrSC(attr, (ssize_t)sym);
+    id->idIns(INS_ldr);
+    id->idInsFmt(IF_LS_2A);
+    id->idInsOpt(INS_OPTS_NONE);
+    id->idReg1(reg1);
+    id->idReg2(reg1);
+    id->idSetIsDspReloc();
+
+    dispIns(id);
+    appendToCurIG(id);
+    return true;
+}
+
 #if defined(FEATURE_SIMD)
 //-----------------------------------------------------------------------------------
 // emitStoreSimd12ToLclOffset: store SIMD12 value from dataReg to varNum+offset.

src/coreclr/jit/emitarm64.h

@@ -197,6 +197,10 @@ FORCEINLINE bool OptimizeLdrStr(instruction ins,
 
 bool OptimizePostIndexed(instruction ins, regNumber reg, ssize_t imm, emitAttr regAttr);
 
+// Try to fold the page offset of a preceding relocatable "add Rd,Rd,#:lo12:sym" (PAGEOFFSET_12A)
+// into a "ldr Rd,[Rd]" to form "ldr Rd,[Rd,#:lo12:sym]" (PAGEOFFSET_12L), removing the add.
+bool TryFoldPageOffsetIntoLdr(instruction ins, emitAttr attr, regNumber reg1, regNumber reg2);
+
 emitLclVarAddr* emitGetLclVarPairLclVar2(instrDesc* id)
 {
     assert(id->idIsLclVarPair());

src/coreclr/tools/Common/JitInterface/CorInfoImpl.cs

@@ -4296,6 +4296,7 @@ private RelocType GetRelocType(CorInfoReloc reloc)
                 CorInfoReloc.ARM64_BRANCH26 => RelocType.IMAGE_REL_BASED_ARM64_BRANCH26,
                 CorInfoReloc.ARM64_PAGEBASE_REL21 => RelocType.IMAGE_REL_BASED_ARM64_PAGEBASE_REL21,
                 CorInfoReloc.ARM64_PAGEOFFSET_12A => RelocType.IMAGE_REL_BASED_ARM64_PAGEOFFSET_12A,
+                CorInfoReloc.ARM64_PAGEOFFSET_12L => RelocType.IMAGE_REL_BASED_ARM64_PAGEOFFSET_12L,
                 CorInfoReloc.ARM64_LIN_TLSDESC_ADR_PAGE21 => RelocType.IMAGE_REL_AARCH64_TLSDESC_ADR_PAGE21,
                 CorInfoReloc.ARM64_LIN_TLSDESC_LD64_LO12 => RelocType.IMAGE_REL_AARCH64_TLSDESC_LD64_LO12,
                 CorInfoReloc.ARM64_LIN_TLSDESC_ADD_LO12 => RelocType.IMAGE_REL_AARCH64_TLSDESC_ADD_LO12,

src/coreclr/tools/Common/JitInterface/CorInfoTypes.cs

@@ -479,6 +479,7 @@ public enum CorInfoReloc
         ARM64_BRANCH26,                        // Arm64: B, BL
         ARM64_PAGEBASE_REL21,                  // ADRP
         ARM64_PAGEOFFSET_12A,                  // ADD/ADDS (immediate) with zero shift, for page offset
+        ARM64_PAGEOFFSET_12L,                  // LDR (indexed, unsigned immediate), for page offset
         // Linux arm64
         ARM64_LIN_TLSDESC_ADR_PAGE21,
         ARM64_LIN_TLSDESC_LD64_LO12,

src/coreclr/tools/superpmi/superpmi-shared/compileresult.cpp

@@ -681,6 +681,7 @@ const char* relocationTypeToString(CorInfoReloc fRelocType)
         ADD_CASE(ARM64_BRANCH26);
         ADD_CASE(ARM64_PAGEBASE_REL21);
         ADD_CASE(ARM64_PAGEOFFSET_12A);
+        ADD_CASE(ARM64_PAGEOFFSET_12L);
         ADD_CASE(ARM64_LIN_TLSDESC_ADR_PAGE21);
         ADD_CASE(ARM64_LIN_TLSDESC_LD64_LO12);
         ADD_CASE(ARM64_LIN_TLSDESC_ADD_LO12);
@@ -846,7 +847,7 @@ void CompileResult::applyRelocs(RelocContext* rc, unsigned char* block1, ULONG b
 
             DWORDLONG target = tmp.target + (int32_t)tmp.addlDelta;
             if (relocType == CorInfoReloc::ARM64_PAGEBASE_REL21 || relocType == CorInfoReloc::ARM64_LIN_TLSDESC_ADR_PAGE21 ||
-                relocType == CorInfoReloc::ARM64_PAGEOFFSET_12A)
+                relocType == CorInfoReloc::ARM64_PAGEOFFSET_12A || relocType == CorInfoReloc::ARM64_PAGEOFFSET_12L)
             {
                 if ((rc->originalRoDataAddress2 <= (size_t)target) &&
                     ((size_t)target < rc->originalRoDataAddress2 + rc->roDataSize2))
@@ -913,6 +914,17 @@ void CompileResult::applyRelocs(RelocContext* rc, unsigned char* block1, ULONG b
                 }
                 break;
 
+                case CorInfoReloc::ARM64_PAGEOFFSET_12L: // LDR 12 bit page offset
+                {
+                    if ((section_begin <= address) && (address < section_end)) // A reloc for our section?
+                    {
+                        INT32 imm12 = (INT32)(SIZE_T)target & 0xFFFLL;
+                        PutArm64Rel12Ldr((UINT32*)address, imm12);
+                    }
+                    wasRelocHandled = true;
+                }
+                break;
+
                 case CorInfoReloc::ARM64_WIN_TLS_SECREL_HIGH12A: // TLSDESC ADD for High-12 Add
                 case CorInfoReloc::ARM64_WIN_TLS_SECREL_LOW12A:  // TLSDESC ADD for Low-12 Add
                 case CorInfoReloc::ARM64_LIN_TLSDESC_LD64_LO12:

src/coreclr/tools/superpmi/superpmi-shared/spmiutil.cpp

@@ -295,6 +295,16 @@ void PutArm64Rel12(UINT32* pCode, INT32 imm12)
     *pCode = addInstr;
 }
 
+void PutArm64Rel12Ldr(UINT32* pCode, INT32 imm12)
+{
+    // For a 64-bit LDR the encoded immediate is scaled by the access size (8).
+    INT32  scaledImm12 = imm12 >> 3;
+    UINT32 ldrInstr    = *pCode;
+    ldrInstr &= 0xFFC003FF;
+    ldrInstr |= (scaledImm12 << 10);
+    *pCode = ldrInstr;
+}
+
 void PutThumb2Imm16(UINT16* p, UINT16 imm16)
 {
     USHORT Opcode0 = p[0];

src/coreclr/tools/superpmi/superpmi-shared/spmiutil.h

@@ -93,6 +93,7 @@ inline size_t SpmiTargetPointerSize()
 void PutArm64Rel28(UINT32* pCode, INT32 imm28);
 void PutArm64Rel21(UINT32* pCode, INT32 imm21);
 void PutArm64Rel12(UINT32* pCode, INT32 imm12);
+void PutArm64Rel12Ldr(UINT32* pCode, INT32 imm12);
 
 void PutThumb2Mov32(UINT16* p, UINT32 imm32);
 void PutThumb2BlRel24(UINT16* p, INT32 imm24);

src/coreclr/utilcode/util.cpp

@@ -2109,6 +2109,49 @@ void PutArm64Rel12(UINT32 * pCode, INT32 imm12)
     _ASSERTE(GetArm64Rel12(pCode) == imm12);
 }
 
+//*****************************************************************************
+//  Extract the 12-bit page offset from an LDR instruction (unsigned immediate).
+//  For a 64-bit LDR the encoded immediate is scaled by 8 bytes.
+//*****************************************************************************
+INT32 GetArm64Rel12Ldr(UINT32 * pCode)
+{
+    LIMITED_METHOD_CONTRACT;
+
+    UINT32 ldrInstr = *pCode;
+
+    // 21-10 contains the scaled immediate. Mask 12 bits and shift by 10 bits.
+    INT32 scaledImm12 = (INT32)(ldrInstr & 0x003FFC00) >> 10;
+
+    // Scale back to a byte offset (multiply by 8).
+    return scaledImm12 << 3;
+}
+
+//*****************************************************************************
+//  Deposit the PC-Relative page offset 'imm12' into an LDR instruction (unsigned
+//  immediate). For a 64-bit LDR the immediate represents offset/8 (scaled by 8).
+//*****************************************************************************
+void PutArm64Rel12Ldr(UINT32 * pCode, INT32 imm12)
+{
+    LIMITED_METHOD_CONTRACT;
+
+    // Verify that we got a valid offset that is aligned to 8 bytes.
+    _ASSERTE(FitsInRel12(imm12));
+    _ASSERTE((imm12 & 7) == 0);
+
+    UINT32 ldrInstr = *pCode;
+    // Check ldr opcode: 1111 1001 0100 .... (LDR 64-bit, unsigned immediate)
+    _ASSERTE((ldrInstr & 0xFFC00000) == 0xF9400000);
+
+    INT32 scaledImm12 = imm12 >> 3;       // scale the offset by the access size (8)
+
+    ldrInstr &= 0xFFC003FF;               // keep bits 31-22, 9-0
+    ldrInstr |= (scaledImm12 << 10);      // Occupy 21-10.
+
+    *pCode = ldrInstr;                    // write the assembled instruction
+
+    _ASSERTE(GetArm64Rel12Ldr(pCode) == imm12);
+}
+
 //*****************************************************************************
 //  Extract the PC-Relative page address and page offset from pcalau12i+add/ld
 //*****************************************************************************

src/coreclr/vm/jitinterface.cpp

@@ -12254,6 +12254,16 @@ void CEEJitInfo::recordRelocation(void *       location,
         break;
     }
 
+    case CorInfoReloc::ARM64_PAGEOFFSET_12L:
+    {
+        _ASSERTE(addlDelta == 0);
+
+        // Write the 12 bits page offset into the ldr instruction.
+        INT32 imm12 = (INT32)(SIZE_T)target & 0xFFFLL;
+        PutArm64Rel12Ldr((UINT32 *)locationRW, imm12);
+        break;
+    }
+
 #endif // TARGET_ARM64
 
 #ifdef TARGET_LOONGARCH64