[Wasm] Write barriers

src/coreclr/inc/jithelpers.h

@@ -170,7 +170,12 @@
     DYNAMICJITHELPER(CORINFO_HELP_ASSIGN_REF,   RhpAssignRef,   METHOD__NIL)
     DYNAMICJITHELPER(CORINFO_HELP_CHECKED_ASSIGN_REF, RhpCheckedAssignRef,METHOD__NIL)
 
+#ifdef TARGET_WASM
+    JITHELPER(CORINFO_HELP_ASSIGN_BYREF, NULL, METHOD__NIL)
+#else
     DYNAMICJITHELPER(CORINFO_HELP_ASSIGN_BYREF, RhpByRefAssignRef,METHOD__NIL)
+#endif // TARGET_WASM
+
     DYNAMICJITHELPER(CORINFO_HELP_BULK_WRITEBARRIER, NULL, METHOD__BUFFER__MEMCOPYGC)
 
     // Accessing fields

src/coreclr/jit/codegenwasm.cpp

@@ -2663,9 +2663,7 @@ void CodeGen::genEmitHelperCall(unsigned helper, int argSize, emitAttr retSize,
         // RhpCheckedAssignRef
         HELPER_SIG(CORINFO_HELP_CHECKED_ASSIGN_REF, UNMANAGED, CORINFO_WASM_TYPE_VOID /* retval */, CORINFO_WASM_TYPE_I,
                    CORINFO_WASM_TYPE_I);
-        // RhpByRefAssignRef
-        HELPER_SIG(CORINFO_HELP_ASSIGN_BYREF, UNMANAGED, CORINFO_WASM_TYPE_VOID /* retval */, CORINFO_WASM_TYPE_I,
-                   CORINFO_WASM_TYPE_I);
+        // RhpByRefAssignRef: Not implemented on Wasm, omitted
         // RhBulkMoveWithWriteBarrier
         HELPER_SIG(CORINFO_HELP_BULK_WRITEBARRIER, UNMANAGED, CORINFO_WASM_TYPE_VOID /* retval */, CORINFO_WASM_TYPE_I,
                    CORINFO_WASM_TYPE_I, CORINFO_WASM_TYPE_I);
@@ -3221,13 +3219,12 @@ void CodeGen::genCodeForStoreBlk(GenTreeBlk* blkOp)
             emit->emitIns_I(INS_local_get, EA_PTRSIZE, WasmRegToIndex(destReg));
             emit->emitIns_I(INS_I_const, EA_PTRSIZE, destOffset);
             emit->emitIns(INS_I_add);
+            // Do an I_load here instead of I_const + I_add because we're using the (Object **, Object *) write barrier,
+            //  not the (Object **, Object **) BYREF write barrier used on other architectures.
             emit->emitIns_I(INS_local_get, EA_PTRSIZE, WasmRegToIndex(srcReg));
-            emit->emitIns_I(INS_I_const, EA_PTRSIZE, srcOffset);
-            emit->emitIns(INS_I_add);
-            // NOTE: This helper's signature omits SP/PEP so all we need on the stack is dst and src.
-            // TODO-WASM-CQ: add a version of CORINFO_HELP_ASSIGN_BYREF that returns the updated dest/src
-            // pointers as a multi-value tuple and use it here.
-            genEmitHelperCall(CORINFO_HELP_ASSIGN_BYREF, 0, EA_PTRSIZE);
+            emit->emitIns_I(INS_I_load, EA_PTRSIZE, srcOffset);
+            // NOTE: This helper's signature omits SP/PEP so all we need on the stack is dst and ref.
+            genEmitHelperCall(CORINFO_HELP_CHECKED_ASSIGN_REF, 0, EA_PTRSIZE);
             gcPtrCount--;
         }
         ++i;

src/coreclr/runtime/arm64/WriteBarriers.S

@@ -187,7 +187,7 @@
         // Exit label
     .endm
 
-// void JIT_ByRefWriteBarrier
+// void RhpByRefAssignRef
 // On entry:
 //   x13  : the source address (points to object reference to write)
 //   x14  : the destination address (object reference written here)
@@ -212,7 +212,7 @@ LEAF_ENTRY RhpByRefAssignRefArm64, _TEXT
 
 LEAF_END RhpByRefAssignRefArm64, _TEXT
 
-// JIT_CheckedWriteBarrier(Object** dst, Object* src)
+// RhpCheckedAssignRef(Object** dst, Object* src)
 //
 // Write barrier for writes to objects that may reside
 // on the managed heap.
@@ -250,7 +250,7 @@ LOCAL_LABEL(NotInHeap):
         ret
 LEAF_END RhpCheckedAssignRefArm64, _TEXT
 
-// JIT_WriteBarrier(Object** dst, Object* src)
+// RhpAssignRef(Object** dst, Object* src)
 //
 // Write barrier for writes to objects that are known to
 // reside on the managed heap.

src/coreclr/runtime/arm64/WriteBarriers.asm

@@ -189,7 +189,7 @@ INVALIDGCVALUE  EQU 0xCCCCCCCD
         ;; Exit label
     MEND
 
-;; void JIT_ByRefWriteBarrier
+;; void RhpByRefAssignRef
 ;; On entry:
 ;;   x13 : the source address (points to object reference to write)
 ;;   x14 : the destination address (object reference written here)
@@ -215,7 +215,7 @@ INVALIDGCVALUE  EQU 0xCCCCCCCD
     LEAF_END RhpByRefAssignRefArm64
 
 
-;; JIT_CheckedWriteBarrier(Object** dst, Object* src)
+;; RhpCheckedAssignRef(Object** dst, Object* src)
 ;;
 ;; Write barrier for writes to objects that may reside
 ;; on the managed heap.
@@ -244,7 +244,7 @@ NotInHeap
 
     LEAF_END RhpCheckedAssignRefArm64
 
-;; JIT_WriteBarrier(Object** dst, Object* src)
+;; RhpAssignRef(Object** dst, Object* src)
 ;;
 ;; Write barrier for writes to objects that are known to
 ;; reside on the managed heap.

src/coreclr/runtime/loongarch64/WriteBarriers.S

@@ -177,7 +177,7 @@
         // Exit label
     .endm
 
-// void JIT_ByRefWriteBarrier
+// void RhpByRefAssignRef
 // On entry:
 //   t8  : the source address (points to object reference to write)
 //   t6  : the destination address (object reference written here)
@@ -203,7 +203,7 @@ LEAF_ENTRY RhpByRefAssignRef, _TEXT
 
 LEAF_END RhpByRefAssignRef, _TEXT
 
-// JIT_CheckedWriteBarrier(Object** dst, Object* src)
+// RhpCheckedAssignRef(Object** dst, Object* src)
 //
 // Write barrier for writes to objects that may reside
 // on the managed heap.
@@ -233,7 +233,7 @@ LOCAL_LABEL(NotInHeap):
 
 LEAF_END RhpCheckedAssignRef, _TEXT
 
-// JIT_WriteBarrier(Object** dst, Object* src)
+// RhpAssignRef(Object** dst, Object* src)
 //
 // Write barrier for writes to objects that are known to
 // reside on the managed heap.

src/coreclr/runtime/portable/WriteBarriers.cpp

@@ -4,22 +4,22 @@
 
 #include <fcall.h>
 
-EXTERN_C FCDECL2(VOID, RhpAssignRef, Object **dst, Object *ref);
-FCIMPL2(VOID, RhpAssignRef, Object **dst, Object *ref)
+EXTERN_C FCDECL2_RAW(VOID, RhpAssignRef, Object **dst, Object *ref);
+FCIMPL2_RAW(VOID, RhpAssignRef, Object **dst, Object *ref)
 {
     PORTABILITY_ASSERT("RhpAssignRef is not yet implemented");
 }
 FCIMPLEND
 
-EXTERN_C FCDECL2(VOID, RhpCheckedAssignRef, Object **dst, Object *ref);
-FCIMPL2(VOID, RhpCheckedAssignRef, Object **dst, Object *ref)
+EXTERN_C FCDECL2_RAW(VOID, RhpCheckedAssignRef, Object **dst, Object *ref);
+FCIMPL2_RAW(VOID, RhpCheckedAssignRef, Object **dst, Object *ref)
 {
     PORTABILITY_ASSERT("RhpCheckedAssignRef is not yet implemented");
 }
 FCIMPLEND
 
-EXTERN_C FCDECL2(VOID, RhpByRefAssignRef, Object **dst, Object *ref);
-FCIMPL2(VOID, RhpByRefAssignRef, Object **dst, Object *ref)
+EXTERN_C FCDECL2_RAW(VOID, RhpByRefAssignRef, Object **dst, Object **ref);
+FCIMPL2_RAW(VOID, RhpByRefAssignRef, Object **dst, Object **ref)
 {
     PORTABILITY_ASSERT("RhpByRefAssignRef is not yet implemented");
 }

src/coreclr/runtime/riscv64/WriteBarriers.S

@@ -187,7 +187,7 @@
         // Exit label
     .endm
 
-// void JIT_ByRefWriteBarrier
+// void RhpByRefAssignRef
 // On entry:
 //   t5  : the source address (points to object reference to write)
 //   t3  : the destination address (object reference written here)
@@ -213,7 +213,7 @@ LEAF_ENTRY RhpByRefAssignRef, _TEXT
 
 LEAF_END RhpByRefAssignRef, _TEXT
 
-// JIT_CheckedWriteBarrier(Object** dst, Object* src)
+// RhpCheckedAssignRef(Object** dst, Object* src)
 //
 // Write barrier for writes to objects that may reside
 // on the managed heap.
@@ -248,7 +248,7 @@ LOCAL_LABEL(NotInHeap):
 
 LEAF_END RhpCheckedAssignRef, _TEXT
 
-// JIT_WriteBarrier(Object** dst, Object* src)
+// RhpAssignRef(Object** dst, Object* src)
 //
 // Write barrier for writes to objects that are known to
 // reside on the managed heap.

src/coreclr/runtime/wasm/writebarriers.cpp

@@ -0,0 +1,126 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+//
+
+#include <fcall.h>
+
+#ifndef FEATURE_NATIVEAOT
+#include "gchelpers.inl"
+#include "gcheaputilities.h"
+#endif
+
+#ifdef FEATURE_MULTITHREADING
+#error The current assembly implementation of write barriers assumes single-threaded Wasm
+#endif
+
+#ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES
+#error The current assembly implementation of write barriers does not implement card bundles
+#endif
+
+// To simplify integration with the rest of the codebase and avoid complicating the build
+// system, Wasm write barriers are implemented using inline assembly inside C functions, below.
+// These write barriers use the Wasm-specific FCDECL2_RAW for a native calling convention instead
+// of a managed calling convention (to omit the sp and pep parameters). This reduces code size
+// considerably and simplifies the JIT. In order to make it safe to call these write barriers from
+// inside of a managed function without manually updating the __stack_pointer global, the barriers
+// *must* be implemented without use of the linear memory stack, and the best way to guarantee that
+// is to implement the barriers by hand in assembly.
+
+#define ASM_HELPER_2(rettype, funcname, a1, a2) \
+    EXTERN_C rettype __attribute__((naked)) F_CALL_CONV funcname(a1, a2)
+
+// Helper to make relevant GC globals and constants visible inside of inline assembly
+#define GC_ASM(text) \
+    asm(text \
+        :: [g_lowest_address] "i" (&g_lowest_address), \
+        [g_highest_address] "i" (&g_highest_address), \
+        [g_ephemeral_low] "i" (&g_ephemeral_low), \
+        [g_ephemeral_high] "i" (&g_ephemeral_high), \
+        [g_card_table] "i" (&g_card_table), \
+        [card_byte_shift] "i" (card_byte_shift) \
+    )
+
+EXTERN_C FCDECL2_RAW(VOID, RhpAssignRef, Object **dst, Object *ref);
+ASM_HELPER_2(VOID, RhpAssignRef, Object **dst, Object *ref)
+{
+    GC_ASM(
+        /* *dst = ref */
+        "local.get 0\n"
+        "local.get 1\n"
+        "i32.store 0\n"
+        /* if ((ref < ephemeral_low) || (ref >= ephemeral_high)) return */
+        "local.get 1\n"
+        "i32.const 0\n i32.load %[g_ephemeral_low]\n"
+        "i32.lt_u\n"
+        "local.get 1\n"
+        "i32.const 0\n i32.load %[g_ephemeral_high]\n"
+        "i32.ge_u\n"
+        "i32.or\n"
+        "if\n return\n end_if\n"
+        /* dst = &g_card_table[(dst >> card_byte_shift)] */
+        "local.get 0\n"
+        "i32.const %[card_byte_shift]\n"
+        "i32.shr_u\n"
+        "i32.const 0\n"
+        "i32.load %[g_card_table]\n"
+        "i32.add\n"
+        "local.tee 0\n"
+        /* if (*dst == 255) return */
+        "i32.load8_u 0\n"
+        "i32.const 255\n"
+        "i32.eq\n"
+        "if\n return\n end_if\n"
+        /* *dst = 255 */
+        "local.get 0\n"
+        "i32.const 255\n"
+        "i32.store8 0\n"
+        "return\n"
+    );
+}
+
+EXTERN_C FCDECL2_RAW(VOID, RhpCheckedAssignRef, Object **dst, Object *ref);
+ASM_HELPER_2(VOID, RhpCheckedAssignRef, Object **dst, Object *ref)
+{
+    GC_ASM(
+        /* *dst = ref */
+        "local.get 0\n"
+        "local.get 1\n"
+        "i32.store 0\n"
+        /* if ((dst < lowest) || (dst >= highest)) return */
+        "local.get 0\n"
+        "i32.const 0\n i32.load %[g_lowest_address]\n"
+        "i32.lt_u\n"
+        "local.get 0\n"
+        "i32.const 0\n i32.load %[g_highest_address]\n"
+        "i32.ge_u\n"
+        "i32.or\n"
+        "if\n return\n end_if\n"
+        /* if ((ref < ephemeral_low) || (ref >= ephemeral_high)) return */
+        "local.get 1\n"
+        "i32.const 0\n i32.load %[g_ephemeral_low]\n"
+        "i32.lt_u\n"
+        "local.get 1\n"
+        "i32.const 0\n i32.load %[g_ephemeral_high]\n"
+        "i32.ge_u\n"
+        "i32.or\n"
+        "if\n return\n end_if\n"
+        /* dst = &g_card_table[(dst >> card_byte_shift)] */
+        "local.get 0\n"
+        "i32.const %[card_byte_shift]\n"
+        "i32.shr_u\n"
+        "i32.const 0\n"
+        "i32.load %[g_card_table]\n"
+        "i32.add\n"
+        "local.tee 0\n"
+        /* if (*dst == 255) return */
+        "i32.load8_u 0\n"
+        "i32.const 255\n"
+        "i32.eq\n"
+        "if\n return\n end_if\n"
+        /* *dst = 255 */
+        "local.get 0\n"
+        "i32.const 255\n"
+        "i32.store8 0\n"
+        "return\n"
+    );
+}

src/coreclr/vm/CMakeLists.txt

@@ -652,8 +652,15 @@ if (FEATURE_PORTABLE_HELPERS)
     list(APPEND VM_SOURCES_WKS
         ${PORTABLE_SOURCES_DIR}/AllocFast.cpp
         ${PORTABLE_VM_SOURCES_DIR}/AllocSlow.cpp
-        ${PORTABLE_SOURCES_DIR}/WriteBarriers.cpp
     )
+
+    if(CLR_CMAKE_TARGET_ARCH_WASM)
+    else()
+        list(APPEND VM_SOURCES_WKS
+            ${PORTABLE_SOURCES_DIR}/WriteBarriers.cpp
+        )
+    endif()
+
 elseif(CLR_CMAKE_TARGET_WIN32)
     if(CLR_CMAKE_TARGET_ARCH_AMD64)
         set(VM_SOURCES_WKS_ARCH_ASM
@@ -940,6 +947,7 @@ elseif(CLR_CMAKE_TARGET_ARCH_WASM)
         ${ARCH_SOURCES_DIR}/entrypoints.h
     )
     set(VM_SOURCES_WKS_ARCH
+        ${RUNTIME_DIR}/${ARCH_SOURCES_DIR}/writebarriers.cpp
         ${ARCH_SOURCES_DIR}/calldescrworkerwasm.cpp
         ${ARCH_SOURCES_DIR}/profiler.cpp
         ${ARCH_SOURCES_DIR}/helpers.cpp

src/coreclr/vm/fcall.h

@@ -148,6 +148,7 @@
 #define FCDECL1(rettype, funcname, a1) rettype F_CALL_CONV funcname(uintptr_t callersStackPointer, a1, PCODE portableEntryPointContext); rettype F_CALL_CONV funcname ## _IMPL(uintptr_t callersStackPointer, a1, PCODE portableEntryPointContext)
 #define FCDECL1_V(rettype, funcname, a1) rettype F_CALL_CONV funcname(uintptr_t callersStackPointer, a1, PCODE portableEntryPointContext); rettype F_CALL_CONV funcname ## _IMPL(uintptr_t callersStackPointer, a1, PCODE portableEntryPointContext)
 #define FCDECL2(rettype, funcname, a1, a2) rettype F_CALL_CONV funcname(uintptr_t callersStackPointer, a1, a2, PCODE portableEntryPointContext); rettype F_CALL_CONV funcname ## _IMPL(uintptr_t callersStackPointer, a1, a2, PCODE portableEntryPointContext)
+#define FCDECL2_RAW(rettype, funcname, a1, a2) rettype F_CALL_CONV funcname(a1, a2)
 #define FCDECL2_VV(rettype, funcname, a1, a2) rettype F_CALL_CONV funcname(uintptr_t callersStackPointer, a1, a2, PCODE portableEntryPointContext); rettype F_CALL_CONV funcname ## _IMPL(uintptr_t callersStackPointer, a1, a2, PCODE portableEntryPointContext)
 #define FCDECL2_VI(rettype, funcname, a1, a2) rettype F_CALL_CONV funcname(uintptr_t callersStackPointer, a1, a2, PCODE portableEntryPointContext); rettype F_CALL_CONV funcname ## _IMPL(uintptr_t callersStackPointer, a1, a2, PCODE portableEntryPointContext)
 #define FCDECL2_IV(rettype, funcname, a1, a2) rettype F_CALL_CONV funcname(uintptr_t callersStackPointer, a1, a2, PCODE portableEntryPointContext); rettype F_CALL_CONV funcname ## _IMPL(uintptr_t callersStackPointer, a1, a2, PCODE portableEntryPointContext)
@@ -190,6 +191,10 @@
 
 #endif // !SWIZZLE_STKARG_ORDER
 
+#ifndef FCDECL2_RAW
+#define FCDECL2_RAW(rettype, funcname, a1, a2) FCDECL2(rettype, funcname, a1, a2)
+#endif
+
 #if defined(ENABLE_CONTRACTS)
 #define FC_CAN_TRIGGER_GC()         FCallGCCanTrigger::Enter()
 #define FC_CAN_TRIGGER_GC_END()     FCallGCCanTrigger::Leave(__FUNCTION__, __FILE__, __LINE__)
@@ -410,6 +415,8 @@ struct FCSigCheck {
 #define FCIMPL1(rettype, funcname, a1) WASM_CALLABLE_FUNC_2(rettype, funcname, a1, PCODE portableEntryPointContext) {  FCIMPL_PROLOG(funcname)
 #define FCIMPL1_V(rettype, funcname, a1) WASM_CALLABLE_FUNC_2(rettype, funcname, a1, PCODE portableEntryPointContext) {  FCIMPL_PROLOG(funcname)
 #define FCIMPL2(rettype, funcname, a1, a2) WASM_CALLABLE_FUNC_3(rettype, funcname, a1, a2, PCODE portableEntryPointContext) {  FCIMPL_PROLOG(funcname)
+#define FCIMPL2_RAW(rettype, funcname, a1, a2) FCSIGCHECK(funcname, #rettype "," #a1 "," #a2) \
+    rettype F_CALL_CONV funcname(a1, a2) { FCIMPL_PROLOG(funcname)
 #define FCIMPL2VA(rettype, funcname, a1, a2) WASM_CALLABLE_FUNC_3(rettype, funcname, a1, a2, PCODE portableEntryPointContext) {  FCIMPL_PROLOG(funcname)
 #define FCIMPL2_VV(rettype, funcname, a1, a2) WASM_CALLABLE_FUNC_3(rettype, funcname, a1, a2, PCODE portableEntryPointContext) {  FCIMPL_PROLOG(funcname)
 #define FCIMPL2_VI(rettype, funcname, a1, a2) WASM_CALLABLE_FUNC_3(rettype, funcname, a1, a2, PCODE portableEntryPointContext) {  FCIMPL_PROLOG(funcname)
@@ -515,6 +522,10 @@ struct FCSigCheck {
 #define HCIMPLEND_RAW }
 #define HCIMPLEND }
 
+#ifndef FCIMPL2_RAW
+#define FCIMPL2_RAW(rettype, funcname, a1, a2) FCIMPL2(rettype, funcname, a1, a2)
+#endif
+
 // The managed calling convention expects returned small types (e.g. bool) to be
 // widened to 32-bit on return. The C/C++ calling convention does not guarantee returned
 // small types to be widened on most platforms. The small types have to be artificially