diff --git a/src/coreclr/gc/env/gcenv.os.h b/src/coreclr/gc/env/gcenv.os.h index 08e9b39e36eb1a..6a73ba6d378bc6 100644 --- a/src/coreclr/gc/env/gcenv.os.h +++ b/src/coreclr/gc/env/gcenv.os.h @@ -451,9 +451,6 @@ class GCToOSInterface // Misc // - // Flush write buffers of processors that are executing threads of the current process - static void FlushProcessWriteBuffers(); - // Break into a debugger static void DebugBreak(); diff --git a/src/coreclr/gc/gc.cpp b/src/coreclr/gc/gc.cpp index cacae581ae019a..025fe97bec77f5 100644 --- a/src/coreclr/gc/gc.cpp +++ b/src/coreclr/gc/gc.cpp @@ -27,6 +27,7 @@ #include "handletable.inl" #include "gcenv.inl" #include "gceventstatus.h" +#include #ifdef __INTELLISENSE__ #if defined(FEATURE_SVR_GC) @@ -9871,7 +9872,7 @@ int gc_heap::grow_brick_card_tables (uint8_t* start, if (!write_barrier_updated) { seg_mapping_table = new_seg_mapping_table; - GCToOSInterface::FlushProcessWriteBuffers(); + minipal_memory_barrier_process_wide(); g_gc_lowest_address = saved_g_lowest_address; g_gc_highest_address = saved_g_highest_address; diff --git a/src/coreclr/gc/softwarewritewatch.cpp b/src/coreclr/gc/softwarewritewatch.cpp index c72e2c6fcb9db5..c42c63dcd0d47a 100644 --- a/src/coreclr/gc/softwarewritewatch.cpp +++ b/src/coreclr/gc/softwarewritewatch.cpp @@ -5,6 +5,7 @@ #include "gcenv.h" #include "env/gcenv.os.h" #include "softwarewritewatch.h" +#include #ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP #ifndef DACCESS_COMPILE @@ -131,7 +132,7 @@ void SoftwareWriteWatch::GetDirty( { // When a page is marked as dirty, a memory barrier is not issued after the write most of the time. Issue a memory // barrier on all active threads of the process now to make recent changes to dirty state visible to this thread. - GCToOSInterface::FlushProcessWriteBuffers(); + minipal_memory_barrier_process_wide(); } uint8_t *tableRegionStart; @@ -233,7 +234,7 @@ void SoftwareWriteWatch::GetDirty( // that the GC will not miss marking through dirtied objects in the page. Issue a memory barrier on all active threads // of the process now. MemoryBarrier(); // flush writes from this thread first to guarantee ordering - GCToOSInterface::FlushProcessWriteBuffers(); + minipal_memory_barrier_process_wide(); } } diff --git a/src/coreclr/gc/unix/gcenv.unix.cpp b/src/coreclr/gc/unix/gcenv.unix.cpp index 33855147497c2b..682c6adac7807c 100644 --- a/src/coreclr/gc/unix/gcenv.unix.cpp +++ b/src/coreclr/gc/unix/gcenv.unix.cpp @@ -23,6 +23,7 @@ #include "volatile.h" #include "gcconfig.h" #include "numasupport.h" +#include #include #include @@ -30,17 +31,6 @@ #include #endif -#ifdef __linux__ -#include -#include -#define membarrier(...) syscall(__NR_membarrier, __VA_ARGS__) -#elif HAVE_SYS_MEMBARRIER_H -#include -#ifdef TARGET_BROWSER -#define membarrier(cmd, flags, cpu_id) 0 // browser/wasm is currently single threaded -#endif -#endif - #include #undef min @@ -85,21 +75,6 @@ #include #include -extern "C" -{ -# include -} - -#define CHECK_MACH(_msg, machret) do { \ - if (machret != KERN_SUCCESS) \ - { \ - char _szError[1024]; \ - snprintf(_szError, ARRAY_SIZE(_szError), "%s: %u: %s", __FUNCTION__, __LINE__, _msg); \ - mach_error(_szError, machret); \ - abort(); \ - } \ - } while (false) - #endif // __APPLE__ #ifdef __HAIKU__ @@ -140,48 +115,6 @@ typedef cpuset_t cpu_set_t; // The cached total number of CPUs that can be used in the OS. static uint32_t g_totalCpuCount = 0; -bool CanFlushUsingMembarrier() -{ -#if defined(__linux__) || HAVE_SYS_MEMBARRIER_H - -#ifdef TARGET_ANDROID - // Avoid calling membarrier on older Android versions where membarrier - // may be barred by seccomp causing the process to be killed. - int apiLevel = android_get_device_api_level(); - if (apiLevel < __ANDROID_API_Q__) - { - return false; - } -#endif - - // Starting with Linux kernel 4.14, process memory barriers can be generated - // using MEMBARRIER_CMD_PRIVATE_EXPEDITED. - - int mask = membarrier(MEMBARRIER_CMD_QUERY, 0, 0); - - if (mask >= 0 && - mask & MEMBARRIER_CMD_PRIVATE_EXPEDITED && - // Register intent to use the private expedited command. - membarrier(MEMBARRIER_CMD_REGISTER_PRIVATE_EXPEDITED, 0, 0) == 0) - { - return true; - } -#endif - - return false; -} - -// -// Tracks if the OS supports FlushProcessWriteBuffers using membarrier -// -static int s_flushUsingMemBarrier = 0; - -// Helper memory page used by the FlushProcessWriteBuffers -static uint8_t* g_helperPage = 0; - -// Mutex to make the FlushProcessWriteBuffersMutex thread safe -static pthread_mutex_t g_flushProcessWriteBuffersMutex; - size_t GetRestrictedPhysicalMemoryLimit(); bool GetPhysicalMemoryUsed(size_t* val); @@ -219,50 +152,10 @@ bool GCToOSInterface::Initialize() g_totalCpuCount = cpuCount; - // - // support for FlusProcessWriteBuffers - // -#ifndef TARGET_WASM - assert(s_flushUsingMemBarrier == 0); - - if (CanFlushUsingMembarrier()) - { - s_flushUsingMemBarrier = TRUE; - } -#ifndef TARGET_APPLE - else + if (!minipal_initialize_memory_barrier_process_wide()) { - assert(g_helperPage == 0); - - g_helperPage = static_cast(mmap(0, OS_PAGE_SIZE, PROT_READ | PROT_WRITE, MAP_ANONYMOUS | MAP_PRIVATE, -1, 0)); - - if (g_helperPage == MAP_FAILED) - { - return false; - } - - // Verify that the s_helperPage is really aligned to the g_SystemInfo.dwPageSize - assert((((size_t)g_helperPage) & (OS_PAGE_SIZE - 1)) == 0); - - // Locking the page ensures that it stays in memory during the two mprotect - // calls in the FlushProcessWriteBuffers below. If the page was unmapped between - // those calls, they would not have the expected effect of generating IPI. - int status = mlock(g_helperPage, OS_PAGE_SIZE); - - if (status != 0) - { - return false; - } - - status = pthread_mutex_init(&g_flushProcessWriteBuffersMutex, NULL); - if (status != 0) - { - munlock(g_helperPage, OS_PAGE_SIZE); - return false; - } + return false; } -#endif // !TARGET_APPLE -#endif // !TARGET_WASM InitializeCGroup(); @@ -354,13 +247,6 @@ bool GCToOSInterface::Initialize() // Shutdown the interface implementation void GCToOSInterface::Shutdown() { - int ret = munlock(g_helperPage, OS_PAGE_SIZE); - assert(ret == 0); - ret = pthread_mutex_destroy(&g_flushProcessWriteBuffersMutex); - assert(ret == 0); - - munmap(g_helperPage, OS_PAGE_SIZE); - CleanupCGroup(); } @@ -410,91 +296,6 @@ bool GCToOSInterface::CanGetCurrentProcessorNumber() return HAVE_SCHED_GETCPU; } -// Flush write buffers of processors that are executing threads of the current process -void GCToOSInterface::FlushProcessWriteBuffers() -{ -#ifndef TARGET_WASM -#if defined(__linux__) || HAVE_SYS_MEMBARRIER_H - if (s_flushUsingMemBarrier) - { - int status = membarrier(MEMBARRIER_CMD_PRIVATE_EXPEDITED, 0, 0); - assert(status == 0 && "Failed to flush using membarrier"); - } - else -#endif - if (g_helperPage != 0) - { - int status = pthread_mutex_lock(&g_flushProcessWriteBuffersMutex); - assert(status == 0 && "Failed to lock the flushProcessWriteBuffersMutex lock"); - - // Changing a helper memory page protection from read / write to no access - // causes the OS to issue IPI to flush TLBs on all processors. This also - // results in flushing the processor buffers. - status = mprotect(g_helperPage, OS_PAGE_SIZE, PROT_READ | PROT_WRITE); - assert(status == 0 && "Failed to change helper page protection to read / write"); - - // Ensure that the page is dirty before we change the protection so that - // we prevent the OS from skipping the global TLB flush. - __sync_add_and_fetch((size_t*)g_helperPage, 1); - - status = mprotect(g_helperPage, OS_PAGE_SIZE, PROT_NONE); - assert(status == 0 && "Failed to change helper page protection to no access"); - - status = pthread_mutex_unlock(&g_flushProcessWriteBuffersMutex); - assert(status == 0 && "Failed to unlock the flushProcessWriteBuffersMutex lock"); - } -#ifdef TARGET_APPLE - else - { - mach_msg_type_number_t cThreads; - thread_act_t *pThreads; - kern_return_t machret = task_threads(mach_task_self(), &pThreads, &cThreads); - CHECK_MACH("task_threads()", machret); - - uintptr_t sp; - uintptr_t registerValues[128]; - - // Iterate through each of the threads in the list. - for (mach_msg_type_number_t i = 0; i < cThreads; i++) - { - if (__builtin_available (macOS 10.14, iOS 12, tvOS 9, *)) - { - // Request the threads pointer values to force the thread to emit a memory barrier - size_t registers = 128; - machret = thread_get_register_pointer_values(pThreads[i], &sp, ®isters, registerValues); - } - else - { - // fallback implementation for older OS versions -#if defined(HOST_AMD64) - x86_thread_state64_t threadState; - mach_msg_type_number_t count = x86_THREAD_STATE64_COUNT; - machret = thread_get_state(pThreads[i], x86_THREAD_STATE64, (thread_state_t)&threadState, &count); -#elif defined(HOST_ARM64) - arm_thread_state64_t threadState; - mach_msg_type_number_t count = ARM_THREAD_STATE64_COUNT; - machret = thread_get_state(pThreads[i], ARM_THREAD_STATE64, (thread_state_t)&threadState, &count); -#else - #error Unexpected architecture -#endif - } - - if (machret == KERN_INSUFFICIENT_BUFFER_SIZE) - { - CHECK_MACH("thread_get_register_pointer_values()", machret); - } - - machret = mach_port_deallocate(mach_task_self(), pThreads[i]); - CHECK_MACH("mach_port_deallocate()", machret); - } - // Deallocate the thread list now we're done with it. - machret = vm_deallocate(mach_task_self(), (vm_address_t)pThreads, cThreads * sizeof(thread_act_t)); - CHECK_MACH("vm_deallocate()", machret); - } -#endif // TARGET_APPLE -#endif // !TARGET_WASM -} - // Break into a debugger. Uses a compiler intrinsic if one is available, // otherwise raises a SIGTRAP. void GCToOSInterface::DebugBreak() diff --git a/src/coreclr/gc/windows/gcenv.windows.cpp b/src/coreclr/gc/windows/gcenv.windows.cpp index 3e8040be0bcbb1..8285e71d354aaf 100644 --- a/src/coreclr/gc/windows/gcenv.windows.cpp +++ b/src/coreclr/gc/windows/gcenv.windows.cpp @@ -640,12 +640,6 @@ bool GCToOSInterface::CanGetCurrentProcessorNumber() return true; } -// Flush write buffers of processors that are executing threads of the current process -void GCToOSInterface::FlushProcessWriteBuffers() -{ - ::FlushProcessWriteBuffers(); -} - // Break into a debugger void GCToOSInterface::DebugBreak() { diff --git a/src/coreclr/nativeaot/Runtime/MiscHelpers.cpp b/src/coreclr/nativeaot/Runtime/MiscHelpers.cpp index 138c3917198ec4..b8d3b4f797839d 100644 --- a/src/coreclr/nativeaot/Runtime/MiscHelpers.cpp +++ b/src/coreclr/nativeaot/Runtime/MiscHelpers.cpp @@ -73,7 +73,7 @@ EXTERN_C void QCALLTYPE RhFlushProcessWriteBuffers() ASSERT_MSG(!ThreadStore::GetCurrentThread()->IsCurrentThreadInCooperativeMode(), "You must p/invoke to RhFlushProcessWriteBuffers"); - PalFlushProcessWriteBuffers(); + minipal_memory_barrier_process_wide(); } // Get the list of currently loaded NativeAOT modules (as OS HMODULE handles). The caller provides a reference diff --git a/src/coreclr/nativeaot/Runtime/Pal.h b/src/coreclr/nativeaot/Runtime/Pal.h index 6a4c57ce7307a1..0a9f9c2ea45599 100644 --- a/src/coreclr/nativeaot/Runtime/Pal.h +++ b/src/coreclr/nativeaot/Runtime/Pal.h @@ -23,6 +23,7 @@ #include #endif +#include #include #include "CommonTypes.h" @@ -290,7 +291,6 @@ int32_t _stricmp(const char *string1, const char *string2); uint16_t PalCaptureStackBackTrace(uint32_t arg1, uint32_t arg2, void* arg3, uint32_t* arg4); UInt32_BOOL PalCloseHandle(HANDLE arg1); -void PalFlushProcessWriteBuffers(); uint32_t PalGetCurrentProcessId(); #ifdef UNICODE diff --git a/src/coreclr/nativeaot/Runtime/gcenv.ee.cpp b/src/coreclr/nativeaot/Runtime/gcenv.ee.cpp index e2c31db91e328b..58d7ffc0bfc963 100644 --- a/src/coreclr/nativeaot/Runtime/gcenv.ee.cpp +++ b/src/coreclr/nativeaot/Runtime/gcenv.ee.cpp @@ -63,7 +63,7 @@ void GCToEEInterface::RestartEE(bool /*bFinishedGC*/) // This is needed to synchronize threads that were running in preemptive mode while // the runtime was suspended and that will return to cooperative mode after the runtime // is restarted. - PalFlushProcessWriteBuffers(); + minipal_memory_barrier_process_wide(); #endif // !defined(TARGET_X86) && !defined(TARGET_AMD64) SyncClean::CleanUp(); @@ -404,7 +404,7 @@ void GCToEEInterface::StompWriteBarrier(WriteBarrierParameters* args) { // If runtime is not suspended, force all threads to see the changed table before seeing updated heap boundaries. // See: http://vstfdevdiv:8080/DevDiv2/DevDiv/_workitems/edit/346765 - PalFlushProcessWriteBuffers(); + minipal_memory_barrier_process_wide(); } #endif @@ -415,7 +415,7 @@ void GCToEEInterface::StompWriteBarrier(WriteBarrierParameters* args) if (!is_runtime_suspended) { // If runtime is not suspended, force all threads to see the changed state before observing future allocations. - PalFlushProcessWriteBuffers(); + minipal_memory_barrier_process_wide(); } #endif return; diff --git a/src/coreclr/nativeaot/Runtime/threadstore.cpp b/src/coreclr/nativeaot/Runtime/threadstore.cpp index daf745ba0b65ef..fd0c221f0725b4 100644 --- a/src/coreclr/nativeaot/Runtime/threadstore.cpp +++ b/src/coreclr/nativeaot/Runtime/threadstore.cpp @@ -237,7 +237,7 @@ void ThreadStore::SuspendAllThreads(bool waitForGCEvent) // Our lock-free algorithm depends on flushing write buffers of all processors running RH code. The // reason for this is that we essentially implement Dekker's algorithm, which requires write ordering. - PalFlushProcessWriteBuffers(); + minipal_memory_barrier_process_wide(); int prevRemaining = INT32_MAX; bool observeOnly = true; @@ -310,7 +310,7 @@ void ThreadStore::SuspendAllThreads(bool waitForGCEvent) // This is needed to synchronize threads that were running in preemptive mode thus were // left alone by suspension to flush their writes that they made before they switched to // preemptive mode. - PalFlushProcessWriteBuffers(); + minipal_memory_barrier_process_wide(); #endif //TARGET_ARM || TARGET_ARM64 || TARGET_LOONGARCH64 } @@ -329,7 +329,7 @@ void ThreadStore::ResumeAllThreads(bool waitForGCEvent) // This is needed to synchronize threads that were running in preemptive mode while // the runtime was suspended and that will return to cooperative mode after the runtime // is restarted. - PalFlushProcessWriteBuffers(); + minipal_memory_barrier_process_wide(); #endif //TARGET_ARM || TARGET_ARM64 || TARGET_LOONGARCH64 RhpTrapThreads &= ~(uint32_t)TrapThreadsFlags::TrapThreads; diff --git a/src/coreclr/nativeaot/Runtime/unix/PalUnix.cpp b/src/coreclr/nativeaot/Runtime/unix/PalUnix.cpp index 76b18bb0ad8e2c..be67447030e85a 100644 --- a/src/coreclr/nativeaot/Runtime/unix/PalUnix.cpp +++ b/src/coreclr/nativeaot/Runtime/unix/PalUnix.cpp @@ -40,6 +40,7 @@ #include #include #include +#include #include #ifdef TARGET_LINUX @@ -1233,11 +1234,6 @@ int32_t PalGetModuleFileName(_Out_ const TCHAR** pModuleNameOut, HANDLE moduleBa #endif // defined(HOST_WASM) } -void PalFlushProcessWriteBuffers() -{ - GCToOSInterface::FlushProcessWriteBuffers(); -} - static const int64_t SECS_BETWEEN_1601_AND_1970_EPOCHS = 11644473600LL; static const int64_t SECS_TO_100NS = 10000000; /* 10^7 */ diff --git a/src/coreclr/nativeaot/Runtime/windows/PalMinWin.cpp b/src/coreclr/nativeaot/Runtime/windows/PalMinWin.cpp index 8df2e75d1aaf81..839e0943877a65 100644 --- a/src/coreclr/nativeaot/Runtime/windows/PalMinWin.cpp +++ b/src/coreclr/nativeaot/Runtime/windows/PalMinWin.cpp @@ -1039,11 +1039,6 @@ UInt32_BOOL PalCloseHandle(HANDLE arg1) return ::CloseHandle(arg1); } -void PalFlushProcessWriteBuffers() -{ - ::FlushProcessWriteBuffers(); -} - uint32_t PalGetCurrentProcessId() { return static_cast(::GetCurrentProcessId()); diff --git a/src/coreclr/pal/inc/pal.h b/src/coreclr/pal/inc/pal.h index 4c1b089882eccf..d9f338c40e7e5c 100644 --- a/src/coreclr/pal/inc/pal.h +++ b/src/coreclr/pal/inc/pal.h @@ -3577,11 +3577,6 @@ RtlCaptureContext( OUT PCONTEXT ContextRecord ); -PALIMPORT -VOID -PALAPI -FlushProcessWriteBuffers(); - typedef void (*PAL_ActivationFunction)(CONTEXT *context); typedef BOOL (*PAL_SafeActivationCheckFunction)(SIZE_T ip); diff --git a/src/coreclr/pal/src/include/pal/process.h b/src/coreclr/pal/src/include/pal/process.h index 1c48093af219da..45ef76577ad197 100644 --- a/src/coreclr/pal/src/include/pal/process.h +++ b/src/coreclr/pal/src/include/pal/process.h @@ -188,17 +188,6 @@ VOID PROCNotifyProcessShutdown(bool isExecutingOnAltStack = false); --*/ VOID PROCCreateCrashDumpIfEnabled(int signal, siginfo_t* siginfo, bool serialize); -/*++ -Function: - InitializeFlushProcessWriteBuffers - -Abstract - This function initializes data structures needed for the FlushProcessWriteBuffers -Return - TRUE if it succeeded, FALSE otherwise ---*/ -BOOL InitializeFlushProcessWriteBuffers(); - #ifdef __cplusplus } #endif // __cplusplus diff --git a/src/coreclr/pal/src/init/pal.cpp b/src/coreclr/pal/src/init/pal.cpp index 64fffbef6e0f2e..4d02460f16d51a 100644 --- a/src/coreclr/pal/src/init/pal.cpp +++ b/src/coreclr/pal/src/init/pal.cpp @@ -36,6 +36,7 @@ SET_DEFAULT_DEBUG_CHANNEL(PAL); // some headers have code with asserts, so do th #include "pal/stackstring.hpp" #include "pal/cgroup.h" #include +#include #if HAVE_MACH_EXCEPTIONS #include "../exception/machexception.h" @@ -580,7 +581,7 @@ Initialize( if (flags & PAL_INITIALIZE_FLUSH_PROCESS_WRITE_BUFFERS) { // Initialize before first thread is created for faster load on Linux - if (!InitializeFlushProcessWriteBuffers()) + if (!minipal_initialize_memory_barrier_process_wide()) { ERROR("Unable to initialize flush process write buffers\n"); palError = ERROR_PALINIT_INITIALIZE_FLUSH_PROCESS_WRITE_BUFFERS; diff --git a/src/coreclr/pal/src/thread/process.cpp b/src/coreclr/pal/src/thread/process.cpp index 9b779c0689c5c2..206a9629189b5b 100644 --- a/src/coreclr/pal/src/thread/process.cpp +++ b/src/coreclr/pal/src/thread/process.cpp @@ -134,21 +134,6 @@ CObjectType CorUnix::otProcess( CObjectType::NoOwner ); -// -// Tracks if the OS supports FlushProcessWriteBuffers using membarrier -// -static int s_flushUsingMemBarrier = 0; - -// -// Helper memory page used by the FlushProcessWriteBuffers -// -static int* s_helperPage = 0; - -// -// Mutex to make the FlushProcessWriteBuffersMutex thread safe -// -pthread_mutex_t flushProcessWriteBuffersMutex; - CAllowedObjectTypes aotProcess(otiProcess); // @@ -2810,70 +2795,6 @@ PROCAbort(int signal, siginfo_t* siginfo) abort(); } -/*++ -Function: - InitializeFlushProcessWriteBuffers - -Abstract - This function initializes data structures needed for the FlushProcessWriteBuffers -Return - TRUE if it succeeded, FALSE otherwise ---*/ -BOOL -InitializeFlushProcessWriteBuffers() -{ - _ASSERTE(s_helperPage == 0); - _ASSERTE(s_flushUsingMemBarrier == 0); - -#if defined(__linux__) || HAVE_SYS_MEMBARRIER_H - // Starting with Linux kernel 4.14, process memory barriers can be generated - // using MEMBARRIER_CMD_PRIVATE_EXPEDITED. - int mask = membarrier(MEMBARRIER_CMD_QUERY, 0, 0); - if (mask >= 0 && - mask & MEMBARRIER_CMD_PRIVATE_EXPEDITED) - { - // Register intent to use the private expedited command. - if (membarrier(MEMBARRIER_CMD_REGISTER_PRIVATE_EXPEDITED, 0, 0) == 0) - { - s_flushUsingMemBarrier = TRUE; - return TRUE; - } - } -#endif - -#if defined(TARGET_APPLE) || defined(TARGET_WASM) - return TRUE; -#else - s_helperPage = static_cast(mmap(0, GetVirtualPageSize(), PROT_READ | PROT_WRITE, MAP_ANONYMOUS | MAP_PRIVATE, -1, 0)); - - if(s_helperPage == MAP_FAILED) - { - return FALSE; - } - - // Verify that the s_helperPage is really aligned to the GetVirtualPageSize() - _ASSERTE((((SIZE_T)s_helperPage) & (GetVirtualPageSize() - 1)) == 0); - - // Locking the page ensures that it stays in memory during the two mprotect - // calls in the FlushProcessWriteBuffers below. If the page was unmapped between - // those calls, they would not have the expected effect of generating IPI. - int status = mlock(s_helperPage, GetVirtualPageSize()); - - if (status != 0) - { - return FALSE; - } - - status = pthread_mutex_init(&flushProcessWriteBuffersMutex, NULL); - if (status != 0) - { - munlock(s_helperPage, GetVirtualPageSize()); - } - - return status == 0; -#endif // TARGET_APPLE || TARGET_WASM -} - #define FATAL_ASSERT(e, msg) \ do \ { \ @@ -2885,79 +2806,6 @@ InitializeFlushProcessWriteBuffers() } \ while(0) -/*++ -Function: - FlushProcessWriteBuffers - -See MSDN doc. ---*/ -VOID -PALAPI -FlushProcessWriteBuffers() -{ -#ifndef TARGET_WASM -#if defined(__linux__) || HAVE_SYS_MEMBARRIER_H - if (s_flushUsingMemBarrier) - { - int status = membarrier(MEMBARRIER_CMD_PRIVATE_EXPEDITED, 0, 0); - FATAL_ASSERT(status == 0, "Failed to flush using membarrier"); - } - else -#endif - if (s_helperPage != 0) - { - int status = pthread_mutex_lock(&flushProcessWriteBuffersMutex); - FATAL_ASSERT(status == 0, "Failed to lock the flushProcessWriteBuffersMutex lock"); - - // Changing a helper memory page protection from read / write to no access - // causes the OS to issue IPI to flush TLBs on all processors. This also - // results in flushing the processor buffers. - status = mprotect(s_helperPage, GetVirtualPageSize(), PROT_READ | PROT_WRITE); - FATAL_ASSERT(status == 0, "Failed to change helper page protection to read / write"); - - // Ensure that the page is dirty before we change the protection so that - // we prevent the OS from skipping the global TLB flush. - InterlockedIncrement(s_helperPage); - - status = mprotect(s_helperPage, GetVirtualPageSize(), PROT_NONE); - FATAL_ASSERT(status == 0, "Failed to change helper page protection to no access"); - - status = pthread_mutex_unlock(&flushProcessWriteBuffersMutex); - FATAL_ASSERT(status == 0, "Failed to unlock the flushProcessWriteBuffersMutex lock"); - } -#ifdef TARGET_APPLE - else - { - mach_msg_type_number_t cThreads; - thread_act_t *pThreads; - kern_return_t machret = task_threads(mach_task_self(), &pThreads, &cThreads); - CHECK_MACH("task_threads()", machret); - - uintptr_t sp; - uintptr_t registerValues[128]; - - // Iterate through each of the threads in the list. - for (mach_msg_type_number_t i = 0; i < cThreads; i++) - { - // Request the threads pointer values to force the thread to emit a memory barrier - size_t registers = 128; - machret = thread_get_register_pointer_values(pThreads[i], &sp, ®isters, registerValues); - if (machret == KERN_INSUFFICIENT_BUFFER_SIZE) - { - CHECK_MACH("thread_get_register_pointer_values()", machret); - } - - machret = mach_port_deallocate(mach_task_self(), pThreads[i]); - CHECK_MACH("mach_port_deallocate()", machret); - } - // Deallocate the thread list now we're done with it. - machret = vm_deallocate(mach_task_self(), (vm_address_t)pThreads, cThreads * sizeof(thread_act_t)); - CHECK_MACH("vm_deallocate()", machret); - } -#endif // TARGET_APPLE -#endif // !TARGET_WASM -} - /*++ Function: PROCGetProcessIDFromHandle diff --git a/src/coreclr/vm/callcounting.cpp b/src/coreclr/vm/callcounting.cpp index f654cc34b0aed0..dea3750dea0aef 100644 --- a/src/coreclr/vm/callcounting.cpp +++ b/src/coreclr/vm/callcounting.cpp @@ -7,6 +7,7 @@ #include "callcounting.h" #include "threadsuspend.h" +#include #ifndef DACCESS_COMPILE extern "C" void STDCALL OnCallCountThresholdReachedStub(); @@ -1025,7 +1026,7 @@ void CallCountingManager::StopAndDeleteAllCallCountingStubs() // will not be used after resuming the runtime. The following ensures that other threads will not use an old cached // entry point value that will not be valid. Do this here in case of exception later. MemoryBarrier(); // flush writes from this thread first to guarantee ordering - FlushProcessWriteBuffers(); + minipal_memory_barrier_process_wide(); // At this point, allocated call counting stubs won't be used anymore. Call counting stubs and corresponding infos may // now be safely deleted. Note that call counting infos may not be deleted prior to this point because call counting diff --git a/src/coreclr/vm/comutilnative.cpp b/src/coreclr/vm/comutilnative.cpp index e8b08578d97d79..95a615ca36207d 100644 --- a/src/coreclr/vm/comutilnative.cpp +++ b/src/coreclr/vm/comutilnative.cpp @@ -33,6 +33,7 @@ #include "typestring.h" #include "finalizerthread.h" #include "threadsuspend.h" +#include #ifdef FEATURE_COMINTEROP #include "comcallablewrapper.h" @@ -1561,7 +1562,7 @@ extern "C" void QCALLTYPE Interlocked_MemoryBarrierProcessWide() { QCALL_CONTRACT; - FlushProcessWriteBuffers(); + minipal_memory_barrier_process_wide(); } static BOOL HasOverriddenMethod(MethodTable* mt, MethodTable* classMT, WORD methodSlot) diff --git a/src/coreclr/vm/gcenv.ee.cpp b/src/coreclr/vm/gcenv.ee.cpp index aebced6bd4388b..c16f6a1b31c9a2 100644 --- a/src/coreclr/vm/gcenv.ee.cpp +++ b/src/coreclr/vm/gcenv.ee.cpp @@ -25,6 +25,7 @@ #include "configuration.h" #include "genanalysis.h" #include "eventpipeadapter.h" +#include // Finalizes a weak reference directly. extern void FinalizeWeakReference(Object* obj); @@ -1006,7 +1007,7 @@ void GCToEEInterface::StompWriteBarrier(WriteBarrierParameters* args) { // If runtime is not suspended, force all threads to see the changed table before seeing updated heap boundaries. // See: http://vstfdevdiv:8080/DevDiv2/DevDiv/_workitems/edit/346765 - FlushProcessWriteBuffers(); + minipal_memory_barrier_process_wide(); } #endif @@ -1058,7 +1059,7 @@ void GCToEEInterface::StompWriteBarrier(WriteBarrierParameters* args) if (!is_runtime_suspended) { // If runtime is not suspended, force all threads to see the changed state before observing future allocations. - FlushProcessWriteBuffers(); + minipal_memory_barrier_process_wide(); } #endif diff --git a/src/coreclr/vm/profilinghelper.cpp b/src/coreclr/vm/profilinghelper.cpp index 99eb7eb0921ec0..ccf23ed65ef1be 100644 --- a/src/coreclr/vm/profilinghelper.cpp +++ b/src/coreclr/vm/profilinghelper.cpp @@ -85,7 +85,7 @@ // these actions: // * (a) Set the profiler's status to a non-active state like kProfStatusDetaching or // kProfStatusNone -// * (b) Call FlushProcessWriteBuffers() +// * (b) Call minipal_memory_barrier_process_wide() // * (c) Grab thread store lock, iterate through all threads, and verify each per-thread // evacuation counter is zero. // @@ -110,6 +110,7 @@ #include "proftoeeinterfaceimpl.inl" #include "profilinghelper.h" #include "profilinghelper.inl" +#include #ifdef FEATURE_PROFAPI_ATTACH_DETACH @@ -208,7 +209,7 @@ void CurrentProfilerStatus::Set(ProfilerStatus newProfStatus) // can safely perform catchup at that time (see // code:#ProfCatchUpSynchronization). // - ::FlushProcessWriteBuffers(); + minipal_memory_barrier_process_wide(); } #endif // !defined(DACCESS_COMPILE) } diff --git a/src/coreclr/vm/threadsuspend.cpp b/src/coreclr/vm/threadsuspend.cpp index fd6bbdaede369a..1411e56ff39e7f 100644 --- a/src/coreclr/vm/threadsuspend.cpp +++ b/src/coreclr/vm/threadsuspend.cpp @@ -14,6 +14,7 @@ #include "finalizerthread.h" #include "dbginterface.h" +#include #include #ifdef FEATURE_EH_FUNCLETS @@ -3310,7 +3311,7 @@ void ThreadSuspend::SuspendAllThreads() // - we get a reliable reading of the threads' m_fPreemptiveGCDisabled state // - other threads see that g_TrapReturningThreads is set // See VSW 475315 and 488918 for details. - ::FlushProcessWriteBuffers(); + minipal_memory_barrier_process_wide(); int prevRemaining = INT32_MAX; bool observeOnly = true; @@ -3381,7 +3382,7 @@ void ThreadSuspend::SuspendAllThreads() // This is needed to synchronize threads that were running in preemptive mode thus were // left alone by suspension to flush their writes that they made before they switched to // preemptive mode. - ::FlushProcessWriteBuffers(); + minipal_memory_barrier_process_wide(); #endif //TARGET_ARM || TARGET_ARM64 STRESS_LOG0(LF_SYNC, LL_INFO1000, "Thread::SuspendAllThreads() - Success\n"); @@ -3606,7 +3607,7 @@ void ThreadSuspend::ResumeAllThreads(BOOL SuspendSucceeded) // This is needed to synchronize threads that were running in preemptive mode while // the runtime was suspended and that will return to cooperative mode after the runtime // is restarted. - ::FlushProcessWriteBuffers(); + minipal_memory_barrier_process_wide(); #endif //TARGET_ARM || TARGET_ARM64 // @@ -5388,7 +5389,7 @@ void ThreadSuspend::RestartEE(BOOL bFinishedGC, BOOL SuspendSucceeded) // This is needed to synchronize threads that were running in preemptive mode while // the runtime was suspended and that will return to cooperative mode after the runtime // is restarted. - ::FlushProcessWriteBuffers(); + minipal_memory_barrier_process_wide(); #endif //TARGET_ARM || TARGET_ARM64 // diff --git a/src/native/minipal/CMakeLists.txt b/src/native/minipal/CMakeLists.txt index 78011d4779694a..f32281a56bb365 100644 --- a/src/native/minipal/CMakeLists.txt +++ b/src/native/minipal/CMakeLists.txt @@ -2,6 +2,7 @@ include(configure.cmake) set(SOURCES cpufeatures.c + memorybarrierprocesswide.c mutex.c guid.c random.c diff --git a/src/native/minipal/memorybarrierprocesswide.c b/src/native/minipal/memorybarrierprocesswide.c new file mode 100644 index 00000000000000..b2fcad9ef789c0 --- /dev/null +++ b/src/native/minipal/memorybarrierprocesswide.c @@ -0,0 +1,240 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +#ifndef HOST_WINDOWS +#include +#include +#include +#include +#include +#include +#include +#include + +#ifdef __APPLE__ +#include +#include + +#define CHECK_MACH(_msg, machret) do { \ + if (machret != KERN_SUCCESS) \ + { \ + char _szError[1024]; \ + snprintf(_szError, ARRAY_SIZE(_szError), "%s: %u: %s", __FUNCTION__, __LINE__, _msg); \ + mach_error(_szError, machret); \ + abort(); \ + } \ + } while (false) + +#endif // __APPLE__ + +#ifdef __linux__ +#include +#include +#define membarrier(...) syscall(__NR_membarrier, __VA_ARGS__) +#undef HAVE_SYS_MEMBARRIER_H +#define HAVE_SYS_MEMBARRIER_H 1 +#elif HAVE_SYS_MEMBARRIER_H +#include +#endif + +#if HAVE_SYS_MEMBARRIER_H +static bool CanFlushUsingMembarrier(void) +{ +#ifdef TARGET_ANDROID + // Avoid calling membarrier on older Android versions where membarrier + // may be barred by seccomp causing the process to be killed. + int apiLevel = android_get_device_api_level(); + if (apiLevel < __ANDROID_API_Q__) + { + return false; + } +#endif + + // Starting with Linux kernel 4.14, process memory barriers can be generated + // using MEMBARRIER_CMD_PRIVATE_EXPEDITED. + + int mask = membarrier(MEMBARRIER_CMD_QUERY, 0, 0); + + if (mask >= 0 && + mask & MEMBARRIER_CMD_PRIVATE_EXPEDITED && + // Register intent to use the private expedited command. + membarrier(MEMBARRIER_CMD_REGISTER_PRIVATE_EXPEDITED, 0, 0) == 0) + { + return true; + } + + return false; +} + +// +// Tracks if the OS supports membarrier syscall +// +static bool s_flushUsingMemBarrier = false; +#endif // HAVE_SYS_MEMBARRIER_H + +#if !defined(HOST_APPLE) && !defined(HOST_WASM) +// Helper memory page used by the fallback path +static uint8_t* g_helperPage = NULL; + +// Mutex to make the fallback path thread safe +static pthread_mutex_t g_flushProcessWriteBuffersMutex; + +static size_t s_pageSize = 0; +#endif // !TARGET_APPLE && !HOST_WASM + +static bool s_initializedMemoryBarrierSuccessfullyInitialized = false; + +bool minipal_initialize_memory_barrier_process_wide(void) +{ + if (s_initializedMemoryBarrierSuccessfullyInitialized) + { + return true; + } + +#ifdef HOST_WASM + // browser/wasm is currently single threaded +#elif defined(HOST_APPLE) + // Apple platforms do not support membarrier, so we use a different mechanism +#else +#if HAVE_SYS_MEMBARRIER_H + if (CanFlushUsingMembarrier()) + { + s_flushUsingMemBarrier = true; + } + else +#endif // HAVE_SYS_MEMBARRIER_H + { + // Fallback implementation + assert(g_helperPage == NULL); + + int pageSize = sysconf( _SC_PAGE_SIZE ); + + s_pageSize = (size_t)((pageSize > 0) ? pageSize : 0x1000); + g_helperPage = (uint8_t*)(mmap(0, s_pageSize, PROT_READ | PROT_WRITE, MAP_ANONYMOUS | MAP_PRIVATE, -1, 0)); + + if (g_helperPage == MAP_FAILED) + { + return false; + } + + // Verify that the s_helperPage is really aligned to the s_pageSize + assert((((size_t)g_helperPage) & (s_pageSize - 1)) == 0); + + // Locking the page ensures that it stays in memory during the two mprotect + // calls in the FlushProcessWriteBuffers below. If the page was unmapped between + // those calls, they would not have the expected effect of generating IPI. + int status = mlock(g_helperPage, s_pageSize); + + if (status != 0) + { + return false; + } + + status = pthread_mutex_init(&g_flushProcessWriteBuffersMutex, NULL); + if (status != 0) + { + munlock(g_helperPage, s_pageSize); + return false; + } + } +#endif // !HOST_WASM && !HOST_APPLE + + s_initializedMemoryBarrierSuccessfullyInitialized = true; + return true; +} + +// Flush write buffers of processors that are executing threads of the current process +void minipal_memory_barrier_process_wide(void) +{ + assert(s_initializedMemoryBarrierSuccessfullyInitialized); + +#ifdef HOST_WASM + // browser/wasm is currently single threaded +#elif defined(HOST_APPLE) + mach_msg_type_number_t cThreads; + thread_act_t *pThreads; + kern_return_t machret = task_threads(mach_task_self(), &pThreads, &cThreads); + CHECK_MACH("task_threads()", machret); + + uintptr_t sp; + uintptr_t registerValues[128]; + + // Iterate through each of the threads in the list. + for (mach_msg_type_number_t i = 0; i < cThreads; i++) + { + if (__builtin_available (macOS 10.14, iOS 12, tvOS 9, *)) + { + // Request the threads pointer values to force the thread to emit a memory barrier + size_t registers = 128; + machret = thread_get_register_pointer_values(pThreads[i], &sp, ®isters, registerValues); + } + else + { + // fallback implementation for older OS versions +#if defined(HOST_AMD64) + x86_thread_state64_t threadState; + mach_msg_type_number_t count = x86_THREAD_STATE64_COUNT; + machret = thread_get_state(pThreads[i], x86_THREAD_STATE64, (thread_state_t)&threadState, &count); +#elif defined(HOST_ARM64) + arm_thread_state64_t threadState; + mach_msg_type_number_t count = ARM_THREAD_STATE64_COUNT; + machret = thread_get_state(pThreads[i], ARM_THREAD_STATE64, (thread_state_t)&threadState, &count); +#else + #error Unexpected architecture +#endif + } + + if (machret == KERN_INSUFFICIENT_BUFFER_SIZE) + { + CHECK_MACH("thread_get_register_pointer_values()", machret); + } + + machret = mach_port_deallocate(mach_task_self(), pThreads[i]); + CHECK_MACH("mach_port_deallocate()", machret); + } + // Deallocate the thread list now we're done with it. + machret = vm_deallocate(mach_task_self(), (vm_address_t)pThreads, cThreads * sizeof(thread_act_t)); + CHECK_MACH("vm_deallocate()", machret); +#else // !HOST_APPLE && !HOST_WASM +#if HAVE_SYS_MEMBARRIER_H + if (s_flushUsingMemBarrier) + { + int status = membarrier(MEMBARRIER_CMD_PRIVATE_EXPEDITED, 0, 0); + assert(status == 0 && "Failed to flush using membarrier"); + } + else +#endif // !HAVE_SYS_MEMBARRIER_H + { + assert(g_helperPage != NULL); + + int status = pthread_mutex_lock(&g_flushProcessWriteBuffersMutex); + (void)status; // unused in release config + assert(status == 0 && "Failed to lock the flushProcessWriteBuffersMutex lock"); + + // causes the OS to issue IPI to flush TLBs on all processors. This also + // results in flushing the processor buffers. + // Changing a helper memory page protection from read / write to no access + status = mprotect(g_helperPage, s_pageSize, PROT_READ | PROT_WRITE); + assert(status == 0 && "Failed to change helper page protection to read / write"); + + // Ensure that the page is dirty before we change the protection so that + // we prevent the OS from skipping the global TLB flush. + __sync_add_and_fetch((size_t*)g_helperPage, 1); + + status = mprotect(g_helperPage, s_pageSize, PROT_NONE); + assert(status == 0 && "Failed to change helper page protection to no access"); + + status = pthread_mutex_unlock(&g_flushProcessWriteBuffersMutex); + assert(status == 0 && "Failed to unlock the flushProcessWriteBuffersMutex lock"); + } +#endif // !HOST_APPLE && !HOST_WASM +} +#else // !HOST_WINDOWS + +#include + +void minipal_memory_barrier_process_wide(void) +{ + FlushProcessWriteBuffers(); +} +#endif // !HOST_WINDOWS diff --git a/src/native/minipal/memorybarrierprocesswide.h b/src/native/minipal/memorybarrierprocesswide.h new file mode 100644 index 00000000000000..99a0d03922e6db --- /dev/null +++ b/src/native/minipal/memorybarrierprocesswide.h @@ -0,0 +1,21 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +#ifndef HAVE_MINIPAL_MEMORY_H +#define HAVE_MINIPAL_MEMORY_H + +#include + +#ifdef __cplusplus +extern "C" +{ +#endif // __cplusplus + + bool minipal_initialize_memory_barrier_process_wide(void); + void minipal_memory_barrier_process_wide(void); + +#ifdef __cplusplus +} +#endif // __cplusplus + +#endif // HAVE_MINIPAL_MEMORY_H